Gao Xiang | 29b24f6 | 2019-07-31 23:57:31 +0800 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0-only |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 2 | /* |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 3 | * Copyright (C) 2019 HUAWEI, Inc. |
Alexander A. Klimov | 592e7cd | 2020-07-13 15:09:44 +0200 | [diff] [blame] | 4 | * https://www.huawei.com/ |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 5 | */ |
| 6 | #include "compress.h" |
| 7 | #include <linux/lz4.h> |
| 8 | |
| 9 | #ifndef LZ4_DISTANCE_MAX /* history window size */ |
| 10 | #define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */ |
| 11 | #endif |
| 12 | |
Gao Xiang | af89bce | 2019-07-03 14:52:09 +0800 | [diff] [blame] | 13 | #define LZ4_MAX_DISTANCE_PAGES (DIV_ROUND_UP(LZ4_DISTANCE_MAX, PAGE_SIZE) + 1) |
Gao Xiang | 0ffd71b | 2019-06-24 15:22:56 +0800 | [diff] [blame] | 14 | #ifndef LZ4_DECOMPRESS_INPLACE_MARGIN |
| 15 | #define LZ4_DECOMPRESS_INPLACE_MARGIN(srcsize) (((srcsize) >> 8) + 32) |
| 16 | #endif |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 17 | |
Gao Xiang | d67aee7 | 2021-12-28 13:46:00 +0800 | [diff] [blame] | 18 | struct z_erofs_lz4_decompress_ctx { |
| 19 | struct z_erofs_decompress_req *rq; |
| 20 | /* # of encoded, decoded pages */ |
| 21 | unsigned int inpages, outpages; |
| 22 | /* decoded block total length (used for in-place decompression) */ |
| 23 | unsigned int oend; |
| 24 | }; |
| 25 | |
Gao Xiang | efb4fb0 | 2023-10-22 21:09:57 +0800 | [diff] [blame] | 26 | static int z_erofs_load_lz4_config(struct super_block *sb, |
| 27 | struct erofs_super_block *dsb, void *data, int size) |
Huang Jianan | 5d50538 | 2021-03-29 09:23:06 +0800 | [diff] [blame] | 28 | { |
Gao Xiang | 4fea63f | 2021-04-07 12:39:23 +0800 | [diff] [blame] | 29 | struct erofs_sb_info *sbi = EROFS_SB(sb); |
Gao Xiang | efb4fb0 | 2023-10-22 21:09:57 +0800 | [diff] [blame] | 30 | struct z_erofs_lz4_cfgs *lz4 = data; |
Gao Xiang | 46249cd | 2021-03-29 09:23:07 +0800 | [diff] [blame] | 31 | u16 distance; |
| 32 | |
| 33 | if (lz4) { |
| 34 | if (size < sizeof(struct z_erofs_lz4_cfgs)) { |
| 35 | erofs_err(sb, "invalid lz4 cfgs, size=%u", size); |
| 36 | return -EINVAL; |
| 37 | } |
| 38 | distance = le16_to_cpu(lz4->max_distance); |
Gao Xiang | 4fea63f | 2021-04-07 12:39:23 +0800 | [diff] [blame] | 39 | |
| 40 | sbi->lz4.max_pclusterblks = le16_to_cpu(lz4->max_pclusterblks); |
| 41 | if (!sbi->lz4.max_pclusterblks) { |
| 42 | sbi->lz4.max_pclusterblks = 1; /* reserved case */ |
| 43 | } else if (sbi->lz4.max_pclusterblks > |
Jingbo Xu | 3acea5f | 2023-03-13 21:53:08 +0800 | [diff] [blame] | 44 | erofs_blknr(sb, Z_EROFS_PCLUSTER_MAX_SIZE)) { |
Gao Xiang | 4fea63f | 2021-04-07 12:39:23 +0800 | [diff] [blame] | 45 | erofs_err(sb, "too large lz4 pclusterblks %u", |
| 46 | sbi->lz4.max_pclusterblks); |
| 47 | return -EINVAL; |
Gao Xiang | 4fea63f | 2021-04-07 12:39:23 +0800 | [diff] [blame] | 48 | } |
Gao Xiang | 46249cd | 2021-03-29 09:23:07 +0800 | [diff] [blame] | 49 | } else { |
Gao Xiang | 1437371 | 2021-03-29 18:00:12 +0800 | [diff] [blame] | 50 | distance = le16_to_cpu(dsb->u1.lz4_max_distance); |
Gao Xiang | 4fea63f | 2021-04-07 12:39:23 +0800 | [diff] [blame] | 51 | sbi->lz4.max_pclusterblks = 1; |
Gao Xiang | 46249cd | 2021-03-29 09:23:07 +0800 | [diff] [blame] | 52 | } |
Huang Jianan | 5d50538 | 2021-03-29 09:23:06 +0800 | [diff] [blame] | 53 | |
Gao Xiang | 4fea63f | 2021-04-07 12:39:23 +0800 | [diff] [blame] | 54 | sbi->lz4.max_distance_pages = distance ? |
Huang Jianan | 5d50538 | 2021-03-29 09:23:06 +0800 | [diff] [blame] | 55 | DIV_ROUND_UP(distance, PAGE_SIZE) + 1 : |
| 56 | LZ4_MAX_DISTANCE_PAGES; |
Chunhai Guo | f36f301 | 2024-04-02 04:00:36 -0600 | [diff] [blame] | 57 | return z_erofs_gbuf_growsize(sbi->lz4.max_pclusterblks); |
Huang Jianan | 5d50538 | 2021-03-29 09:23:06 +0800 | [diff] [blame] | 58 | } |
| 59 | |
Gao Xiang | 966edfb | 2021-10-11 05:31:44 +0800 | [diff] [blame] | 60 | /* |
| 61 | * Fill all gaps with bounce pages if it's a sparse page list. Also check if |
| 62 | * all physical pages are consecutive, which can be seen for moderate CR. |
| 63 | */ |
Gao Xiang | d67aee7 | 2021-12-28 13:46:00 +0800 | [diff] [blame] | 64 | static int z_erofs_lz4_prepare_dstpages(struct z_erofs_lz4_decompress_ctx *ctx, |
Gao Xiang | eaa9172 | 2021-10-22 17:01:20 +0800 | [diff] [blame] | 65 | struct page **pagepool) |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 66 | { |
Gao Xiang | d67aee7 | 2021-12-28 13:46:00 +0800 | [diff] [blame] | 67 | struct z_erofs_decompress_req *rq = ctx->rq; |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 68 | struct page *availables[LZ4_MAX_DISTANCE_PAGES] = { NULL }; |
Gao Xiang | af89bce | 2019-07-03 14:52:09 +0800 | [diff] [blame] | 69 | unsigned long bounced[DIV_ROUND_UP(LZ4_MAX_DISTANCE_PAGES, |
| 70 | BITS_PER_LONG)] = { 0 }; |
Huang Jianan | 5d50538 | 2021-03-29 09:23:06 +0800 | [diff] [blame] | 71 | unsigned int lz4_max_distance_pages = |
| 72 | EROFS_SB(rq->sb)->lz4.max_distance_pages; |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 73 | void *kaddr = NULL; |
Gao Xiang | af89bce | 2019-07-03 14:52:09 +0800 | [diff] [blame] | 74 | unsigned int i, j, top; |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 75 | |
Gao Xiang | af89bce | 2019-07-03 14:52:09 +0800 | [diff] [blame] | 76 | top = 0; |
Gao Xiang | d67aee7 | 2021-12-28 13:46:00 +0800 | [diff] [blame] | 77 | for (i = j = 0; i < ctx->outpages; ++i, ++j) { |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 78 | struct page *const page = rq->out[i]; |
Gao Xiang | af89bce | 2019-07-03 14:52:09 +0800 | [diff] [blame] | 79 | struct page *victim; |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 80 | |
Huang Jianan | 5d50538 | 2021-03-29 09:23:06 +0800 | [diff] [blame] | 81 | if (j >= lz4_max_distance_pages) |
Gao Xiang | af89bce | 2019-07-03 14:52:09 +0800 | [diff] [blame] | 82 | j = 0; |
| 83 | |
| 84 | /* 'valid' bounced can only be tested after a complete round */ |
Gao Xiang | 267f249 | 2022-07-15 23:42:03 +0800 | [diff] [blame] | 85 | if (!rq->fillgaps && test_bit(j, bounced)) { |
Huang Jianan | 5d50538 | 2021-03-29 09:23:06 +0800 | [diff] [blame] | 86 | DBG_BUGON(i < lz4_max_distance_pages); |
| 87 | DBG_BUGON(top >= lz4_max_distance_pages); |
| 88 | availables[top++] = rq->out[i - lz4_max_distance_pages]; |
Gao Xiang | af89bce | 2019-07-03 14:52:09 +0800 | [diff] [blame] | 89 | } |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 90 | |
| 91 | if (page) { |
Gao Xiang | af89bce | 2019-07-03 14:52:09 +0800 | [diff] [blame] | 92 | __clear_bit(j, bounced); |
Gao Xiang | 448b5a1 | 2022-07-08 18:10:01 +0800 | [diff] [blame] | 93 | if (!PageHighMem(page)) { |
| 94 | if (!i) { |
| 95 | kaddr = page_address(page); |
| 96 | continue; |
| 97 | } |
| 98 | if (kaddr && |
| 99 | kaddr + PAGE_SIZE == page_address(page)) { |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 100 | kaddr += PAGE_SIZE; |
Gao Xiang | 448b5a1 | 2022-07-08 18:10:01 +0800 | [diff] [blame] | 101 | continue; |
| 102 | } |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 103 | } |
Gao Xiang | 448b5a1 | 2022-07-08 18:10:01 +0800 | [diff] [blame] | 104 | kaddr = NULL; |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 105 | continue; |
| 106 | } |
| 107 | kaddr = NULL; |
Gao Xiang | af89bce | 2019-07-03 14:52:09 +0800 | [diff] [blame] | 108 | __set_bit(j, bounced); |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 109 | |
Gao Xiang | af89bce | 2019-07-03 14:52:09 +0800 | [diff] [blame] | 110 | if (top) { |
| 111 | victim = availables[--top]; |
| 112 | get_page(victim); |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 113 | } else { |
Chunhai Guo | 0f6273a | 2024-04-02 07:15:23 -0600 | [diff] [blame] | 114 | victim = __erofs_allocpage(pagepool, rq->gfp, true); |
Chunhai Guo | d928166 | 2024-01-26 22:01:42 +0800 | [diff] [blame] | 115 | if (!victim) |
| 116 | return -ENOMEM; |
Gao Xiang | 6aaa7b0 | 2020-12-08 17:58:32 +0800 | [diff] [blame] | 117 | set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE); |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 118 | } |
Gao Xiang | af89bce | 2019-07-03 14:52:09 +0800 | [diff] [blame] | 119 | rq->out[i] = victim; |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 120 | } |
| 121 | return kaddr ? 1 : 0; |
| 122 | } |
| 123 | |
Gao Xiang | d67aee7 | 2021-12-28 13:46:00 +0800 | [diff] [blame] | 124 | static void *z_erofs_lz4_handle_overlap(struct z_erofs_lz4_decompress_ctx *ctx, |
Gao Xiang | 3c12466 | 2023-12-06 12:55:34 +0800 | [diff] [blame] | 125 | void *inpage, void *out, unsigned int *inputmargin, |
| 126 | int *maptype, bool may_inplace) |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 127 | { |
Gao Xiang | d67aee7 | 2021-12-28 13:46:00 +0800 | [diff] [blame] | 128 | struct z_erofs_decompress_req *rq = ctx->rq; |
Gao Xiang | 3c12466 | 2023-12-06 12:55:34 +0800 | [diff] [blame] | 129 | unsigned int omargin, total, i; |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 130 | struct page **in; |
| 131 | void *src, *tmp; |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 132 | |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 133 | if (rq->inplace_io) { |
Gao Xiang | d67aee7 | 2021-12-28 13:46:00 +0800 | [diff] [blame] | 134 | omargin = PAGE_ALIGN(ctx->oend) - ctx->oend; |
Gao Xiang | ab749ba | 2021-12-28 13:46:02 +0800 | [diff] [blame] | 135 | if (rq->partial_decoding || !may_inplace || |
Gao Xiang | d67aee7 | 2021-12-28 13:46:00 +0800 | [diff] [blame] | 136 | omargin < LZ4_DECOMPRESS_INPLACE_MARGIN(rq->inputsize)) |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 137 | goto docopy; |
| 138 | |
Gao Xiang | 3c12466 | 2023-12-06 12:55:34 +0800 | [diff] [blame] | 139 | for (i = 0; i < ctx->inpages; ++i) |
| 140 | if (rq->out[ctx->outpages - ctx->inpages + i] != |
| 141 | rq->in[i]) |
| 142 | goto docopy; |
| 143 | kunmap_local(inpage); |
| 144 | *maptype = 3; |
| 145 | return out + ((ctx->outpages - ctx->inpages) << PAGE_SHIFT); |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 146 | } |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 147 | |
Gao Xiang | d67aee7 | 2021-12-28 13:46:00 +0800 | [diff] [blame] | 148 | if (ctx->inpages <= 1) { |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 149 | *maptype = 0; |
| 150 | return inpage; |
| 151 | } |
Gao Xiang | 123ec24 | 2023-06-28 00:12:39 +0800 | [diff] [blame] | 152 | kunmap_local(inpage); |
Gao Xiang | d67aee7 | 2021-12-28 13:46:00 +0800 | [diff] [blame] | 153 | src = erofs_vm_map_ram(rq->in, ctx->inpages); |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 154 | if (!src) |
| 155 | return ERR_PTR(-ENOMEM); |
| 156 | *maptype = 1; |
| 157 | return src; |
| 158 | |
| 159 | docopy: |
| 160 | /* Or copy compressed data which can be overlapped to per-CPU buffer */ |
| 161 | in = rq->in; |
Chunhai Guo | f36f301 | 2024-04-02 04:00:36 -0600 | [diff] [blame] | 162 | src = z_erofs_get_gbuf(ctx->inpages); |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 163 | if (!src) { |
| 164 | DBG_BUGON(1); |
Gao Xiang | 123ec24 | 2023-06-28 00:12:39 +0800 | [diff] [blame] | 165 | kunmap_local(inpage); |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 166 | return ERR_PTR(-EFAULT); |
| 167 | } |
| 168 | |
| 169 | tmp = src; |
| 170 | total = rq->inputsize; |
| 171 | while (total) { |
| 172 | unsigned int page_copycnt = |
| 173 | min_t(unsigned int, total, PAGE_SIZE - *inputmargin); |
| 174 | |
| 175 | if (!inpage) |
Gao Xiang | 123ec24 | 2023-06-28 00:12:39 +0800 | [diff] [blame] | 176 | inpage = kmap_local_page(*in); |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 177 | memcpy(tmp, inpage + *inputmargin, page_copycnt); |
Gao Xiang | 123ec24 | 2023-06-28 00:12:39 +0800 | [diff] [blame] | 178 | kunmap_local(inpage); |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 179 | inpage = NULL; |
| 180 | tmp += page_copycnt; |
| 181 | total -= page_copycnt; |
| 182 | ++in; |
| 183 | *inputmargin = 0; |
| 184 | } |
| 185 | *maptype = 2; |
| 186 | return src; |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 187 | } |
| 188 | |
Gao Xiang | 10e5f6e | 2021-12-28 13:46:01 +0800 | [diff] [blame] | 189 | /* |
| 190 | * Get the exact inputsize with zero_padding feature. |
| 191 | * - For LZ4, it should work if zero_padding feature is on (5.3+); |
| 192 | * - For MicroLZMA, it'd be enabled all the time. |
| 193 | */ |
| 194 | int z_erofs_fixup_insize(struct z_erofs_decompress_req *rq, const char *padbuf, |
| 195 | unsigned int padbufsize) |
| 196 | { |
| 197 | const char *padend; |
| 198 | |
| 199 | padend = memchr_inv(padbuf, 0, padbufsize); |
| 200 | if (!padend) |
| 201 | return -EFSCORRUPTED; |
| 202 | rq->inputsize -= padend - padbuf; |
| 203 | rq->pageofs_in += padend - padbuf; |
| 204 | return 0; |
| 205 | } |
| 206 | |
Gao Xiang | d67aee7 | 2021-12-28 13:46:00 +0800 | [diff] [blame] | 207 | static int z_erofs_lz4_decompress_mem(struct z_erofs_lz4_decompress_ctx *ctx, |
Gao Xiang | 3c12466 | 2023-12-06 12:55:34 +0800 | [diff] [blame] | 208 | u8 *dst) |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 209 | { |
Gao Xiang | d67aee7 | 2021-12-28 13:46:00 +0800 | [diff] [blame] | 210 | struct z_erofs_decompress_req *rq = ctx->rq; |
Gao Xiang | ab749ba | 2021-12-28 13:46:02 +0800 | [diff] [blame] | 211 | bool support_0padding = false, may_inplace = false; |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 212 | unsigned int inputmargin; |
Gao Xiang | 3c12466 | 2023-12-06 12:55:34 +0800 | [diff] [blame] | 213 | u8 *out, *headpage, *src; |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 214 | int ret, maptype; |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 215 | |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 216 | DBG_BUGON(*rq->in == NULL); |
Gao Xiang | 123ec24 | 2023-06-28 00:12:39 +0800 | [diff] [blame] | 217 | headpage = kmap_local_page(*rq->in); |
Gao Xiang | 0ffd71b | 2019-06-24 15:22:56 +0800 | [diff] [blame] | 218 | |
Gao Xiang | 10e5f6e | 2021-12-28 13:46:01 +0800 | [diff] [blame] | 219 | /* LZ4 decompression inplace is only safe if zero_padding is enabled */ |
Huang Jianan | 7e508f2 | 2021-11-13 00:09:33 +0800 | [diff] [blame] | 220 | if (erofs_sb_has_zero_padding(EROFS_SB(rq->sb))) { |
Gao Xiang | 0ffd71b | 2019-06-24 15:22:56 +0800 | [diff] [blame] | 221 | support_0padding = true; |
Gao Xiang | 10e5f6e | 2021-12-28 13:46:01 +0800 | [diff] [blame] | 222 | ret = z_erofs_fixup_insize(rq, headpage + rq->pageofs_in, |
| 223 | min_t(unsigned int, rq->inputsize, |
Jingbo Xu | 3acea5f | 2023-03-13 21:53:08 +0800 | [diff] [blame] | 224 | rq->sb->s_blocksize - rq->pageofs_in)); |
Gao Xiang | 10e5f6e | 2021-12-28 13:46:01 +0800 | [diff] [blame] | 225 | if (ret) { |
Gao Xiang | 123ec24 | 2023-06-28 00:12:39 +0800 | [diff] [blame] | 226 | kunmap_local(headpage); |
Gao Xiang | 10e5f6e | 2021-12-28 13:46:01 +0800 | [diff] [blame] | 227 | return ret; |
Gao Xiang | 0ffd71b | 2019-06-24 15:22:56 +0800 | [diff] [blame] | 228 | } |
Gao Xiang | ab749ba | 2021-12-28 13:46:02 +0800 | [diff] [blame] | 229 | may_inplace = !((rq->pageofs_in + rq->inputsize) & |
Jingbo Xu | 3acea5f | 2023-03-13 21:53:08 +0800 | [diff] [blame] | 230 | (rq->sb->s_blocksize - 1)); |
Gao Xiang | 0ffd71b | 2019-06-24 15:22:56 +0800 | [diff] [blame] | 231 | } |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 232 | |
Gao Xiang | 10e5f6e | 2021-12-28 13:46:01 +0800 | [diff] [blame] | 233 | inputmargin = rq->pageofs_in; |
Gao Xiang | 3c12466 | 2023-12-06 12:55:34 +0800 | [diff] [blame] | 234 | src = z_erofs_lz4_handle_overlap(ctx, headpage, dst, &inputmargin, |
Gao Xiang | ab749ba | 2021-12-28 13:46:02 +0800 | [diff] [blame] | 235 | &maptype, may_inplace); |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 236 | if (IS_ERR(src)) |
| 237 | return PTR_ERR(src); |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 238 | |
Gao Xiang | 3c12466 | 2023-12-06 12:55:34 +0800 | [diff] [blame] | 239 | out = dst + rq->pageofs_out; |
Gao Xiang | af1038ab | 2020-02-26 16:10:07 +0800 | [diff] [blame] | 240 | /* legacy format could compress extra data in a pcluster. */ |
| 241 | if (rq->partial_decoding || !support_0padding) |
| 242 | ret = LZ4_decompress_safe_partial(src + inputmargin, out, |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 243 | rq->inputsize, rq->outputsize, rq->outputsize); |
Gao Xiang | af1038ab | 2020-02-26 16:10:07 +0800 | [diff] [blame] | 244 | else |
| 245 | ret = LZ4_decompress_safe(src + inputmargin, out, |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 246 | rq->inputsize, rq->outputsize); |
Gao Xiang | af1038ab | 2020-02-26 16:10:07 +0800 | [diff] [blame] | 247 | |
Gao Xiang | aa99a76 | 2020-02-26 16:10:08 +0800 | [diff] [blame] | 248 | if (ret != rq->outputsize) { |
| 249 | erofs_err(rq->sb, "failed to decompress %d in[%u, %u] out[%u]", |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 250 | ret, rq->inputsize, inputmargin, rq->outputsize); |
Gao Xiang | aa99a76 | 2020-02-26 16:10:08 +0800 | [diff] [blame] | 251 | if (ret >= 0) |
| 252 | memset(out + ret, 0, rq->outputsize - ret); |
Gao Xiang | 496530c | 2023-12-27 23:19:03 +0800 | [diff] [blame] | 253 | ret = -EFSCORRUPTED; |
Yue Hu | 5b6e7e1 | 2021-10-14 14:57:44 +0800 | [diff] [blame] | 254 | } else { |
| 255 | ret = 0; |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 256 | } |
| 257 | |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 258 | if (maptype == 0) { |
Gao Xiang | 123ec24 | 2023-06-28 00:12:39 +0800 | [diff] [blame] | 259 | kunmap_local(headpage); |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 260 | } else if (maptype == 1) { |
Gao Xiang | d67aee7 | 2021-12-28 13:46:00 +0800 | [diff] [blame] | 261 | vm_unmap_ram(src, ctx->inpages); |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 262 | } else if (maptype == 2) { |
Chunhai Guo | f36f301 | 2024-04-02 04:00:36 -0600 | [diff] [blame] | 263 | z_erofs_put_gbuf(src); |
Gao Xiang | 3c12466 | 2023-12-06 12:55:34 +0800 | [diff] [blame] | 264 | } else if (maptype != 3) { |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 265 | DBG_BUGON(1); |
| 266 | return -EFAULT; |
| 267 | } |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 268 | return ret; |
| 269 | } |
| 270 | |
Gao Xiang | 966edfb | 2021-10-11 05:31:44 +0800 | [diff] [blame] | 271 | static int z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq, |
Gao Xiang | eaa9172 | 2021-10-22 17:01:20 +0800 | [diff] [blame] | 272 | struct page **pagepool) |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 273 | { |
Gao Xiang | d67aee7 | 2021-12-28 13:46:00 +0800 | [diff] [blame] | 274 | struct z_erofs_lz4_decompress_ctx ctx; |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 275 | unsigned int dst_maptype; |
| 276 | void *dst; |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 277 | int ret; |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 278 | |
Gao Xiang | d67aee7 | 2021-12-28 13:46:00 +0800 | [diff] [blame] | 279 | ctx.rq = rq; |
| 280 | ctx.oend = rq->pageofs_out + rq->outputsize; |
| 281 | ctx.outpages = PAGE_ALIGN(ctx.oend) >> PAGE_SHIFT; |
| 282 | ctx.inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT; |
| 283 | |
Yue Hu | 5b6e7e1 | 2021-10-14 14:57:44 +0800 | [diff] [blame] | 284 | /* one optimized fast path only for non bigpcluster cases yet */ |
Gao Xiang | d67aee7 | 2021-12-28 13:46:00 +0800 | [diff] [blame] | 285 | if (ctx.inpages == 1 && ctx.outpages == 1 && !rq->inplace_io) { |
Yue Hu | 5b6e7e1 | 2021-10-14 14:57:44 +0800 | [diff] [blame] | 286 | DBG_BUGON(!*rq->out); |
Gao Xiang | 123ec24 | 2023-06-28 00:12:39 +0800 | [diff] [blame] | 287 | dst = kmap_local_page(*rq->out); |
Yue Hu | 5b6e7e1 | 2021-10-14 14:57:44 +0800 | [diff] [blame] | 288 | dst_maptype = 0; |
| 289 | goto dstmap_out; |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 290 | } |
| 291 | |
Gao Xiang | 598162d | 2021-04-07 12:39:26 +0800 | [diff] [blame] | 292 | /* general decoding path which can be used for all cases */ |
Gao Xiang | d67aee7 | 2021-12-28 13:46:00 +0800 | [diff] [blame] | 293 | ret = z_erofs_lz4_prepare_dstpages(&ctx, pagepool); |
| 294 | if (ret < 0) { |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 295 | return ret; |
Gao Xiang | d67aee7 | 2021-12-28 13:46:00 +0800 | [diff] [blame] | 296 | } else if (ret > 0) { |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 297 | dst = page_address(*rq->out); |
| 298 | dst_maptype = 1; |
Gao Xiang | d67aee7 | 2021-12-28 13:46:00 +0800 | [diff] [blame] | 299 | } else { |
| 300 | dst = erofs_vm_map_ram(rq->out, ctx.outpages); |
| 301 | if (!dst) |
| 302 | return -ENOMEM; |
| 303 | dst_maptype = 2; |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 304 | } |
| 305 | |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 306 | dstmap_out: |
Gao Xiang | 3c12466 | 2023-12-06 12:55:34 +0800 | [diff] [blame] | 307 | ret = z_erofs_lz4_decompress_mem(&ctx, dst); |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 308 | if (!dst_maptype) |
Gao Xiang | 123ec24 | 2023-06-28 00:12:39 +0800 | [diff] [blame] | 309 | kunmap_local(dst); |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 310 | else if (dst_maptype == 2) |
Gao Xiang | d67aee7 | 2021-12-28 13:46:00 +0800 | [diff] [blame] | 311 | vm_unmap_ram(dst, ctx.outpages); |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 312 | return ret; |
| 313 | } |
| 314 | |
Yue Hu | fdffc09 | 2022-09-23 10:11:21 +0800 | [diff] [blame] | 315 | static int z_erofs_transform_plain(struct z_erofs_decompress_req *rq, |
| 316 | struct page **pagepool) |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 317 | { |
Gao Xiang | 1ca0152 | 2023-12-06 17:10:56 +0800 | [diff] [blame] | 318 | const unsigned int nrpages_in = |
| 319 | PAGE_ALIGN(rq->pageofs_in + rq->inputsize) >> PAGE_SHIFT; |
| 320 | const unsigned int nrpages_out = |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 321 | PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT; |
Gao Xiang | 1ca0152 | 2023-12-06 17:10:56 +0800 | [diff] [blame] | 322 | const unsigned int bs = rq->sb->s_blocksize; |
| 323 | unsigned int cur = 0, ni = 0, no, pi, po, insz, cnt; |
| 324 | u8 *kin; |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 325 | |
Gao Xiang | 893e5e9 | 2024-03-04 11:53:39 +0800 | [diff] [blame] | 326 | if (rq->outputsize > rq->inputsize) |
| 327 | return -EOPNOTSUPP; |
Gao Xiang | 1ca0152 | 2023-12-06 17:10:56 +0800 | [diff] [blame] | 328 | if (rq->alg == Z_EROFS_COMPRESSION_INTERLACED) { |
| 329 | cur = bs - (rq->pageofs_out & (bs - 1)); |
| 330 | pi = (rq->pageofs_in + rq->inputsize - cur) & ~PAGE_MASK; |
| 331 | cur = min(cur, rq->outputsize); |
| 332 | if (cur && rq->out[0]) { |
| 333 | kin = kmap_local_page(rq->in[nrpages_in - 1]); |
| 334 | if (rq->out[0] == rq->in[nrpages_in - 1]) { |
| 335 | memmove(kin + rq->pageofs_out, kin + pi, cur); |
| 336 | flush_dcache_page(rq->out[0]); |
| 337 | } else { |
| 338 | memcpy_to_page(rq->out[0], rq->pageofs_out, |
| 339 | kin + pi, cur); |
| 340 | } |
| 341 | kunmap_local(kin); |
Gao Xiang | 4d20243 | 2020-01-07 10:25:46 +0800 | [diff] [blame] | 342 | } |
Gao Xiang | 1ca0152 | 2023-12-06 17:10:56 +0800 | [diff] [blame] | 343 | rq->outputsize -= cur; |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 344 | } |
Gao Xiang | 1ca0152 | 2023-12-06 17:10:56 +0800 | [diff] [blame] | 345 | |
| 346 | for (; rq->outputsize; rq->pageofs_in = 0, cur += PAGE_SIZE, ni++) { |
| 347 | insz = min(PAGE_SIZE - rq->pageofs_in, rq->outputsize); |
| 348 | rq->outputsize -= insz; |
| 349 | if (!rq->in[ni]) |
| 350 | continue; |
| 351 | kin = kmap_local_page(rq->in[ni]); |
| 352 | pi = 0; |
| 353 | do { |
| 354 | no = (rq->pageofs_out + cur + pi) >> PAGE_SHIFT; |
| 355 | po = (rq->pageofs_out + cur + pi) & ~PAGE_MASK; |
| 356 | DBG_BUGON(no >= nrpages_out); |
| 357 | cnt = min(insz - pi, PAGE_SIZE - po); |
| 358 | if (rq->out[no] == rq->in[ni]) { |
| 359 | memmove(kin + po, |
| 360 | kin + rq->pageofs_in + pi, cnt); |
| 361 | flush_dcache_page(rq->out[no]); |
| 362 | } else if (rq->out[no]) { |
| 363 | memcpy_to_page(rq->out[no], po, |
| 364 | kin + rq->pageofs_in + pi, cnt); |
| 365 | } |
| 366 | pi += cnt; |
| 367 | } while (pi < insz); |
| 368 | kunmap_local(kin); |
| 369 | } |
| 370 | DBG_BUGON(ni > nrpages_in); |
Gao Xiang | 7fc45db | 2019-06-24 15:22:55 +0800 | [diff] [blame] | 371 | return 0; |
| 372 | } |
| 373 | |
Yue Hu | 597e295 | 2023-04-26 16:44:49 +0800 | [diff] [blame] | 374 | const struct z_erofs_decompressor erofs_decompressors[] = { |
Gao Xiang | 966edfb | 2021-10-11 05:31:44 +0800 | [diff] [blame] | 375 | [Z_EROFS_COMPRESSION_SHIFTED] = { |
Yue Hu | fdffc09 | 2022-09-23 10:11:21 +0800 | [diff] [blame] | 376 | .decompress = z_erofs_transform_plain, |
Gao Xiang | 966edfb | 2021-10-11 05:31:44 +0800 | [diff] [blame] | 377 | .name = "shifted" |
| 378 | }, |
Yue Hu | fdffc09 | 2022-09-23 10:11:21 +0800 | [diff] [blame] | 379 | [Z_EROFS_COMPRESSION_INTERLACED] = { |
| 380 | .decompress = z_erofs_transform_plain, |
| 381 | .name = "interlaced" |
| 382 | }, |
Gao Xiang | 966edfb | 2021-10-11 05:31:44 +0800 | [diff] [blame] | 383 | [Z_EROFS_COMPRESSION_LZ4] = { |
Gao Xiang | efb4fb0 | 2023-10-22 21:09:57 +0800 | [diff] [blame] | 384 | .config = z_erofs_load_lz4_config, |
Gao Xiang | 966edfb | 2021-10-11 05:31:44 +0800 | [diff] [blame] | 385 | .decompress = z_erofs_lz4_decompress, |
| 386 | .name = "lz4" |
| 387 | }, |
Gao Xiang | 622cead | 2021-10-11 05:31:45 +0800 | [diff] [blame] | 388 | #ifdef CONFIG_EROFS_FS_ZIP_LZMA |
| 389 | [Z_EROFS_COMPRESSION_LZMA] = { |
Gao Xiang | efb4fb0 | 2023-10-22 21:09:57 +0800 | [diff] [blame] | 390 | .config = z_erofs_load_lzma_config, |
Gao Xiang | 622cead | 2021-10-11 05:31:45 +0800 | [diff] [blame] | 391 | .decompress = z_erofs_lzma_decompress, |
| 392 | .name = "lzma" |
| 393 | }, |
| 394 | #endif |
Gao Xiang | ffa09b3bd | 2023-08-10 23:48:59 +0800 | [diff] [blame] | 395 | #ifdef CONFIG_EROFS_FS_ZIP_DEFLATE |
| 396 | [Z_EROFS_COMPRESSION_DEFLATE] = { |
Gao Xiang | efb4fb0 | 2023-10-22 21:09:57 +0800 | [diff] [blame] | 397 | .config = z_erofs_load_deflate_config, |
Gao Xiang | ffa09b3bd | 2023-08-10 23:48:59 +0800 | [diff] [blame] | 398 | .decompress = z_erofs_deflate_decompress, |
| 399 | .name = "deflate" |
| 400 | }, |
| 401 | #endif |
Gao Xiang | 7c35de4 | 2024-05-09 07:44:53 +0800 | [diff] [blame] | 402 | #ifdef CONFIG_EROFS_FS_ZIP_ZSTD |
| 403 | [Z_EROFS_COMPRESSION_ZSTD] = { |
| 404 | .config = z_erofs_load_zstd_config, |
| 405 | .decompress = z_erofs_zstd_decompress, |
| 406 | .name = "zstd" |
| 407 | }, |
| 408 | #endif |
Gao Xiang | 966edfb | 2021-10-11 05:31:44 +0800 | [diff] [blame] | 409 | }; |
Gao Xiang | efb4fb0 | 2023-10-22 21:09:57 +0800 | [diff] [blame] | 410 | |
| 411 | int z_erofs_parse_cfgs(struct super_block *sb, struct erofs_super_block *dsb) |
| 412 | { |
| 413 | struct erofs_sb_info *sbi = EROFS_SB(sb); |
| 414 | struct erofs_buf buf = __EROFS_BUF_INITIALIZER; |
| 415 | unsigned int algs, alg; |
| 416 | erofs_off_t offset; |
| 417 | int size, ret = 0; |
| 418 | |
| 419 | if (!erofs_sb_has_compr_cfgs(sbi)) { |
Gao Xiang | 118a8cf | 2024-01-13 23:06:02 +0800 | [diff] [blame] | 420 | sbi->available_compr_algs = 1 << Z_EROFS_COMPRESSION_LZ4; |
Gao Xiang | efb4fb0 | 2023-10-22 21:09:57 +0800 | [diff] [blame] | 421 | return z_erofs_load_lz4_config(sb, dsb, NULL, 0); |
| 422 | } |
| 423 | |
| 424 | sbi->available_compr_algs = le16_to_cpu(dsb->u1.available_compr_algs); |
| 425 | if (sbi->available_compr_algs & ~Z_EROFS_ALL_COMPR_ALGS) { |
| 426 | erofs_err(sb, "unidentified algorithms %x, please upgrade kernel", |
| 427 | sbi->available_compr_algs & ~Z_EROFS_ALL_COMPR_ALGS); |
| 428 | return -EOPNOTSUPP; |
| 429 | } |
| 430 | |
| 431 | erofs_init_metabuf(&buf, sb); |
| 432 | offset = EROFS_SUPER_OFFSET + sbi->sb_size; |
| 433 | alg = 0; |
| 434 | for (algs = sbi->available_compr_algs; algs; algs >>= 1, ++alg) { |
| 435 | void *data; |
| 436 | |
| 437 | if (!(algs & 1)) |
| 438 | continue; |
| 439 | |
| 440 | data = erofs_read_metadata(sb, &buf, &offset, &size); |
| 441 | if (IS_ERR(data)) { |
| 442 | ret = PTR_ERR(data); |
| 443 | break; |
| 444 | } |
| 445 | |
| 446 | if (alg >= ARRAY_SIZE(erofs_decompressors) || |
| 447 | !erofs_decompressors[alg].config) { |
| 448 | erofs_err(sb, "algorithm %d isn't enabled on this kernel", |
| 449 | alg); |
| 450 | ret = -EOPNOTSUPP; |
| 451 | } else { |
| 452 | ret = erofs_decompressors[alg].config(sb, |
| 453 | dsb, data, size); |
| 454 | } |
| 455 | |
| 456 | kfree(data); |
| 457 | if (ret) |
| 458 | break; |
| 459 | } |
| 460 | erofs_put_metabuf(&buf); |
| 461 | return ret; |
| 462 | } |