1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2018-2019 HUAWEI, Inc.
4 * https://www.huawei.com/
5 */
6 #include "internal.h"
7 #include <asm/unaligned.h>
8 #include <trace/events/erofs.h>
9
z_erofs_fill_inode(struct inode * inode)10 int z_erofs_fill_inode(struct inode *inode)
11 {
12 struct erofs_inode *const vi = EROFS_I(inode);
13 struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb);
14
15 if (!erofs_sb_has_big_pcluster(sbi) &&
16 !erofs_sb_has_ztailpacking(sbi) && !erofs_sb_has_fragments(sbi) &&
17 vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) {
18 vi->z_advise = 0;
19 vi->z_algorithmtype[0] = 0;
20 vi->z_algorithmtype[1] = 0;
21 vi->z_logical_clusterbits = LOG_BLOCK_SIZE;
22 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
23 }
24 inode->i_mapping->a_ops = &z_erofs_aops;
25 return 0;
26 }
27
28 struct z_erofs_maprecorder {
29 struct inode *inode;
30 struct erofs_map_blocks *map;
31 void *kaddr;
32
33 unsigned long lcn;
34 /* compression extent information gathered */
35 u8 type, headtype;
36 u16 clusterofs;
37 u16 delta[2];
38 erofs_blk_t pblk, compressedblks;
39 erofs_off_t nextpackoff;
40 bool partialref;
41 };
42
legacy_load_cluster_from_disk(struct z_erofs_maprecorder * m,unsigned long lcn)43 static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m,
44 unsigned long lcn)
45 {
46 struct inode *const inode = m->inode;
47 struct erofs_inode *const vi = EROFS_I(inode);
48 const erofs_off_t pos =
49 Z_EROFS_VLE_LEGACY_INDEX_ALIGN(erofs_iloc(inode) +
50 vi->inode_isize + vi->xattr_isize) +
51 lcn * sizeof(struct z_erofs_vle_decompressed_index);
52 struct z_erofs_vle_decompressed_index *di;
53 unsigned int advise, type;
54
55 m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb,
56 erofs_blknr(pos), EROFS_KMAP);
57 if (IS_ERR(m->kaddr))
58 return PTR_ERR(m->kaddr);
59
60 m->nextpackoff = pos + sizeof(struct z_erofs_vle_decompressed_index);
61 m->lcn = lcn;
62 di = m->kaddr + erofs_blkoff(pos);
63
64 advise = le16_to_cpu(di->di_advise);
65 type = (advise >> Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT) &
66 ((1 << Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) - 1);
67 switch (type) {
68 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
69 m->clusterofs = 1 << vi->z_logical_clusterbits;
70 m->delta[0] = le16_to_cpu(di->di_u.delta[0]);
71 if (m->delta[0] & Z_EROFS_VLE_DI_D0_CBLKCNT) {
72 if (!(vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 |
73 Z_EROFS_ADVISE_BIG_PCLUSTER_2))) {
74 DBG_BUGON(1);
75 return -EFSCORRUPTED;
76 }
77 m->compressedblks = m->delta[0] &
78 ~Z_EROFS_VLE_DI_D0_CBLKCNT;
79 m->delta[0] = 1;
80 }
81 m->delta[1] = le16_to_cpu(di->di_u.delta[1]);
82 break;
83 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
84 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1:
85 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2:
86 if (advise & Z_EROFS_VLE_DI_PARTIAL_REF)
87 m->partialref = true;
88 m->clusterofs = le16_to_cpu(di->di_clusterofs);
89 m->pblk = le32_to_cpu(di->di_u.blkaddr);
90 break;
91 default:
92 DBG_BUGON(1);
93 return -EOPNOTSUPP;
94 }
95 m->type = type;
96 return 0;
97 }
98
decode_compactedbits(unsigned int lobits,unsigned int lomask,u8 * in,unsigned int pos,u8 * type)99 static unsigned int decode_compactedbits(unsigned int lobits,
100 unsigned int lomask,
101 u8 *in, unsigned int pos, u8 *type)
102 {
103 const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7);
104 const unsigned int lo = v & lomask;
105
106 *type = (v >> lobits) & 3;
107 return lo;
108 }
109
get_compacted_la_distance(unsigned int lclusterbits,unsigned int encodebits,unsigned int vcnt,u8 * in,int i)110 static int get_compacted_la_distance(unsigned int lclusterbits,
111 unsigned int encodebits,
112 unsigned int vcnt, u8 *in, int i)
113 {
114 const unsigned int lomask = (1 << lclusterbits) - 1;
115 unsigned int lo, d1 = 0;
116 u8 type;
117
118 DBG_BUGON(i >= vcnt);
119
120 do {
121 lo = decode_compactedbits(lclusterbits, lomask,
122 in, encodebits * i, &type);
123
124 if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
125 return d1;
126 ++d1;
127 } while (++i < vcnt);
128
129 /* vcnt - 1 (Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) item */
130 if (!(lo & Z_EROFS_VLE_DI_D0_CBLKCNT))
131 d1 += lo - 1;
132 return d1;
133 }
134
unpack_compacted_index(struct z_erofs_maprecorder * m,unsigned int amortizedshift,erofs_off_t pos,bool lookahead)135 static int unpack_compacted_index(struct z_erofs_maprecorder *m,
136 unsigned int amortizedshift,
137 erofs_off_t pos, bool lookahead)
138 {
139 struct erofs_inode *const vi = EROFS_I(m->inode);
140 const unsigned int lclusterbits = vi->z_logical_clusterbits;
141 const unsigned int lomask = (1 << lclusterbits) - 1;
142 unsigned int vcnt, base, lo, encodebits, nblk, eofs;
143 int i;
144 u8 *in, type;
145 bool big_pcluster;
146
147 if (1 << amortizedshift == 4)
148 vcnt = 2;
149 else if (1 << amortizedshift == 2 && lclusterbits == 12)
150 vcnt = 16;
151 else
152 return -EOPNOTSUPP;
153
154 /* it doesn't equal to round_up(..) */
155 m->nextpackoff = round_down(pos, vcnt << amortizedshift) +
156 (vcnt << amortizedshift);
157 big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1;
158 encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt;
159 eofs = erofs_blkoff(pos);
160 base = round_down(eofs, vcnt << amortizedshift);
161 in = m->kaddr + base;
162
163 i = (eofs - base) >> amortizedshift;
164
165 lo = decode_compactedbits(lclusterbits, lomask,
166 in, encodebits * i, &type);
167 m->type = type;
168 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) {
169 m->clusterofs = 1 << lclusterbits;
170
171 /* figure out lookahead_distance: delta[1] if needed */
172 if (lookahead)
173 m->delta[1] = get_compacted_la_distance(lclusterbits,
174 encodebits, vcnt, in, i);
175 if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) {
176 if (!big_pcluster) {
177 DBG_BUGON(1);
178 return -EFSCORRUPTED;
179 }
180 m->compressedblks = lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT;
181 m->delta[0] = 1;
182 return 0;
183 } else if (i + 1 != (int)vcnt) {
184 m->delta[0] = lo;
185 return 0;
186 }
187 /*
188 * since the last lcluster in the pack is special,
189 * of which lo saves delta[1] rather than delta[0].
190 * Hence, get delta[0] by the previous lcluster indirectly.
191 */
192 lo = decode_compactedbits(lclusterbits, lomask,
193 in, encodebits * (i - 1), &type);
194 if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
195 lo = 0;
196 else if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT)
197 lo = 1;
198 m->delta[0] = lo + 1;
199 return 0;
200 }
201 m->clusterofs = lo;
202 m->delta[0] = 0;
203 /* figout out blkaddr (pblk) for HEAD lclusters */
204 if (!big_pcluster) {
205 nblk = 1;
206 while (i > 0) {
207 --i;
208 lo = decode_compactedbits(lclusterbits, lomask,
209 in, encodebits * i, &type);
210 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
211 i -= lo;
212
213 if (i >= 0)
214 ++nblk;
215 }
216 } else {
217 nblk = 0;
218 while (i > 0) {
219 --i;
220 lo = decode_compactedbits(lclusterbits, lomask,
221 in, encodebits * i, &type);
222 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) {
223 if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) {
224 --i;
225 nblk += lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT;
226 continue;
227 }
228 /* bigpcluster shouldn't have plain d0 == 1 */
229 if (lo <= 1) {
230 DBG_BUGON(1);
231 return -EFSCORRUPTED;
232 }
233 i -= lo - 2;
234 continue;
235 }
236 ++nblk;
237 }
238 }
239 in += (vcnt << amortizedshift) - sizeof(__le32);
240 m->pblk = le32_to_cpu(*(__le32 *)in) + nblk;
241 return 0;
242 }
243
compacted_load_cluster_from_disk(struct z_erofs_maprecorder * m,unsigned long lcn,bool lookahead)244 static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m,
245 unsigned long lcn, bool lookahead)
246 {
247 struct inode *const inode = m->inode;
248 struct erofs_inode *const vi = EROFS_I(inode);
249 const unsigned int lclusterbits = vi->z_logical_clusterbits;
250 const erofs_off_t ebase = sizeof(struct z_erofs_map_header) +
251 ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8);
252 const unsigned int totalidx = DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ);
253 unsigned int compacted_4b_initial, compacted_2b;
254 unsigned int amortizedshift;
255 erofs_off_t pos;
256
257 if (lclusterbits != 12)
258 return -EOPNOTSUPP;
259
260 if (lcn >= totalidx)
261 return -EINVAL;
262
263 m->lcn = lcn;
264 /* used to align to 32-byte (compacted_2b) alignment */
265 compacted_4b_initial = (32 - ebase % 32) / 4;
266 if (compacted_4b_initial == 32 / 4)
267 compacted_4b_initial = 0;
268
269 if ((vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) &&
270 compacted_4b_initial < totalidx)
271 compacted_2b = rounddown(totalidx - compacted_4b_initial, 16);
272 else
273 compacted_2b = 0;
274
275 pos = ebase;
276 if (lcn < compacted_4b_initial) {
277 amortizedshift = 2;
278 goto out;
279 }
280 pos += compacted_4b_initial * 4;
281 lcn -= compacted_4b_initial;
282
283 if (lcn < compacted_2b) {
284 amortizedshift = 1;
285 goto out;
286 }
287 pos += compacted_2b * 2;
288 lcn -= compacted_2b;
289 amortizedshift = 2;
290 out:
291 pos += lcn * (1 << amortizedshift);
292 m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb,
293 erofs_blknr(pos), EROFS_KMAP);
294 if (IS_ERR(m->kaddr))
295 return PTR_ERR(m->kaddr);
296 return unpack_compacted_index(m, amortizedshift, pos, lookahead);
297 }
298
z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder * m,unsigned int lcn,bool lookahead)299 static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m,
300 unsigned int lcn, bool lookahead)
301 {
302 const unsigned int datamode = EROFS_I(m->inode)->datalayout;
303
304 if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY)
305 return legacy_load_cluster_from_disk(m, lcn);
306
307 if (datamode == EROFS_INODE_FLAT_COMPRESSION)
308 return compacted_load_cluster_from_disk(m, lcn, lookahead);
309
310 return -EINVAL;
311 }
312
z_erofs_extent_lookback(struct z_erofs_maprecorder * m,unsigned int lookback_distance)313 static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m,
314 unsigned int lookback_distance)
315 {
316 struct erofs_inode *const vi = EROFS_I(m->inode);
317 const unsigned int lclusterbits = vi->z_logical_clusterbits;
318
319 while (m->lcn >= lookback_distance) {
320 unsigned long lcn = m->lcn - lookback_distance;
321 int err;
322
323 /* load extent head logical cluster if needed */
324 err = z_erofs_load_cluster_from_disk(m, lcn, false);
325 if (err)
326 return err;
327
328 switch (m->type) {
329 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
330 if (!m->delta[0]) {
331 erofs_err(m->inode->i_sb,
332 "invalid lookback distance 0 @ nid %llu",
333 vi->nid);
334 DBG_BUGON(1);
335 return -EFSCORRUPTED;
336 }
337 lookback_distance = m->delta[0];
338 continue;
339 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
340 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1:
341 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2:
342 m->headtype = m->type;
343 m->map->m_la = (lcn << lclusterbits) | m->clusterofs;
344 return 0;
345 default:
346 erofs_err(m->inode->i_sb,
347 "unknown type %u @ lcn %lu of nid %llu",
348 m->type, lcn, vi->nid);
349 DBG_BUGON(1);
350 return -EOPNOTSUPP;
351 }
352 }
353
354 erofs_err(m->inode->i_sb, "bogus lookback distance @ nid %llu",
355 vi->nid);
356 DBG_BUGON(1);
357 return -EFSCORRUPTED;
358 }
359
z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder * m,unsigned int initial_lcn)360 static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m,
361 unsigned int initial_lcn)
362 {
363 struct erofs_inode *const vi = EROFS_I(m->inode);
364 struct erofs_map_blocks *const map = m->map;
365 const unsigned int lclusterbits = vi->z_logical_clusterbits;
366 unsigned long lcn;
367 int err;
368
369 DBG_BUGON(m->type != Z_EROFS_VLE_CLUSTER_TYPE_PLAIN &&
370 m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD1 &&
371 m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD2);
372 DBG_BUGON(m->type != m->headtype);
373
374 if (m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN ||
375 ((m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD1) &&
376 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) ||
377 ((m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) &&
378 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2))) {
379 map->m_plen = 1ULL << lclusterbits;
380 return 0;
381 }
382 lcn = m->lcn + 1;
383 if (m->compressedblks)
384 goto out;
385
386 err = z_erofs_load_cluster_from_disk(m, lcn, false);
387 if (err)
388 return err;
389
390 /*
391 * If the 1st NONHEAD lcluster has already been handled initially w/o
392 * valid compressedblks, which means at least it mustn't be CBLKCNT, or
393 * an internal implemenatation error is detected.
394 *
395 * The following code can also handle it properly anyway, but let's
396 * BUG_ON in the debugging mode only for developers to notice that.
397 */
398 DBG_BUGON(lcn == initial_lcn &&
399 m->type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD);
400
401 switch (m->type) {
402 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
403 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1:
404 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2:
405 /*
406 * if the 1st NONHEAD lcluster is actually PLAIN or HEAD type
407 * rather than CBLKCNT, it's a 1 lcluster-sized pcluster.
408 */
409 m->compressedblks = 1 << (lclusterbits - LOG_BLOCK_SIZE);
410 break;
411 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
412 if (m->delta[0] != 1)
413 goto err_bonus_cblkcnt;
414 if (m->compressedblks)
415 break;
416 fallthrough;
417 default:
418 erofs_err(m->inode->i_sb,
419 "cannot found CBLKCNT @ lcn %lu of nid %llu",
420 lcn, vi->nid);
421 DBG_BUGON(1);
422 return -EFSCORRUPTED;
423 }
424 out:
425 map->m_plen = (u64)m->compressedblks << LOG_BLOCK_SIZE;
426 return 0;
427 err_bonus_cblkcnt:
428 erofs_err(m->inode->i_sb,
429 "bogus CBLKCNT @ lcn %lu of nid %llu",
430 lcn, vi->nid);
431 DBG_BUGON(1);
432 return -EFSCORRUPTED;
433 }
434
z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder * m)435 static int z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m)
436 {
437 struct inode *inode = m->inode;
438 struct erofs_inode *vi = EROFS_I(inode);
439 struct erofs_map_blocks *map = m->map;
440 unsigned int lclusterbits = vi->z_logical_clusterbits;
441 u64 lcn = m->lcn, headlcn = map->m_la >> lclusterbits;
442 int err;
443
444 do {
445 /* handle the last EOF pcluster (no next HEAD lcluster) */
446 if ((lcn << lclusterbits) >= inode->i_size) {
447 map->m_llen = inode->i_size - map->m_la;
448 return 0;
449 }
450
451 err = z_erofs_load_cluster_from_disk(m, lcn, true);
452 if (err)
453 return err;
454
455 if (m->type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) {
456 DBG_BUGON(!m->delta[1] &&
457 m->clusterofs != 1 << lclusterbits);
458 } else if (m->type == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN ||
459 m->type == Z_EROFS_VLE_CLUSTER_TYPE_HEAD1 ||
460 m->type == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) {
461 /* go on until the next HEAD lcluster */
462 if (lcn != headlcn)
463 break;
464 m->delta[1] = 1;
465 } else {
466 erofs_err(inode->i_sb, "unknown type %u @ lcn %llu of nid %llu",
467 m->type, lcn, vi->nid);
468 DBG_BUGON(1);
469 return -EOPNOTSUPP;
470 }
471 lcn += m->delta[1];
472 } while (m->delta[1]);
473
474 map->m_llen = (lcn << lclusterbits) + m->clusterofs - map->m_la;
475 return 0;
476 }
477
z_erofs_do_map_blocks(struct inode * inode,struct erofs_map_blocks * map,int flags)478 static int z_erofs_do_map_blocks(struct inode *inode,
479 struct erofs_map_blocks *map,
480 int flags)
481 {
482 struct erofs_inode *const vi = EROFS_I(inode);
483 bool ztailpacking = vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER;
484 bool fragment = vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER;
485 struct z_erofs_maprecorder m = {
486 .inode = inode,
487 .map = map,
488 };
489 int err = 0;
490 unsigned int lclusterbits, endoff;
491 unsigned long initial_lcn;
492 unsigned long long ofs, end;
493
494 lclusterbits = vi->z_logical_clusterbits;
495 ofs = flags & EROFS_GET_BLOCKS_FINDTAIL ? inode->i_size - 1 : map->m_la;
496 initial_lcn = ofs >> lclusterbits;
497 endoff = ofs & ((1 << lclusterbits) - 1);
498
499 err = z_erofs_load_cluster_from_disk(&m, initial_lcn, false);
500 if (err)
501 goto unmap_out;
502
503 if (ztailpacking && (flags & EROFS_GET_BLOCKS_FINDTAIL))
504 vi->z_idataoff = m.nextpackoff;
505
506 map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_ENCODED;
507 end = (m.lcn + 1ULL) << lclusterbits;
508
509 switch (m.type) {
510 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
511 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1:
512 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2:
513 if (endoff >= m.clusterofs) {
514 m.headtype = m.type;
515 map->m_la = (m.lcn << lclusterbits) | m.clusterofs;
516 /*
517 * For ztailpacking files, in order to inline data more
518 * effectively, special EOF lclusters are now supported
519 * which can have three parts at most.
520 */
521 if (ztailpacking && end > inode->i_size)
522 end = inode->i_size;
523 break;
524 }
525 /* m.lcn should be >= 1 if endoff < m.clusterofs */
526 if (!m.lcn) {
527 erofs_err(inode->i_sb,
528 "invalid logical cluster 0 at nid %llu",
529 vi->nid);
530 err = -EFSCORRUPTED;
531 goto unmap_out;
532 }
533 end = (m.lcn << lclusterbits) | m.clusterofs;
534 map->m_flags |= EROFS_MAP_FULL_MAPPED;
535 m.delta[0] = 1;
536 fallthrough;
537 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
538 /* get the corresponding first chunk */
539 err = z_erofs_extent_lookback(&m, m.delta[0]);
540 if (err)
541 goto unmap_out;
542 break;
543 default:
544 erofs_err(inode->i_sb,
545 "unknown type %u @ offset %llu of nid %llu",
546 m.type, ofs, vi->nid);
547 err = -EOPNOTSUPP;
548 goto unmap_out;
549 }
550 if (m.partialref)
551 map->m_flags |= EROFS_MAP_PARTIAL_REF;
552 map->m_llen = end - map->m_la;
553
554 if (flags & EROFS_GET_BLOCKS_FINDTAIL) {
555 vi->z_tailextent_headlcn = m.lcn;
556 /* for non-compact indexes, fragmentoff is 64 bits */
557 if (fragment &&
558 vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY)
559 vi->z_fragmentoff |= (u64)m.pblk << 32;
560 }
561 if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) {
562 map->m_flags |= EROFS_MAP_META;
563 map->m_pa = vi->z_idataoff;
564 map->m_plen = vi->z_idata_size;
565 } else if (fragment && m.lcn == vi->z_tailextent_headlcn) {
566 map->m_flags |= EROFS_MAP_FRAGMENT;
567 } else {
568 map->m_pa = blknr_to_addr(m.pblk);
569 err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
570 if (err)
571 goto unmap_out;
572 }
573
574 if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN) {
575 if (map->m_llen > map->m_plen) {
576 DBG_BUGON(1);
577 err = -EFSCORRUPTED;
578 goto unmap_out;
579 }
580 if (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER)
581 map->m_algorithmformat =
582 Z_EROFS_COMPRESSION_INTERLACED;
583 else
584 map->m_algorithmformat =
585 Z_EROFS_COMPRESSION_SHIFTED;
586 } else if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) {
587 map->m_algorithmformat = vi->z_algorithmtype[1];
588 } else {
589 map->m_algorithmformat = vi->z_algorithmtype[0];
590 }
591
592 if ((flags & EROFS_GET_BLOCKS_FIEMAP) ||
593 ((flags & EROFS_GET_BLOCKS_READMORE) &&
594 map->m_algorithmformat == Z_EROFS_COMPRESSION_LZMA &&
595 map->m_llen >= EROFS_BLKSIZ)) {
596 err = z_erofs_get_extent_decompressedlen(&m);
597 if (!err)
598 map->m_flags |= EROFS_MAP_FULL_MAPPED;
599 }
600
601 unmap_out:
602 erofs_unmap_metabuf(&m.map->buf);
603 erofs_dbg("%s, m_la %llu m_pa %llu m_llen %llu m_plen %llu m_flags 0%o",
604 __func__, map->m_la, map->m_pa,
605 map->m_llen, map->m_plen, map->m_flags);
606 return err;
607 }
608
z_erofs_fill_inode_lazy(struct inode * inode)609 static int z_erofs_fill_inode_lazy(struct inode *inode)
610 {
611 struct erofs_inode *const vi = EROFS_I(inode);
612 struct super_block *const sb = inode->i_sb;
613 int err, headnr;
614 erofs_off_t pos;
615 struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
616 void *kaddr;
617 struct z_erofs_map_header *h;
618
619 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) {
620 /*
621 * paired with smp_mb() at the end of the function to ensure
622 * fields will only be observed after the bit is set.
623 */
624 smp_mb();
625 return 0;
626 }
627
628 if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE))
629 return -ERESTARTSYS;
630
631 err = 0;
632 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags))
633 goto out_unlock;
634
635 pos = ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8);
636 kaddr = erofs_read_metabuf(&buf, sb, erofs_blknr(pos), EROFS_KMAP);
637 if (IS_ERR(kaddr)) {
638 err = PTR_ERR(kaddr);
639 goto out_unlock;
640 }
641
642 h = kaddr + erofs_blkoff(pos);
643 /*
644 * if the highest bit of the 8-byte map header is set, the whole file
645 * is stored in the packed inode. The rest bits keeps z_fragmentoff.
646 */
647 if (h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT) {
648 vi->z_advise = Z_EROFS_ADVISE_FRAGMENT_PCLUSTER;
649 vi->z_fragmentoff = le64_to_cpu(*(__le64 *)h) ^ (1ULL << 63);
650 vi->z_tailextent_headlcn = 0;
651 goto done;
652 }
653 vi->z_advise = le16_to_cpu(h->h_advise);
654 vi->z_algorithmtype[0] = h->h_algorithmtype & 15;
655 vi->z_algorithmtype[1] = h->h_algorithmtype >> 4;
656
657 headnr = 0;
658 if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX ||
659 vi->z_algorithmtype[++headnr] >= Z_EROFS_COMPRESSION_MAX) {
660 erofs_err(sb, "unknown HEAD%u format %u for nid %llu, please upgrade kernel",
661 headnr + 1, vi->z_algorithmtype[headnr], vi->nid);
662 err = -EOPNOTSUPP;
663 goto out_put_metabuf;
664 }
665
666 vi->z_logical_clusterbits = LOG_BLOCK_SIZE + (h->h_clusterbits & 7);
667 if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) &&
668 vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 |
669 Z_EROFS_ADVISE_BIG_PCLUSTER_2)) {
670 erofs_err(sb, "per-inode big pcluster without sb feature for nid %llu",
671 vi->nid);
672 err = -EFSCORRUPTED;
673 goto out_put_metabuf;
674 }
675 if (vi->datalayout == EROFS_INODE_FLAT_COMPRESSION &&
676 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) ^
677 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2)) {
678 erofs_err(sb, "big pcluster head1/2 of compact indexes should be consistent for nid %llu",
679 vi->nid);
680 err = -EFSCORRUPTED;
681 goto out_put_metabuf;
682 }
683
684 if (vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) {
685 struct erofs_map_blocks map = {
686 .buf = __EROFS_BUF_INITIALIZER
687 };
688
689 vi->z_idata_size = le16_to_cpu(h->h_idata_size);
690 err = z_erofs_do_map_blocks(inode, &map,
691 EROFS_GET_BLOCKS_FINDTAIL);
692 erofs_put_metabuf(&map.buf);
693
694 if (!map.m_plen ||
695 erofs_blkoff(map.m_pa) + map.m_plen > EROFS_BLKSIZ) {
696 erofs_err(sb, "invalid tail-packing pclustersize %llu",
697 map.m_plen);
698 err = -EFSCORRUPTED;
699 }
700 if (err < 0)
701 goto out_put_metabuf;
702 }
703
704 if (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER &&
705 !(h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT)) {
706 struct erofs_map_blocks map = {
707 .buf = __EROFS_BUF_INITIALIZER
708 };
709
710 vi->z_fragmentoff = le32_to_cpu(h->h_fragmentoff);
711 err = z_erofs_do_map_blocks(inode, &map,
712 EROFS_GET_BLOCKS_FINDTAIL);
713 erofs_put_metabuf(&map.buf);
714 if (err < 0)
715 goto out_put_metabuf;
716 }
717 done:
718 /* paired with smp_mb() at the beginning of the function */
719 smp_mb();
720 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
721 out_put_metabuf:
722 erofs_put_metabuf(&buf);
723 out_unlock:
724 clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags);
725 return err;
726 }
727
z_erofs_map_blocks_iter(struct inode * inode,struct erofs_map_blocks * map,int flags)728 int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map,
729 int flags)
730 {
731 struct erofs_inode *const vi = EROFS_I(inode);
732 int err = 0;
733
734 trace_z_erofs_map_blocks_iter_enter(inode, map, flags);
735
736 /* when trying to read beyond EOF, leave it unmapped */
737 if (map->m_la >= inode->i_size) {
738 map->m_llen = map->m_la + 1 - inode->i_size;
739 map->m_la = inode->i_size;
740 map->m_flags = 0;
741 goto out;
742 }
743
744 err = z_erofs_fill_inode_lazy(inode);
745 if (err)
746 goto out;
747
748 if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) &&
749 !vi->z_tailextent_headlcn) {
750 map->m_la = 0;
751 map->m_llen = inode->i_size;
752 map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_FULL_MAPPED |
753 EROFS_MAP_FRAGMENT;
754 goto out;
755 }
756
757 err = z_erofs_do_map_blocks(inode, map, flags);
758 out:
759 trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err);
760 return err;
761 }
762
z_erofs_iomap_begin_report(struct inode * inode,loff_t offset,loff_t length,unsigned int flags,struct iomap * iomap,struct iomap * srcmap)763 static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset,
764 loff_t length, unsigned int flags,
765 struct iomap *iomap, struct iomap *srcmap)
766 {
767 int ret;
768 struct erofs_map_blocks map = { .m_la = offset };
769
770 ret = z_erofs_map_blocks_iter(inode, &map, EROFS_GET_BLOCKS_FIEMAP);
771 erofs_put_metabuf(&map.buf);
772 if (ret < 0)
773 return ret;
774
775 iomap->bdev = inode->i_sb->s_bdev;
776 iomap->offset = map.m_la;
777 iomap->length = map.m_llen;
778 if (map.m_flags & EROFS_MAP_MAPPED) {
779 iomap->type = IOMAP_MAPPED;
780 iomap->addr = map.m_flags & EROFS_MAP_FRAGMENT ?
781 IOMAP_NULL_ADDR : map.m_pa;
782 } else {
783 iomap->type = IOMAP_HOLE;
784 iomap->addr = IOMAP_NULL_ADDR;
785 /*
786 * No strict rule on how to describe extents for post EOF, yet
787 * we need to do like below. Otherwise, iomap itself will get
788 * into an endless loop on post EOF.
789 *
790 * Calculate the effective offset by subtracting extent start
791 * (map.m_la) from the requested offset, and add it to length.
792 * (NB: offset >= map.m_la always)
793 */
794 if (iomap->offset >= inode->i_size)
795 iomap->length = length + offset - map.m_la;
796 }
797 iomap->flags = 0;
798 return 0;
799 }
800
801 const struct iomap_ops z_erofs_iomap_report_ops = {
802 .iomap_begin = z_erofs_iomap_begin_report,
803 };
804