Lines Matching refs:ctl
37 static int link_free_space(struct btrfs_free_space_ctl *ctl,
39 static void unlink_free_space(struct btrfs_free_space_ctl *ctl,
41 static int search_bitmap(struct btrfs_free_space_ctl *ctl,
44 static void free_bitmap(struct btrfs_free_space_ctl *ctl,
46 static void bitmap_clear_bits(struct btrfs_free_space_ctl *ctl,
667 static void recalculate_thresholds(struct btrfs_free_space_ctl *ctl) in recalculate_thresholds() argument
669 struct btrfs_block_group *block_group = ctl->private; in recalculate_thresholds()
674 u64 bytes_per_bg = BITS_PER_BITMAP * ctl->unit; in recalculate_thresholds()
679 ASSERT(ctl->total_bitmaps <= max_bitmaps); in recalculate_thresholds()
692 bitmap_bytes = ctl->total_bitmaps * ctl->unit; in recalculate_thresholds()
701 ctl->extents_thresh = in recalculate_thresholds()
706 struct btrfs_free_space_ctl *ctl, in __load_free_space_cache() argument
804 spin_lock(&ctl->tree_lock); in __load_free_space_cache()
805 ret = link_free_space(ctl, e); in __load_free_space_cache()
806 spin_unlock(&ctl->tree_lock); in __load_free_space_cache()
824 spin_lock(&ctl->tree_lock); in __load_free_space_cache()
825 ret = link_free_space(ctl, e); in __load_free_space_cache()
826 ctl->total_bitmaps++; in __load_free_space_cache()
827 recalculate_thresholds(ctl); in __load_free_space_cache()
828 spin_unlock(&ctl->tree_lock); in __load_free_space_cache()
861 __btrfs_remove_free_space_cache(ctl); in __load_free_space_cache()
866 struct btrfs_free_space_ctl *ctl) in copy_free_space_cache() argument
872 while (!ret && (n = rb_first(&ctl->free_space_offset)) != NULL) { in copy_free_space_cache()
875 unlink_free_space(ctl, info); in copy_free_space_cache()
881 u64 bytes = ctl->unit; in copy_free_space_cache()
883 while (search_bitmap(ctl, info, &offset, &bytes, in copy_free_space_cache()
889 bitmap_clear_bits(ctl, info, offset, bytes); in copy_free_space_cache()
891 bytes = ctl->unit; in copy_free_space_cache()
893 free_bitmap(ctl, info); in copy_free_space_cache()
903 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in load_free_space_cache() local
1006 spin_lock(&ctl->tree_lock); in load_free_space_cache()
1008 spin_unlock(&ctl->tree_lock); in load_free_space_cache()
1015 struct btrfs_free_space_ctl *ctl, in write_cache_extent_entries() argument
1023 struct rb_node *node = rb_first(&ctl->free_space_offset); in write_cache_extent_entries()
1075 list_for_each_entry(trim_entry, &ctl->trimming_ranges, list) { in write_cache_extent_entries()
1324 struct btrfs_free_space_ctl *ctl, in __btrfs_write_out_cache() argument
1369 mutex_lock(&ctl->cache_writeout_mutex); in __btrfs_write_out_cache()
1371 spin_lock(&ctl->tree_lock); in __btrfs_write_out_cache()
1372 ret = write_cache_extent_entries(io_ctl, ctl, in __btrfs_write_out_cache()
1396 spin_unlock(&ctl->tree_lock); in __btrfs_write_out_cache()
1397 mutex_unlock(&ctl->cache_writeout_mutex); in __btrfs_write_out_cache()
1439 spin_unlock(&ctl->tree_lock); in __btrfs_write_out_cache()
1440 mutex_unlock(&ctl->cache_writeout_mutex); in __btrfs_write_out_cache()
1467 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_write_out_cache() local
1482 ret = __btrfs_write_out_cache(fs_info->tree_root, inode, ctl, in btrfs_write_out_cache()
1517 static inline u64 offset_to_bitmap(struct btrfs_free_space_ctl *ctl, in offset_to_bitmap() argument
1523 bytes_per_bitmap = BITS_PER_BITMAP * ctl->unit; in offset_to_bitmap()
1524 bitmap_start = offset - ctl->start; in offset_to_bitmap()
1527 bitmap_start += ctl->start; in offset_to_bitmap()
1591 tree_search_offset(struct btrfs_free_space_ctl *ctl, in tree_search_offset() argument
1594 struct rb_node *n = ctl->free_space_offset.rb_node; in tree_search_offset()
1680 if (entry->offset + BITS_PER_BITMAP * ctl->unit > offset) in tree_search_offset()
1691 ctl->unit > offset) in tree_search_offset()
1707 __unlink_free_space(struct btrfs_free_space_ctl *ctl, in __unlink_free_space() argument
1710 rb_erase(&info->offset_index, &ctl->free_space_offset); in __unlink_free_space()
1711 ctl->free_extents--; in __unlink_free_space()
1714 ctl->discardable_extents[BTRFS_STAT_CURR]--; in __unlink_free_space()
1715 ctl->discardable_bytes[BTRFS_STAT_CURR] -= info->bytes; in __unlink_free_space()
1719 static void unlink_free_space(struct btrfs_free_space_ctl *ctl, in unlink_free_space() argument
1722 __unlink_free_space(ctl, info); in unlink_free_space()
1723 ctl->free_space -= info->bytes; in unlink_free_space()
1726 static int link_free_space(struct btrfs_free_space_ctl *ctl, in link_free_space() argument
1732 ret = tree_insert_offset(&ctl->free_space_offset, info->offset, in link_free_space()
1738 ctl->discardable_extents[BTRFS_STAT_CURR]++; in link_free_space()
1739 ctl->discardable_bytes[BTRFS_STAT_CURR] += info->bytes; in link_free_space()
1742 ctl->free_space += info->bytes; in link_free_space()
1743 ctl->free_extents++; in link_free_space()
1747 static inline void __bitmap_clear_bits(struct btrfs_free_space_ctl *ctl, in __bitmap_clear_bits() argument
1754 start = offset_to_bit(info->offset, ctl->unit, offset); in __bitmap_clear_bits()
1755 count = bytes_to_bits(bytes, ctl->unit); in __bitmap_clear_bits()
1762 if (info->max_extent_size > ctl->unit) in __bitmap_clear_bits()
1773 ctl->discardable_extents[BTRFS_STAT_CURR] += extent_delta; in __bitmap_clear_bits()
1774 ctl->discardable_bytes[BTRFS_STAT_CURR] -= bytes; in __bitmap_clear_bits()
1778 static void bitmap_clear_bits(struct btrfs_free_space_ctl *ctl, in bitmap_clear_bits() argument
1782 __bitmap_clear_bits(ctl, info, offset, bytes); in bitmap_clear_bits()
1783 ctl->free_space -= bytes; in bitmap_clear_bits()
1786 static void bitmap_set_bits(struct btrfs_free_space_ctl *ctl, in bitmap_set_bits() argument
1793 start = offset_to_bit(info->offset, ctl->unit, offset); in bitmap_set_bits()
1794 count = bytes_to_bits(bytes, ctl->unit); in bitmap_set_bits()
1801 ctl->free_space += bytes; in bitmap_set_bits()
1811 ctl->discardable_extents[BTRFS_STAT_CURR] += extent_delta; in bitmap_set_bits()
1812 ctl->discardable_bytes[BTRFS_STAT_CURR] += bytes; in bitmap_set_bits()
1820 static int search_bitmap(struct btrfs_free_space_ctl *ctl, in search_bitmap() argument
1841 i = offset_to_bit(bitmap_info->offset, ctl->unit, in search_bitmap()
1843 bits = bytes_to_bits(*bytes, ctl->unit); in search_bitmap()
1863 *offset = (u64)(i * ctl->unit) + bitmap_info->offset; in search_bitmap()
1864 *bytes = (u64)(found_bits) * ctl->unit; in search_bitmap()
1868 *bytes = (u64)(max_bits) * ctl->unit; in search_bitmap()
1882 find_free_space(struct btrfs_free_space_ctl *ctl, u64 *offset, u64 *bytes, in find_free_space() argument
1891 if (!ctl->free_space_offset.rb_node) in find_free_space()
1894 entry = tree_search_offset(ctl, offset_to_bitmap(ctl, *offset), 0, 1); in find_free_space()
1910 tmp = entry->offset - ctl->start + align - 1; in find_free_space()
1912 tmp = tmp * align + ctl->start; in find_free_space()
1928 ret = search_bitmap(ctl, entry, &tmp, &size, true); in find_free_space()
1949 static void add_new_bitmap(struct btrfs_free_space_ctl *ctl, in add_new_bitmap() argument
1952 info->offset = offset_to_bitmap(ctl, offset); in add_new_bitmap()
1956 link_free_space(ctl, info); in add_new_bitmap()
1957 ctl->total_bitmaps++; in add_new_bitmap()
1958 recalculate_thresholds(ctl); in add_new_bitmap()
1961 static void free_bitmap(struct btrfs_free_space_ctl *ctl, in free_bitmap() argument
1971 ctl->discardable_extents[BTRFS_STAT_CURR] -= in free_bitmap()
1973 ctl->discardable_bytes[BTRFS_STAT_CURR] -= bitmap_info->bytes; in free_bitmap()
1976 unlink_free_space(ctl, bitmap_info); in free_bitmap()
1979 ctl->total_bitmaps--; in free_bitmap()
1980 recalculate_thresholds(ctl); in free_bitmap()
1983 static noinline int remove_from_bitmap(struct btrfs_free_space_ctl *ctl, in remove_from_bitmap() argument
1992 end = bitmap_info->offset + (u64)(BITS_PER_BITMAP * ctl->unit) - 1; in remove_from_bitmap()
2001 search_bytes = ctl->unit; in remove_from_bitmap()
2003 ret = search_bitmap(ctl, bitmap_info, &search_start, &search_bytes, in remove_from_bitmap()
2014 bitmap_clear_bits(ctl, bitmap_info, search_start, search_bytes); in remove_from_bitmap()
2021 free_bitmap(ctl, bitmap_info); in remove_from_bitmap()
2047 search_bytes = ctl->unit; in remove_from_bitmap()
2048 ret = search_bitmap(ctl, bitmap_info, &search_start, in remove_from_bitmap()
2055 free_bitmap(ctl, bitmap_info); in remove_from_bitmap()
2060 static u64 add_bytes_to_bitmap(struct btrfs_free_space_ctl *ctl, in add_bytes_to_bitmap() argument
2073 ctl->discardable_extents[BTRFS_STAT_CURR] += in add_bytes_to_bitmap()
2075 ctl->discardable_bytes[BTRFS_STAT_CURR] += info->bytes; in add_bytes_to_bitmap()
2080 end = info->offset + (u64)(BITS_PER_BITMAP * ctl->unit); in add_bytes_to_bitmap()
2084 bitmap_set_bits(ctl, info, offset, bytes_to_set); in add_bytes_to_bitmap()
2096 static bool use_bitmap(struct btrfs_free_space_ctl *ctl, in use_bitmap() argument
2099 struct btrfs_block_group *block_group = ctl->private; in use_bitmap()
2116 if (!forced && ctl->free_extents < ctl->extents_thresh) { in use_bitmap()
2125 if (ctl->free_extents * 3 <= ctl->extents_thresh) in use_bitmap()
2140 if (((BITS_PER_BITMAP * ctl->unit) >> 1) > block_group->length) in use_bitmap()
2150 static int insert_into_bitmap(struct btrfs_free_space_ctl *ctl, in insert_into_bitmap() argument
2164 if (!ctl->op->use_bitmap(ctl, info)) in insert_into_bitmap()
2167 if (ctl->op == &free_space_op) in insert_into_bitmap()
2168 block_group = ctl->private; in insert_into_bitmap()
2196 if (entry->offset == offset_to_bitmap(ctl, offset)) { in insert_into_bitmap()
2197 bytes_added = add_bytes_to_bitmap(ctl, entry, offset, in insert_into_bitmap()
2210 bitmap_info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in insert_into_bitmap()
2217 bytes_added = add_bytes_to_bitmap(ctl, bitmap_info, offset, bytes, in insert_into_bitmap()
2231 add_new_bitmap(ctl, info, offset); in insert_into_bitmap()
2236 spin_unlock(&ctl->tree_lock); in insert_into_bitmap()
2243 spin_lock(&ctl->tree_lock); in insert_into_bitmap()
2253 spin_lock(&ctl->tree_lock); in insert_into_bitmap()
2288 static bool try_merge_free_space(struct btrfs_free_space_ctl *ctl, in try_merge_free_space() argument
2303 right_info = tree_search_offset(ctl, offset + bytes, 0, 0); in try_merge_free_space()
2308 left_info = tree_search_offset(ctl, offset - 1, 0, 0); in try_merge_free_space()
2314 unlink_free_space(ctl, right_info); in try_merge_free_space()
2316 __unlink_free_space(ctl, right_info); in try_merge_free_space()
2327 unlink_free_space(ctl, left_info); in try_merge_free_space()
2329 __unlink_free_space(ctl, left_info); in try_merge_free_space()
2339 static bool steal_from_bitmap_to_end(struct btrfs_free_space_ctl *ctl, in steal_from_bitmap_to_end() argument
2347 const u64 bitmap_offset = offset_to_bitmap(ctl, end); in steal_from_bitmap_to_end()
2350 bitmap = tree_search_offset(ctl, bitmap_offset, 1, 0); in steal_from_bitmap_to_end()
2354 i = offset_to_bit(bitmap->offset, ctl->unit, end); in steal_from_bitmap_to_end()
2358 bytes = (j - i) * ctl->unit; in steal_from_bitmap_to_end()
2366 bitmap_clear_bits(ctl, bitmap, end, bytes); in steal_from_bitmap_to_end()
2368 __bitmap_clear_bits(ctl, bitmap, end, bytes); in steal_from_bitmap_to_end()
2371 free_bitmap(ctl, bitmap); in steal_from_bitmap_to_end()
2376 static bool steal_from_bitmap_to_front(struct btrfs_free_space_ctl *ctl, in steal_from_bitmap_to_front() argument
2387 bitmap_offset = offset_to_bitmap(ctl, info->offset); in steal_from_bitmap_to_front()
2392 bitmap_offset = offset_to_bitmap(ctl, info->offset - 1); in steal_from_bitmap_to_front()
2395 bitmap = tree_search_offset(ctl, bitmap_offset, 1, 0); in steal_from_bitmap_to_front()
2399 i = offset_to_bit(bitmap->offset, ctl->unit, info->offset) - 1; in steal_from_bitmap_to_front()
2411 bytes = (i + 1) * ctl->unit; in steal_from_bitmap_to_front()
2413 bytes = (i - prev_j) * ctl->unit; in steal_from_bitmap_to_front()
2423 bitmap_clear_bits(ctl, bitmap, info->offset, bytes); in steal_from_bitmap_to_front()
2425 __bitmap_clear_bits(ctl, bitmap, info->offset, bytes); in steal_from_bitmap_to_front()
2428 free_bitmap(ctl, bitmap); in steal_from_bitmap_to_front()
2444 static void steal_from_bitmap(struct btrfs_free_space_ctl *ctl, in steal_from_bitmap() argument
2455 if (ctl->total_bitmaps > 0) { in steal_from_bitmap()
2459 stole_end = steal_from_bitmap_to_end(ctl, info, update_stat); in steal_from_bitmap()
2460 if (ctl->total_bitmaps > 0) in steal_from_bitmap()
2461 stole_front = steal_from_bitmap_to_front(ctl, info, in steal_from_bitmap()
2465 try_merge_free_space(ctl, info, update_stat); in steal_from_bitmap()
2470 struct btrfs_free_space_ctl *ctl, in __btrfs_add_free_space() argument
2474 struct btrfs_block_group *block_group = ctl->private; in __btrfs_add_free_space()
2490 spin_lock(&ctl->tree_lock); in __btrfs_add_free_space()
2492 if (try_merge_free_space(ctl, info, true)) in __btrfs_add_free_space()
2500 ret = insert_into_bitmap(ctl, info); in __btrfs_add_free_space()
2514 steal_from_bitmap(ctl, info, true); in __btrfs_add_free_space()
2518 ret = link_free_space(ctl, info); in __btrfs_add_free_space()
2523 spin_unlock(&ctl->tree_lock); in __btrfs_add_free_space()
2542 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in __btrfs_add_free_space_zoned() local
2551 spin_lock(&ctl->tree_lock); in __btrfs_add_free_space_zoned()
2564 ctl->free_space += to_free; in __btrfs_add_free_space_zoned()
2571 spin_unlock(&ctl->tree_lock); in __btrfs_add_free_space_zoned()
2646 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_remove_free_space() local
2671 spin_lock(&ctl->tree_lock); in btrfs_remove_free_space()
2678 info = tree_search_offset(ctl, offset, 0, 0); in btrfs_remove_free_space()
2684 info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in btrfs_remove_free_space()
2699 unlink_free_space(ctl, info); in btrfs_remove_free_space()
2706 ret = link_free_space(ctl, info); in btrfs_remove_free_space()
2719 ret = link_free_space(ctl, info); in btrfs_remove_free_space()
2733 spin_unlock(&ctl->tree_lock); in btrfs_remove_free_space()
2735 ret = __btrfs_add_free_space(block_group->fs_info, ctl, in btrfs_remove_free_space()
2744 ret = remove_from_bitmap(ctl, info, &offset, &bytes); in btrfs_remove_free_space()
2751 spin_unlock(&ctl->tree_lock); in btrfs_remove_free_space()
2760 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_dump_free_space() local
2776 spin_lock(&ctl->tree_lock); in btrfs_dump_free_space()
2777 for (n = rb_first(&ctl->free_space_offset); n; n = rb_next(n)) { in btrfs_dump_free_space()
2785 spin_unlock(&ctl->tree_lock); in btrfs_dump_free_space()
2793 struct btrfs_free_space_ctl *ctl) in btrfs_init_free_space_ctl() argument
2797 spin_lock_init(&ctl->tree_lock); in btrfs_init_free_space_ctl()
2798 ctl->unit = fs_info->sectorsize; in btrfs_init_free_space_ctl()
2799 ctl->start = block_group->start; in btrfs_init_free_space_ctl()
2800 ctl->private = block_group; in btrfs_init_free_space_ctl()
2801 ctl->op = &free_space_op; in btrfs_init_free_space_ctl()
2802 INIT_LIST_HEAD(&ctl->trimming_ranges); in btrfs_init_free_space_ctl()
2803 mutex_init(&ctl->cache_writeout_mutex); in btrfs_init_free_space_ctl()
2810 ctl->extents_thresh = (SZ_32K / 2) / sizeof(struct btrfs_free_space); in btrfs_init_free_space_ctl()
2823 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in __btrfs_return_cluster_to_free_space() local
2850 ctl->discardable_extents[BTRFS_STAT_CURR]--; in __btrfs_return_cluster_to_free_space()
2851 ctl->discardable_bytes[BTRFS_STAT_CURR] -= in __btrfs_return_cluster_to_free_space()
2855 try_merge_free_space(ctl, entry, false); in __btrfs_return_cluster_to_free_space()
2856 steal_from_bitmap(ctl, entry, false); in __btrfs_return_cluster_to_free_space()
2860 ctl->discardable_extents[BTRFS_STAT_CURR]++; in __btrfs_return_cluster_to_free_space()
2861 ctl->discardable_bytes[BTRFS_STAT_CURR] += in __btrfs_return_cluster_to_free_space()
2865 tree_insert_offset(&ctl->free_space_offset, in __btrfs_return_cluster_to_free_space()
2874 struct btrfs_free_space_ctl *ctl) in __btrfs_remove_free_space_cache_locked() argument
2879 while ((node = rb_last(&ctl->free_space_offset)) != NULL) { in __btrfs_remove_free_space_cache_locked()
2882 unlink_free_space(ctl, info); in __btrfs_remove_free_space_cache_locked()
2885 free_bitmap(ctl, info); in __btrfs_remove_free_space_cache_locked()
2888 cond_resched_lock(&ctl->tree_lock); in __btrfs_remove_free_space_cache_locked()
2892 void __btrfs_remove_free_space_cache(struct btrfs_free_space_ctl *ctl) in __btrfs_remove_free_space_cache() argument
2894 spin_lock(&ctl->tree_lock); in __btrfs_remove_free_space_cache()
2895 __btrfs_remove_free_space_cache_locked(ctl); in __btrfs_remove_free_space_cache()
2896 if (ctl->private) in __btrfs_remove_free_space_cache()
2897 btrfs_discard_update_discardable(ctl->private); in __btrfs_remove_free_space_cache()
2898 spin_unlock(&ctl->tree_lock); in __btrfs_remove_free_space_cache()
2903 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_remove_free_space_cache() local
2907 spin_lock(&ctl->tree_lock); in btrfs_remove_free_space_cache()
2916 cond_resched_lock(&ctl->tree_lock); in btrfs_remove_free_space_cache()
2918 __btrfs_remove_free_space_cache_locked(ctl); in btrfs_remove_free_space_cache()
2920 spin_unlock(&ctl->tree_lock); in btrfs_remove_free_space_cache()
2932 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_is_free_space_trimmed() local
2937 spin_lock(&ctl->tree_lock); in btrfs_is_free_space_trimmed()
2938 node = rb_first(&ctl->free_space_offset); in btrfs_is_free_space_trimmed()
2951 spin_unlock(&ctl->tree_lock); in btrfs_is_free_space_trimmed()
2959 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_find_space_for_alloc() local
2971 spin_lock(&ctl->tree_lock); in btrfs_find_space_for_alloc()
2972 entry = find_free_space(ctl, &offset, &bytes_search, in btrfs_find_space_for_alloc()
2979 bitmap_clear_bits(ctl, entry, offset, bytes); in btrfs_find_space_for_alloc()
2985 free_bitmap(ctl, entry); in btrfs_find_space_for_alloc()
2987 unlink_free_space(ctl, entry); in btrfs_find_space_for_alloc()
3002 link_free_space(ctl, entry); in btrfs_find_space_for_alloc()
3006 spin_unlock(&ctl->tree_lock); in btrfs_find_space_for_alloc()
3009 __btrfs_add_free_space(block_group->fs_info, ctl, in btrfs_find_space_for_alloc()
3027 struct btrfs_free_space_ctl *ctl; in btrfs_return_cluster_to_free_space() local
3045 ctl = block_group->free_space_ctl; in btrfs_return_cluster_to_free_space()
3048 spin_lock(&ctl->tree_lock); in btrfs_return_cluster_to_free_space()
3050 spin_unlock(&ctl->tree_lock); in btrfs_return_cluster_to_free_space()
3064 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_alloc_from_bitmap() local
3073 err = search_bitmap(ctl, entry, &search_start, &search_bytes, true); in btrfs_alloc_from_bitmap()
3081 __bitmap_clear_bits(ctl, entry, ret, bytes); in btrfs_alloc_from_bitmap()
3095 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_alloc_from_cluster() local
3160 spin_lock(&ctl->tree_lock); in btrfs_alloc_from_cluster()
3165 ctl->free_space -= bytes; in btrfs_alloc_from_cluster()
3167 ctl->discardable_bytes[BTRFS_STAT_CURR] -= bytes; in btrfs_alloc_from_cluster()
3172 ctl->free_extents--; in btrfs_alloc_from_cluster()
3176 ctl->total_bitmaps--; in btrfs_alloc_from_cluster()
3177 recalculate_thresholds(ctl); in btrfs_alloc_from_cluster()
3179 ctl->discardable_extents[BTRFS_STAT_CURR]--; in btrfs_alloc_from_cluster()
3185 spin_unlock(&ctl->tree_lock); in btrfs_alloc_from_cluster()
3196 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_bitmap_cluster() local
3207 i = offset_to_bit(entry->offset, ctl->unit, in btrfs_bitmap_cluster()
3209 want_bits = bytes_to_bits(bytes, ctl->unit); in btrfs_bitmap_cluster()
3210 min_bits = bytes_to_bits(min_bytes, ctl->unit); in btrfs_bitmap_cluster()
3236 entry->max_extent_size = (u64)max_bits * ctl->unit; in btrfs_bitmap_cluster()
3247 if (cluster->max_size < found_bits * ctl->unit) in btrfs_bitmap_cluster()
3248 cluster->max_size = found_bits * ctl->unit; in btrfs_bitmap_cluster()
3255 cluster->window_start = start * ctl->unit + entry->offset; in btrfs_bitmap_cluster()
3256 rb_erase(&entry->offset_index, &ctl->free_space_offset); in btrfs_bitmap_cluster()
3262 total_found * ctl->unit, 1); in btrfs_bitmap_cluster()
3277 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in setup_cluster_no_bitmap() local
3286 entry = tree_search_offset(ctl, offset, 0, 1); in setup_cluster_no_bitmap()
3346 rb_erase(&entry->offset_index, &ctl->free_space_offset); in setup_cluster_no_bitmap()
3368 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in setup_cluster_bitmap() local
3371 u64 bitmap_offset = offset_to_bitmap(ctl, offset); in setup_cluster_bitmap()
3373 if (ctl->total_bitmaps == 0) in setup_cluster_bitmap()
3384 entry = tree_search_offset(ctl, bitmap_offset, 1, 0); in setup_cluster_bitmap()
3418 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_find_space_cluster() local
3441 spin_lock(&ctl->tree_lock); in btrfs_find_space_cluster()
3447 if (ctl->free_space < bytes) { in btrfs_find_space_cluster()
3448 spin_unlock(&ctl->tree_lock); in btrfs_find_space_cluster()
3485 spin_unlock(&ctl->tree_lock); in btrfs_find_space_cluster()
3512 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in do_trimming() local
3536 mutex_lock(&ctl->cache_writeout_mutex); in do_trimming()
3538 __btrfs_add_free_space(fs_info, ctl, reserved_start, in do_trimming()
3542 __btrfs_add_free_space(fs_info, ctl, end, reserved_end - end, in do_trimming()
3544 __btrfs_add_free_space(fs_info, ctl, start, bytes, trim_state); in do_trimming()
3546 mutex_unlock(&ctl->cache_writeout_mutex); in do_trimming()
3571 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in trim_no_bitmap() local
3584 mutex_lock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3585 spin_lock(&ctl->tree_lock); in trim_no_bitmap()
3587 if (ctl->free_space < minlen) in trim_no_bitmap()
3590 entry = tree_search_offset(ctl, start, 0, 1); in trim_no_bitmap()
3614 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3615 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3618 unlink_free_space(ctl, entry); in trim_no_bitmap()
3631 link_free_space(ctl, entry); in trim_no_bitmap()
3639 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3640 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3644 unlink_free_space(ctl, entry); in trim_no_bitmap()
3648 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3651 list_add_tail(&trim_entry.list, &ctl->trimming_ranges); in trim_no_bitmap()
3652 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3679 spin_unlock(&ctl->tree_lock); in trim_no_bitmap()
3680 mutex_unlock(&ctl->cache_writeout_mutex); in trim_no_bitmap()
3699 static void reset_trimming_bitmap(struct btrfs_free_space_ctl *ctl, u64 offset) in reset_trimming_bitmap() argument
3703 spin_lock(&ctl->tree_lock); in reset_trimming_bitmap()
3704 entry = tree_search_offset(ctl, offset, 1, 0); in reset_trimming_bitmap()
3707 ctl->discardable_extents[BTRFS_STAT_CURR] += in reset_trimming_bitmap()
3709 ctl->discardable_bytes[BTRFS_STAT_CURR] += entry->bytes; in reset_trimming_bitmap()
3714 spin_unlock(&ctl->tree_lock); in reset_trimming_bitmap()
3717 static void end_trimming_bitmap(struct btrfs_free_space_ctl *ctl, in end_trimming_bitmap() argument
3722 ctl->discardable_extents[BTRFS_STAT_CURR] -= in end_trimming_bitmap()
3724 ctl->discardable_bytes[BTRFS_STAT_CURR] -= entry->bytes; in end_trimming_bitmap()
3737 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in trim_bitmaps() local
3742 u64 offset = offset_to_bitmap(ctl, start); in trim_bitmaps()
3749 mutex_lock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3750 spin_lock(&ctl->tree_lock); in trim_bitmaps()
3752 if (ctl->free_space < minlen) { in trim_bitmaps()
3755 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3756 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3760 entry = tree_search_offset(ctl, offset, 1, 0); in trim_bitmaps()
3771 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3772 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3787 ret2 = search_bitmap(ctl, entry, &start, &bytes, false); in trim_bitmaps()
3794 end_trimming_bitmap(ctl, entry); in trim_bitmaps()
3797 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3798 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3808 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3809 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3815 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3816 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3831 bitmap_clear_bits(ctl, entry, start, bytes); in trim_bitmaps()
3833 free_bitmap(ctl, entry); in trim_bitmaps()
3835 spin_unlock(&ctl->tree_lock); in trim_bitmaps()
3838 list_add_tail(&trim_entry.list, &ctl->trimming_ranges); in trim_bitmaps()
3839 mutex_unlock(&ctl->cache_writeout_mutex); in trim_bitmaps()
3844 reset_trimming_bitmap(ctl, offset); in trim_bitmaps()
3851 offset += BITS_PER_BITMAP * ctl->unit; in trim_bitmaps()
3860 reset_trimming_bitmap(ctl, offset); in trim_bitmaps()
3878 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in btrfs_trim_block_group() local
3899 div64_u64_rem(end, BITS_PER_BITMAP * ctl->unit, &rem); in btrfs_trim_block_group()
3902 reset_trimming_bitmap(ctl, offset_to_bitmap(ctl, end)); in btrfs_trim_block_group()
4024 struct btrfs_free_space_ctl *ctl = cache->free_space_ctl; in test_add_free_space_entry() local
4039 spin_lock(&ctl->tree_lock); in test_add_free_space_entry()
4043 ret = link_free_space(ctl, info); in test_add_free_space_entry()
4044 spin_unlock(&ctl->tree_lock); in test_add_free_space_entry()
4058 spin_lock(&ctl->tree_lock); in test_add_free_space_entry()
4059 bitmap_info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in test_add_free_space_entry()
4064 add_new_bitmap(ctl, info, offset); in test_add_free_space_entry()
4069 bytes_added = add_bytes_to_bitmap(ctl, bitmap_info, offset, bytes, in test_add_free_space_entry()
4074 spin_unlock(&ctl->tree_lock); in test_add_free_space_entry()
4094 struct btrfs_free_space_ctl *ctl = cache->free_space_ctl; in test_check_exists() local
4098 spin_lock(&ctl->tree_lock); in test_check_exists()
4099 info = tree_search_offset(ctl, offset, 0, 0); in test_check_exists()
4101 info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), in test_check_exists()
4114 bit_bytes = ctl->unit; in test_check_exists()
4115 ret = search_bitmap(ctl, info, &bit_off, &bit_bytes, false); in test_check_exists()
4167 spin_unlock(&ctl->tree_lock); in test_check_exists()