Lines Matching refs:folio
852 struct folio *fold = page_folio(old); in replace_page_cache_page()
853 struct folio *fnew = page_folio(new); in replace_page_cache_page()
890 struct folio *folio, pgoff_t index, gfp_t gfp, void **shadowp) in __filemap_add_folio() argument
893 int huge = folio_test_hugetlb(folio); in __filemap_add_folio()
897 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); in __filemap_add_folio()
898 VM_BUG_ON_FOLIO(folio_test_swapbacked(folio), folio); in __filemap_add_folio()
901 folio_get(folio); in __filemap_add_folio()
902 folio->mapping = mapping; in __filemap_add_folio()
903 folio->index = index; in __filemap_add_folio()
906 error = mem_cgroup_charge(folio, NULL, gfp); in __filemap_add_folio()
907 VM_BUG_ON_FOLIO(index & (folio_nr_pages(folio) - 1), folio); in __filemap_add_folio()
919 if (order > folio_order(folio)) in __filemap_add_folio()
936 if (order > folio_order(folio)) { in __filemap_add_folio()
942 xas_store(&xas, folio); in __filemap_add_folio()
950 __lruvec_stat_add_folio(folio, NR_FILE_PAGES); in __filemap_add_folio()
958 mem_cgroup_uncharge(folio); in __filemap_add_folio()
962 trace_mm_filemap_add_to_page_cache(&folio->page); in __filemap_add_folio()
965 folio->mapping = NULL; in __filemap_add_folio()
967 folio_put(folio); in __filemap_add_folio()
992 int filemap_add_folio(struct address_space *mapping, struct folio *folio, in filemap_add_folio() argument
998 __folio_set_locked(folio); in filemap_add_folio()
999 ret = __filemap_add_folio(mapping, folio, index, gfp, &shadow); in filemap_add_folio()
1001 __folio_clear_locked(folio); in filemap_add_folio()
1011 WARN_ON_ONCE(folio_test_active(folio)); in filemap_add_folio()
1013 workingset_refault(folio, shadow); in filemap_add_folio()
1014 folio_add_lru(folio); in filemap_add_folio()
1021 struct folio *filemap_alloc_folio(gfp_t gfp, unsigned int order) in filemap_alloc_folio()
1024 struct folio *folio; in filemap_alloc_folio() local
1031 folio = __folio_alloc_node(gfp, order, n); in filemap_alloc_folio()
1032 } while (!folio && read_mems_allowed_retry(cpuset_mems_cookie)); in filemap_alloc_folio()
1034 return folio; in filemap_alloc_folio()
1093 static wait_queue_head_t *folio_waitqueue(struct folio *folio) in folio_waitqueue() argument
1095 return &folio_wait_table[hash_ptr(folio, PAGE_WAIT_TABLE_BITS)]; in folio_waitqueue()
1158 if (test_bit(key->bit_nr, &key->folio->flags)) in wake_page_function()
1161 if (test_and_set_bit(key->bit_nr, &key->folio->flags)) in wake_page_function()
1193 static void folio_wake_bit(struct folio *folio, int bit_nr) in folio_wake_bit() argument
1195 wait_queue_head_t *q = folio_waitqueue(folio); in folio_wake_bit()
1200 key.folio = folio; in folio_wake_bit()
1235 folio_clear_waiters(folio); in folio_wake_bit()
1247 static void folio_wake(struct folio *folio, int bit) in folio_wake() argument
1249 if (!folio_test_waiters(folio)) in folio_wake()
1251 folio_wake_bit(folio, bit); in folio_wake()
1273 static inline bool folio_trylock_flag(struct folio *folio, int bit_nr, in folio_trylock_flag() argument
1277 if (test_and_set_bit(bit_nr, &folio->flags)) in folio_trylock_flag()
1279 } else if (test_bit(bit_nr, &folio->flags)) in folio_trylock_flag()
1289 static inline int folio_wait_bit_common(struct folio *folio, int bit_nr, in folio_wait_bit_common() argument
1292 wait_queue_head_t *q = folio_waitqueue(folio); in folio_wait_bit_common()
1301 !folio_test_uptodate(folio) && folio_test_workingset(folio)) { in folio_wait_bit_common()
1302 if (!folio_test_swapbacked(folio)) { in folio_wait_bit_common()
1312 wait_page.folio = folio; in folio_wait_bit_common()
1338 folio_set_waiters(folio); in folio_wait_bit_common()
1339 if (!folio_trylock_flag(folio, bit_nr, wait)) in folio_wait_bit_common()
1352 folio_put(folio); in folio_wait_bit_common()
1389 if (unlikely(test_and_set_bit(bit_nr, folio_flags(folio, 0)))) in folio_wait_bit_common()
1429 void folio_wait_bit(struct folio *folio, int bit_nr) in folio_wait_bit() argument
1431 folio_wait_bit_common(folio, bit_nr, TASK_UNINTERRUPTIBLE, SHARED); in folio_wait_bit()
1435 int folio_wait_bit_killable(struct folio *folio, int bit_nr) in folio_wait_bit_killable() argument
1437 return folio_wait_bit_common(folio, bit_nr, TASK_KILLABLE, SHARED); in folio_wait_bit_killable()
1467 void folio_add_wait_queue(struct folio *folio, wait_queue_entry_t *waiter) in folio_add_wait_queue() argument
1469 wait_queue_head_t *q = folio_waitqueue(folio); in folio_add_wait_queue()
1474 folio_set_waiters(folio); in folio_add_wait_queue()
1511 void folio_unlock(struct folio *folio) in folio_unlock() argument
1516 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); in folio_unlock()
1517 if (clear_bit_unlock_is_negative_byte(PG_locked, folio_flags(folio, 0))) in folio_unlock()
1518 folio_wake_bit(folio, PG_locked); in folio_unlock()
1533 void folio_end_private_2(struct folio *folio) in folio_end_private_2() argument
1535 VM_BUG_ON_FOLIO(!folio_test_private_2(folio), folio); in folio_end_private_2()
1536 clear_bit_unlock(PG_private_2, folio_flags(folio, 0)); in folio_end_private_2()
1537 folio_wake_bit(folio, PG_private_2); in folio_end_private_2()
1538 folio_put(folio); in folio_end_private_2()
1548 void folio_wait_private_2(struct folio *folio) in folio_wait_private_2() argument
1550 while (folio_test_private_2(folio)) in folio_wait_private_2()
1551 folio_wait_bit(folio, PG_private_2); in folio_wait_private_2()
1566 int folio_wait_private_2_killable(struct folio *folio) in folio_wait_private_2_killable() argument
1570 while (folio_test_private_2(folio)) { in folio_wait_private_2_killable()
1571 ret = folio_wait_bit_killable(folio, PG_private_2); in folio_wait_private_2_killable()
1584 void folio_end_writeback(struct folio *folio) in folio_end_writeback() argument
1593 if (folio_test_reclaim(folio)) { in folio_end_writeback()
1594 folio_clear_reclaim(folio); in folio_end_writeback()
1595 folio_rotate_reclaimable(folio); in folio_end_writeback()
1604 folio_get(folio); in folio_end_writeback()
1605 if (!__folio_end_writeback(folio)) in folio_end_writeback()
1609 folio_wake(folio, PG_writeback); in folio_end_writeback()
1610 acct_reclaim_writeback(folio); in folio_end_writeback()
1611 folio_put(folio); in folio_end_writeback()
1647 void __folio_lock(struct folio *folio) in __folio_lock() argument
1649 folio_wait_bit_common(folio, PG_locked, TASK_UNINTERRUPTIBLE, in __folio_lock()
1654 int __folio_lock_killable(struct folio *folio) in __folio_lock_killable() argument
1656 return folio_wait_bit_common(folio, PG_locked, TASK_KILLABLE, in __folio_lock_killable()
1661 static int __folio_lock_async(struct folio *folio, struct wait_page_queue *wait) in __folio_lock_async() argument
1663 struct wait_queue_head *q = folio_waitqueue(folio); in __folio_lock_async()
1666 wait->folio = folio; in __folio_lock_async()
1671 folio_set_waiters(folio); in __folio_lock_async()
1672 ret = !folio_trylock(folio); in __folio_lock_async()
1698 bool __folio_lock_or_retry(struct folio *folio, struct mm_struct *mm, in __folio_lock_or_retry() argument
1711 folio_wait_locked_killable(folio); in __folio_lock_or_retry()
1713 folio_wait_locked(folio); in __folio_lock_or_retry()
1719 ret = __folio_lock_killable(folio); in __folio_lock_or_retry()
1725 __folio_lock(folio); in __folio_lock_or_retry()
1838 struct folio *folio; in mapping_get_entry() local
1843 folio = xas_load(&xas); in mapping_get_entry()
1844 if (xas_retry(&xas, folio)) in mapping_get_entry()
1850 if (!folio || xa_is_value(folio)) in mapping_get_entry()
1853 if (!folio_try_get_rcu(folio)) in mapping_get_entry()
1856 if (unlikely(folio != xas_reload(&xas))) { in mapping_get_entry()
1857 folio_put(folio); in mapping_get_entry()
1863 return folio; in mapping_get_entry()
1899 struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index, in __filemap_get_folio()
1902 struct folio *folio; in __filemap_get_folio() local
1905 folio = mapping_get_entry(mapping, index); in __filemap_get_folio()
1906 if (xa_is_value(folio)) { in __filemap_get_folio()
1908 return folio; in __filemap_get_folio()
1909 folio = NULL; in __filemap_get_folio()
1911 if (!folio) in __filemap_get_folio()
1916 if (!folio_trylock(folio)) { in __filemap_get_folio()
1917 folio_put(folio); in __filemap_get_folio()
1921 folio_lock(folio); in __filemap_get_folio()
1925 if (unlikely(folio->mapping != mapping)) { in __filemap_get_folio()
1926 folio_unlock(folio); in __filemap_get_folio()
1927 folio_put(folio); in __filemap_get_folio()
1930 VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio); in __filemap_get_folio()
1934 folio_mark_accessed(folio); in __filemap_get_folio()
1937 if (folio_test_idle(folio)) in __filemap_get_folio()
1938 folio_clear_idle(folio); in __filemap_get_folio()
1942 folio_wait_stable(folio); in __filemap_get_folio()
1944 if (!folio && (fgp_flags & FGP_CREAT)) { in __filemap_get_folio()
1951 folio = filemap_alloc_folio(gfp, 0); in __filemap_get_folio()
1952 if (!folio) in __filemap_get_folio()
1960 __folio_set_referenced(folio); in __filemap_get_folio()
1962 err = filemap_add_folio(mapping, folio, index, gfp); in __filemap_get_folio()
1964 folio_put(folio); in __filemap_get_folio()
1965 folio = NULL; in __filemap_get_folio()
1974 if (folio && (fgp_flags & FGP_FOR_MMAP)) in __filemap_get_folio()
1975 folio_unlock(folio); in __filemap_get_folio()
1978 return folio; in __filemap_get_folio()
2434 struct folio *folio = page_folio(page); in filemap_update_page() local
2444 if (!folio_trylock(folio)) { in filemap_update_page()
2450 put_and_wait_on_page_locked(&folio->page, TASK_KILLABLE); in filemap_update_page()
2453 error = __folio_lock_async(folio, iocb->ki_waitq); in filemap_update_page()
2459 if (!folio->mapping) in filemap_update_page()
2463 if (filemap_range_uptodate(mapping, iocb->ki_pos, iter, &folio->page)) in filemap_update_page()
2470 error = filemap_read_page(iocb->ki_filp, mapping, &folio->page); in filemap_update_page()
2473 folio_unlock(folio); in filemap_update_page()
2477 folio_put(folio); in filemap_update_page()
2916 struct folio *folio = page_folio(page); in lock_page_maybe_drop_mmap() local
2918 if (folio_trylock(folio)) in lock_page_maybe_drop_mmap()
2931 if (__folio_lock_killable(folio)) { in lock_page_maybe_drop_mmap()
2943 __folio_lock(folio); in lock_page_maybe_drop_mmap()