Lines Matching refs:inode
70 static inline struct inode *wb_inode(struct list_head *head) in wb_inode()
72 return list_entry(head, struct inode, i_io_list); in wb_inode()
118 static bool inode_io_list_move_locked(struct inode *inode, in inode_io_list_move_locked() argument
124 list_move(&inode->i_io_list, head); in inode_io_list_move_locked()
233 / sizeof(struct inode *))
238 void __inode_attach_wb(struct inode *inode, struct page *page) in __inode_attach_wb() argument
240 struct backing_dev_info *bdi = inode_to_bdi(inode); in __inode_attach_wb()
243 if (inode_cgwb_enabled(inode)) { in __inode_attach_wb()
264 if (unlikely(cmpxchg(&inode->i_wb, NULL, wb))) in __inode_attach_wb()
277 static void inode_cgwb_move_to_attached(struct inode *inode, in inode_cgwb_move_to_attached() argument
281 assert_spin_locked(&inode->i_lock); in inode_cgwb_move_to_attached()
283 inode->i_state &= ~I_SYNC_QUEUED; in inode_cgwb_move_to_attached()
285 list_move(&inode->i_io_list, &wb->b_attached); in inode_cgwb_move_to_attached()
287 list_del_init(&inode->i_io_list); in inode_cgwb_move_to_attached()
300 locked_inode_to_wb_and_lock_list(struct inode *inode) in locked_inode_to_wb_and_lock_list() argument
301 __releases(&inode->i_lock) in locked_inode_to_wb_and_lock_list()
305 struct bdi_writeback *wb = inode_to_wb(inode); in locked_inode_to_wb_and_lock_list()
314 spin_unlock(&inode->i_lock); in locked_inode_to_wb_and_lock_list()
318 if (likely(wb == inode->i_wb)) { in locked_inode_to_wb_and_lock_list()
326 spin_lock(&inode->i_lock); in locked_inode_to_wb_and_lock_list()
337 static struct bdi_writeback *inode_to_wb_and_lock_list(struct inode *inode) in inode_to_wb_and_lock_list() argument
340 spin_lock(&inode->i_lock); in inode_to_wb_and_lock_list()
341 return locked_inode_to_wb_and_lock_list(inode); in inode_to_wb_and_lock_list()
356 struct inode *inodes[];
369 static bool inode_do_switch_wbs(struct inode *inode, in inode_do_switch_wbs() argument
373 struct address_space *mapping = inode->i_mapping; in inode_do_switch_wbs()
378 spin_lock(&inode->i_lock); in inode_do_switch_wbs()
385 if (unlikely(inode->i_state & (I_FREEING | I_WILL_FREE))) in inode_do_switch_wbs()
388 trace_inode_switch_wbs(inode, old_wb, new_wb); in inode_do_switch_wbs()
424 if (!list_empty(&inode->i_io_list)) { in inode_do_switch_wbs()
425 inode->i_wb = new_wb; in inode_do_switch_wbs()
427 if (inode->i_state & I_DIRTY_ALL) { in inode_do_switch_wbs()
428 struct inode *pos; in inode_do_switch_wbs()
431 if (time_after_eq(inode->dirtied_when, in inode_do_switch_wbs()
434 inode_io_list_move_locked(inode, new_wb, in inode_do_switch_wbs()
437 inode_cgwb_move_to_attached(inode, new_wb); in inode_do_switch_wbs()
440 inode->i_wb = new_wb; in inode_do_switch_wbs()
444 inode->i_wb_frn_winner = 0; in inode_do_switch_wbs()
445 inode->i_wb_frn_avg_time = 0; in inode_do_switch_wbs()
446 inode->i_wb_frn_history = 0; in inode_do_switch_wbs()
453 smp_store_release(&inode->i_state, inode->i_state & ~I_WB_SWITCH); in inode_do_switch_wbs()
456 spin_unlock(&inode->i_lock); in inode_do_switch_wbs()
469 struct inode **inodep; in inode_switch_wbs_work_fn()
518 static bool inode_prepare_wbs_switch(struct inode *inode, in inode_prepare_wbs_switch() argument
529 if (IS_DAX(inode)) in inode_prepare_wbs_switch()
533 spin_lock(&inode->i_lock); in inode_prepare_wbs_switch()
534 if (!(inode->i_sb->s_flags & SB_ACTIVE) || in inode_prepare_wbs_switch()
535 inode->i_state & (I_WB_SWITCH | I_FREEING | I_WILL_FREE) || in inode_prepare_wbs_switch()
536 inode_to_wb(inode) == new_wb) { in inode_prepare_wbs_switch()
537 spin_unlock(&inode->i_lock); in inode_prepare_wbs_switch()
540 inode->i_state |= I_WB_SWITCH; in inode_prepare_wbs_switch()
541 __iget(inode); in inode_prepare_wbs_switch()
542 spin_unlock(&inode->i_lock); in inode_prepare_wbs_switch()
555 static void inode_switch_wbs(struct inode *inode, int new_wb_id) in inode_switch_wbs() argument
557 struct backing_dev_info *bdi = inode_to_bdi(inode); in inode_switch_wbs()
562 if (inode->i_state & I_WB_SWITCH) in inode_switch_wbs()
589 if (!inode_prepare_wbs_switch(inode, isw->new_wb)) in inode_switch_wbs()
592 isw->inodes[0] = inode; in inode_switch_wbs()
623 struct inode *inode; in cleanup_offline_cgwb() local
645 list_for_each_entry(inode, &wb->b_attached, i_io_list) { in cleanup_offline_cgwb()
646 if (!inode_prepare_wbs_switch(inode, isw->new_wb)) in cleanup_offline_cgwb()
649 isw->inodes[nr++] = inode; in cleanup_offline_cgwb()
689 struct inode *inode) in wbc_attach_and_unlock_inode() argument
691 if (!inode_cgwb_enabled(inode)) { in wbc_attach_and_unlock_inode()
692 spin_unlock(&inode->i_lock); in wbc_attach_and_unlock_inode()
696 wbc->wb = inode_to_wb(inode); in wbc_attach_and_unlock_inode()
697 wbc->inode = inode; in wbc_attach_and_unlock_inode()
700 wbc->wb_lcand_id = inode->i_wb_frn_winner; in wbc_attach_and_unlock_inode()
707 spin_unlock(&inode->i_lock); in wbc_attach_and_unlock_inode()
717 inode_switch_wbs(inode, wbc->wb_id); in wbc_attach_and_unlock_inode()
761 struct inode *inode = wbc->inode; in wbc_detach_inode() local
769 history = inode->i_wb_frn_history; in wbc_detach_inode()
770 avg_time = inode->i_wb_frn_avg_time; in wbc_detach_inode()
818 trace_inode_foreign_history(inode, wbc, history); in wbc_detach_inode()
828 inode_switch_wbs(inode, max_id); in wbc_detach_inode()
835 inode->i_wb_frn_winner = max_id; in wbc_detach_inode()
836 inode->i_wb_frn_avg_time = min(avg_time, (unsigned long)U16_MAX); in wbc_detach_inode()
837 inode->i_wb_frn_history = history; in wbc_detach_inode()
910 int inode_congested(struct inode *inode, int cong_bits) in inode_congested() argument
916 if (inode && inode_to_wb_is_valid(inode)) { in inode_congested()
921 wb = unlocked_inode_to_wb_begin(inode, &lock_cookie); in inode_congested()
923 unlocked_inode_to_wb_end(inode, &lock_cookie); in inode_congested()
927 return wb_congested(&inode_to_bdi(inode)->wb, cong_bits); in inode_congested()
1161 static void inode_cgwb_move_to_attached(struct inode *inode, in inode_cgwb_move_to_attached() argument
1165 assert_spin_locked(&inode->i_lock); in inode_cgwb_move_to_attached()
1167 inode->i_state &= ~I_SYNC_QUEUED; in inode_cgwb_move_to_attached()
1168 list_del_init(&inode->i_io_list); in inode_cgwb_move_to_attached()
1173 locked_inode_to_wb_and_lock_list(struct inode *inode) in locked_inode_to_wb_and_lock_list() argument
1174 __releases(&inode->i_lock) in locked_inode_to_wb_and_lock_list()
1177 struct bdi_writeback *wb = inode_to_wb(inode); in locked_inode_to_wb_and_lock_list()
1179 spin_unlock(&inode->i_lock); in locked_inode_to_wb_and_lock_list()
1184 static struct bdi_writeback *inode_to_wb_and_lock_list(struct inode *inode) in inode_to_wb_and_lock_list() argument
1187 struct bdi_writeback *wb = inode_to_wb(inode); in inode_to_wb_and_lock_list()
1266 void inode_io_list_del(struct inode *inode) in inode_io_list_del() argument
1270 wb = inode_to_wb_and_lock_list(inode); in inode_io_list_del()
1271 spin_lock(&inode->i_lock); in inode_io_list_del()
1273 inode->i_state &= ~I_SYNC_QUEUED; in inode_io_list_del()
1274 list_del_init(&inode->i_io_list); in inode_io_list_del()
1277 spin_unlock(&inode->i_lock); in inode_io_list_del()
1285 void sb_mark_inode_writeback(struct inode *inode) in sb_mark_inode_writeback() argument
1287 struct super_block *sb = inode->i_sb; in sb_mark_inode_writeback()
1290 if (list_empty(&inode->i_wb_list)) { in sb_mark_inode_writeback()
1292 if (list_empty(&inode->i_wb_list)) { in sb_mark_inode_writeback()
1293 list_add_tail(&inode->i_wb_list, &sb->s_inodes_wb); in sb_mark_inode_writeback()
1294 trace_sb_mark_inode_writeback(inode); in sb_mark_inode_writeback()
1303 void sb_clear_inode_writeback(struct inode *inode) in sb_clear_inode_writeback() argument
1305 struct super_block *sb = inode->i_sb; in sb_clear_inode_writeback()
1308 if (!list_empty(&inode->i_wb_list)) { in sb_clear_inode_writeback()
1310 if (!list_empty(&inode->i_wb_list)) { in sb_clear_inode_writeback()
1311 list_del_init(&inode->i_wb_list); in sb_clear_inode_writeback()
1312 trace_sb_clear_inode_writeback(inode); in sb_clear_inode_writeback()
1327 static void redirty_tail_locked(struct inode *inode, struct bdi_writeback *wb) in redirty_tail_locked() argument
1329 assert_spin_locked(&inode->i_lock); in redirty_tail_locked()
1332 struct inode *tail; in redirty_tail_locked()
1335 if (time_before(inode->dirtied_when, tail->dirtied_when)) in redirty_tail_locked()
1336 inode->dirtied_when = jiffies; in redirty_tail_locked()
1338 inode_io_list_move_locked(inode, wb, &wb->b_dirty); in redirty_tail_locked()
1339 inode->i_state &= ~I_SYNC_QUEUED; in redirty_tail_locked()
1342 static void redirty_tail(struct inode *inode, struct bdi_writeback *wb) in redirty_tail() argument
1344 spin_lock(&inode->i_lock); in redirty_tail()
1345 redirty_tail_locked(inode, wb); in redirty_tail()
1346 spin_unlock(&inode->i_lock); in redirty_tail()
1352 static void requeue_io(struct inode *inode, struct bdi_writeback *wb) in requeue_io() argument
1354 inode_io_list_move_locked(inode, wb, &wb->b_more_io); in requeue_io()
1357 static void inode_sync_complete(struct inode *inode) in inode_sync_complete() argument
1359 inode->i_state &= ~I_SYNC; in inode_sync_complete()
1361 inode_add_lru(inode); in inode_sync_complete()
1364 wake_up_bit(&inode->i_state, __I_SYNC); in inode_sync_complete()
1367 static bool inode_dirtied_after(struct inode *inode, unsigned long t) in inode_dirtied_after() argument
1369 bool ret = time_after(inode->dirtied_when, t); in inode_dirtied_after()
1377 ret = ret && time_before_eq(inode->dirtied_when, jiffies); in inode_dirtied_after()
1395 struct inode *inode; in move_expired_inodes() local
1400 inode = wb_inode(delaying_queue->prev); in move_expired_inodes()
1401 if (inode_dirtied_after(inode, dirtied_before)) in move_expired_inodes()
1403 list_move(&inode->i_io_list, &tmp); in move_expired_inodes()
1405 spin_lock(&inode->i_lock); in move_expired_inodes()
1406 inode->i_state |= I_SYNC_QUEUED; in move_expired_inodes()
1407 spin_unlock(&inode->i_lock); in move_expired_inodes()
1408 if (sb_is_blkdev_sb(inode->i_sb)) in move_expired_inodes()
1410 if (sb && sb != inode->i_sb) in move_expired_inodes()
1412 sb = inode->i_sb; in move_expired_inodes()
1425 inode = wb_inode(pos); in move_expired_inodes()
1426 if (inode->i_sb == sb) in move_expired_inodes()
1427 list_move(&inode->i_io_list, dispatch_queue); in move_expired_inodes()
1463 static int write_inode(struct inode *inode, struct writeback_control *wbc) in write_inode() argument
1467 if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode)) { in write_inode()
1468 trace_writeback_write_inode_start(inode, wbc); in write_inode()
1469 ret = inode->i_sb->s_op->write_inode(inode, wbc); in write_inode()
1470 trace_writeback_write_inode(inode, wbc); in write_inode()
1480 static void __inode_wait_for_writeback(struct inode *inode) in __inode_wait_for_writeback() argument
1481 __releases(inode->i_lock) in __inode_wait_for_writeback()
1482 __acquires(inode->i_lock) in __inode_wait_for_writeback()
1484 DEFINE_WAIT_BIT(wq, &inode->i_state, __I_SYNC); in __inode_wait_for_writeback()
1487 wqh = bit_waitqueue(&inode->i_state, __I_SYNC); in __inode_wait_for_writeback()
1488 while (inode->i_state & I_SYNC) { in __inode_wait_for_writeback()
1489 spin_unlock(&inode->i_lock); in __inode_wait_for_writeback()
1492 spin_lock(&inode->i_lock); in __inode_wait_for_writeback()
1499 void inode_wait_for_writeback(struct inode *inode) in inode_wait_for_writeback() argument
1501 spin_lock(&inode->i_lock); in inode_wait_for_writeback()
1502 __inode_wait_for_writeback(inode); in inode_wait_for_writeback()
1503 spin_unlock(&inode->i_lock); in inode_wait_for_writeback()
1511 static void inode_sleep_on_writeback(struct inode *inode) in inode_sleep_on_writeback() argument
1512 __releases(inode->i_lock) in inode_sleep_on_writeback()
1515 wait_queue_head_t *wqh = bit_waitqueue(&inode->i_state, __I_SYNC); in inode_sleep_on_writeback()
1519 sleep = inode->i_state & I_SYNC; in inode_sleep_on_writeback()
1520 spin_unlock(&inode->i_lock); in inode_sleep_on_writeback()
1534 static void requeue_inode(struct inode *inode, struct bdi_writeback *wb, in requeue_inode() argument
1537 if (inode->i_state & I_FREEING) in requeue_inode()
1545 if ((inode->i_state & I_DIRTY) && in requeue_inode()
1547 inode->dirtied_when = jiffies; in requeue_inode()
1554 redirty_tail_locked(inode, wb); in requeue_inode()
1558 if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY)) { in requeue_inode()
1565 requeue_io(inode, wb); in requeue_inode()
1574 redirty_tail_locked(inode, wb); in requeue_inode()
1576 } else if (inode->i_state & I_DIRTY) { in requeue_inode()
1582 redirty_tail_locked(inode, wb); in requeue_inode()
1583 } else if (inode->i_state & I_DIRTY_TIME) { in requeue_inode()
1584 inode->dirtied_when = jiffies; in requeue_inode()
1585 inode_io_list_move_locked(inode, wb, &wb->b_dirty_time); in requeue_inode()
1586 inode->i_state &= ~I_SYNC_QUEUED; in requeue_inode()
1589 inode_cgwb_move_to_attached(inode, wb); in requeue_inode()
1605 __writeback_single_inode(struct inode *inode, struct writeback_control *wbc) in __writeback_single_inode() argument
1607 struct address_space *mapping = inode->i_mapping; in __writeback_single_inode()
1612 WARN_ON(!(inode->i_state & I_SYNC)); in __writeback_single_inode()
1614 trace_writeback_single_inode_start(inode, wbc, nr_to_write); in __writeback_single_inode()
1636 if ((inode->i_state & I_DIRTY_TIME) && in __writeback_single_inode()
1638 time_after(jiffies, inode->dirtied_time_when + in __writeback_single_inode()
1640 trace_writeback_lazytime(inode); in __writeback_single_inode()
1641 mark_inode_dirty_sync(inode); in __writeback_single_inode()
1650 spin_lock(&inode->i_lock); in __writeback_single_inode()
1651 dirty = inode->i_state & I_DIRTY; in __writeback_single_inode()
1652 inode->i_state &= ~dirty; in __writeback_single_inode()
1668 inode->i_state |= I_DIRTY_PAGES; in __writeback_single_inode()
1670 spin_unlock(&inode->i_lock); in __writeback_single_inode()
1674 int err = write_inode(inode, wbc); in __writeback_single_inode()
1678 trace_writeback_single_inode(inode, wbc, nr_to_write); in __writeback_single_inode()
1691 static int writeback_single_inode(struct inode *inode, in writeback_single_inode() argument
1697 spin_lock(&inode->i_lock); in writeback_single_inode()
1698 if (!atomic_read(&inode->i_count)) in writeback_single_inode()
1699 WARN_ON(!(inode->i_state & (I_WILL_FREE|I_FREEING))); in writeback_single_inode()
1701 WARN_ON(inode->i_state & I_WILL_FREE); in writeback_single_inode()
1703 if (inode->i_state & I_SYNC) { in writeback_single_inode()
1712 __inode_wait_for_writeback(inode); in writeback_single_inode()
1714 WARN_ON(inode->i_state & I_SYNC); in writeback_single_inode()
1722 if (!(inode->i_state & I_DIRTY_ALL) && in writeback_single_inode()
1724 !mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK))) in writeback_single_inode()
1726 inode->i_state |= I_SYNC; in writeback_single_inode()
1727 wbc_attach_and_unlock_inode(wbc, inode); in writeback_single_inode()
1729 ret = __writeback_single_inode(inode, wbc); in writeback_single_inode()
1733 wb = inode_to_wb_and_lock_list(inode); in writeback_single_inode()
1734 spin_lock(&inode->i_lock); in writeback_single_inode()
1740 if (!(inode->i_state & I_DIRTY_ALL)) in writeback_single_inode()
1741 inode_cgwb_move_to_attached(inode, wb); in writeback_single_inode()
1743 inode_sync_complete(inode); in writeback_single_inode()
1745 spin_unlock(&inode->i_lock); in writeback_single_inode()
1808 struct inode *inode = wb_inode(wb->b_io.prev); in writeback_sb_inodes() local
1811 if (inode->i_sb != sb) { in writeback_sb_inodes()
1818 redirty_tail(inode, wb); in writeback_sb_inodes()
1835 spin_lock(&inode->i_lock); in writeback_sb_inodes()
1836 if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) { in writeback_sb_inodes()
1837 redirty_tail_locked(inode, wb); in writeback_sb_inodes()
1838 spin_unlock(&inode->i_lock); in writeback_sb_inodes()
1841 if ((inode->i_state & I_SYNC) && wbc.sync_mode != WB_SYNC_ALL) { in writeback_sb_inodes()
1851 spin_unlock(&inode->i_lock); in writeback_sb_inodes()
1852 requeue_io(inode, wb); in writeback_sb_inodes()
1853 trace_writeback_sb_inodes_requeue(inode); in writeback_sb_inodes()
1863 if (inode->i_state & I_SYNC) { in writeback_sb_inodes()
1865 inode_sleep_on_writeback(inode); in writeback_sb_inodes()
1870 inode->i_state |= I_SYNC; in writeback_sb_inodes()
1871 wbc_attach_and_unlock_inode(&wbc, inode); in writeback_sb_inodes()
1881 __writeback_single_inode(inode, &wbc); in writeback_sb_inodes()
1905 tmp_wb = inode_to_wb_and_lock_list(inode); in writeback_sb_inodes()
1906 spin_lock(&inode->i_lock); in writeback_sb_inodes()
1907 if (!(inode->i_state & I_DIRTY_ALL)) in writeback_sb_inodes()
1909 requeue_inode(inode, tmp_wb, &wbc); in writeback_sb_inodes()
1910 inode_sync_complete(inode); in writeback_sb_inodes()
1911 spin_unlock(&inode->i_lock); in writeback_sb_inodes()
1939 struct inode *inode = wb_inode(wb->b_io.prev); in __writeback_inodes_wb() local
1940 struct super_block *sb = inode->i_sb; in __writeback_inodes_wb()
1948 redirty_tail(inode, wb); in __writeback_inodes_wb()
2008 struct inode *inode; in wb_writeback() local
2080 inode = wb_inode(wb->b_more_io.prev); in wb_writeback()
2081 spin_lock(&inode->i_lock); in wb_writeback()
2084 inode_sleep_on_writeback(inode); in wb_writeback()
2382 void __mark_inode_dirty(struct inode *inode, int flags) in __mark_inode_dirty() argument
2384 struct super_block *sb = inode->i_sb; in __mark_inode_dirty()
2387 trace_writeback_mark_inode_dirty(inode, flags); in __mark_inode_dirty()
2397 trace_writeback_dirty_inode_start(inode, flags); in __mark_inode_dirty()
2399 sb->s_op->dirty_inode(inode, flags & I_DIRTY_INODE); in __mark_inode_dirty()
2400 trace_writeback_dirty_inode(inode, flags); in __mark_inode_dirty()
2420 if (((inode->i_state & flags) == flags) || in __mark_inode_dirty()
2421 (dirtytime && (inode->i_state & I_DIRTY_INODE))) in __mark_inode_dirty()
2424 spin_lock(&inode->i_lock); in __mark_inode_dirty()
2425 if (dirtytime && (inode->i_state & I_DIRTY_INODE)) in __mark_inode_dirty()
2427 if ((inode->i_state & flags) != flags) { in __mark_inode_dirty()
2428 const int was_dirty = inode->i_state & I_DIRTY; in __mark_inode_dirty()
2430 inode_attach_wb(inode, NULL); in __mark_inode_dirty()
2434 inode->i_state &= ~I_DIRTY_TIME; in __mark_inode_dirty()
2435 inode->i_state |= flags; in __mark_inode_dirty()
2443 if (inode->i_state & I_SYNC_QUEUED) in __mark_inode_dirty()
2450 if (!S_ISBLK(inode->i_mode)) { in __mark_inode_dirty()
2451 if (inode_unhashed(inode)) in __mark_inode_dirty()
2454 if (inode->i_state & I_FREEING) in __mark_inode_dirty()
2466 wb = locked_inode_to_wb_and_lock_list(inode); in __mark_inode_dirty()
2468 inode->dirtied_when = jiffies; in __mark_inode_dirty()
2470 inode->dirtied_time_when = jiffies; in __mark_inode_dirty()
2472 if (inode->i_state & I_DIRTY) in __mark_inode_dirty()
2477 wakeup_bdi = inode_io_list_move_locked(inode, wb, in __mark_inode_dirty()
2481 trace_writeback_dirty_inode_enqueue(inode); in __mark_inode_dirty()
2496 spin_unlock(&inode->i_lock); in __mark_inode_dirty()
2542 struct inode *inode = list_first_entry(&sync_list, struct inode, in wait_sb_inodes() local
2544 struct address_space *mapping = inode->i_mapping; in wait_sb_inodes()
2552 list_move_tail(&inode->i_wb_list, &sb->s_inodes_wb); in wait_sb_inodes()
2564 spin_lock(&inode->i_lock); in wait_sb_inodes()
2565 if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) { in wait_sb_inodes()
2566 spin_unlock(&inode->i_lock); in wait_sb_inodes()
2571 __iget(inode); in wait_sb_inodes()
2572 spin_unlock(&inode->i_lock); in wait_sb_inodes()
2584 iput(inode); in wait_sb_inodes()
2716 int write_inode_now(struct inode *inode, int sync) in write_inode_now() argument
2725 if (!mapping_can_writeback(inode->i_mapping)) in write_inode_now()
2729 return writeback_single_inode(inode, &wbc); in write_inode_now()
2742 int sync_inode_metadata(struct inode *inode, int wait) in sync_inode_metadata() argument
2749 return writeback_single_inode(inode, &wbc); in sync_inode_metadata()