Lines Matching refs:cblist
317 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */ in rcu_nocb_do_flush_bypass()
319 rcu_segcblist_insert_pend_cbs(&rdp->cblist, &rcl); in rcu_nocb_do_flush_bypass()
387 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
393 if (!rcu_segcblist_completely_offloaded(&rdp->cblist)) { in rcu_nocb_try_bypass()
395 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
403 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
427 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
442 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
450 rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in rcu_nocb_try_bypass()
463 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */ in rcu_nocb_try_bypass()
476 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) { in rcu_nocb_try_bypass()
513 len = rcu_segcblist_n_cbs(&rdp->cblist); in __call_rcu_nocb_wake()
532 rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in __call_rcu_nocb_wake()
539 !rcu_segcblist_ready_cbs(&rdp->cblist)) && in __call_rcu_nocb_wake()
572 return rcu_segcblist_test_flags(&rdp->cblist, flags); in nocb_gp_enabled_cb()
578 struct rcu_segcblist *cblist = &rdp->cblist; in nocb_gp_update_state_deoffloading() local
580 if (rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED)) { in nocb_gp_update_state_deoffloading()
581 if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)) { in nocb_gp_update_state_deoffloading()
582 rcu_segcblist_set_flags(cblist, SEGCBLIST_KTHREAD_GP); in nocb_gp_update_state_deoffloading()
583 if (rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB)) in nocb_gp_update_state_deoffloading()
593 WARN_ON_ONCE(!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)); in nocb_gp_update_state_deoffloading()
594 rcu_segcblist_clear_flags(cblist, SEGCBLIST_KTHREAD_GP); in nocb_gp_update_state_deoffloading()
595 if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB)) in nocb_gp_update_state_deoffloading()
648 } else if (!bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) { in nocb_gp_wait()
663 if (!rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
665 (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in nocb_gp_wait()
669 wasempty = rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
675 !rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
677 if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) { in nocb_gp_wait()
685 if (rcu_segcblist_ready_cbs(&rdp->cblist)) { in nocb_gp_wait()
770 return rcu_segcblist_test_flags(&rdp->cblist, flags); in nocb_cb_can_run()
784 struct rcu_segcblist *cblist = &rdp->cblist; in nocb_cb_wait() local
806 if (rcu_segcblist_nextgp(cblist, &cur_gp_seq) && in nocb_cb_wait()
813 if (rcu_segcblist_test_flags(cblist, SEGCBLIST_OFFLOADED)) { in nocb_cb_wait()
814 if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB)) { in nocb_cb_wait()
815 rcu_segcblist_set_flags(cblist, SEGCBLIST_KTHREAD_CB); in nocb_cb_wait()
816 if (rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)) in nocb_cb_wait()
819 if (rcu_segcblist_ready_cbs(cblist)) in nocb_cb_wait()
827 WARN_ON_ONCE(!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB)); in nocb_cb_wait()
828 rcu_segcblist_clear_flags(cblist, SEGCBLIST_KTHREAD_CB); in nocb_cb_wait()
829 if (!rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)) in nocb_cb_wait()
942 struct rcu_segcblist *cblist = &rdp->cblist; in rdp_offload_toggle() local
946 rcu_segcblist_offload(cblist, offload); in rdp_offload_toggle()
974 struct rcu_segcblist *cblist = &rdp->cblist; in rcu_nocb_rdp_deoffload() local
995 !rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB | in rcu_nocb_rdp_deoffload()
1006 rcu_segcblist_set_flags(cblist, SEGCBLIST_SOFTIRQ_ONLY); in rcu_nocb_rdp_deoffload()
1047 struct rcu_segcblist *cblist = &rdp->cblist; in rcu_nocb_rdp_offload() local
1084 rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_CB) && in rcu_nocb_rdp_offload()
1085 rcu_segcblist_test_flags(cblist, SEGCBLIST_KTHREAD_GP)); in rcu_nocb_rdp_offload()
1154 if (rcu_segcblist_empty(&rdp->cblist)) in rcu_init_nohz()
1155 rcu_segcblist_init(&rdp->cblist); in rcu_init_nohz()
1156 rcu_segcblist_offload(&rdp->cblist, true); in rcu_init_nohz()
1157 rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_KTHREAD_CB); in rcu_init_nohz()
1158 rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_KTHREAD_GP); in rcu_init_nohz()
1355 struct rcu_segcblist *rsclp = &rdp->cblist; in show_rcu_nocb_state()
1383 rcu_segcblist_n_cbs(&rdp->cblist), in show_rcu_nocb_state()