Lines Matching refs:cblist
82 .cblist.flags = SEGCBLIST_SOFTIRQ_ONLY,
242 if (rcu_segcblist_is_enabled(&rdp->cblist)) in rcu_get_n_cbs_cpu()
243 return rcu_segcblist_n_cbs(&rdp->cblist); in rcu_get_n_cbs_cpu()
1499 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) in rcu_accelerate_cbs()
1502 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbPreAcc")); in rcu_accelerate_cbs()
1515 if (rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req)) in rcu_accelerate_cbs()
1519 if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL)) in rcu_accelerate_cbs()
1524 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbPostAcc")); in rcu_accelerate_cbs()
1546 (void)rcu_segcblist_accelerate(&rdp->cblist, c); in rcu_accelerate_cbs_unlocked()
1572 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) in rcu_advance_cbs()
1579 rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq); in rcu_advance_cbs()
2454 if (!rcu_segcblist_ready_cbs(&rdp->cblist)) { in rcu_do_batch()
2456 rcu_segcblist_n_cbs(&rdp->cblist), 0); in rcu_do_batch()
2458 !rcu_segcblist_empty(&rdp->cblist), in rcu_do_batch()
2472 pending = rcu_segcblist_n_cbs(&rdp->cblist); in rcu_do_batch()
2483 rcu_segcblist_n_cbs(&rdp->cblist), bl); in rcu_do_batch()
2484 rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl); in rcu_do_batch()
2486 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); in rcu_do_batch()
2488 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbDequeued")); in rcu_do_batch()
2540 rcu_segcblist_insert_done_cbs(&rdp->cblist, &rcl); in rcu_do_batch()
2541 rcu_segcblist_add_len(&rdp->cblist, -count); in rcu_do_batch()
2544 count = rcu_segcblist_n_cbs(&rdp->cblist); in rcu_do_batch()
2559 empty = rcu_segcblist_empty(&rdp->cblist); in rcu_do_batch()
2563 WARN_ON_ONCE(count == 0 && rcu_segcblist_n_segment_cbs(&rdp->cblist) != 0); in rcu_do_batch()
2564 WARN_ON_ONCE(!empty && rcu_segcblist_n_segment_cbs(&rdp->cblist) == 0); in rcu_do_batch()
2569 if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist)) in rcu_do_batch()
2709 const bool do_batch = !rcu_segcblist_completely_offloaded(&rdp->cblist); in rcu_core()
2729 rcu_segcblist_is_enabled(&rdp->cblist) && do_batch) { in rcu_core()
2731 if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) in rcu_core()
2739 if (do_batch && rcu_segcblist_ready_cbs(&rdp->cblist) && in rcu_core()
2887 if (unlikely(rcu_segcblist_n_cbs(&rdp->cblist) > in __call_rcu_core()
2900 rcu_segcblist_first_pend_cb(&rdp->cblist) != head) in __call_rcu_core()
2903 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); in __call_rcu_core()
2926 if (rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) in check_cb_ovld_locked()
2949 ((rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) == in check_cb_ovld()
2989 if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist))) { in __call_rcu()
2995 if (rcu_segcblist_empty(&rdp->cblist)) in __call_rcu()
2996 rcu_segcblist_init(&rdp->cblist); in __call_rcu()
3003 rcu_segcblist_enqueue(&rdp->cblist, head); in __call_rcu()
3007 rcu_segcblist_n_cbs(&rdp->cblist)); in __call_rcu()
3010 rcu_segcblist_n_cbs(&rdp->cblist)); in __call_rcu()
3012 trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCBQueued")); in __call_rcu()
3895 rcu_segcblist_ready_cbs(&rdp->cblist)) in rcu_pending()
3899 if (!gp_in_progress && rcu_segcblist_is_enabled(&rdp->cblist) && in rcu_pending()
3901 !rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)) in rcu_pending()
3958 if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head)) { in rcu_barrier_func()
4021 if (rcu_segcblist_n_cbs(&rdp->cblist) && cpu_online(cpu)) { in rcu_barrier()
4025 } else if (rcu_segcblist_n_cbs(&rdp->cblist) && in rcu_barrier()
4138 if (!rcu_segcblist_is_enabled(&rdp->cblist)) in rcutree_prepare_cpu()
4139 rcu_segcblist_init(&rdp->cblist); /* Re-enable callbacks. */ in rcutree_prepare_cpu()
4339 rcu_segcblist_empty(&rdp->cblist)) in rcutree_migrate_callbacks()
4351 rcu_segcblist_merge(&my_rdp->cblist, &rdp->cblist); in rcutree_migrate_callbacks()
4353 rcu_segcblist_disable(&rdp->cblist); in rcutree_migrate_callbacks()
4354 WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp->cblist) != in rcutree_migrate_callbacks()
4355 !rcu_segcblist_n_cbs(&my_rdp->cblist)); in rcutree_migrate_callbacks()
4366 WARN_ONCE(rcu_segcblist_n_cbs(&rdp->cblist) != 0 || in rcutree_migrate_callbacks()
4367 !rcu_segcblist_empty(&rdp->cblist), in rcutree_migrate_callbacks()
4369 cpu, rcu_segcblist_n_cbs(&rdp->cblist), in rcutree_migrate_callbacks()
4370 rcu_segcblist_first_cb(&rdp->cblist)); in rcutree_migrate_callbacks()