Lines Matching refs:rdp
16 static bool rcu_rdp_is_offloaded(struct rcu_data *rdp) in rcu_rdp_is_offloaded() argument
30 rcu_lockdep_is_held_nocb(rdp) || in rcu_rdp_is_offloaded()
31 (rdp == this_cpu_ptr(&rcu_data) && in rcu_rdp_is_offloaded()
33 rcu_current_is_nocb_kthread(rdp)), in rcu_rdp_is_offloaded()
37 return rcu_segcblist_is_offloaded(&rdp->cblist); in rcu_rdp_is_offloaded()
153 static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp) in rcu_preempt_ctxt_queue() argument
158 (rnp->qsmask & rdp->grpmask ? RCU_GP_BLKD : 0) + in rcu_preempt_ctxt_queue()
159 (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0); in rcu_preempt_ctxt_queue()
163 WARN_ON_ONCE(rdp->mynode != rnp); in rcu_preempt_ctxt_queue()
167 rdp->grpmask); in rcu_preempt_ctxt_queue()
252 !(rnp->qsmask & rdp->grpmask)); in rcu_preempt_ctxt_queue()
254 !(rnp->expmask & rdp->grpmask)); in rcu_preempt_ctxt_queue()
263 if (blkd_state & RCU_EXP_BLKD && rdp->exp_deferred_qs) in rcu_preempt_ctxt_queue()
264 rcu_report_exp_rdp(rdp); in rcu_preempt_ctxt_queue()
266 WARN_ON_ONCE(rdp->exp_deferred_qs); in rcu_preempt_ctxt_queue()
311 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_note_context_switch() local
321 rnp = rdp->mynode; in rcu_note_context_switch()
331 WARN_ON_ONCE((rdp->grpmask & rcu_rnp_online_cpus(rnp)) == 0); in rcu_note_context_switch()
335 (rnp->qsmask & rdp->grpmask) in rcu_note_context_switch()
338 rcu_preempt_ctxt_queue(rnp, rdp); in rcu_note_context_switch()
353 if (rdp->exp_deferred_qs) in rcu_note_context_switch()
354 rcu_report_exp_rdp(rdp); in rcu_note_context_switch()
469 struct rcu_data *rdp; in rcu_preempt_deferred_qs_irqrestore() local
479 rdp = this_cpu_ptr(&rcu_data); in rcu_preempt_deferred_qs_irqrestore()
480 if (!special.s && !rdp->exp_deferred_qs) { in rcu_preempt_deferred_qs_irqrestore()
487 rcu_report_qs_rdp(rdp); in rcu_preempt_deferred_qs_irqrestore()
500 if (rdp->exp_deferred_qs) in rcu_preempt_deferred_qs_irqrestore()
501 rcu_report_exp_rdp(rdp); in rcu_preempt_deferred_qs_irqrestore()
610 struct rcu_data *rdp; in rcu_preempt_deferred_qs_handler() local
612 rdp = container_of(iwp, struct rcu_data, defer_qs_iw); in rcu_preempt_deferred_qs_handler()
613 rdp->defer_qs_iw_pending = false; in rcu_preempt_deferred_qs_handler()
636 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_read_unlock_special() local
637 struct rcu_node *rnp = rdp->mynode; in rcu_read_unlock_special()
640 (rdp->grpmask & READ_ONCE(rnp->expmask)) || in rcu_read_unlock_special()
658 expboost && !rdp->defer_qs_iw_pending && cpu_online(rdp->cpu)) { in rcu_read_unlock_special()
661 init_irq_work(&rdp->defer_qs_iw, rcu_preempt_deferred_qs_handler); in rcu_read_unlock_special()
662 rdp->defer_qs_iw_pending = true; in rcu_read_unlock_special()
663 irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu); in rcu_read_unlock_special()
775 struct rcu_data *rdp; in dump_blkd_tasks() local
797 rdp = per_cpu_ptr(&rcu_data, cpu); in dump_blkd_tasks()
798 onl = !!(rdp->grpmask & rcu_rnp_online_cpus(rnp)); in dump_blkd_tasks()
801 (long)rdp->rcu_onl_gp_seq, rdp->rcu_onl_gp_flags, in dump_blkd_tasks()
802 (long)rdp->rcu_ofl_gp_seq, rdp->rcu_ofl_gp_flags); in dump_blkd_tasks()
815 struct rcu_data *rdp; in rcu_read_unlock_strict() local
819 rdp = this_cpu_ptr(&rcu_data); in rcu_read_unlock_strict()
820 rcu_report_qs_rdp(rdp); in rcu_read_unlock_strict()
1324 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_try_advance_all_cbs() local
1328 if (jiffies == rdp->last_advance_all) in rcu_try_advance_all_cbs()
1330 rdp->last_advance_all = jiffies; in rcu_try_advance_all_cbs()
1332 rnp = rdp->mynode; in rcu_try_advance_all_cbs()
1339 if ((rcu_seq_completed_gp(rdp->gp_seq, in rcu_try_advance_all_cbs()
1341 unlikely(READ_ONCE(rdp->gpwrap))) && in rcu_try_advance_all_cbs()
1342 rcu_segcblist_pend_cbs(&rdp->cblist)) in rcu_try_advance_all_cbs()
1343 note_gp_changes(rdp); in rcu_try_advance_all_cbs()
1345 if (rcu_segcblist_ready_cbs(&rdp->cblist)) in rcu_try_advance_all_cbs()
1359 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_needs_cpu() local
1365 if (rcu_segcblist_empty(&rdp->cblist) || in rcu_needs_cpu()
1366 rcu_rdp_is_offloaded(rdp)) { in rcu_needs_cpu()
1377 rdp->last_accelerate = jiffies; in rcu_needs_cpu()
1397 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_prepare_for_idle() local
1402 if (rcu_rdp_is_offloaded(rdp)) in rcu_prepare_for_idle()
1407 if (tne != rdp->tick_nohz_enabled_snap) { in rcu_prepare_for_idle()
1408 if (!rcu_segcblist_empty(&rdp->cblist)) in rcu_prepare_for_idle()
1410 rdp->tick_nohz_enabled_snap = tne; in rcu_prepare_for_idle()
1420 if (rdp->last_accelerate == jiffies) in rcu_prepare_for_idle()
1422 rdp->last_accelerate = jiffies; in rcu_prepare_for_idle()
1423 if (rcu_segcblist_pend_cbs(&rdp->cblist)) { in rcu_prepare_for_idle()
1424 rnp = rdp->mynode; in rcu_prepare_for_idle()
1426 needwake = rcu_accelerate_cbs(rnp, rdp); in rcu_prepare_for_idle()
1440 struct rcu_data *rdp = this_cpu_ptr(&rcu_data); in rcu_cleanup_after_idle() local
1443 if (rcu_rdp_is_offloaded(rdp)) in rcu_cleanup_after_idle()