Lines Matching refs:rdp
19 static inline int rcu_lockdep_is_held_nocb(struct rcu_data *rdp) in rcu_lockdep_is_held_nocb() argument
21 return lockdep_is_held(&rdp->nocb_lock); in rcu_lockdep_is_held_nocb()
24 static inline bool rcu_current_is_nocb_kthread(struct rcu_data *rdp) in rcu_current_is_nocb_kthread() argument
27 if (!rdp->nocb_cb_kthread || !rdp->nocb_gp_kthread) in rcu_current_is_nocb_kthread()
30 if (current == rdp->nocb_cb_kthread || current == rdp->nocb_gp_kthread) in rcu_current_is_nocb_kthread()
94 static void rcu_nocb_bypass_lock(struct rcu_data *rdp) in rcu_nocb_bypass_lock() argument
95 __acquires(&rdp->nocb_bypass_lock) in rcu_nocb_bypass_lock()
98 if (raw_spin_trylock(&rdp->nocb_bypass_lock)) in rcu_nocb_bypass_lock()
100 atomic_inc(&rdp->nocb_lock_contended); in rcu_nocb_bypass_lock()
101 WARN_ON_ONCE(smp_processor_id() != rdp->cpu); in rcu_nocb_bypass_lock()
103 raw_spin_lock(&rdp->nocb_bypass_lock); in rcu_nocb_bypass_lock()
105 atomic_dec(&rdp->nocb_lock_contended); in rcu_nocb_bypass_lock()
118 static void rcu_nocb_wait_contended(struct rcu_data *rdp) in rcu_nocb_wait_contended() argument
120 WARN_ON_ONCE(smp_processor_id() != rdp->cpu); in rcu_nocb_wait_contended()
121 while (WARN_ON_ONCE(atomic_read(&rdp->nocb_lock_contended))) in rcu_nocb_wait_contended()
129 static bool rcu_nocb_bypass_trylock(struct rcu_data *rdp) in rcu_nocb_bypass_trylock() argument
132 return raw_spin_trylock(&rdp->nocb_bypass_lock); in rcu_nocb_bypass_trylock()
138 static void rcu_nocb_bypass_unlock(struct rcu_data *rdp) in rcu_nocb_bypass_unlock() argument
139 __releases(&rdp->nocb_bypass_lock) in rcu_nocb_bypass_unlock()
142 raw_spin_unlock(&rdp->nocb_bypass_lock); in rcu_nocb_bypass_unlock()
149 static void rcu_nocb_lock(struct rcu_data *rdp) in rcu_nocb_lock() argument
152 if (!rcu_rdp_is_offloaded(rdp)) in rcu_nocb_lock()
154 raw_spin_lock(&rdp->nocb_lock); in rcu_nocb_lock()
161 static void rcu_nocb_unlock(struct rcu_data *rdp) in rcu_nocb_unlock() argument
163 if (rcu_rdp_is_offloaded(rdp)) { in rcu_nocb_unlock()
165 raw_spin_unlock(&rdp->nocb_lock); in rcu_nocb_unlock()
173 static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, in rcu_nocb_unlock_irqrestore() argument
176 if (rcu_rdp_is_offloaded(rdp)) { in rcu_nocb_unlock_irqrestore()
178 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in rcu_nocb_unlock_irqrestore()
185 static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp) in rcu_lockdep_assert_cblist_protected() argument
188 if (rcu_rdp_is_offloaded(rdp)) in rcu_lockdep_assert_cblist_protected()
189 lockdep_assert_held(&rdp->nocb_lock); in rcu_lockdep_assert_cblist_protected()
221 struct rcu_data *rdp, in __wake_nocb_gp()
229 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __wake_nocb_gp()
245 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DoWake")); in __wake_nocb_gp()
255 static bool wake_nocb_gp(struct rcu_data *rdp, bool force) in wake_nocb_gp() argument
258 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in wake_nocb_gp()
261 return __wake_nocb_gp(rdp_gp, rdp, force, flags); in wake_nocb_gp()
268 static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype, in wake_nocb_gp_defer() argument
272 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in wake_nocb_gp_defer()
292 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, reason); in wake_nocb_gp_defer()
303 static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_do_flush_bypass() argument
308 WARN_ON_ONCE(!rcu_rdp_is_offloaded(rdp)); in rcu_nocb_do_flush_bypass()
309 rcu_lockdep_assert_cblist_protected(rdp); in rcu_nocb_do_flush_bypass()
310 lockdep_assert_held(&rdp->nocb_bypass_lock); in rcu_nocb_do_flush_bypass()
311 if (rhp && !rcu_cblist_n_cbs(&rdp->nocb_bypass)) { in rcu_nocb_do_flush_bypass()
312 raw_spin_unlock(&rdp->nocb_bypass_lock); in rcu_nocb_do_flush_bypass()
317 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */ in rcu_nocb_do_flush_bypass()
318 rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp); in rcu_nocb_do_flush_bypass()
319 rcu_segcblist_insert_pend_cbs(&rdp->cblist, &rcl); in rcu_nocb_do_flush_bypass()
320 WRITE_ONCE(rdp->nocb_bypass_first, j); in rcu_nocb_do_flush_bypass()
321 rcu_nocb_bypass_unlock(rdp); in rcu_nocb_do_flush_bypass()
333 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_flush_bypass() argument
336 if (!rcu_rdp_is_offloaded(rdp)) in rcu_nocb_flush_bypass()
338 rcu_lockdep_assert_cblist_protected(rdp); in rcu_nocb_flush_bypass()
339 rcu_nocb_bypass_lock(rdp); in rcu_nocb_flush_bypass()
340 return rcu_nocb_do_flush_bypass(rdp, rhp, j); in rcu_nocb_flush_bypass()
347 static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j) in rcu_nocb_try_flush_bypass() argument
349 rcu_lockdep_assert_cblist_protected(rdp); in rcu_nocb_try_flush_bypass()
350 if (!rcu_rdp_is_offloaded(rdp) || in rcu_nocb_try_flush_bypass()
351 !rcu_nocb_bypass_trylock(rdp)) in rcu_nocb_try_flush_bypass()
353 WARN_ON_ONCE(!rcu_nocb_do_flush_bypass(rdp, NULL, j)); in rcu_nocb_try_flush_bypass()
374 static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_try_bypass() argument
380 long ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in rcu_nocb_try_bypass()
386 if (!rcu_rdp_is_offloaded(rdp)) { in rcu_nocb_try_bypass()
387 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
393 if (!rcu_segcblist_completely_offloaded(&rdp->cblist)) { in rcu_nocb_try_bypass()
394 rcu_nocb_lock(rdp); in rcu_nocb_try_bypass()
395 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
401 rcu_nocb_lock(rdp); in rcu_nocb_try_bypass()
402 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_try_bypass()
403 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
409 if (j == rdp->nocb_nobypass_last) { in rcu_nocb_try_bypass()
410 c = rdp->nocb_nobypass_count + 1; in rcu_nocb_try_bypass()
412 WRITE_ONCE(rdp->nocb_nobypass_last, j); in rcu_nocb_try_bypass()
413 c = rdp->nocb_nobypass_count - nocb_nobypass_lim_per_jiffy; in rcu_nocb_try_bypass()
414 if (ULONG_CMP_LT(rdp->nocb_nobypass_count, in rcu_nocb_try_bypass()
420 WRITE_ONCE(rdp->nocb_nobypass_count, c); in rcu_nocb_try_bypass()
425 if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy) { in rcu_nocb_try_bypass()
426 rcu_nocb_lock(rdp); in rcu_nocb_try_bypass()
427 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
429 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
431 WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, j)); in rcu_nocb_try_bypass()
432 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_try_bypass()
438 if ((ncbs && j != READ_ONCE(rdp->nocb_bypass_first)) || in rcu_nocb_try_bypass()
440 rcu_nocb_lock(rdp); in rcu_nocb_try_bypass()
441 if (!rcu_nocb_flush_bypass(rdp, rhp, j)) { in rcu_nocb_try_bypass()
442 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist); in rcu_nocb_try_bypass()
444 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
446 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_try_bypass()
449 if (j != rdp->nocb_gp_adv_time && in rcu_nocb_try_bypass()
450 rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in rcu_nocb_try_bypass()
451 rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) { in rcu_nocb_try_bypass()
452 rcu_advance_cbs_nowake(rdp->mynode, rdp); in rcu_nocb_try_bypass()
453 rdp->nocb_gp_adv_time = j; in rcu_nocb_try_bypass()
455 rcu_nocb_unlock_irqrestore(rdp, flags); in rcu_nocb_try_bypass()
460 rcu_nocb_wait_contended(rdp); in rcu_nocb_try_bypass()
461 rcu_nocb_bypass_lock(rdp); in rcu_nocb_try_bypass()
462 ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in rcu_nocb_try_bypass()
463 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */ in rcu_nocb_try_bypass()
464 rcu_cblist_enqueue(&rdp->nocb_bypass, rhp); in rcu_nocb_try_bypass()
466 WRITE_ONCE(rdp->nocb_bypass_first, j); in rcu_nocb_try_bypass()
467 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FirstBQ")); in rcu_nocb_try_bypass()
469 rcu_nocb_bypass_unlock(rdp); in rcu_nocb_try_bypass()
475 rcu_nocb_lock(rdp); // Rare during call_rcu() flood. in rcu_nocb_try_bypass()
476 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) { in rcu_nocb_try_bypass()
477 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
479 __call_rcu_nocb_wake(rdp, true, flags); in rcu_nocb_try_bypass()
481 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in rcu_nocb_try_bypass()
483 rcu_nocb_unlock_irqrestore(rdp, flags); in rcu_nocb_try_bypass()
495 static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, in __call_rcu_nocb_wake() argument
497 __releases(rdp->nocb_lock) in __call_rcu_nocb_wake()
505 t = READ_ONCE(rdp->nocb_gp_kthread); in __call_rcu_nocb_wake()
507 rcu_nocb_unlock_irqrestore(rdp, flags); in __call_rcu_nocb_wake()
508 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __call_rcu_nocb_wake()
513 len = rcu_segcblist_n_cbs(&rdp->cblist); in __call_rcu_nocb_wake()
515 rdp->qlen_last_fqs_check = len; in __call_rcu_nocb_wake()
518 rcu_nocb_unlock_irqrestore(rdp, flags); in __call_rcu_nocb_wake()
519 wake_nocb_gp(rdp, false); in __call_rcu_nocb_wake()
520 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in __call_rcu_nocb_wake()
523 rcu_nocb_unlock_irqrestore(rdp, flags); in __call_rcu_nocb_wake()
524 wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE, in __call_rcu_nocb_wake()
527 } else if (len > rdp->qlen_last_fqs_check + qhimark) { in __call_rcu_nocb_wake()
529 rdp->qlen_last_fqs_check = len; in __call_rcu_nocb_wake()
531 if (j != rdp->nocb_gp_adv_time && in __call_rcu_nocb_wake()
532 rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in __call_rcu_nocb_wake()
533 rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) { in __call_rcu_nocb_wake()
534 rcu_advance_cbs_nowake(rdp->mynode, rdp); in __call_rcu_nocb_wake()
535 rdp->nocb_gp_adv_time = j; in __call_rcu_nocb_wake()
538 if ((rdp->nocb_cb_sleep || in __call_rcu_nocb_wake()
539 !rcu_segcblist_ready_cbs(&rdp->cblist)) && in __call_rcu_nocb_wake()
540 !timer_pending(&rdp->nocb_timer)) { in __call_rcu_nocb_wake()
541 rcu_nocb_unlock_irqrestore(rdp, flags); in __call_rcu_nocb_wake()
542 wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_FORCE, in __call_rcu_nocb_wake()
545 rcu_nocb_unlock_irqrestore(rdp, flags); in __call_rcu_nocb_wake()
546 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot")); in __call_rcu_nocb_wake()
549 rcu_nocb_unlock_irqrestore(rdp, flags); in __call_rcu_nocb_wake()
550 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot")); in __call_rcu_nocb_wake()
568 static inline bool nocb_gp_enabled_cb(struct rcu_data *rdp) in nocb_gp_enabled_cb() argument
572 return rcu_segcblist_test_flags(&rdp->cblist, flags); in nocb_gp_enabled_cb()
575 static inline bool nocb_gp_update_state_deoffloading(struct rcu_data *rdp, in nocb_gp_update_state_deoffloading() argument
578 struct rcu_segcblist *cblist = &rdp->cblist; in nocb_gp_update_state_deoffloading()
617 struct rcu_data *rdp; in nocb_gp_wait() local
628 for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_cb_rdp) { in nocb_gp_wait()
631 if (!nocb_gp_enabled_cb(rdp)) in nocb_gp_wait()
633 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Check")); in nocb_gp_wait()
634 rcu_nocb_lock_irqsave(rdp, flags); in nocb_gp_wait()
635 if (nocb_gp_update_state_deoffloading(rdp, &needwake_state)) { in nocb_gp_wait()
636 rcu_nocb_unlock_irqrestore(rdp, flags); in nocb_gp_wait()
638 swake_up_one(&rdp->nocb_state_wq); in nocb_gp_wait()
641 bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in nocb_gp_wait()
643 (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + 1) || in nocb_gp_wait()
646 (void)rcu_nocb_try_flush_bypass(rdp, j); in nocb_gp_wait()
647 bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); in nocb_gp_wait()
648 } else if (!bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) { in nocb_gp_wait()
649 rcu_nocb_unlock_irqrestore(rdp, flags); in nocb_gp_wait()
651 swake_up_one(&rdp->nocb_state_wq); in nocb_gp_wait()
655 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in nocb_gp_wait()
659 rnp = rdp->mynode; in nocb_gp_wait()
663 if (!rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
665 (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && in nocb_gp_wait()
668 needwake_gp = rcu_advance_cbs(rnp, rdp); in nocb_gp_wait()
669 wasempty = rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
675 !rcu_segcblist_restempty(&rdp->cblist, in nocb_gp_wait()
677 if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) { in nocb_gp_wait()
682 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, in nocb_gp_wait()
685 if (rcu_segcblist_ready_cbs(&rdp->cblist)) { in nocb_gp_wait()
686 needwake = rdp->nocb_cb_sleep; in nocb_gp_wait()
687 WRITE_ONCE(rdp->nocb_cb_sleep, false); in nocb_gp_wait()
692 rcu_nocb_unlock_irqrestore(rdp, flags); in nocb_gp_wait()
694 swake_up_one(&rdp->nocb_cb_wq); in nocb_gp_wait()
700 swake_up_one(&rdp->nocb_state_wq); in nocb_gp_wait()
756 struct rcu_data *rdp = arg; in rcu_nocb_gp_kthread() local
759 WRITE_ONCE(rdp->nocb_gp_loops, rdp->nocb_gp_loops + 1); in rcu_nocb_gp_kthread()
760 nocb_gp_wait(rdp); in rcu_nocb_gp_kthread()
766 static inline bool nocb_cb_can_run(struct rcu_data *rdp) in nocb_cb_can_run() argument
770 return rcu_segcblist_test_flags(&rdp->cblist, flags); in nocb_cb_can_run()
773 static inline bool nocb_cb_wait_cond(struct rcu_data *rdp) in nocb_cb_wait_cond() argument
775 return nocb_cb_can_run(rdp) && !READ_ONCE(rdp->nocb_cb_sleep); in nocb_cb_wait_cond()
782 static void nocb_cb_wait(struct rcu_data *rdp) in nocb_cb_wait() argument
784 struct rcu_segcblist *cblist = &rdp->cblist; in nocb_cb_wait()
790 struct rcu_node *rnp = rdp->mynode; in nocb_cb_wait()
802 rcu_do_batch(rdp); in nocb_cb_wait()
805 rcu_nocb_lock_irqsave(rdp, flags); in nocb_cb_wait()
809 needwake_gp = rcu_advance_cbs(rdp->mynode, rdp); in nocb_cb_wait()
833 WRITE_ONCE(rdp->nocb_cb_sleep, can_sleep); in nocb_cb_wait()
835 if (rdp->nocb_cb_sleep) in nocb_cb_wait()
836 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("CBSleep")); in nocb_cb_wait()
838 rcu_nocb_unlock_irqrestore(rdp, flags); in nocb_cb_wait()
843 swake_up_one(&rdp->nocb_state_wq); in nocb_cb_wait()
846 swait_event_interruptible_exclusive(rdp->nocb_cb_wq, in nocb_cb_wait()
847 nocb_cb_wait_cond(rdp)); in nocb_cb_wait()
850 if (smp_load_acquire(&rdp->nocb_cb_sleep)) { // ^^^ in nocb_cb_wait()
852 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty")); in nocb_cb_wait()
854 } while (!nocb_cb_can_run(rdp)); in nocb_cb_wait()
863 struct rcu_data *rdp = arg; in rcu_nocb_cb_kthread() local
868 nocb_cb_wait(rdp); in rcu_nocb_cb_kthread()
875 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level) in rcu_nocb_need_deferred_wakeup() argument
877 return READ_ONCE(rdp->nocb_defer_wakeup) >= level; in rcu_nocb_need_deferred_wakeup()
882 struct rcu_data *rdp, int level, in do_nocb_deferred_wakeup_common()
895 ret = __wake_nocb_gp(rdp_gp, rdp, ndw == RCU_NOCB_WAKE_FORCE, flags); in do_nocb_deferred_wakeup_common()
896 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DeferredWake")); in do_nocb_deferred_wakeup_common()
905 struct rcu_data *rdp = from_timer(rdp, t, nocb_timer); in do_nocb_deferred_wakeup_timer() local
907 WARN_ON_ONCE(rdp->nocb_gp_rdp != rdp); in do_nocb_deferred_wakeup_timer()
908 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Timer")); in do_nocb_deferred_wakeup_timer()
910 raw_spin_lock_irqsave(&rdp->nocb_gp_lock, flags); in do_nocb_deferred_wakeup_timer()
912 do_nocb_deferred_wakeup_common(rdp, rdp, RCU_NOCB_WAKE_BYPASS, flags); in do_nocb_deferred_wakeup_timer()
920 static bool do_nocb_deferred_wakeup(struct rcu_data *rdp) in do_nocb_deferred_wakeup() argument
923 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in do_nocb_deferred_wakeup()
929 return do_nocb_deferred_wakeup_common(rdp_gp, rdp, RCU_NOCB_WAKE, flags); in do_nocb_deferred_wakeup()
938 static int rdp_offload_toggle(struct rcu_data *rdp, in rdp_offload_toggle() argument
940 __releases(rdp->nocb_lock) in rdp_offload_toggle()
942 struct rcu_segcblist *cblist = &rdp->cblist; in rdp_offload_toggle()
943 struct rcu_data *rdp_gp = rdp->nocb_gp_rdp; in rdp_offload_toggle()
948 if (rdp->nocb_cb_sleep) in rdp_offload_toggle()
949 rdp->nocb_cb_sleep = false; in rdp_offload_toggle()
950 rcu_nocb_unlock_irqrestore(rdp, flags); in rdp_offload_toggle()
956 swake_up_one(&rdp->nocb_cb_wq); in rdp_offload_toggle()
973 struct rcu_data *rdp = arg; in rcu_nocb_rdp_deoffload() local
974 struct rcu_segcblist *cblist = &rdp->cblist; in rcu_nocb_rdp_deoffload()
978 WARN_ON_ONCE(rdp->cpu != raw_smp_processor_id()); in rcu_nocb_rdp_deoffload()
980 pr_info("De-offloading %d\n", rdp->cpu); in rcu_nocb_rdp_deoffload()
982 rcu_nocb_lock_irqsave(rdp, flags); in rcu_nocb_rdp_deoffload()
992 WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies)); in rcu_nocb_rdp_deoffload()
993 ret = rdp_offload_toggle(rdp, false, flags); in rcu_nocb_rdp_deoffload()
994 swait_event_exclusive(rdp->nocb_state_wq, in rcu_nocb_rdp_deoffload()
1001 rcu_nocb_lock_irqsave(rdp, flags); in rcu_nocb_rdp_deoffload()
1011 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags); in rcu_nocb_rdp_deoffload()
1014 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass)); in rcu_nocb_rdp_deoffload()
1022 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_nocb_cpu_deoffload() local
1027 if (rcu_rdp_is_offloaded(rdp)) { in rcu_nocb_cpu_deoffload()
1029 ret = work_on_cpu(cpu, rcu_nocb_rdp_deoffload, rdp); in rcu_nocb_cpu_deoffload()
1046 struct rcu_data *rdp = arg; in rcu_nocb_rdp_offload() local
1047 struct rcu_segcblist *cblist = &rdp->cblist; in rcu_nocb_rdp_offload()
1051 WARN_ON_ONCE(rdp->cpu != raw_smp_processor_id()); in rcu_nocb_rdp_offload()
1056 if (!rdp->nocb_gp_rdp) in rcu_nocb_rdp_offload()
1059 pr_info("Offloading %d\n", rdp->cpu); in rcu_nocb_rdp_offload()
1064 raw_spin_lock_irqsave(&rdp->nocb_lock, flags); in rcu_nocb_rdp_offload()
1082 ret = rdp_offload_toggle(rdp, true, flags); in rcu_nocb_rdp_offload()
1083 swait_event_exclusive(rdp->nocb_state_wq, in rcu_nocb_rdp_offload()
1092 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_nocb_cpu_offload() local
1097 if (!rcu_rdp_is_offloaded(rdp)) { in rcu_nocb_cpu_offload()
1099 ret = work_on_cpu(cpu, rcu_nocb_rdp_offload, rdp); in rcu_nocb_cpu_offload()
1118 struct rcu_data *rdp; in rcu_init_nohz() local
1153 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_init_nohz()
1154 if (rcu_segcblist_empty(&rdp->cblist)) in rcu_init_nohz()
1155 rcu_segcblist_init(&rdp->cblist); in rcu_init_nohz()
1156 rcu_segcblist_offload(&rdp->cblist, true); in rcu_init_nohz()
1157 rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_KTHREAD_CB); in rcu_init_nohz()
1158 rcu_segcblist_set_flags(&rdp->cblist, SEGCBLIST_KTHREAD_GP); in rcu_init_nohz()
1164 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) in rcu_boot_init_nocb_percpu_data() argument
1166 init_swait_queue_head(&rdp->nocb_cb_wq); in rcu_boot_init_nocb_percpu_data()
1167 init_swait_queue_head(&rdp->nocb_gp_wq); in rcu_boot_init_nocb_percpu_data()
1168 init_swait_queue_head(&rdp->nocb_state_wq); in rcu_boot_init_nocb_percpu_data()
1169 raw_spin_lock_init(&rdp->nocb_lock); in rcu_boot_init_nocb_percpu_data()
1170 raw_spin_lock_init(&rdp->nocb_bypass_lock); in rcu_boot_init_nocb_percpu_data()
1171 raw_spin_lock_init(&rdp->nocb_gp_lock); in rcu_boot_init_nocb_percpu_data()
1172 timer_setup(&rdp->nocb_timer, do_nocb_deferred_wakeup_timer, 0); in rcu_boot_init_nocb_percpu_data()
1173 rcu_cblist_init(&rdp->nocb_bypass); in rcu_boot_init_nocb_percpu_data()
1183 struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_spawn_one_nocb_kthread() local
1191 if (!rcu_is_nocb_cpu(cpu) || rdp->nocb_cb_kthread) in rcu_spawn_one_nocb_kthread()
1195 rdp_gp = rdp->nocb_gp_rdp; in rcu_spawn_one_nocb_kthread()
1205 t = kthread_run(rcu_nocb_cb_kthread, rdp, in rcu_spawn_one_nocb_kthread()
1209 WRITE_ONCE(rdp->nocb_cb_kthread, t); in rcu_spawn_one_nocb_kthread()
1210 WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread); in rcu_spawn_one_nocb_kthread()
1252 struct rcu_data *rdp; in rcu_organize_nocb_kthreads() local
1269 rdp = per_cpu_ptr(&rcu_data, cpu); in rcu_organize_nocb_kthreads()
1270 if (rdp->cpu >= nl) { in rcu_organize_nocb_kthreads()
1273 nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls; in rcu_organize_nocb_kthreads()
1274 rdp->nocb_gp_rdp = rdp; in rcu_organize_nocb_kthreads()
1275 rdp_gp = rdp; in rcu_organize_nocb_kthreads()
1288 rdp->nocb_gp_rdp = rdp_gp; in rcu_organize_nocb_kthreads()
1289 rdp_prev->nocb_next_cb_rdp = rdp; in rcu_organize_nocb_kthreads()
1293 rdp_prev = rdp; in rcu_organize_nocb_kthreads()
1327 static void show_rcu_nocb_gp_state(struct rcu_data *rdp) in show_rcu_nocb_gp_state() argument
1329 struct rcu_node *rnp = rdp->mynode; in show_rcu_nocb_gp_state()
1332 rdp->cpu, in show_rcu_nocb_gp_state()
1333 "kK"[!!rdp->nocb_gp_kthread], in show_rcu_nocb_gp_state()
1334 "lL"[raw_spin_is_locked(&rdp->nocb_gp_lock)], in show_rcu_nocb_gp_state()
1335 "dD"[!!rdp->nocb_defer_wakeup], in show_rcu_nocb_gp_state()
1336 "tT"[timer_pending(&rdp->nocb_timer)], in show_rcu_nocb_gp_state()
1337 "sS"[!!rdp->nocb_gp_sleep], in show_rcu_nocb_gp_state()
1338 ".W"[swait_active(&rdp->nocb_gp_wq)], in show_rcu_nocb_gp_state()
1341 ".B"[!!rdp->nocb_gp_bypass], in show_rcu_nocb_gp_state()
1342 ".G"[!!rdp->nocb_gp_gp], in show_rcu_nocb_gp_state()
1343 (long)rdp->nocb_gp_seq, in show_rcu_nocb_gp_state()
1344 rnp->grplo, rnp->grphi, READ_ONCE(rdp->nocb_gp_loops), in show_rcu_nocb_gp_state()
1345 rdp->nocb_gp_kthread ? task_state_to_char(rdp->nocb_gp_kthread) : '.', in show_rcu_nocb_gp_state()
1346 rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1, in show_rcu_nocb_gp_state()
1347 show_rcu_should_be_on_cpu(rdp->nocb_cb_kthread)); in show_rcu_nocb_gp_state()
1351 static void show_rcu_nocb_state(struct rcu_data *rdp) in show_rcu_nocb_state() argument
1355 struct rcu_segcblist *rsclp = &rdp->cblist; in show_rcu_nocb_state()
1359 if (rdp->nocb_gp_rdp == rdp) in show_rcu_nocb_state()
1360 show_rcu_nocb_gp_state(rdp); in show_rcu_nocb_state()
1365 rdp->cpu, rdp->nocb_gp_rdp->cpu, in show_rcu_nocb_state()
1366 rdp->nocb_next_cb_rdp ? rdp->nocb_next_cb_rdp->cpu : -1, in show_rcu_nocb_state()
1367 "kK"[!!rdp->nocb_cb_kthread], in show_rcu_nocb_state()
1368 "bB"[raw_spin_is_locked(&rdp->nocb_bypass_lock)], in show_rcu_nocb_state()
1369 "cC"[!!atomic_read(&rdp->nocb_lock_contended)], in show_rcu_nocb_state()
1370 "lL"[raw_spin_is_locked(&rdp->nocb_lock)], in show_rcu_nocb_state()
1371 "sS"[!!rdp->nocb_cb_sleep], in show_rcu_nocb_state()
1372 ".W"[swait_active(&rdp->nocb_cb_wq)], in show_rcu_nocb_state()
1373 jiffies - rdp->nocb_bypass_first, in show_rcu_nocb_state()
1374 jiffies - rdp->nocb_nobypass_last, in show_rcu_nocb_state()
1375 rdp->nocb_nobypass_count, in show_rcu_nocb_state()
1382 ".B"[!!rcu_cblist_n_cbs(&rdp->nocb_bypass)], in show_rcu_nocb_state()
1383 rcu_segcblist_n_cbs(&rdp->cblist), in show_rcu_nocb_state()
1384 rdp->nocb_cb_kthread ? task_state_to_char(rdp->nocb_cb_kthread) : '.', in show_rcu_nocb_state()
1385 rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1, in show_rcu_nocb_state()
1386 show_rcu_should_be_on_cpu(rdp->nocb_cb_kthread)); in show_rcu_nocb_state()
1389 if (rdp->nocb_gp_rdp == rdp) in show_rcu_nocb_state()
1392 waslocked = raw_spin_is_locked(&rdp->nocb_gp_lock); in show_rcu_nocb_state()
1393 wassleep = swait_active(&rdp->nocb_gp_wq); in show_rcu_nocb_state()
1394 if (!rdp->nocb_gp_sleep && !waslocked && !wassleep) in show_rcu_nocb_state()
1399 "dD"[!!rdp->nocb_defer_wakeup], in show_rcu_nocb_state()
1400 "sS"[!!rdp->nocb_gp_sleep], in show_rcu_nocb_state()
1406 static inline int rcu_lockdep_is_held_nocb(struct rcu_data *rdp) in rcu_lockdep_is_held_nocb() argument
1411 static inline bool rcu_current_is_nocb_kthread(struct rcu_data *rdp) in rcu_current_is_nocb_kthread() argument
1417 static void rcu_nocb_lock(struct rcu_data *rdp) in rcu_nocb_lock() argument
1422 static void rcu_nocb_unlock(struct rcu_data *rdp) in rcu_nocb_unlock() argument
1427 static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, in rcu_nocb_unlock_irqrestore() argument
1434 static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp) in rcu_lockdep_assert_cblist_protected() argument
1452 static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_flush_bypass() argument
1458 static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, in rcu_nocb_try_bypass() argument
1464 static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty, in __call_rcu_nocb_wake() argument
1470 static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) in rcu_boot_init_nocb_percpu_data() argument
1474 static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level) in rcu_nocb_need_deferred_wakeup() argument
1479 static bool do_nocb_deferred_wakeup(struct rcu_data *rdp) in do_nocb_deferred_wakeup() argument
1492 static void show_rcu_nocb_state(struct rcu_data *rdp) in show_rcu_nocb_state() argument