Lines Matching refs:sbq
398 static unsigned int sbq_calc_wake_batch(struct sbitmap_queue *sbq, in sbq_calc_wake_batch() argument
420 shallow_depth = min(1U << sbq->sb.shift, sbq->min_shallow_depth); in sbq_calc_wake_batch()
421 depth = ((depth >> sbq->sb.shift) * shallow_depth + in sbq_calc_wake_batch()
422 min(depth & ((1U << sbq->sb.shift) - 1), shallow_depth)); in sbq_calc_wake_batch()
429 int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth, in sbitmap_queue_init_node() argument
435 ret = sbitmap_init_node(&sbq->sb, depth, shift, flags, node, in sbitmap_queue_init_node()
440 sbq->min_shallow_depth = UINT_MAX; in sbitmap_queue_init_node()
441 sbq->wake_batch = sbq_calc_wake_batch(sbq, depth); in sbitmap_queue_init_node()
442 atomic_set(&sbq->wake_index, 0); in sbitmap_queue_init_node()
443 atomic_set(&sbq->ws_active, 0); in sbitmap_queue_init_node()
445 sbq->ws = kzalloc_node(SBQ_WAIT_QUEUES * sizeof(*sbq->ws), flags, node); in sbitmap_queue_init_node()
446 if (!sbq->ws) { in sbitmap_queue_init_node()
447 sbitmap_free(&sbq->sb); in sbitmap_queue_init_node()
452 init_waitqueue_head(&sbq->ws[i].wait); in sbitmap_queue_init_node()
453 atomic_set(&sbq->ws[i].wait_cnt, sbq->wake_batch); in sbitmap_queue_init_node()
460 static void sbitmap_queue_update_wake_batch(struct sbitmap_queue *sbq, in sbitmap_queue_update_wake_batch() argument
463 unsigned int wake_batch = sbq_calc_wake_batch(sbq, depth); in sbitmap_queue_update_wake_batch()
466 if (sbq->wake_batch != wake_batch) { in sbitmap_queue_update_wake_batch()
467 WRITE_ONCE(sbq->wake_batch, wake_batch); in sbitmap_queue_update_wake_batch()
475 atomic_set(&sbq->ws[i].wait_cnt, 1); in sbitmap_queue_update_wake_batch()
479 void sbitmap_queue_resize(struct sbitmap_queue *sbq, unsigned int depth) in sbitmap_queue_resize() argument
481 sbitmap_queue_update_wake_batch(sbq, depth); in sbitmap_queue_resize()
482 sbitmap_resize(&sbq->sb, depth); in sbitmap_queue_resize()
486 int __sbitmap_queue_get(struct sbitmap_queue *sbq) in __sbitmap_queue_get() argument
488 return sbitmap_get(&sbq->sb); in __sbitmap_queue_get()
492 unsigned long __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags, in __sbitmap_queue_get_batch() argument
495 struct sbitmap *sb = &sbq->sb; in __sbitmap_queue_get_batch()
543 int __sbitmap_queue_get_shallow(struct sbitmap_queue *sbq, in __sbitmap_queue_get_shallow() argument
546 WARN_ON_ONCE(shallow_depth < sbq->min_shallow_depth); in __sbitmap_queue_get_shallow()
548 return sbitmap_get_shallow(&sbq->sb, shallow_depth); in __sbitmap_queue_get_shallow()
552 void sbitmap_queue_min_shallow_depth(struct sbitmap_queue *sbq, in sbitmap_queue_min_shallow_depth() argument
555 sbq->min_shallow_depth = min_shallow_depth; in sbitmap_queue_min_shallow_depth()
556 sbitmap_queue_update_wake_batch(sbq, sbq->sb.depth); in sbitmap_queue_min_shallow_depth()
560 static struct sbq_wait_state *sbq_wake_ptr(struct sbitmap_queue *sbq) in sbq_wake_ptr() argument
564 if (!atomic_read(&sbq->ws_active)) in sbq_wake_ptr()
567 wake_index = atomic_read(&sbq->wake_index); in sbq_wake_ptr()
569 struct sbq_wait_state *ws = &sbq->ws[wake_index]; in sbq_wake_ptr()
572 if (wake_index != atomic_read(&sbq->wake_index)) in sbq_wake_ptr()
573 atomic_set(&sbq->wake_index, wake_index); in sbq_wake_ptr()
583 static bool __sbq_wake_up(struct sbitmap_queue *sbq) in __sbq_wake_up() argument
589 ws = sbq_wake_ptr(sbq); in __sbq_wake_up()
597 wake_batch = READ_ONCE(sbq->wake_batch); in __sbq_wake_up()
613 sbq_index_atomic_inc(&sbq->wake_index); in __sbq_wake_up()
624 void sbitmap_queue_wake_up(struct sbitmap_queue *sbq) in sbitmap_queue_wake_up() argument
626 while (__sbq_wake_up(sbq)) in sbitmap_queue_wake_up()
637 void sbitmap_queue_clear_batch(struct sbitmap_queue *sbq, int offset, in sbitmap_queue_clear_batch() argument
640 struct sbitmap *sb = &sbq->sb; in sbitmap_queue_clear_batch()
666 sbitmap_queue_wake_up(sbq); in sbitmap_queue_clear_batch()
667 sbitmap_update_cpu_hint(&sbq->sb, raw_smp_processor_id(), in sbitmap_queue_clear_batch()
671 void sbitmap_queue_clear(struct sbitmap_queue *sbq, unsigned int nr, in sbitmap_queue_clear() argument
685 sbitmap_deferred_clear_bit(&sbq->sb, nr); in sbitmap_queue_clear()
694 sbitmap_queue_wake_up(sbq); in sbitmap_queue_clear()
695 sbitmap_update_cpu_hint(&sbq->sb, cpu, nr); in sbitmap_queue_clear()
699 void sbitmap_queue_wake_all(struct sbitmap_queue *sbq) in sbitmap_queue_wake_all() argument
708 wake_index = atomic_read(&sbq->wake_index); in sbitmap_queue_wake_all()
710 struct sbq_wait_state *ws = &sbq->ws[wake_index]; in sbitmap_queue_wake_all()
720 void sbitmap_queue_show(struct sbitmap_queue *sbq, struct seq_file *m) in sbitmap_queue_show() argument
725 sbitmap_show(&sbq->sb, m); in sbitmap_queue_show()
733 seq_printf(m, "%u", *per_cpu_ptr(sbq->sb.alloc_hint, i)); in sbitmap_queue_show()
737 seq_printf(m, "wake_batch=%u\n", sbq->wake_batch); in sbitmap_queue_show()
738 seq_printf(m, "wake_index=%d\n", atomic_read(&sbq->wake_index)); in sbitmap_queue_show()
739 seq_printf(m, "ws_active=%d\n", atomic_read(&sbq->ws_active)); in sbitmap_queue_show()
743 struct sbq_wait_state *ws = &sbq->ws[i]; in sbitmap_queue_show()
751 seq_printf(m, "round_robin=%d\n", sbq->sb.round_robin); in sbitmap_queue_show()
752 seq_printf(m, "min_shallow_depth=%u\n", sbq->min_shallow_depth); in sbitmap_queue_show()
756 void sbitmap_add_wait_queue(struct sbitmap_queue *sbq, in sbitmap_add_wait_queue() argument
760 if (!sbq_wait->sbq) { in sbitmap_add_wait_queue()
761 sbq_wait->sbq = sbq; in sbitmap_add_wait_queue()
762 atomic_inc(&sbq->ws_active); in sbitmap_add_wait_queue()
771 if (sbq_wait->sbq) { in sbitmap_del_wait_queue()
772 atomic_dec(&sbq_wait->sbq->ws_active); in sbitmap_del_wait_queue()
773 sbq_wait->sbq = NULL; in sbitmap_del_wait_queue()
778 void sbitmap_prepare_to_wait(struct sbitmap_queue *sbq, in sbitmap_prepare_to_wait() argument
782 if (!sbq_wait->sbq) { in sbitmap_prepare_to_wait()
783 atomic_inc(&sbq->ws_active); in sbitmap_prepare_to_wait()
784 sbq_wait->sbq = sbq; in sbitmap_prepare_to_wait()
790 void sbitmap_finish_wait(struct sbitmap_queue *sbq, struct sbq_wait_state *ws, in sbitmap_finish_wait() argument
794 if (sbq_wait->sbq) { in sbitmap_finish_wait()
795 atomic_dec(&sbq->ws_active); in sbitmap_finish_wait()
796 sbq_wait->sbq = NULL; in sbitmap_finish_wait()