Lines Matching refs:split
193 struct vring_virtqueue_split split; member
445 struct vring_desc_extra *extra = vq->split.desc_extra; in vring_unmap_one_split()
503 struct vring_desc_extra *extra = vring->split.desc_extra; in virtqueue_add_desc_split()
559 WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect); in virtqueue_add_split()
570 desc = vq->split.vring.desc; in virtqueue_add_split()
624 vq->split.desc_extra[prev & (vq->split.vring.num - 1)].flags &= in virtqueue_add_split()
635 virtqueue_add_desc_split(_vq, vq->split.vring.desc, in virtqueue_add_split()
647 vq->free_head = vq->split.desc_extra[head].next; in virtqueue_add_split()
652 vq->split.desc_state[head].data = data; in virtqueue_add_split()
654 vq->split.desc_state[head].indir_desc = desc; in virtqueue_add_split()
656 vq->split.desc_state[head].indir_desc = ctx; in virtqueue_add_split()
660 avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1); in virtqueue_add_split()
661 vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head); in virtqueue_add_split()
666 vq->split.avail_idx_shadow++; in virtqueue_add_split()
667 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev, in virtqueue_add_split()
668 vq->split.avail_idx_shadow); in virtqueue_add_split()
717 old = vq->split.avail_idx_shadow - vq->num_added; in virtqueue_kick_prepare_split()
718 new = vq->split.avail_idx_shadow; in virtqueue_kick_prepare_split()
726 vring_avail_event(&vq->split.vring)), in virtqueue_kick_prepare_split()
729 needs_kick = !(vq->split.vring.used->flags & in virtqueue_kick_prepare_split()
744 vq->split.desc_state[head].data = NULL; in detach_buf_split()
749 while (vq->split.vring.desc[i].flags & nextflag) { in detach_buf_split()
751 i = vq->split.desc_extra[i].next; in detach_buf_split()
756 vq->split.desc_extra[i].next = vq->free_head; in detach_buf_split()
764 vq->split.desc_state[head].indir_desc; in detach_buf_split()
771 len = vq->split.desc_extra[head].len; in detach_buf_split()
773 BUG_ON(!(vq->split.desc_extra[head].flags & in detach_buf_split()
781 vq->split.desc_state[head].indir_desc = NULL; in detach_buf_split()
783 *ctx = vq->split.desc_state[head].indir_desc; in detach_buf_split()
790 vq->split.vring.used->idx); in more_used_split()
818 last_used = (vq->last_used_idx & (vq->split.vring.num - 1)); in virtqueue_get_buf_ctx_split()
820 vq->split.vring.used->ring[last_used].id); in virtqueue_get_buf_ctx_split()
822 vq->split.vring.used->ring[last_used].len); in virtqueue_get_buf_ctx_split()
824 if (unlikely(i >= vq->split.vring.num)) { in virtqueue_get_buf_ctx_split()
828 if (unlikely(!vq->split.desc_state[i].data)) { in virtqueue_get_buf_ctx_split()
834 ret = vq->split.desc_state[i].data; in virtqueue_get_buf_ctx_split()
840 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) in virtqueue_get_buf_ctx_split()
842 &vring_used_event(&vq->split.vring), in virtqueue_get_buf_ctx_split()
855 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) { in virtqueue_disable_cb_split()
856 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; in virtqueue_disable_cb_split()
859 vring_used_event(&vq->split.vring) = 0x0; in virtqueue_disable_cb_split()
861 vq->split.vring.avail->flags = in virtqueue_disable_cb_split()
863 vq->split.avail_flags_shadow); in virtqueue_disable_cb_split()
879 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { in virtqueue_enable_cb_prepare_split()
880 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; in virtqueue_enable_cb_prepare_split()
882 vq->split.vring.avail->flags = in virtqueue_enable_cb_prepare_split()
884 vq->split.avail_flags_shadow); in virtqueue_enable_cb_prepare_split()
886 vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev, in virtqueue_enable_cb_prepare_split()
897 vq->split.vring.used->idx); in virtqueue_poll_split()
912 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { in virtqueue_enable_cb_delayed_split()
913 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; in virtqueue_enable_cb_delayed_split()
915 vq->split.vring.avail->flags = in virtqueue_enable_cb_delayed_split()
917 vq->split.avail_flags_shadow); in virtqueue_enable_cb_delayed_split()
920 bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4; in virtqueue_enable_cb_delayed_split()
923 &vring_used_event(&vq->split.vring), in virtqueue_enable_cb_delayed_split()
926 if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx) in virtqueue_enable_cb_delayed_split()
944 for (i = 0; i < vq->split.vring.num; i++) { in virtqueue_detach_unused_buf_split()
945 if (!vq->split.desc_state[i].data) in virtqueue_detach_unused_buf_split()
948 buf = vq->split.desc_state[i].data; in virtqueue_detach_unused_buf_split()
950 vq->split.avail_idx_shadow--; in virtqueue_detach_unused_buf_split()
951 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev, in virtqueue_detach_unused_buf_split()
952 vq->split.avail_idx_shadow); in virtqueue_detach_unused_buf_split()
957 BUG_ON(vq->vq.num_free != vq->split.vring.num); in virtqueue_detach_unused_buf_split()
986 num = vq->split.vring.num; in virtqueue_reinit_split()
988 vq->split.vring.avail->flags = 0; in virtqueue_reinit_split()
989 vq->split.vring.avail->idx = 0; in virtqueue_reinit_split()
992 vq->split.vring.avail->ring[num] = 0; in virtqueue_reinit_split()
994 vq->split.vring.used->flags = 0; in virtqueue_reinit_split()
995 vq->split.vring.used->idx = 0; in virtqueue_reinit_split()
998 *(__virtio16 *)&(vq->split.vring.used->ring[num]) = 0; in virtqueue_reinit_split()
1002 virtqueue_vring_init_split(&vq->split, vq); in virtqueue_reinit_split()
1008 vq->split = *vring_split; in virtqueue_vring_attach_split()
1145 vq->split.vring_align, in virtqueue_resize_split()
1146 vq->split.may_reduce_num, in virtqueue_resize_split()
2649 if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num) in virtqueue_resize()
2729 vq->split.queue_size_in_bytes, in vring_free()
2730 vq->split.vring.desc, in vring_free()
2731 vq->split.queue_dma_addr, in vring_free()
2736 kfree(vq->split.desc_state); in vring_free()
2737 kfree(vq->split.desc_extra); in vring_free()
2794 return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num; in virtqueue_get_vring_size()
2880 return vq->split.queue_dma_addr; in virtqueue_get_desc_addr()
2893 return vq->split.queue_dma_addr + in virtqueue_get_avail_addr()
2894 ((char *)vq->split.vring.avail - (char *)vq->split.vring.desc); in virtqueue_get_avail_addr()
2907 return vq->split.queue_dma_addr + in virtqueue_get_used_addr()
2908 ((char *)vq->split.vring.used - (char *)vq->split.vring.desc); in virtqueue_get_used_addr()
2915 return &to_vvq(vq)->split.vring; in virtqueue_get_vring()