Lines Matching refs:imr
100 struct mlx5_ib_mr *imr, int flags) in populate_klm() argument
107 pklm->key = cpu_to_be32(mr_to_mdev(imr)->null_mkey); in populate_klm()
130 lockdep_assert_held(&to_ib_umem_odp(imr->umem)->umem_mutex); in populate_klm()
133 struct mlx5_ib_mr *mtt = xa_load(&imr->implicit_children, idx); in populate_klm()
140 pklm->key = cpu_to_be32(mr_to_mdev(imr)->null_mkey); in populate_klm()
194 struct mlx5_ib_mr *imr = mr->parent; in free_implicit_child_mr_work() local
195 struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem); in free_implicit_child_mr_work()
207 mlx5r_deref_odp_mkey(&imr->mmkey); in free_implicit_child_mr_work()
214 struct mlx5_ib_mr *imr = mr->parent; in destroy_unused_implicit_child_mr() local
216 if (!refcount_inc_not_zero(&imr->mmkey.usecount)) in destroy_unused_implicit_child_mr()
219 xa_erase(&imr->implicit_children, idx); in destroy_unused_implicit_child_mr()
407 static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr, in implicit_get_child_mr() argument
415 odp = ib_umem_odp_alloc_child(to_ib_umem_odp(imr->umem), in implicit_get_child_mr()
422 mr_to_mdev(imr), MLX5_IMR_MTT_CACHE_ENTRY, imr->access_flags); in implicit_get_child_mr()
428 mr->ibmr.pd = imr->ibmr.pd; in implicit_get_child_mr()
429 mr->ibmr.device = &mr_to_mdev(imr)->ib_dev; in implicit_get_child_mr()
434 mr->parent = imr; in implicit_get_child_mr()
453 xa_lock(&imr->implicit_children); in implicit_get_child_mr()
454 ret = __xa_cmpxchg(&imr->implicit_children, idx, NULL, mr, in implicit_get_child_mr()
468 xa_unlock(&imr->implicit_children); in implicit_get_child_mr()
470 mlx5_ib_dbg(mr_to_mdev(imr), "key %x mr %p\n", mr->mmkey.key, mr); in implicit_get_child_mr()
474 xa_unlock(&imr->implicit_children); in implicit_get_child_mr()
485 struct mlx5_ib_mr *imr; in mlx5_ib_alloc_implicit_mr() local
496 imr = mlx5_mr_cache_alloc(dev, MLX5_IMR_KSM_CACHE_ENTRY, access_flags); in mlx5_ib_alloc_implicit_mr()
497 if (IS_ERR(imr)) { in mlx5_ib_alloc_implicit_mr()
499 return imr; in mlx5_ib_alloc_implicit_mr()
502 imr->ibmr.pd = &pd->ibpd; in mlx5_ib_alloc_implicit_mr()
503 imr->ibmr.iova = 0; in mlx5_ib_alloc_implicit_mr()
504 imr->umem = &umem_odp->umem; in mlx5_ib_alloc_implicit_mr()
505 imr->ibmr.lkey = imr->mmkey.key; in mlx5_ib_alloc_implicit_mr()
506 imr->ibmr.rkey = imr->mmkey.key; in mlx5_ib_alloc_implicit_mr()
507 imr->ibmr.device = &dev->ib_dev; in mlx5_ib_alloc_implicit_mr()
508 imr->umem = &umem_odp->umem; in mlx5_ib_alloc_implicit_mr()
509 imr->is_odp_implicit = true; in mlx5_ib_alloc_implicit_mr()
510 xa_init(&imr->implicit_children); in mlx5_ib_alloc_implicit_mr()
512 err = mlx5_ib_update_xlt(imr, 0, in mlx5_ib_alloc_implicit_mr()
521 err = mlx5r_store_odp_mkey(dev, &imr->mmkey); in mlx5_ib_alloc_implicit_mr()
525 mlx5_ib_dbg(dev, "key %x mr %p\n", imr->mmkey.key, imr); in mlx5_ib_alloc_implicit_mr()
526 return imr; in mlx5_ib_alloc_implicit_mr()
529 mlx5_ib_dereg_mr(&imr->ibmr, NULL); in mlx5_ib_alloc_implicit_mr()
603 static int pagefault_implicit_mr(struct mlx5_ib_mr *imr, in pagefault_implicit_mr() argument
625 xa_lock(&imr->implicit_children); in pagefault_implicit_mr()
626 mtt = xa_load(&imr->implicit_children, idx); in pagefault_implicit_mr()
628 xa_unlock(&imr->implicit_children); in pagefault_implicit_mr()
629 mtt = implicit_get_child_mr(imr, idx); in pagefault_implicit_mr()
638 xa_unlock(&imr->implicit_children); in pagefault_implicit_mr()
678 err = mlx5_ib_update_xlt(imr, upd_start_idx, upd_len, 0, in pagefault_implicit_mr()
683 mlx5_ib_err(mr_to_mdev(imr), "Failed to update PAS\n"); in pagefault_implicit_mr()