Lines Matching refs:tr
64 struct bpf_trampoline *tr; in bpf_trampoline_lookup() local
70 hlist_for_each_entry(tr, head, hlist) { in bpf_trampoline_lookup()
71 if (tr->key == key) { in bpf_trampoline_lookup()
72 refcount_inc(&tr->refcnt); in bpf_trampoline_lookup()
76 tr = kzalloc(sizeof(*tr), GFP_KERNEL); in bpf_trampoline_lookup()
77 if (!tr) in bpf_trampoline_lookup()
80 tr->key = key; in bpf_trampoline_lookup()
81 INIT_HLIST_NODE(&tr->hlist); in bpf_trampoline_lookup()
82 hlist_add_head(&tr->hlist, head); in bpf_trampoline_lookup()
83 refcount_set(&tr->refcnt, 1); in bpf_trampoline_lookup()
84 mutex_init(&tr->mutex); in bpf_trampoline_lookup()
86 INIT_HLIST_HEAD(&tr->progs_hlist[i]); in bpf_trampoline_lookup()
89 return tr; in bpf_trampoline_lookup()
92 static int bpf_trampoline_module_get(struct bpf_trampoline *tr) in bpf_trampoline_module_get() argument
98 mod = __module_text_address((unsigned long) tr->func.addr); in bpf_trampoline_module_get()
102 tr->mod = mod; in bpf_trampoline_module_get()
106 static void bpf_trampoline_module_put(struct bpf_trampoline *tr) in bpf_trampoline_module_put() argument
108 module_put(tr->mod); in bpf_trampoline_module_put()
109 tr->mod = NULL; in bpf_trampoline_module_put()
124 static int unregister_fentry(struct bpf_trampoline *tr, void *old_addr) in unregister_fentry() argument
126 void *ip = tr->func.addr; in unregister_fentry()
129 if (tr->func.ftrace_managed) in unregister_fentry()
135 bpf_trampoline_module_put(tr); in unregister_fentry()
139 static int modify_fentry(struct bpf_trampoline *tr, void *old_addr, void *new_addr) in modify_fentry() argument
141 void *ip = tr->func.addr; in modify_fentry()
144 if (tr->func.ftrace_managed) in modify_fentry()
152 static int register_fentry(struct bpf_trampoline *tr, void *new_addr) in register_fentry() argument
154 void *ip = tr->func.addr; in register_fentry()
160 tr->func.ftrace_managed = ret; in register_fentry()
162 if (bpf_trampoline_module_get(tr)) in register_fentry()
165 if (tr->func.ftrace_managed) in register_fentry()
171 bpf_trampoline_module_put(tr); in register_fentry()
176 bpf_trampoline_get_progs(const struct bpf_trampoline *tr, int *total, bool *ip_arg) in bpf_trampoline_get_progs() argument
189 tprogs[kind].nr_progs = tr->progs_cnt[kind]; in bpf_trampoline_get_progs()
190 *total += tr->progs_cnt[kind]; in bpf_trampoline_get_progs()
193 hlist_for_each_entry(aux, &tr->progs_hlist[kind], tramp_hlist) { in bpf_trampoline_get_progs()
334 static int bpf_trampoline_update(struct bpf_trampoline *tr) in bpf_trampoline_update() argument
342 tprogs = bpf_trampoline_get_progs(tr, &total, &ip_arg); in bpf_trampoline_update()
347 err = unregister_fentry(tr, tr->cur_image->image); in bpf_trampoline_update()
348 bpf_tramp_image_put(tr->cur_image); in bpf_trampoline_update()
349 tr->cur_image = NULL; in bpf_trampoline_update()
350 tr->selector = 0; in bpf_trampoline_update()
354 im = bpf_tramp_image_alloc(tr->key, tr->selector); in bpf_trampoline_update()
368 &tr->func.model, flags, tprogs, in bpf_trampoline_update()
369 tr->func.addr); in bpf_trampoline_update()
373 WARN_ON(tr->cur_image && tr->selector == 0); in bpf_trampoline_update()
374 WARN_ON(!tr->cur_image && tr->selector); in bpf_trampoline_update()
375 if (tr->cur_image) in bpf_trampoline_update()
377 err = modify_fentry(tr, tr->cur_image->image, im->image); in bpf_trampoline_update()
380 err = register_fentry(tr, im->image); in bpf_trampoline_update()
383 if (tr->cur_image) in bpf_trampoline_update()
384 bpf_tramp_image_put(tr->cur_image); in bpf_trampoline_update()
385 tr->cur_image = im; in bpf_trampoline_update()
386 tr->selector++; in bpf_trampoline_update()
414 int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr) in bpf_trampoline_link_prog() argument
421 mutex_lock(&tr->mutex); in bpf_trampoline_link_prog()
422 if (tr->extension_prog) { in bpf_trampoline_link_prog()
429 cnt = tr->progs_cnt[BPF_TRAMP_FENTRY] + tr->progs_cnt[BPF_TRAMP_FEXIT]; in bpf_trampoline_link_prog()
436 tr->extension_prog = prog; in bpf_trampoline_link_prog()
437 err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, NULL, in bpf_trampoline_link_prog()
450 hlist_add_head(&prog->aux->tramp_hlist, &tr->progs_hlist[kind]); in bpf_trampoline_link_prog()
451 tr->progs_cnt[kind]++; in bpf_trampoline_link_prog()
452 err = bpf_trampoline_update(tr); in bpf_trampoline_link_prog()
455 tr->progs_cnt[kind]--; in bpf_trampoline_link_prog()
458 mutex_unlock(&tr->mutex); in bpf_trampoline_link_prog()
463 int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr) in bpf_trampoline_unlink_prog() argument
469 mutex_lock(&tr->mutex); in bpf_trampoline_unlink_prog()
471 WARN_ON_ONCE(!tr->extension_prog); in bpf_trampoline_unlink_prog()
472 err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, in bpf_trampoline_unlink_prog()
473 tr->extension_prog->bpf_func, NULL); in bpf_trampoline_unlink_prog()
474 tr->extension_prog = NULL; in bpf_trampoline_unlink_prog()
478 tr->progs_cnt[kind]--; in bpf_trampoline_unlink_prog()
479 err = bpf_trampoline_update(tr); in bpf_trampoline_unlink_prog()
481 mutex_unlock(&tr->mutex); in bpf_trampoline_unlink_prog()
488 struct bpf_trampoline *tr; in bpf_trampoline_get() local
490 tr = bpf_trampoline_lookup(key); in bpf_trampoline_get()
491 if (!tr) in bpf_trampoline_get()
494 mutex_lock(&tr->mutex); in bpf_trampoline_get()
495 if (tr->func.addr) in bpf_trampoline_get()
498 memcpy(&tr->func.model, &tgt_info->fmodel, sizeof(tgt_info->fmodel)); in bpf_trampoline_get()
499 tr->func.addr = (void *)tgt_info->tgt_addr; in bpf_trampoline_get()
501 mutex_unlock(&tr->mutex); in bpf_trampoline_get()
502 return tr; in bpf_trampoline_get()
505 void bpf_trampoline_put(struct bpf_trampoline *tr) in bpf_trampoline_put() argument
507 if (!tr) in bpf_trampoline_put()
510 if (!refcount_dec_and_test(&tr->refcnt)) in bpf_trampoline_put()
512 WARN_ON_ONCE(mutex_is_locked(&tr->mutex)); in bpf_trampoline_put()
513 if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[BPF_TRAMP_FENTRY]))) in bpf_trampoline_put()
515 if (WARN_ON_ONCE(!hlist_empty(&tr->progs_hlist[BPF_TRAMP_FEXIT]))) in bpf_trampoline_put()
523 hlist_del(&tr->hlist); in bpf_trampoline_put()
524 kfree(tr); in bpf_trampoline_put()
628 void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr) in __bpf_tramp_enter() argument
630 percpu_ref_get(&tr->pcref); in __bpf_tramp_enter()
633 void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr) in __bpf_tramp_exit() argument
635 percpu_ref_put(&tr->pcref); in __bpf_tramp_exit()
639 arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *image_end, in arch_prepare_bpf_trampoline() argument