Lines Matching refs:objcg
264 static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg,
269 struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt); in obj_cgroup_release() local
294 nr_bytes = atomic_read(&objcg->nr_charged_bytes); in obj_cgroup_release()
299 obj_cgroup_uncharge_pages(objcg, nr_pages); in obj_cgroup_release()
302 list_del(&objcg->list); in obj_cgroup_release()
306 kfree_rcu(objcg, rcu); in obj_cgroup_release()
311 struct obj_cgroup *objcg; in obj_cgroup_alloc() local
314 objcg = kzalloc(sizeof(struct obj_cgroup), GFP_KERNEL); in obj_cgroup_alloc()
315 if (!objcg) in obj_cgroup_alloc()
318 ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0, in obj_cgroup_alloc()
321 kfree(objcg); in obj_cgroup_alloc()
324 INIT_LIST_HEAD(&objcg->list); in obj_cgroup_alloc()
325 return objcg; in obj_cgroup_alloc()
331 struct obj_cgroup *objcg, *iter; in memcg_reparent_objcgs() local
333 objcg = rcu_replace_pointer(memcg->objcg, NULL, true); in memcg_reparent_objcgs()
338 list_add(&objcg->list, &memcg->objcg_list); in memcg_reparent_objcgs()
347 percpu_ref_kill(&objcg->refcnt); in memcg_reparent_objcgs()
2744 static struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *objcg) in get_mem_cgroup_from_objcg() argument
2750 memcg = obj_cgroup_memcg(objcg); in get_mem_cgroup_from_objcg()
2805 static inline void mod_objcg_mlstate(struct obj_cgroup *objcg, in mod_objcg_mlstate() argument
2813 memcg = obj_cgroup_memcg(objcg); in mod_objcg_mlstate()
2881 struct obj_cgroup *objcg; in mem_cgroup_from_obj() local
2885 objcg = page_objcgs(page)[off]; in mem_cgroup_from_obj()
2886 if (objcg) in mem_cgroup_from_obj()
2887 return obj_cgroup_memcg(objcg); in mem_cgroup_from_obj()
2904 struct obj_cgroup *objcg = NULL; in get_obj_cgroup_from_current() local
2917 objcg = rcu_dereference(memcg->objcg); in get_obj_cgroup_from_current()
2918 if (objcg && obj_cgroup_tryget(objcg)) in get_obj_cgroup_from_current()
2920 objcg = NULL; in get_obj_cgroup_from_current()
2924 return objcg; in get_obj_cgroup_from_current()
2975 static void obj_cgroup_uncharge_pages(struct obj_cgroup *objcg, in obj_cgroup_uncharge_pages() argument
2980 memcg = get_mem_cgroup_from_objcg(objcg); in obj_cgroup_uncharge_pages()
2997 static int obj_cgroup_charge_pages(struct obj_cgroup *objcg, gfp_t gfp, in obj_cgroup_charge_pages() argument
3003 memcg = get_mem_cgroup_from_objcg(objcg); in obj_cgroup_charge_pages()
3027 struct obj_cgroup *objcg; in __memcg_kmem_charge_page() local
3030 objcg = get_obj_cgroup_from_current(); in __memcg_kmem_charge_page()
3031 if (objcg) { in __memcg_kmem_charge_page()
3032 ret = obj_cgroup_charge_pages(objcg, gfp, 1 << order); in __memcg_kmem_charge_page()
3034 page->memcg_data = (unsigned long)objcg | in __memcg_kmem_charge_page()
3038 obj_cgroup_put(objcg); in __memcg_kmem_charge_page()
3051 struct obj_cgroup *objcg; in __memcg_kmem_uncharge_page() local
3057 objcg = __folio_objcg(folio); in __memcg_kmem_uncharge_page()
3058 obj_cgroup_uncharge_pages(objcg, nr_pages); in __memcg_kmem_uncharge_page()
3060 obj_cgroup_put(objcg); in __memcg_kmem_uncharge_page()
3063 void mod_objcg_state(struct obj_cgroup *objcg, struct pglist_data *pgdat, in mod_objcg_state() argument
3075 if (stock->cached_objcg != objcg) { in mod_objcg_state()
3077 obj_cgroup_get(objcg); in mod_objcg_state()
3078 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes) in mod_objcg_state()
3079 ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0; in mod_objcg_state()
3080 stock->cached_objcg = objcg; in mod_objcg_state()
3087 mod_objcg_mlstate(objcg, oldpg, NR_SLAB_RECLAIMABLE_B, in mod_objcg_state()
3092 mod_objcg_mlstate(objcg, oldpg, NR_SLAB_UNRECLAIMABLE_B, in mod_objcg_state()
3118 mod_objcg_mlstate(objcg, pgdat, idx, nr); in mod_objcg_state()
3123 static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes) in consume_obj_stock() argument
3129 if (objcg == stock->cached_objcg && stock->nr_bytes >= nr_bytes) { in consume_obj_stock()
3209 static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes, in refill_obj_stock() argument
3216 if (stock->cached_objcg != objcg) { /* reset if necessary */ in refill_obj_stock()
3218 obj_cgroup_get(objcg); in refill_obj_stock()
3219 stock->cached_objcg = objcg; in refill_obj_stock()
3220 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes) in refill_obj_stock()
3221 ? atomic_xchg(&objcg->nr_charged_bytes, 0) : 0; in refill_obj_stock()
3234 obj_cgroup_uncharge_pages(objcg, nr_pages); in refill_obj_stock()
3237 int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size) in obj_cgroup_charge() argument
3242 if (consume_obj_stock(objcg, size)) in obj_cgroup_charge()
3274 ret = obj_cgroup_charge_pages(objcg, gfp, nr_pages); in obj_cgroup_charge()
3276 refill_obj_stock(objcg, PAGE_SIZE - nr_bytes, false); in obj_cgroup_charge()
3281 void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size) in obj_cgroup_uncharge() argument
3283 refill_obj_stock(objcg, size, true); in obj_cgroup_uncharge()
3615 struct obj_cgroup *objcg; in memcg_online_kmem() local
3627 objcg = obj_cgroup_alloc(); in memcg_online_kmem()
3628 if (!objcg) { in memcg_online_kmem()
3632 objcg->memcg = memcg; in memcg_online_kmem()
3633 rcu_assign_pointer(memcg->objcg, objcg); in memcg_online_kmem()
6802 struct obj_cgroup *objcg; in uncharge_folio() local
6813 objcg = __folio_objcg(folio); in uncharge_folio()
6818 memcg = get_mem_cgroup_from_objcg(objcg); in uncharge_folio()
6845 obj_cgroup_put(objcg); in uncharge_folio()