Lines Matching refs:pg
424 struct ftrace_profile_page *pg; in function_stat_next() local
426 pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK); in function_stat_next()
432 if ((void *)rec >= (void *)&pg->records[pg->index]) { in function_stat_next()
433 pg = pg->next; in function_stat_next()
434 if (!pg) in function_stat_next()
436 rec = &pg->records[0]; in function_stat_next()
565 struct ftrace_profile_page *pg; in ftrace_profile_reset() local
567 pg = stat->pages = stat->start; in ftrace_profile_reset()
569 while (pg) { in ftrace_profile_reset()
570 memset(pg->records, 0, PROFILE_RECORDS_SIZE); in ftrace_profile_reset()
571 pg->index = 0; in ftrace_profile_reset()
572 pg = pg->next; in ftrace_profile_reset()
581 struct ftrace_profile_page *pg; in ftrace_profile_pages_init() local
607 pg = stat->start = stat->pages; in ftrace_profile_pages_init()
612 pg->next = (void *)get_zeroed_page(GFP_KERNEL); in ftrace_profile_pages_init()
613 if (!pg->next) in ftrace_profile_pages_init()
615 pg = pg->next; in ftrace_profile_pages_init()
621 pg = stat->start; in ftrace_profile_pages_init()
622 while (pg) { in ftrace_profile_pages_init()
623 unsigned long tmp = (unsigned long)pg; in ftrace_profile_pages_init()
625 pg = pg->next; in ftrace_profile_pages_init()
1502 #define do_for_each_ftrace_rec(pg, rec) \ argument
1503 for (pg = ftrace_pages_start; pg; pg = pg->next) { \
1505 for (_____i = 0; _____i < pg->index; _____i++) { \
1506 rec = &pg->records[_____i];
1527 struct ftrace_page *pg; in lookup_rec() local
1534 for (pg = ftrace_pages_start; pg; pg = pg->next) { in lookup_rec()
1535 if (end < pg->records[0].ip || in lookup_rec()
1536 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE)) in lookup_rec()
1538 rec = bsearch(&key, pg->records, pg->index, in lookup_rec()
1637 struct ftrace_page *pg; in __ftrace_hash_rec_update() local
1675 do_for_each_ftrace_rec(pg, rec) { in __ftrace_hash_rec_update()
1861 struct ftrace_page *pg; in __ftrace_hash_update_ipmodify() local
1880 do_for_each_ftrace_rec(pg, rec) { in __ftrace_hash_update_ipmodify()
1906 do_for_each_ftrace_rec(pg, rec) { in __ftrace_hash_update_ipmodify()
2576 struct ftrace_page *pg; in ftrace_replace_code() local
2584 do_for_each_ftrace_rec(pg, rec) { in ftrace_replace_code()
2601 struct ftrace_page *pg; member
2623 iter->pg = ftrace_pages_start; in ftrace_rec_iter_start()
2627 while (iter->pg && !iter->pg->index) in ftrace_rec_iter_start()
2628 iter->pg = iter->pg->next; in ftrace_rec_iter_start()
2630 if (!iter->pg) in ftrace_rec_iter_start()
2646 if (iter->index >= iter->pg->index) { in ftrace_rec_iter_next()
2647 iter->pg = iter->pg->next; in ftrace_rec_iter_next()
2651 while (iter->pg && !iter->pg->index) in ftrace_rec_iter_next()
2652 iter->pg = iter->pg->next; in ftrace_rec_iter_next()
2655 if (!iter->pg) in ftrace_rec_iter_next()
2669 return &iter->pg->records[iter->index]; in ftrace_rec_iter_record()
3001 struct ftrace_page *pg; in ftrace_shutdown() local
3004 do_for_each_ftrace_rec(pg, rec) { in ftrace_shutdown()
3133 struct ftrace_page *pg; in ftrace_update_code() local
3156 for (pg = new_pgs; pg; pg = pg->next) { in ftrace_update_code()
3158 for (i = 0; i < pg->index; i++) { in ftrace_update_code()
3164 p = &pg->records[i]; in ftrace_update_code()
3185 static int ftrace_allocate_records(struct ftrace_page *pg, int count) in ftrace_allocate_records() argument
3199 pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); in ftrace_allocate_records()
3201 if (!pg->records) { in ftrace_allocate_records()
3213 pg->order = order; in ftrace_allocate_records()
3225 struct ftrace_page *pg; in ftrace_allocate_pages() local
3231 start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL); in ftrace_allocate_pages()
3232 if (!pg) in ftrace_allocate_pages()
3241 cnt = ftrace_allocate_records(pg, num_to_init); in ftrace_allocate_pages()
3249 pg->next = kzalloc(sizeof(*pg), GFP_KERNEL); in ftrace_allocate_pages()
3250 if (!pg->next) in ftrace_allocate_pages()
3253 pg = pg->next; in ftrace_allocate_pages()
3259 pg = start_pg; in ftrace_allocate_pages()
3260 while (pg) { in ftrace_allocate_pages()
3261 if (pg->records) { in ftrace_allocate_pages()
3262 free_pages((unsigned long)pg->records, pg->order); in ftrace_allocate_pages()
3263 ftrace_number_of_pages -= 1 << pg->order; in ftrace_allocate_pages()
3265 start_pg = pg->next; in ftrace_allocate_pages()
3266 kfree(pg); in ftrace_allocate_pages()
3267 pg = start_pg; in ftrace_allocate_pages()
3280 struct ftrace_page *pg; member
3510 if (iter->idx >= iter->pg->index) { in t_func_next()
3511 if (iter->pg->next) { in t_func_next()
3512 iter->pg = iter->pg->next; in t_func_next()
3517 rec = &iter->pg->records[iter->idx++]; in t_func_next()
3616 iter->pg = ftrace_pages_start; in t_start()
3737 iter->pg = ftrace_pages_start; in ftrace_avail_open()
3761 iter->pg = ftrace_pages_start; in ftrace_enabled_open()
3845 iter->pg = ftrace_pages_start; in ftrace_regex_open()
3973 struct ftrace_page *pg; in add_rec_by_index() local
3980 do_for_each_ftrace_rec(pg, rec) { in add_rec_by_index()
3981 if (pg->index <= index) { in add_rec_by_index()
3982 index -= pg->index; in add_rec_by_index()
3986 rec = &pg->records[index]; in add_rec_by_index()
4034 struct ftrace_page *pg; in match_records() local
4066 do_for_each_ftrace_rec(pg, rec) { in match_records()
6229 struct ftrace_page *pg; in ftrace_graph_set_hash() local
6247 do_for_each_ftrace_rec(pg, rec) { in ftrace_graph_set_hash()
6402 struct ftrace_page *pg; in ftrace_process_locs() local
6447 pg = start_pg; in ftrace_process_locs()
6460 end_offset = (pg->index+1) * sizeof(pg->records[0]); in ftrace_process_locs()
6461 if (end_offset > PAGE_SIZE << pg->order) { in ftrace_process_locs()
6463 if (WARN_ON(!pg->next)) in ftrace_process_locs()
6465 pg = pg->next; in ftrace_process_locs()
6468 rec = &pg->records[pg->index++]; in ftrace_process_locs()
6473 WARN_ON(pg->next); in ftrace_process_locs()
6476 ftrace_pages = pg; in ftrace_process_locs()
6567 clear_mod_from_hash(struct ftrace_page *pg, struct ftrace_hash *hash) in clear_mod_from_hash() argument
6576 for (i = 0; i < pg->index; i++) { in clear_mod_from_hash()
6577 rec = &pg->records[i]; in clear_mod_from_hash()
6590 static void clear_mod_from_hashes(struct ftrace_page *pg) in clear_mod_from_hashes() argument
6599 clear_mod_from_hash(pg, tr->ops->func_hash->filter_hash); in clear_mod_from_hashes()
6600 clear_mod_from_hash(pg, tr->ops->func_hash->notrace_hash); in clear_mod_from_hashes()
6629 struct ftrace_page *pg; in ftrace_release_mod() local
6649 for (pg = ftrace_pages_start; pg; pg = *last_pg) { in ftrace_release_mod()
6650 rec = &pg->records[0]; in ftrace_release_mod()
6657 if (WARN_ON(pg == ftrace_pages_start)) in ftrace_release_mod()
6661 if (pg == ftrace_pages) in ftrace_release_mod()
6664 ftrace_update_tot_cnt -= pg->index; in ftrace_release_mod()
6665 *last_pg = pg->next; in ftrace_release_mod()
6667 pg->next = tmp_page; in ftrace_release_mod()
6668 tmp_page = pg; in ftrace_release_mod()
6670 last_pg = &pg->next; in ftrace_release_mod()
6675 for (pg = tmp_page; pg; pg = tmp_page) { in ftrace_release_mod()
6678 clear_mod_from_hashes(pg); in ftrace_release_mod()
6680 if (pg->records) { in ftrace_release_mod()
6681 free_pages((unsigned long)pg->records, pg->order); in ftrace_release_mod()
6682 ftrace_number_of_pages -= 1 << pg->order; in ftrace_release_mod()
6684 tmp_page = pg->next; in ftrace_release_mod()
6685 kfree(pg); in ftrace_release_mod()
6693 struct ftrace_page *pg; in ftrace_module_enable() local
6716 do_for_each_ftrace_rec(pg, rec) { in ftrace_module_enable()
6997 struct ftrace_page *pg; in ftrace_free_mem() local
7019 for (pg = ftrace_pages_start; pg; last_pg = &pg->next, pg = *last_pg) { in ftrace_free_mem()
7020 if (end < pg->records[0].ip || in ftrace_free_mem()
7021 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE)) in ftrace_free_mem()
7024 rec = bsearch(&key, pg->records, pg->index, in ftrace_free_mem()
7036 pg->index--; in ftrace_free_mem()
7038 if (!pg->index) { in ftrace_free_mem()
7039 *last_pg = pg->next; in ftrace_free_mem()
7040 if (pg->records) { in ftrace_free_mem()
7041 free_pages((unsigned long)pg->records, pg->order); in ftrace_free_mem()
7042 ftrace_number_of_pages -= 1 << pg->order; in ftrace_free_mem()
7045 kfree(pg); in ftrace_free_mem()
7046 pg = container_of(last_pg, struct ftrace_page, next); in ftrace_free_mem()
7048 ftrace_pages = pg; in ftrace_free_mem()
7052 (pg->index - (rec - pg->records)) * sizeof(*rec)); in ftrace_free_mem()