Release 4.14 tools/perf/builtin-kmem.c

// SPDX-License-Identifier: GPL-2.0
#include "builtin.h"
#include "perf.h"

#include "util/evlist.h"
#include "util/evsel.h"
#include "util/util.h"
#include "util/config.h"
#include "util/symbol.h"
#include "util/thread.h"
#include "util/header.h"
#include "util/session.h"
#include "util/tool.h"
#include "util/callchain.h"
#include "util/time-utils.h"

#include <subcmd/parse-options.h>
#include "util/trace-event.h"
#include "util/data.h"
#include "util/cpumap.h"

#include "util/debug.h"

#include <linux/kernel.h>
#include <linux/rbtree.h>
#include <linux/string.h>
#include <errno.h>
#include <inttypes.h>
#include <locale.h>
#include <regex.h>

#include "sane_ctype.h"


static int	kmem_slab;

static int	kmem_page;


static long	kmem_page_size;
static enum {
	
KMEM_SLAB,
	
KMEM_PAGE,
} 
kmem_default = KMEM_SLAB;  
/* for backward compatibility */

struct alloc_stat;

typedef int (*sort_fn_t)(void *, void *);


static int			alloc_flag;

static int			caller_flag;


static int			alloc_lines = -1;

static int			caller_lines = -1;


static bool			raw_ip;


struct alloc_stat {
	
u64	call_site;
	
u64	ptr;
	
u64	bytes_req;
	
u64	bytes_alloc;
	
u64	last_alloc;
	
u32	hit;
	
u32	pingpong;

	
short	alloc_cpu;

	
struct rb_node node;
};


static struct rb_root root_alloc_stat;

static struct rb_root root_alloc_sorted;

static struct rb_root root_caller_stat;

static struct rb_root root_caller_sorted;




static unsigned long total_requested, total_allocated, total_freed;


static unsigned long nr_allocs, nr_cross_allocs;

/* filters for controlling start and stop of time of analysis */

static struct perf_time_interval ptime;

const char *time_str;



static int insert_alloc_stat(unsigned long call_site, unsigned long ptr,
			     int bytes_req, int bytes_alloc, int cpu)
{
	struct rb_node **node = &root_alloc_stat.rb_node;
	struct rb_node *parent = NULL;
	struct alloc_stat *data = NULL;

	while (*node) {
		parent = *node;
		data = rb_entry(*node, struct alloc_stat, node);

		if (ptr > data->ptr)
			node = &(*node)->rb_right;
		else if (ptr < data->ptr)
			node = &(*node)->rb_left;
		else
			break;
	}

	if (data && data->ptr == ptr) {
		data->hit++;
		data->bytes_req += bytes_req;
		data->bytes_alloc += bytes_alloc;
	} else {
		data = malloc(sizeof(*data));
		if (!data) {
			pr_err("%s: malloc failed\n", __func__);
			return -1;
		}
		data->ptr = ptr;
		data->pingpong = 0;
		data->hit = 1;
		data->bytes_req = bytes_req;
		data->bytes_alloc = bytes_alloc;

		rb_link_node(&data->node, parent, node);
		rb_insert_color(&data->node, &root_alloc_stat);
	}
	data->call_site = call_site;
	data->alloc_cpu = cpu;
	data->last_alloc = bytes_alloc;

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Li Zefan
226
91.50%
2
40.00%
Arnaldo Carvalho de Melo
14
5.67%
1
20.00%
David Ahern
6
2.43%
1
20.00%
Wenji Huang
1
0.40%
1
20.00%
Total
247
100.00%
5
100.00%



static int insert_caller_stat(unsigned long call_site,
			      int bytes_req, int bytes_alloc)
{
	struct rb_node **node = &root_caller_stat.rb_node;
	struct rb_node *parent = NULL;
	struct alloc_stat *data = NULL;

	while (*node) {
		parent = *node;
		data = rb_entry(*node, struct alloc_stat, node);

		if (call_site > data->call_site)
			node = &(*node)->rb_right;
		else if (call_site < data->call_site)
			node = &(*node)->rb_left;
		else
			break;
	}

	if (data && data->call_site == call_site) {
		data->hit++;
		data->bytes_req += bytes_req;
		data->bytes_alloc += bytes_alloc;
	} else {
		data = malloc(sizeof(*data));
		if (!data) {
			pr_err("%s: malloc failed\n", __func__);
			return -1;
		}
		data->call_site = call_site;
		data->pingpong = 0;
		data->hit = 1;
		data->bytes_req = bytes_req;
		data->bytes_alloc = bytes_alloc;

		rb_link_node(&data->node, parent, node);
		rb_insert_color(&data->node, &root_caller_stat);
	}

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Li Zefan
207
93.24%
2
50.00%
Arnaldo Carvalho de Melo
14
6.31%
1
25.00%
Wenji Huang
1
0.45%
1
25.00%
Total
222
100.00%
4
100.00%



static int perf_evsel__process_alloc_event(struct perf_evsel *evsel,
					   struct perf_sample *sample)
{
	unsigned long ptr = perf_evsel__intval(evsel, sample, "ptr"),
		      call_site = perf_evsel__intval(evsel, sample, "call_site");
	int bytes_req = perf_evsel__intval(evsel, sample, "bytes_req"),
	    bytes_alloc = perf_evsel__intval(evsel, sample, "bytes_alloc");

	if (insert_alloc_stat(call_site, ptr, bytes_req, bytes_alloc, sample->cpu) ||
	    insert_caller_stat(call_site, bytes_req, bytes_alloc))
		return -1;

	total_requested += bytes_req;
	total_allocated += bytes_alloc;

	nr_allocs++;
	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Li Zefan
59
55.14%
2
40.00%
Arnaldo Carvalho de Melo
48
44.86%
3
60.00%
Total
107
100.00%
5
100.00%



static int perf_evsel__process_alloc_node_event(struct perf_evsel *evsel,
						struct perf_sample *sample)
{
	int ret = perf_evsel__process_alloc_event(evsel, sample);

	if (!ret) {
		int node1 = cpu__get_node(sample->cpu),
		    node2 = perf_evsel__intval(evsel, sample, "node");

		if (node1 != node2)
			nr_cross_allocs++;
	}

	return ret;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Arnaldo Carvalho de Melo
41
62.12%
2
40.00%
Li Zefan
22
33.33%
2
40.00%
Don Zickus
3
4.55%
1
20.00%
Total
66
100.00%
5
100.00%

static int ptr_cmp(void *, void *);
static int slab_callsite_cmp(void *, void *);



static struct alloc_stat *search_alloc_stat(unsigned long ptr,
					    unsigned long call_site,
					    struct rb_root *root,
					    sort_fn_t sort_fn)
{
	struct rb_node *node = root->rb_node;
	struct alloc_stat key = { .ptr = ptr, .call_site = call_site };

	while (node) {
		struct alloc_stat *data;
		int cmp;

		data = rb_entry(node, struct alloc_stat, node);

		cmp = sort_fn(&key, data);
		if (cmp < 0)
			node = node->rb_left;
		else if (cmp > 0)
			node = node->rb_right;
		else
			return data;
	}
	return NULL;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Li Zefan
117
100.00%
1
100.00%
Total
117
100.00%
1
100.00%



static int perf_evsel__process_free_event(struct perf_evsel *evsel,
					  struct perf_sample *sample)
{
	unsigned long ptr = perf_evsel__intval(evsel, sample, "ptr");
	struct alloc_stat *s_alloc, *s_caller;

	s_alloc = search_alloc_stat(ptr, 0, &root_alloc_stat, ptr_cmp);
	if (!s_alloc)
		return 0;

	total_freed += s_alloc->last_alloc;

	if ((short)sample->cpu != s_alloc->alloc_cpu) {
		s_alloc->pingpong++;

		s_caller = search_alloc_stat(0, s_alloc->call_site,
					     &root_caller_stat,
					     slab_callsite_cmp);
		if (!s_caller)
			return -1;
		s_caller->pingpong++;
	}
	s_alloc->alloc_cpu = -1;

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Li Zefan
84
67.20%
2
28.57%
Arnaldo Carvalho de Melo
34
27.20%
3
42.86%
David Ahern
6
4.80%
1
14.29%
Namhyung Kim
1
0.80%
1
14.29%
Total
125
100.00%
7
100.00%


static u64 total_page_alloc_bytes;

static u64 total_page_free_bytes;

static u64 total_page_nomatch_bytes;

static u64 total_page_fail_bytes;

static unsigned long nr_page_allocs;

static unsigned long nr_page_frees;

static unsigned long nr_page_fails;

static unsigned long nr_page_nomatch;


static bool use_pfn;

static bool live_page;

static struct perf_session *kmem_session;


#define MAX_MIGRATE_TYPES  6

#define MAX_PAGE_ORDER     11


static int order_stats[MAX_PAGE_ORDER][MAX_MIGRATE_TYPES];


struct page_stat {
	
struct rb_node 	node;
	
u64 		page;
	
u64 		callsite;
	
int 		order;
	
unsigned 	gfp_flags;
	
unsigned 	migrate_type;
	
u64		alloc_bytes;
	
u64 		free_bytes;
	
int 		nr_alloc;
	
int 		nr_free;
};


static struct rb_root page_live_tree;

static struct rb_root page_alloc_tree;

static struct rb_root page_alloc_sorted;

static struct rb_root page_caller_tree;

static struct rb_root page_caller_sorted;


struct alloc_func {
	
u64 start;
	
u64 end;
	
char *name;
};


static int nr_alloc_funcs;

static struct alloc_func *alloc_func_list;



static int funcmp(const void *a, const void *b)
{
	const struct alloc_func *fa = a;
	const struct alloc_func *fb = b;

	if (fa->start > fb->start)
		return 1;
	else
		return -1;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Namhyung Kim
45
90.00%
2
50.00%
Arnaldo Carvalho de Melo
5
10.00%
2
50.00%
Total
50
100.00%
4
100.00%



static int callcmp(const void *a, const void *b)
{
	const struct alloc_func *fa = a;
	const struct alloc_func *fb = b;

	if (fb->start <= fa->start && fa->end < fb->end)
		return 0;

	if (fa->start > fb->start)
		return 1;
	else
		return -1;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Namhyung Kim
71
100.00%
2
100.00%
Total
71
100.00%
2
100.00%



static int build_alloc_func_list(void)
{
	int ret;
	struct map *kernel_map;
	struct symbol *sym;
	struct rb_node *node;
	struct alloc_func *func;
	struct machine *machine = &kmem_session->machines.host;
	regex_t alloc_func_regex;
	const char pattern[] = "^_?_?(alloc|get_free|get_zeroed)_pages?";

	ret = regcomp(&alloc_func_regex, pattern, REG_EXTENDED);
	if (ret) {
		char err[BUFSIZ];

		regerror(ret, &alloc_func_regex, err, sizeof(err));
		pr_err("Invalid regex: %s\n%s", pattern, err);
		return -EINVAL;
	}

	kernel_map = machine__kernel_map(machine);
	if (map__load(kernel_map) < 0) {
		pr_err("cannot load kernel map\n");
		return -ENOENT;
	}

	map__for_each_symbol(kernel_map, sym, node) {
		if (regexec(&alloc_func_regex, sym->name, 0, NULL, 0))
			continue;

		func = realloc(alloc_func_list,
			       (nr_alloc_funcs + 1) * sizeof(*func));
		if (func == NULL)
			return -ENOMEM;

		pr_debug("alloc func: %s\n", sym->name);
		func[nr_alloc_funcs].start = sym->start;
		func[nr_alloc_funcs].end   = sym->end;
		func[nr_alloc_funcs].name  = sym->name;

		alloc_func_list = func;
		nr_alloc_funcs++;
	}

	qsort(alloc_func_list, nr_alloc_funcs, sizeof(*func), funcmp);

	regfree(&alloc_func_regex);
	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Namhyung Kim
258
98.10%
2
50.00%
Arnaldo Carvalho de Melo
5
1.90%
2
50.00%
Total
263
100.00%
4
100.00%

/*
 * Find first non-memory allocation function from callchain.
 * The allocation functions are in the 'alloc_func_list'.
 */


static u64 find_callsite(struct perf_evsel *evsel, struct perf_sample *sample)
{
	struct addr_location al;
	struct machine *machine = &kmem_session->machines.host;
	struct callchain_cursor_node *node;

	if (alloc_func_list == NULL) {
		if (build_alloc_func_list() < 0)
			goto out;
	}

	al.thread = machine__findnew_thread(machine, sample->pid, sample->tid);
	sample__resolve_callchain(sample, &callchain_cursor, NULL, evsel, &al, 16);

	callchain_cursor_commit(&callchain_cursor);
	while (true) {
		struct alloc_func key, *caller;
		u64 addr;

		node = callchain_cursor_current(&callchain_cursor);
		if (node == NULL)
			break;

		key.start = key.end = node->ip;
		caller = bsearch(&key, alloc_func_list, nr_alloc_funcs,
				 sizeof(key), callcmp);
		if (!caller) {
			/* found */
			if (node->map)
				addr = map__unmap_ip(node->map, node->ip);
			else
				addr = node->ip;

			return addr;
		} else
			pr_debug3("skipping alloc function: %s\n", caller->name);

		callchain_cursor_advance(&callchain_cursor);
	}

out:
	pr_debug2("unknown callsite: %"PRIx64 "\n", sample->ip);
	return sample->ip;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Namhyung Kim
225
98.68%
2
66.67%
Arnaldo Carvalho de Melo
3
1.32%
1
33.33%
Total
228
100.00%
3
100.00%


struct sort_dimension {
	
const char		name[20];
	
sort_fn_t		cmp;
	
struct list_head	list;
};

static LIST_HEAD(page_alloc_sort_input);
static LIST_HEAD(page_caller_sort_input);


static struct page_stat *

__page_stat__findnew_page(struct page_stat *pstat, bool create)
{
	struct rb_node **node = &page_live_tree.rb_node;
	struct rb_node *parent = NULL;
	struct page_stat *data;

	while (*node) {
		s64 cmp;

		parent = *node;
		data = rb_entry(*node, struct page_stat, node);

		cmp = data->page - pstat->page;
		if (cmp < 0)
			node = &parent->rb_left;
		else if (cmp > 0)
			node = &parent->rb_right;
		else
			return data;
	}

	if (!create)
		return NULL;

	data = zalloc(sizeof(*data));
	if (data != NULL) {
		data->page = pstat->page;
		data->order = pstat->order;
		data->gfp_flags = pstat->gfp_flags;
		data->migrate_type = pstat->migrate_type;

		rb_link_node(&data->node, parent, node);
		rb_insert_color(&data->node, &page_live_tree);
	}

	return data;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Namhyung Kim
193
100.00%
3
100.00%
Total
193
100.00%
3
100.00%



static struct page_stat *page_stat__find_page(struct page_stat *pstat)
{
	return __page_stat__findnew_page(pstat, false);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Namhyung Kim
21
100.00%
3
100.00%
Total
21
100.00%
3
100.00%



static struct page_stat *page_stat__findnew_page(struct page_stat *pstat)
{
	return __page_stat__findnew_page(pstat, true);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Namhyung Kim
21
100.00%
3
100.00%
Total
21
100.00%
3
100.00%


static struct page_stat *

__page_stat__findnew_alloc(struct page_stat *pstat, bool create)
{
	struct rb_node **node = &page_alloc_tree.rb_node;
	struct rb_node *parent = NULL;
	struct page_stat *data;
	struct sort_dimension *sort;

	while (*node) {
		int cmp = 0;

		parent = *node;
		data = rb_entry(*node, struct page_stat, node);

		list_for_each_entry(sort, &page_alloc_sort_input, list) {
			cmp = sort->cmp(pstat, data);
			if (cmp)
				break;
		}

		if (cmp < 0)
			node = &parent->rb_left;
		else if (cmp > 0)
			node = &parent->rb_right;
		else
			return data;
	}

	if (!create)
		return NULL;

	data = zalloc(sizeof(*data));
	if (data != NULL) {
		data->page = pstat->page;
		data->order = pstat->order;
		data->gfp_flags = pstat->gfp_flags;
		data->migrate_type = pstat->migrate_type;

		rb_link_node(&data->node, parent, node);
		rb_insert_color(&data->node, &page_alloc_tree);
	}

	return data;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Namhyung Kim
216
100.00%
2
100.00%
Total
216
100.00%
2
100.00%



static struct page_stat *page_stat__find_alloc(struct page_stat *pstat)
{
	return __page_stat__findnew_alloc(pstat, false);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Namhyung Kim
21
100.00%
1
100.00%
Total
21
100.00%
1
100.00%



static struct page_stat *page_stat__findnew_alloc(struct page_stat *pstat)
{
	return __page_stat__findnew_alloc(pstat, true);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Namhyung Kim
21
100.00%
1
100.00%
Total
21
100.00%
1
100.00%


static struct page_stat *

__page_stat__findnew_caller(struct page_stat *pstat, bool create)
{
	struct rb_node **node = &page_caller_tree.rb_node;
	struct rb_node *parent = NULL;
	struct page_stat *data;
	struct sort_dimension *sort;

	while (*node) {
		int cmp = 0;

		parent = *node;
		data = rb_entry(*node, struct page_stat, node);

		list_for_each_entry(sort, &page_caller_sort_input, list) {
			cmp = sort->cmp(pstat, data);
			if (cmp)
				break;
		}

		if (cmp < 0)
			node = &parent->rb_left;
		else if (cmp > 0)
			node = &parent->rb_right;
		else
			return data;
	}

	if (!create)
		return NULL;

	data = zalloc(sizeof(*data));
	if (data != NULL) {
		data->callsite = pstat->callsite;
		data->order = pstat->order;
		data->gfp_flags = pstat->gfp_flags;
		data->migrate_type = pstat->migrate_type;

		rb_link_node(&data->node, parent, node);
		rb_insert_color(&data->node, &page_caller_tree);
	}

	return data;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Namhyung Kim
216
100.00%
2
100.00%
Total
216
100.00%
2
100.00%



static struct page_stat *page_stat__find_caller(struct page_stat *pstat)
{
	return __page_stat__findnew_caller(pstat, false);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Namhyung Kim
21
100.00%
2
100.00%
Total
21
100.00%
2
100.00%



static struct page_stat *page_stat__findnew_caller(struct page_stat *pstat)
{
	return __page_stat__findnew_caller(pstat, true);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Namhyung Kim
21
100.00%
2
100.00%
Total
21
100.00%
2
100.00%



static bool valid_page(u64 pfn_or_page)
{
	if (use_pfn && pfn_or_page == -1UL)
		return false;
	if (!use_pfn && pfn_or_page == 0)
		return false;
	return true;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Namhyung Kim
36
100.00%
1
100.00%
Total
36
100.00%
1
100.00%


struct gfp_flag {
	
unsigned int flags;
	
char *compact_str;
	
char *human_readable;
};


static struct gfp_flag *gfps;

static int nr_gfps;



static int gfpcmp(const void *a, const void *b)
{
	const struct gfp_flag *fa = a;
	const struct gfp_flag *fb = b;

	return fa->flags - fb->flags;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Namhyung Kim
41
100.00%
1
100.00%
Total
41
100.00%
1
100.00%

/* see include/trace/events/mmflags.h */
static const struct {
	
const char *original;
	
const char *compact;

} gfp_compact_table[] = {
	{ "GFP_TRANSHUGE",		"THP" },
	{ "GFP_TRANSHUGE_LIGHT",	"THL" },
	{ "GFP_HIGHUSER_MOVABLE",	"HUM" },
	{ "GFP_HIGHUSER",		"HU" },
	{ "GFP_USER",			"U" },
	{ "GFP_KERNEL_ACCOUNT",		"KAC" },
	{ "GFP_KERNEL",			"K" },
	{ "GFP_NOFS",			"NF" },
	{ "GFP_ATOMIC",			"A" },
	{ "GFP_NOIO",			"NI" },
	{ "GFP_NOWAIT",			"NW" },
	{ "GFP_DMA",			"D" },
	{ "__GFP_HIGHMEM",		"HM" },
	{ "GFP_DMA32",			"D32" },
	{ "__GFP_HIGH",			"H" },
	{ "__GFP_ATOMIC",		"_A" },
	{ "__GFP_IO",			"I" },
	{ "__GFP_FS",			"F" },
	{ "__GFP_COLD",			"CO" },
	{ "__GFP_NOWARN",		"NWR" },
	{ "__GFP_RETRY_MAYFAIL",	"R" },
	{ "__GFP_NOFAIL",		"NF" },
	{ "__GFP_NORETRY",		"NR" },
	{ "__GFP_COMP",			"C" },
	{ "__GFP_ZERO",			"Z" },
	{ "__GFP_NOMEMALLOC",		"NMA" },
	{ "__GFP_MEMALLOC",		"MA" },
	{ "__GFP_HARDWALL",		"HW" },
	{ "__GFP_THISNODE",		"TN" },
	{ "__GFP_RECLAIMABLE",		"RC" },
	{ "__GFP_MOVABLE",		"M" },
	{ "__GFP_ACCOUNT",		"AC" },
	{ "__GFP_NOTRACK",		"NT" },
	{ "__GFP_WRITE",		"WR" },
	{ "__GFP_RECLAIM",		"R" },
	{ "__GFP_DIRECT_RECLAIM",	"DR" },
	{