Release 4.10 tools/perf/util/auxtrace.c

/*
 * auxtrace.c: AUX area trace support
 * Copyright (c) 2013-2015, Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 */

#include <sys/types.h>
#include <sys/mman.h>
#include <stdbool.h>
#include <ctype.h>
#include <string.h>
#include <limits.h>
#include <errno.h>

#include <linux/kernel.h>
#include <linux/perf_event.h>
#include <linux/types.h>
#include <linux/bitops.h>
#include <linux/log2.h>
#include <linux/string.h>

#include <sys/param.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <limits.h>
#include <errno.h>
#include <linux/list.h>

#include "../perf.h"
#include "util.h"
#include "evlist.h"
#include "dso.h"
#include "map.h"
#include "pmu.h"
#include "evsel.h"
#include "cpumap.h"
#include "thread_map.h"
#include "asm/bug.h"
#include "symbol/kallsyms.h"
#include "auxtrace.h"

#include <linux/hash.h>

#include "event.h"
#include "session.h"
#include "debug.h"
#include <subcmd/parse-options.h>

#include "intel-pt.h"
#include "intel-bts.h"



int auxtrace_mmap__mmap(struct auxtrace_mmap *mm,
			struct auxtrace_mmap_params *mp,
			void *userpg, int fd)
{
	struct perf_event_mmap_page *pc = userpg;

	WARN_ONCE(mm->base, "Uninitialized auxtrace_mmap\n");

	mm->userpg = userpg;
	mm->mask = mp->mask;
	mm->len = mp->len;
	mm->prev = 0;
	mm->idx = mp->idx;
	mm->tid = mp->tid;
	mm->cpu = mp->cpu;

	if (!mp->len) {
		mm->base = NULL;
		return 0;
	}

#if BITS_PER_LONG != 64 && !defined(HAVE_SYNC_COMPARE_AND_SWAP_SUPPORT)
	pr_err("Cannot use AUX area tracing mmaps\n");
	return -1;
#endif

	pc->aux_offset = mp->offset;
	pc->aux_size = mp->len;

	mm->base = mmap(NULL, mp->len, mp->prot, MAP_SHARED, fd, mp->offset);
	if (mm->base == MAP_FAILED) {
		pr_debug2("failed to mmap AUX area\n");
		mm->base = NULL;
		return -1;
	}

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
199
100.00%
2
100.00%
Total
199
100.00%
2
100.00%



void auxtrace_mmap__munmap(struct auxtrace_mmap *mm)
{
	if (mm->base) {
		munmap(mm->base, mm->len);
		mm->base = NULL;
	}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
35
100.00%
1
100.00%
Total
35
100.00%
1
100.00%



void auxtrace_mmap_params__init(struct auxtrace_mmap_params *mp,
				off_t auxtrace_offset,
				unsigned int auxtrace_pages,
				bool auxtrace_overwrite)
{
	if (auxtrace_pages) {
		mp->offset = auxtrace_offset;
		mp->len = auxtrace_pages * (size_t)page_size;
		mp->mask = is_power_of_2(mp->len) ? mp->len - 1 : 0;
		mp->prot = PROT_READ | (auxtrace_overwrite ? 0 : PROT_WRITE);
		pr_debug2("AUX area mmap length %zu\n", mp->len);
	} else {
		mp->len = 0;
	}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
94
100.00%
1
100.00%
Total
94
100.00%
1
100.00%



void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp,
				   struct perf_evlist *evlist, int idx,
				   bool per_cpu)
{
	mp->idx = idx;

	if (per_cpu) {
		mp->cpu = evlist->cpus->map[idx];
		if (evlist->threads)
			mp->tid = thread_map__pid(evlist->threads, 0);
		else
			mp->tid = -1;
	} else {
		mp->cpu = -1;
		mp->tid = thread_map__pid(evlist->threads, idx);
	}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
88
91.67%
1
50.00%
jiri olsa
jiri olsa
8
8.33%
1
50.00%
Total
96
100.00%
2
100.00%


#define AUXTRACE_INIT_NR_QUEUES	32



static struct auxtrace_queue *auxtrace_alloc_queue_array(unsigned int nr_queues)
{
	struct auxtrace_queue *queue_array;
	unsigned int max_nr_queues, i;

	max_nr_queues = UINT_MAX / sizeof(struct auxtrace_queue);
	if (nr_queues > max_nr_queues)
		return NULL;

	queue_array = calloc(nr_queues, sizeof(struct auxtrace_queue));
	if (!queue_array)
		return NULL;

	for (i = 0; i < nr_queues; i++) {
		INIT_LIST_HEAD(&queue_array[i].head);
		queue_array[i].priv = NULL;
	}

	return queue_array;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
101
100.00%
2
100.00%
Total
101
100.00%
2
100.00%



int auxtrace_queues__init(struct auxtrace_queues *queues)
{
	queues->nr_queues = AUXTRACE_INIT_NR_QUEUES;
	queues->queue_array = auxtrace_alloc_queue_array(queues->nr_queues);
	if (!queues->queue_array)
		return -ENOMEM;
	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
41
100.00%
2
100.00%
Total
41
100.00%
2
100.00%



static int auxtrace_queues__grow(struct auxtrace_queues *queues,
				 unsigned int new_nr_queues)
{
	unsigned int nr_queues = queues->nr_queues;
	struct auxtrace_queue *queue_array;
	unsigned int i;

	if (!nr_queues)
		nr_queues = AUXTRACE_INIT_NR_QUEUES;

	while (nr_queues && nr_queues < new_nr_queues)
		nr_queues <<= 1;

	if (nr_queues < queues->nr_queues || nr_queues < new_nr_queues)
		return -EINVAL;

	queue_array = auxtrace_alloc_queue_array(nr_queues);
	if (!queue_array)
		return -ENOMEM;

	for (i = 0; i < queues->nr_queues; i++) {
		list_splice_tail(&queues->queue_array[i].head,
				 &queue_array[i].head);
		queue_array[i].priv = queues->queue_array[i].priv;
	}

	queues->nr_queues = nr_queues;
	queues->queue_array = queue_array;

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
154
100.00%
2
100.00%
Total
154
100.00%
2
100.00%



static void *auxtrace_copy_data(u64 size, struct perf_session *session)
{
	int fd = perf_data_file__fd(session->file);
	void *p;
	ssize_t ret;

	if (size > SSIZE_MAX)
		return NULL;

	p = malloc(size);
	if (!p)
		return NULL;

	ret = readn(fd, p, size);
	if (ret != (ssize_t)size) {
		free(p);
		return NULL;
	}

	return p;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
89
100.00%
2
100.00%
Total
89
100.00%
2
100.00%



static int auxtrace_queues__add_buffer(struct auxtrace_queues *queues,
				       unsigned int idx,
				       struct auxtrace_buffer *buffer)
{
	struct auxtrace_queue *queue;
	int err;

	if (idx >= queues->nr_queues) {
		err = auxtrace_queues__grow(queues, idx + 1);
		if (err)
			return err;
	}

	queue = &queues->queue_array[idx];

	if (!queue->set) {
		queue->set = true;
		queue->tid = buffer->tid;
		queue->cpu = buffer->cpu;
	} else if (buffer->cpu != queue->cpu || buffer->tid != queue->tid) {
		pr_err("auxtrace queue conflict: cpu %d, tid %d vs cpu %d, tid %d\n",
		       queue->cpu, queue->tid, buffer->cpu, buffer->tid);
		return -EINVAL;
	}

	buffer->buffer_nr = queues->next_buffer_nr++;

	list_add_tail(&buffer->list, &queue->head);

	queues->new_data = true;
	queues->populated = true;

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
180
100.00%
1
100.00%
Total
180
100.00%
1
100.00%

/* Limit buffers to 32MiB on 32-bit */

#define BUFFER_LIMIT_FOR_32_BIT (32 * 1024 * 1024)



static int auxtrace_queues__split_buffer(struct auxtrace_queues *queues,
					 unsigned int idx,
					 struct auxtrace_buffer *buffer)
{
	u64 sz = buffer->size;
	bool consecutive = false;
	struct auxtrace_buffer *b;
	int err;

	while (sz > BUFFER_LIMIT_FOR_32_BIT) {
		b = memdup(buffer, sizeof(struct auxtrace_buffer));
		if (!b)
			return -ENOMEM;
		b->size = BUFFER_LIMIT_FOR_32_BIT;
		b->consecutive = consecutive;
		err = auxtrace_queues__add_buffer(queues, idx, b);
		if (err) {
			auxtrace_buffer__free(b);
			return err;
		}
		buffer->data_offset += BUFFER_LIMIT_FOR_32_BIT;
		sz -= BUFFER_LIMIT_FOR_32_BIT;
		consecutive = true;
	}

	buffer->size = sz;
	buffer->consecutive = consecutive;

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
136
100.00%
1
100.00%
Total
136
100.00%
1
100.00%



static int auxtrace_queues__add_event_buffer(struct auxtrace_queues *queues,
					     struct perf_session *session,
					     unsigned int idx,
					     struct auxtrace_buffer *buffer)
{
	if (session->one_mmap) {
		buffer->data = buffer->data_offset - session->one_mmap_offset +
			       session->one_mmap_addr;
	} else if (perf_data_file__is_pipe(session->file)) {
		buffer->data = auxtrace_copy_data(buffer->size, session);
		if (!buffer->data)
			return -ENOMEM;
		buffer->data_needs_freeing = true;
	} else if (BITS_PER_LONG == 32 &&
		   buffer->size > BUFFER_LIMIT_FOR_32_BIT) {
		int err;

		err = auxtrace_queues__split_buffer(queues, idx, buffer);
		if (err)
			return err;
	}

	return auxtrace_queues__add_buffer(queues, idx, buffer);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
137
100.00%
1
100.00%
Total
137
100.00%
1
100.00%



int auxtrace_queues__add_event(struct auxtrace_queues *queues,
			       struct perf_session *session,
			       union perf_event *event, off_t data_offset,
			       struct auxtrace_buffer **buffer_ptr)
{
	struct auxtrace_buffer *buffer;
	unsigned int idx;
	int err;

	buffer = zalloc(sizeof(struct auxtrace_buffer));
	if (!buffer)
		return -ENOMEM;

	buffer->pid = -1;
	buffer->tid = event->auxtrace.tid;
	buffer->cpu = event->auxtrace.cpu;
	buffer->data_offset = data_offset;
	buffer->offset = event->auxtrace.offset;
	buffer->reference = event->auxtrace.reference;
	buffer->size = event->auxtrace.size;
	idx = event->auxtrace.idx;

	err = auxtrace_queues__add_event_buffer(queues, session, idx, buffer);
	if (err)
		goto out_err;

	if (buffer_ptr)
		*buffer_ptr = buffer;

	return 0;

out_err:
	auxtrace_buffer__free(buffer);
	return err;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
174
100.00%
1
100.00%
Total
174
100.00%
1
100.00%



static int auxtrace_queues__add_indexed_event(struct auxtrace_queues *queues,
					      struct perf_session *session,
					      off_t file_offset, size_t sz)
{
	union perf_event *event;
	int err;
	char buf[PERF_SAMPLE_MAX_SIZE];

	err = perf_session__peek_event(session, file_offset, buf,
				       PERF_SAMPLE_MAX_SIZE, &event, NULL);
	if (err)
		return err;

	if (event->header.type == PERF_RECORD_AUXTRACE) {
		if (event->header.size < sizeof(struct auxtrace_event) ||
		    event->header.size != sz) {
			err = -EINVAL;
			goto out;
		}
		file_offset += event->header.size;
		err = auxtrace_queues__add_event(queues, session, event,
						 file_offset, NULL);
	}
out:
	return err;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
133
100.00%
1
100.00%
Total
133
100.00%
1
100.00%



void auxtrace_queues__free(struct auxtrace_queues *queues)
{
	unsigned int i;

	for (i = 0; i < queues->nr_queues; i++) {
		while (!list_empty(&queues->queue_array[i].head)) {
			struct auxtrace_buffer *buffer;

			buffer = list_entry(queues->queue_array[i].head.next,
					    struct auxtrace_buffer, list);
			list_del(&buffer->list);
			auxtrace_buffer__free(buffer);
		}
	}

	zfree(&queues->queue_array);
	queues->nr_queues = 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
102
100.00%
1
100.00%
Total
102
100.00%
1
100.00%



static void auxtrace_heapify(struct auxtrace_heap_item *heap_array,
			     unsigned int pos, unsigned int queue_nr,
			     u64 ordinal)
{
	unsigned int parent;

	while (pos) {
		parent = (pos - 1) >> 1;
		if (heap_array[parent].ordinal <= ordinal)
			break;
		heap_array[pos] = heap_array[parent];
		pos = parent;
	}
	heap_array[pos].queue_nr = queue_nr;
	heap_array[pos].ordinal = ordinal;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
86
100.00%
1
100.00%
Total
86
100.00%
1
100.00%



int auxtrace_heap__add(struct auxtrace_heap *heap, unsigned int queue_nr,
		       u64 ordinal)
{
	struct auxtrace_heap_item *heap_array;

	if (queue_nr >= heap->heap_sz) {
		unsigned int heap_sz = AUXTRACE_INIT_NR_QUEUES;

		while (heap_sz <= queue_nr)
			heap_sz <<= 1;
		heap_array = realloc(heap->heap_array,
				     heap_sz * sizeof(struct auxtrace_heap_item));
		if (!heap_array)
			return -ENOMEM;
		heap->heap_array = heap_array;
		heap->heap_sz = heap_sz;
	}

	auxtrace_heapify(heap->heap_array, heap->heap_cnt++, queue_nr, ordinal);

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
105
100.00%
1
100.00%
Total
105
100.00%
1
100.00%



void auxtrace_heap__free(struct auxtrace_heap *heap)
{
	zfree(&heap->heap_array);
	heap->heap_cnt = 0;
	heap->heap_sz = 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
30
100.00%
1
100.00%
Total
30
100.00%
1
100.00%



void auxtrace_heap__pop(struct auxtrace_heap *heap)
{
	unsigned int pos, last, heap_cnt = heap->heap_cnt;
	struct auxtrace_heap_item *heap_array;

	if (!heap_cnt)
		return;

	heap->heap_cnt -= 1;

	heap_array = heap->heap_array;

	pos = 0;
	while (1) {
		unsigned int left, right;

		left = (pos << 1) + 1;
		if (left >= heap_cnt)
			break;
		right = left + 1;
		if (right >= heap_cnt) {
			heap_array[pos] = heap_array[left];
			return;
		}
		if (heap_array[left].ordinal < heap_array[right].ordinal) {
			heap_array[pos] = heap_array[left];
			pos = left;
		} else {
			heap_array[pos] = heap_array[right];
			pos = right;
		}
	}

	last = heap_cnt - 1;
	auxtrace_heapify(heap_array, pos, heap_array[last].queue_nr,
			 heap_array[last].ordinal);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
179
100.00%
1
100.00%
Total
179
100.00%
1
100.00%



size_t auxtrace_record__info_priv_size(struct auxtrace_record *itr,
				       struct perf_evlist *evlist)
{
	if (itr)
		return itr->info_priv_size(itr, evlist);
	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
25
78.12%
1
50.00%
mathieu j. poirier
mathieu j. poirier
7
21.88%
1
50.00%
Total
32
100.00%
2
100.00%



static int auxtrace_not_supported(void)
{
	pr_err("AUX area tracing is not supported on this architecture\n");
	return -EINVAL;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
17
100.00%
1
100.00%
Total
17
100.00%
1
100.00%



int auxtrace_record__info_fill(struct auxtrace_record *itr,
			       struct perf_session *session,
			       struct auxtrace_info_event *auxtrace_info,
			       size_t priv_size)
{
	if (itr)
		return itr->info_fill(itr, session, auxtrace_info, priv_size);
	return auxtrace_not_supported();
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
45
100.00%
1
100.00%
Total
45
100.00%
1
100.00%



void auxtrace_record__free(struct auxtrace_record *itr)
{
	if (itr)
		itr->free(itr);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
21
100.00%
1
100.00%
Total
21
100.00%
1
100.00%



int auxtrace_record__snapshot_start(struct auxtrace_record *itr)
{
	if (itr && itr->snapshot_start)
		return itr->snapshot_start(itr);
	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
29
100.00%
1
100.00%
Total
29
100.00%
1
100.00%



int auxtrace_record__snapshot_finish(struct auxtrace_record *itr)
{
	if (itr && itr->snapshot_finish)
		return itr->snapshot_finish(itr);
	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
29
100.00%
1
100.00%
Total
29
100.00%
1
100.00%



int auxtrace_record__find_snapshot(struct auxtrace_record *itr, int idx,
				   struct auxtrace_mmap *mm,
				   unsigned char *data, u64 *head, u64 *old)
{
	if (itr && itr->find_snapshot)
		return itr->find_snapshot(itr, idx, mm, data, head, old);
	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
60
100.00%
1
100.00%
Total
60
100.00%
1
100.00%



int auxtrace_record__options(struct auxtrace_record *itr,
			     struct perf_evlist *evlist,
			     struct record_opts *opts)
{
	if (itr)
		return itr->recording_options(itr, evlist, opts);
	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
39
100.00%
1
100.00%
Total
39
100.00%
1
100.00%



u64 auxtrace_record__reference(struct auxtrace_record *itr)
{
	if (itr)
		return itr->reference(itr);
	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
25
100.00%
2
100.00%
Total
25
100.00%
2
100.00%



int auxtrace_parse_snapshot_options(struct auxtrace_record *itr,
				    struct record_opts *opts, const char *str)
{
	if (!str)
		return 0;

	if (itr)
		return itr->parse_snapshot_options(itr, opts, str);

	pr_err("No AUX area tracing to snapshot\n");
	return -EINVAL;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
53
100.00%
1
100.00%
Total
53
100.00%
1
100.00%


struct auxtrace_record *__weak

auxtrace_record__init(struct perf_evlist *evlist __maybe_unused, int *err)
{
	*err = 0;
	return NULL;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
26
100.00%
1
100.00%
Total
26
100.00%
1
100.00%



static int auxtrace_index__alloc(struct list_head *head)
{
	struct auxtrace_index *auxtrace_index;

	auxtrace_index = malloc(sizeof(struct auxtrace_index));
	if (!auxtrace_index)
		return -ENOMEM;

	auxtrace_index->nr = 0;
	INIT_LIST_HEAD(&auxtrace_index->list);

	list_add_tail(&auxtrace_index->list, head);

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
63
100.00%
1
100.00%
Total
63
100.00%
1
100.00%



void auxtrace_index__free(struct list_head *head)
{
	struct auxtrace_index *auxtrace_index, *n;

	list_for_each_entry_safe(auxtrace_index, n, head, list) {
		list_del(&auxtrace_index->list);
		free(auxtrace_index);
	}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
43
100.00%
1
100.00%
Total
43
100.00%
1
100.00%



static struct auxtrace_index *auxtrace_index__last(struct list_head *head)
{
	struct auxtrace_index *auxtrace_index;
	int err;

	if (list_empty(head)) {
		err = auxtrace_index__alloc(head);
		if (err)
			return NULL;
	}

	auxtrace_index = list_entry(head->prev, struct auxtrace_index, list);

	if (auxtrace_index->nr >= PERF_AUXTRACE_INDEX_ENTRY_COUNT) {
		err = auxtrace_index__alloc(head);
		if (err)
			return NULL;
		auxtrace_index = list_entry(head->prev, struct auxtrace_index,
					    list);
	}

	return auxtrace_index;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
adrian hunter
adrian hunter
99
100.00%
1
100.00%
Total
99
100.00%
1
100.00%



int auxtrace_index__auxtrace_event(struct list_head *head,
				   union perf_event *event, off_t file_offset)
{
	struct auxtrace_index *auxtrace_index;
	size_t nr;

	auxtrace_index = auxtrace_index__last(head);
	if (!auxtrace_index)
		return -ENOMEM;

	nr = auxtrace_index->nr;
	auxtrace_index->entries[nr].file_offset = file_offset;
	auxtrace_index->entries[nr].sz =