Release 4.10 tools/lib/bpf/libbpf.c

/*
 * Common eBPF ELF object loading operations.
 *
 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
 * Copyright (C) 2015 Huawei Inc.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation;
 * version 2.1 of the License (not later!)
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this program; if not,  see <http://www.gnu.org/licenses>
 */

#include <stdlib.h>
#include <stdio.h>
#include <stdarg.h>
#include <inttypes.h>
#include <string.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <asm/unistd.h>
#include <linux/kernel.h>
#include <linux/bpf.h>
#include <linux/list.h>
#include <libelf.h>
#include <gelf.h>

#include "libbpf.h"
#include "bpf.h"

#ifndef EM_BPF

#define EM_BPF 247
#endif


#define __printf(a, b)	__attribute__((format(printf, a, b)))

__printf(1, 2)


static int __base_pr(const char *format, ...)
{
	va_list args;
	int err;

	va_start(args, format);
	err = vfprintf(stderr, format, args);
	va_end(args);
	return err;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
wang nan
wang nan
45
100.00%
1
100.00%
Total
45
100.00%
1
100.00%


static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr;

static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr;

static __printf(1, 2) libbpf_print_fn_t __pr_debug;


#define __pr(func, fmt, ...)	\
do {                            \
        if ((func))             \
                (func)("libbpf: " fmt, ##__VA_ARGS__); \
} while (0)


#define pr_warning(fmt, ...)	__pr(__pr_warning, fmt, ##__VA_ARGS__)

#define pr_info(fmt, ...)	__pr(__pr_info, fmt, ##__VA_ARGS__)

#define pr_debug(fmt, ...)	__pr(__pr_debug, fmt, ##__VA_ARGS__)



void libbpf_set_print(libbpf_print_fn_t warn,
		      libbpf_print_fn_t info,
		      libbpf_print_fn_t debug)
{
	__pr_warning = warn;
	__pr_info = info;
	__pr_debug = debug;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
wang nan
wang nan
26
100.00%
1
100.00%
Total
26
100.00%
1
100.00%


#define STRERR_BUFSIZE  128


#define ERRNO_OFFSET(e)		((e) - __LIBBPF_ERRNO__START)

#define ERRCODE_OFFSET(c)	ERRNO_OFFSET(LIBBPF_ERRNO__##c)

#define NR_ERRNO	(__LIBBPF_ERRNO__END - __LIBBPF_ERRNO__START)


static const char *libbpf_strerror_table[NR_ERRNO] = {
	[ERRCODE_OFFSET(LIBELF)]	= "Something wrong in libelf",
	[ERRCODE_OFFSET(FORMAT)]	= "BPF object format invalid",
	[ERRCODE_OFFSET(KVERSION)]	= "'version' section incorrect or lost",
	[ERRCODE_OFFSET(ENDIAN)]	= "Endian mismatch",
	[ERRCODE_OFFSET(INTERNAL)]	= "Internal error in libbpf",
	[ERRCODE_OFFSET(RELOC)]		= "Relocation failed",
	[ERRCODE_OFFSET(VERIFY)]	= "Kernel verifier blocks program loading",
	[ERRCODE_OFFSET(PROG2BIG)]	= "Program too big",
	[ERRCODE_OFFSET(KVER)]		= "Incorrect kernel version",
	[ERRCODE_OFFSET(PROGTYPE)]	= "Kernel doesn't support this program type",
};



int libbpf_strerror(int err, char *buf, size_t size)
{
	if (!buf || !size)
		return -1;

	err = err > 0 ? err : -err;

	if (err < __LIBBPF_ERRNO__START) {
		int ret;

		ret = strerror_r(err, buf, size);
		buf[size - 1] = '\0';
		return ret;
	}

	if (err < __LIBBPF_ERRNO__END) {
		const char *msg;

		msg = libbpf_strerror_table[ERRNO_OFFSET(err)];
		snprintf(buf, size, "%s", msg);
		buf[size - 1] = '\0';
		return 0;
	}

	snprintf(buf, size, "Unknown libbpf error %d", err);
	buf[size - 1] = '\0';
	return -1;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
wang nan
wang nan
142
100.00%
1
100.00%
Total
142
100.00%
1
100.00%


#define CHECK_ERR(action, err, out) do {    \
        err = action;                   \
        if (err)                        \
                goto out;               \
} while(0)


/* Copied from tools/perf/util/util.h */
#ifndef zfree

# define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
#endif

#ifndef zclose

# define zclose(fd) ({                      \
        int ___err = 0;                 \
        if ((fd) >= 0)                  \
                ___err = close((fd));   \
        fd = -1;                        \
        ___err; })
#endif

#ifdef HAVE_LIBELF_MMAP_SUPPORT

# define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
#else

# define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
#endif

/*
 * bpf_prog should be a better name but it has been used in
 * linux/filter.h.
 */

struct bpf_program {
	/* Index in elf obj file, for relocation use. */
	
int idx;
	
char *section_name;
	
struct bpf_insn *insns;
	
size_t insns_cnt;
	
enum bpf_prog_type type;

	struct {
		
int insn_idx;
		
int map_idx;
	} 
*reloc_desc;
	
int nr_reloc;

	struct {
		
int nr;
		
int *fds;
	} 
instances;
	
bpf_program_prep_t preprocessor;

	
struct bpf_object *obj;
	
void *priv;
	
bpf_program_clear_priv_t clear_priv;
};


struct bpf_map {
	
int fd;
	
char *name;
	
size_t offset;
	
struct bpf_map_def def;
	
void *priv;
	
bpf_map_clear_priv_t clear_priv;
};

static LIST_HEAD(bpf_objects_list);


struct bpf_object {
	
char license[64];
	
u32 kern_version;

	
struct bpf_program *programs;
	
size_t nr_programs;
	
struct bpf_map *maps;
	
size_t nr_maps;

	
bool loaded;

	/*
         * Information when doing elf related work. Only valid if fd
         * is valid.
         */
	struct {
		
int fd;
		
void *obj_buf;
		
size_t obj_buf_sz;
		
Elf *elf;
		
GElf_Ehdr ehdr;
		
Elf_Data *symbols;
		
size_t strtabidx;
		struct {
			
GElf_Shdr shdr;
			
Elf_Data *data;
		} 
*reloc;
		
int nr_reloc;
		
int maps_shndx;
	} 
efile;
	/*
         * All loaded bpf_object is linked in a list, which is
         * hidden to caller. bpf_objects__<func> handlers deal with
         * all objects.
         */
	
struct list_head list;

	
void *priv;
	
bpf_object_clear_priv_t clear_priv;

	
char path[];
};

#define obj_elf_valid(o)	((o)->efile.elf)



static void bpf_program__unload(struct bpf_program *prog)
{
	int i;

	if (!prog)
		return;

	/*
         * If the object is opened but the program was never loaded,
         * it is possible that prog->instances.nr == -1.
         */
	if (prog->instances.nr > 0) {
		for (i = 0; i < prog->instances.nr; i++)
			zclose(prog->instances.fds[i]);
	} else if (prog->instances.nr != -1) {
		pr_warning("Internal error: instances.nr is %d\n",
			   prog->instances.nr);
	}

	prog->instances.nr = -1;
	zfree(&prog->instances.fds);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
wang nan
wang nan
106
100.00%
2
100.00%
Total
106
100.00%
2
100.00%



static void bpf_program__exit(struct bpf_program *prog)
{
	if (!prog)
		return;

	if (prog->clear_priv)
		prog->clear_priv(prog, prog->priv);

	prog->priv = NULL;
	prog->clear_priv = NULL;

	bpf_program__unload(prog);
	zfree(&prog->section_name);
	zfree(&prog->insns);
	zfree(&prog->reloc_desc);

	prog->nr_reloc = 0;
	prog->insns_cnt = 0;
	prog->idx = -1;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
wang nan
wang nan
94
100.00%
4
100.00%
Total
94
100.00%
4
100.00%


static int

bpf_program__init(void *data, size_t size, char *name, int idx,
		    struct bpf_program *prog)
{
	if (size < sizeof(struct bpf_insn)) {
		pr_warning("corrupted section '%s'\n", name);
		return -EINVAL;
	}

	bzero(prog, sizeof(*prog));

	prog->section_name = strdup(name);
	if (!prog->section_name) {
		pr_warning("failed to alloc name for prog %s\n",
			   name);
		goto errout;
	}

	prog->insns = malloc(size);
	if (!prog->insns) {
		pr_warning("failed to alloc insns for %s\n", name);
		goto errout;
	}
	prog->insns_cnt = size / sizeof(struct bpf_insn);
	memcpy(prog->insns, data,
	       prog->insns_cnt * sizeof(struct bpf_insn));
	prog->idx = idx;
	prog->instances.fds = NULL;
	prog->instances.nr = -1;
	prog->type = BPF_PROG_TYPE_KPROBE;

	return 0;
errout:
	bpf_program__exit(prog);
	return -ENOMEM;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
wang nan
wang nan
189
100.00%
4
100.00%
Total
189
100.00%
4
100.00%


static int

bpf_object__add_program(struct bpf_object *obj, void *data, size_t size,
			char *name, int idx)
{
	struct bpf_program prog, *progs;
	int nr_progs, err;

	err = bpf_program__init(data, size, name, idx, &prog);
	if (err)
		return err;

	progs = obj->programs;
	nr_progs = obj->nr_programs;

	progs = realloc(progs, sizeof(progs[0]) * (nr_progs + 1));
	if (!progs) {
		/*
                 * In this case the original obj->programs
                 * is still valid, so don't need special treat for
                 * bpf_close_object().
                 */
		pr_warning("failed to alloc a new program '%s'\n",
			   name);
		bpf_program__exit(&prog);
		return -ENOMEM;
	}

	pr_debug("found program %s\n", prog.section_name);
	obj->programs = progs;
	obj->nr_programs = nr_progs + 1;
	prog.obj = obj;
	progs[nr_progs] = prog;
	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
wang nan
wang nan
157
100.00%
2
100.00%
Total
157
100.00%
2
100.00%



static struct bpf_object *bpf_object__new(const char *path,
					  void *obj_buf,
					  size_t obj_buf_sz)
{
	struct bpf_object *obj;

	obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
	if (!obj) {
		pr_warning("alloc memory failed for %s\n", path);
		return ERR_PTR(-ENOMEM);
	}

	strcpy(obj->path, path);
	obj->efile.fd = -1;

	/*
         * Caller of this function should also calls
         * bpf_object__elf_finish() after data collection to return
         * obj_buf to user. If not, we should duplicate the buffer to
         * avoid user freeing them before elf finish.
         */
	obj->efile.obj_buf = obj_buf;
	obj->efile.obj_buf_sz = obj_buf_sz;
	obj->efile.maps_shndx = -1;

	obj->loaded = false;

	INIT_LIST_HEAD(&obj->list);
	list_add(&obj->list, &bpf_objects_list);
	return obj;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
wang nan
wang nan
138
100.00%
6
100.00%
Total
138
100.00%
6
100.00%



static void bpf_object__elf_finish(struct bpf_object *obj)
{
	if (!obj_elf_valid(obj))
		return;

	if (obj->efile.elf) {
		elf_end(obj->efile.elf);
		obj->efile.elf = NULL;
	}
	obj->efile.symbols = NULL;

	zfree(&obj->efile.reloc);
	obj->efile.nr_reloc = 0;
	zclose(obj->efile.fd);
	obj->efile.obj_buf = NULL;
	obj->efile.obj_buf_sz = 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
wang nan
wang nan
98
100.00%
4
100.00%
Total
98
100.00%
4
100.00%



static int bpf_object__elf_init(struct bpf_object *obj)
{
	int err = 0;
	GElf_Ehdr *ep;

	if (obj_elf_valid(obj)) {
		pr_warning("elf init: internal error\n");
		return -LIBBPF_ERRNO__LIBELF;
	}

	if (obj->efile.obj_buf_sz > 0) {
		/*
                 * obj_buf should have been validated by
                 * bpf_object__open_buffer().
                 */
		obj->efile.elf = elf_memory(obj->efile.obj_buf,
					    obj->efile.obj_buf_sz);
	} else {
		obj->efile.fd = open(obj->path, O_RDONLY);
		if (obj->efile.fd < 0) {
			pr_warning("failed to open %s: %s\n", obj->path,
					strerror(errno));
			return -errno;
		}

		obj->efile.elf = elf_begin(obj->efile.fd,
				LIBBPF_ELF_C_READ_MMAP,
				NULL);
	}

	if (!obj->efile.elf) {
		pr_warning("failed to open %s as ELF file\n",
				obj->path);
		err = -LIBBPF_ERRNO__LIBELF;
		goto errout;
	}

	if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
		pr_warning("failed to get EHDR from %s\n",
				obj->path);
		err = -LIBBPF_ERRNO__FORMAT;
		goto errout;
	}
	ep = &obj->efile.ehdr;

	/* Old LLVM set e_machine to EM_NONE */
	if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) {
		pr_warning("%s is not an eBPF object file\n",
			obj->path);
		err = -LIBBPF_ERRNO__FORMAT;
		goto errout;
	}

	return 0;
errout:
	bpf_object__elf_finish(obj);
	return err;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
wang nan
wang nan
271
100.00%
4
100.00%
Total
271
100.00%
4
100.00%


static int

bpf_object__check_endianness(struct bpf_object *obj)
{
	static unsigned int const endian = 1;

	switch (obj->efile.ehdr.e_ident[EI_DATA]) {
	case ELFDATA2LSB:
		/* We are big endian, BPF obj is little endian. */
		if (*(unsigned char const *)&endian != 1)
			goto mismatch;
		break;

	case ELFDATA2MSB:
		/* We are little endian, BPF obj is big endian. */
		if (*(unsigned char const *)&endian != 0)
			goto mismatch;
		break;
	default:
		return -LIBBPF_ERRNO__ENDIAN;
	}

	return 0;

mismatch:
	pr_warning("Error: endianness mismatch.\n");
	return -LIBBPF_ERRNO__ENDIAN;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
wang nan
wang nan
97
100.00%
3
100.00%
Total
97
100.00%
3
100.00%


static int

bpf_object__init_license(struct bpf_object *obj,
			 void *data, size_t size)
{
	memcpy(obj->license, data,
	       min(size, sizeof(obj->license) - 1));
	pr_debug("license of %s is %s\n", obj->path, obj->license);
	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
wang nan
wang nan
57
100.00%
1
100.00%
Total
57
100.00%
1
100.00%


static int

bpf_object__init_kversion(struct bpf_object *obj,
			  void *data, size_t size)
{
	u32 kver;

	if (size != sizeof(kver)) {
		pr_warning("invalid kver section in %s\n", obj->path);
		return -LIBBPF_ERRNO__FORMAT;
	}
	memcpy(&kver, data, sizeof(kver));
	obj->kern_version = kver;
	pr_debug("kernel version of %s is %x\n", obj->path,
		 obj->kern_version);
	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
wang nan
wang nan
80
100.00%
2
100.00%
Total
80
100.00%
2
100.00%


static int

bpf_object__validate_maps(struct bpf_object *obj)
{
	int i;

	/*
         * If there's only 1 map, the only error case should have been
         * catched in bpf_object__init_maps().
         */
	if (!obj->maps || !obj->nr_maps || (obj->nr_maps == 1))
		return 0;

	for (i = 1; i < obj->nr_maps; i++) {
		const struct bpf_map *a = &obj->maps[i - 1];
		const struct bpf_map *b = &obj->maps[i];

		if (b->offset - a->offset < sizeof(struct bpf_map_def)) {
			pr_warning("corrupted map section in %s: map \"%s\" too small\n",
				   obj->path, a->name);
			return -EINVAL;
		}
	}
	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
wang nan
wang nan
64
52.03%
2
66.67%
eric leblond
eric leblond
59
47.97%
1
33.33%
Total
123
100.00%
3
100.00%



static int compare_bpf_map(const void *_a, const void *_b)
{
	const struct bpf_map *a = _a;
	const struct bpf_map *b = _b;

	return a->offset - b->offset;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
eric leblond
eric leblond
41
100.00%
1
100.00%
Total
41
100.00%
1
100.00%


static int

bpf_object__init_maps(struct bpf_object *obj)
{
	int i, map_idx, nr_maps = 0;
	Elf_Scn *scn;
	Elf_Data *data;
	Elf_Data *symbols = obj->efile.symbols;

	if (obj->efile.maps_shndx < 0)
		return -EINVAL;
	if (!symbols)
		return -EINVAL;

	scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx);
	if (scn)
		data = elf_getdata(scn, NULL);
	if (!scn || !data) {
		pr_warning("failed to get Elf_Data from map section %d\n",
			   obj->efile.maps_shndx);
		return -EINVAL;
	}

	/*
         * Count number of maps. Each map has a name.
         * Array of maps is not supported: only the first element is
         * considered.
         *
         * TODO: Detect array of map and report error.
         */
	for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) {
		GElf_Sym sym;

		if (!gelf_getsym(symbols, i, &sym))
			continue;
		if (sym.st_shndx != obj->efile.maps_shndx)
			continue;
		nr_maps++;
	}

	/* Alloc obj->maps and fill nr_maps. */
	pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path,
		 nr_maps, data->d_size);

	if (!nr_maps)
		return 0;

	obj->maps = calloc(nr_maps, sizeof(obj->maps[0]));
	if (!obj->maps) {
		pr_warning("alloc maps for object failed\n");
		return -ENOMEM;
	}
	obj->nr_maps = nr_maps;

	/*
         * fill all fd with -1 so won't close incorrect
         * fd (fd=0 is stdin) when failure (zclose won't close
         * negative fd)).
         */
	for (i = 0; i < nr_maps; i++)
		obj->maps[i].fd = -1;

	/*
         * Fill obj->maps using data in "maps" section.
         */
	for (i = 0, map_idx = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) {
		GElf_Sym sym;
		const char *map_name;
		struct bpf_map_def *def;

		if (!gelf_getsym(symbols, i, &sym))
			continue;
		if (sym.st_shndx != obj->efile.maps_shndx)
			continue;

		map_name = elf_strptr(obj->efile.elf,
				      obj->efile.strtabidx,
				      sym.st_name);
		obj->maps[map_idx].offset = sym.st_value;
		if (sym.st_value + sizeof(struct bpf_map_def) > data->d_size) {
			pr_warning("corrupted maps section in %s: last map \"%s\" too small\n",
				   obj->path, map_name);
			return -EINVAL;
		}

		obj->maps[map_idx].name = strdup(map_name);
		if (!obj->maps[map_idx].name) {
			pr_warning("failed to alloc map name\n");
			return -ENOMEM;
		}
		pr_debug("map %d is \"%s\"\n", map_idx,
			 obj->maps[map_idx].name);
		def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
		obj->maps[map_idx].def = *def;
		map_idx++;
	}

	qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map);
	return bpf_object__validate_maps(obj);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
eric leblond
eric leblond
360
70.18%
1
20.00%
wang nan
wang nan
153
29.82%
4
80.00%
Total
513
100.00%
5
100.00%



static int bpf_object__elf_collect(struct bpf_object *obj)
{
	Elf *elf = obj->efile.elf;
	GElf_Ehdr *ep = &obj->efile.ehdr;
	Elf_Scn *scn = NULL;
	int idx = 0, err = 0;

	/* Elf is corrupted/truncated, avoid calling elf_strptr. */
	if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) {
		pr_warning("failed to get e_shstrndx from %s\n",
			   obj->path);
		return -LIBBPF_ERRNO__FORMAT;
	}

	while ((scn = elf_nextscn(elf, scn)) != NULL) {
		char *name;
		GElf_Shdr sh;
		Elf_Data *data;

		idx++;
		if (gelf_getshdr(scn, &sh) != &sh) {
			pr_warning("failed to get section header from %s\n",
				   obj->path);
			err = -LIBBPF_ERRNO__FORMAT;
			goto out;
		}

		name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
		if (!name) {
			pr_warning("failed to get section name from %s\n",
				   obj->path);
			err = -LIBBPF_ERRNO__FORMAT;
			goto out;
		}

		data = elf_getdata(scn, 0);
		if (!data) {
			pr_warning("failed to get section data from %s(%s)\n",
				   name, obj->path);