Contributors: 6
Author Tokens Token Proportion Commits Commit Proportion
Jens Axboe 1921 98.46% 21 70.00%
Pavel Begunkov 18 0.92% 4 13.33%
Kees Cook 5 0.26% 1 3.33%
Peter Chubb 3 0.15% 1 3.33%
Linus Torvalds (pre-git) 3 0.15% 2 6.67%
Greg Kroah-Hartman 1 0.05% 1 3.33%
Total 1951 30 


// SPDX-License-Identifier: GPL-2.0
/*
 * BPF filter support for io_uring. Supports SQE opcodes for now.
 */
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/io_uring.h>
#include <linux/filter.h>
#include <linux/bpf.h>
#include <uapi/linux/io_uring.h>

#include "io_uring.h"
#include "bpf_filter.h"
#include "net.h"
#include "openclose.h"

struct io_bpf_filter {
	refcount_t		refs;
	struct bpf_prog		*prog;
	struct io_bpf_filter	*next;
};

/* Deny if this is set as the filter */
static const struct io_bpf_filter dummy_filter;

static void io_uring_populate_bpf_ctx(struct io_uring_bpf_ctx *bctx,
				      struct io_kiocb *req)
{
	const struct io_issue_def *def = &io_issue_defs[req->opcode];

	bctx->opcode = req->opcode;
	bctx->sqe_flags = (__force int) req->flags & SQE_VALID_FLAGS;
	bctx->user_data = req->cqe.user_data;
	/* clear residual, anything from pdu_size and below */
	memset((void *) bctx + offsetof(struct io_uring_bpf_ctx, pdu_size), 0,
		sizeof(*bctx) - offsetof(struct io_uring_bpf_ctx, pdu_size));

	/*
	 * Opcodes can provide a handler for populating more data into bctx,
	 * for filters to use.
	 */
	if (def->filter_pdu_size) {
		bctx->pdu_size = def->filter_pdu_size;
		def->filter_populate(bctx, req);
	}
}

/*
 * Run registered filters for a given opcode. For filters, a return of 0 denies
 * execution of the request, a return of 1 allows it. If any filter for an
 * opcode returns 0, filter processing is stopped, and the request is denied.
 * This also stops the processing of filters.
 *
 * __io_uring_run_bpf_filters() returns 0 on success, allow running the
 * request, and -EACCES when a request is denied.
 */
int __io_uring_run_bpf_filters(struct io_bpf_filter __rcu **filters,
			       struct io_kiocb *req)
{
	struct io_bpf_filter *filter;
	struct io_uring_bpf_ctx bpf_ctx;
	int ret;

	/* Fast check for existence of filters outside of RCU */
	if (!rcu_access_pointer(filters[req->opcode]))
		return 0;

	/*
	 * req->opcode has already been validated to be within the range
	 * of what we expect, io_init_req() does this.
	 */
	guard(rcu)();
	filter = rcu_dereference(filters[req->opcode]);
	if (!filter)
		return 0;
	else if (filter == &dummy_filter)
		return -EACCES;

	io_uring_populate_bpf_ctx(&bpf_ctx, req);

	/*
	 * Iterate registered filters. The opcode is allowed IFF all filters
	 * return 1. If any filter returns denied, opcode will be denied.
	 */
	do {
		if (filter == &dummy_filter)
			return -EACCES;
		ret = bpf_prog_run_pin_on_cpu(filter->prog, &bpf_ctx);
		if (!ret)
			return -EACCES;
		filter = filter->next;
	} while (filter);

	return 0;
}

static void io_free_bpf_filters(struct rcu_head *head)
{
	struct io_bpf_filter __rcu **filter;
	struct io_bpf_filters *filters;
	int i;

	filters = container_of(head, struct io_bpf_filters, rcu_head);
	scoped_guard(spinlock, &filters->lock) {
		filter = filters->filters;
		if (!filter)
			return;
	}

	for (i = 0; i < IORING_OP_LAST; i++) {
		struct io_bpf_filter *f;

		rcu_read_lock();
		f = rcu_dereference(filter[i]);
		while (f) {
			struct io_bpf_filter *next = f->next;

			/*
			 * Even if stacked, dummy filter will always be last
			 * as it can only get installed into an empty spot.
			 */
			if (f == &dummy_filter)
				break;

			/* Someone still holds a ref, stop iterating. */
			if (!refcount_dec_and_test(&f->refs))
				break;

			bpf_prog_destroy(f->prog);
			kfree(f);
			f = next;
		}
		rcu_read_unlock();
	}
	kfree(filters->filters);
	kfree(filters);
}

static void __io_put_bpf_filters(struct io_bpf_filters *filters)
{
	if (refcount_dec_and_test(&filters->refs))
		call_rcu(&filters->rcu_head, io_free_bpf_filters);
}

void io_put_bpf_filters(struct io_restriction *res)
{
	if (res->bpf_filters)
		__io_put_bpf_filters(res->bpf_filters);
}

static struct io_bpf_filters *io_new_bpf_filters(void)
{
	struct io_bpf_filters *filters __free(kfree) = NULL;

	filters = kzalloc_obj(*filters, GFP_KERNEL_ACCOUNT);
	if (!filters)
		return ERR_PTR(-ENOMEM);

	filters->filters = kzalloc_objs(struct io_bpf_filter *, IORING_OP_LAST,
					GFP_KERNEL_ACCOUNT);
	if (!filters->filters)
		return ERR_PTR(-ENOMEM);

	refcount_set(&filters->refs, 1);
	spin_lock_init(&filters->lock);
	return no_free_ptr(filters);
}

/*
 * Validate classic BPF filter instructions. Only allow a safe subset of
 * operations - no packet data access, just context field loads and basic
 * ALU/jump operations.
 */
static int io_uring_check_cbpf_filter(struct sock_filter *filter,
				      unsigned int flen)
{
	int pc;

	for (pc = 0; pc < flen; pc++) {
		struct sock_filter *ftest = &filter[pc];
		u16 code = ftest->code;
		u32 k = ftest->k;

		switch (code) {
		case BPF_LD | BPF_W | BPF_ABS:
			ftest->code = BPF_LDX | BPF_W | BPF_ABS;
			/* 32-bit aligned and not out of bounds. */
			if (k >= sizeof(struct io_uring_bpf_ctx) || k & 3)
				return -EINVAL;
			continue;
		case BPF_LD | BPF_W | BPF_LEN:
			ftest->code = BPF_LD | BPF_IMM;
			ftest->k = sizeof(struct io_uring_bpf_ctx);
			continue;
		case BPF_LDX | BPF_W | BPF_LEN:
			ftest->code = BPF_LDX | BPF_IMM;
			ftest->k = sizeof(struct io_uring_bpf_ctx);
			continue;
		/* Explicitly include allowed calls. */
		case BPF_RET | BPF_K:
		case BPF_RET | BPF_A:
		case BPF_ALU | BPF_ADD | BPF_K:
		case BPF_ALU | BPF_ADD | BPF_X:
		case BPF_ALU | BPF_SUB | BPF_K:
		case BPF_ALU | BPF_SUB | BPF_X:
		case BPF_ALU | BPF_MUL | BPF_K:
		case BPF_ALU | BPF_MUL | BPF_X:
		case BPF_ALU | BPF_DIV | BPF_K:
		case BPF_ALU | BPF_DIV | BPF_X:
		case BPF_ALU | BPF_AND | BPF_K:
		case BPF_ALU | BPF_AND | BPF_X:
		case BPF_ALU | BPF_OR | BPF_K:
		case BPF_ALU | BPF_OR | BPF_X:
		case BPF_ALU | BPF_XOR | BPF_K:
		case BPF_ALU | BPF_XOR | BPF_X:
		case BPF_ALU | BPF_LSH | BPF_K:
		case BPF_ALU | BPF_LSH | BPF_X:
		case BPF_ALU | BPF_RSH | BPF_K:
		case BPF_ALU | BPF_RSH | BPF_X:
		case BPF_ALU | BPF_NEG:
		case BPF_LD | BPF_IMM:
		case BPF_LDX | BPF_IMM:
		case BPF_MISC | BPF_TAX:
		case BPF_MISC | BPF_TXA:
		case BPF_LD | BPF_MEM:
		case BPF_LDX | BPF_MEM:
		case BPF_ST:
		case BPF_STX:
		case BPF_JMP | BPF_JA:
		case BPF_JMP | BPF_JEQ | BPF_K:
		case BPF_JMP | BPF_JEQ | BPF_X:
		case BPF_JMP | BPF_JGE | BPF_K:
		case BPF_JMP | BPF_JGE | BPF_X:
		case BPF_JMP | BPF_JGT | BPF_K:
		case BPF_JMP | BPF_JGT | BPF_X:
		case BPF_JMP | BPF_JSET | BPF_K:
		case BPF_JMP | BPF_JSET | BPF_X:
			continue;
		default:
			return -EINVAL;
		}
	}
	return 0;
}

void io_bpf_filter_clone(struct io_restriction *dst, struct io_restriction *src)
{
	if (!src->bpf_filters)
		return;

	rcu_read_lock();
	/*
	 * If the src filter is going away, just ignore it.
	 */
	if (refcount_inc_not_zero(&src->bpf_filters->refs)) {
		dst->bpf_filters = src->bpf_filters;
		dst->bpf_filters_cow = true;
	}
	rcu_read_unlock();
}

/*
 * Allocate a new struct io_bpf_filters. Used when a filter is cloned and
 * modifications need to be made.
 */
static struct io_bpf_filters *io_bpf_filter_cow(struct io_restriction *src)
{
	struct io_bpf_filters *filters;
	struct io_bpf_filter *srcf;
	int i;

	filters = io_new_bpf_filters();
	if (IS_ERR(filters))
		return filters;

	/*
	 * Iterate filters from src and assign in destination. Grabbing
	 * a reference is enough, we don't need to duplicate the memory.
	 * This is safe because filters are only ever appended to the
	 * front of the list, hence the only memory ever touched inside
	 * a filter is the refcount.
	 */
	rcu_read_lock();
	for (i = 0; i < IORING_OP_LAST; i++) {
		srcf = rcu_dereference(src->bpf_filters->filters[i]);
		if (!srcf) {
			continue;
		} else if (srcf == &dummy_filter) {
			rcu_assign_pointer(filters->filters[i], &dummy_filter);
			continue;
		}

		/*
		 * Getting a ref on the first node is enough, putting the
		 * filter and iterating nodes to free will stop on the first
		 * one that doesn't hit zero when dropping.
		 */
		if (!refcount_inc_not_zero(&srcf->refs))
			goto err;
		rcu_assign_pointer(filters->filters[i], srcf);
	}
	rcu_read_unlock();
	return filters;
err:
	rcu_read_unlock();
	__io_put_bpf_filters(filters);
	return ERR_PTR(-EBUSY);
}

#define IO_URING_BPF_FILTER_FLAGS	(IO_URING_BPF_FILTER_DENY_REST | \
					 IO_URING_BPF_FILTER_SZ_STRICT)

static int io_bpf_filter_import(struct io_uring_bpf *reg,
				struct io_uring_bpf __user *arg)
{
	const struct io_issue_def *def;
	int ret;

	if (copy_from_user(reg, arg, sizeof(*reg)))
		return -EFAULT;
	if (reg->cmd_type != IO_URING_BPF_CMD_FILTER)
		return -EINVAL;
	if (reg->cmd_flags || reg->resv)
		return -EINVAL;

	if (reg->filter.opcode >= IORING_OP_LAST)
		return -EINVAL;
	if (reg->filter.flags & ~IO_URING_BPF_FILTER_FLAGS)
		return -EINVAL;
	if (!mem_is_zero(reg->filter.resv, sizeof(reg->filter.resv)))
		return -EINVAL;
	if (!mem_is_zero(reg->filter.resv2, sizeof(reg->filter.resv2)))
		return -EINVAL;
	if (!reg->filter.filter_len || reg->filter.filter_len > BPF_MAXINSNS)
		return -EINVAL;

	/* Verify filter size */
	def = &io_issue_defs[array_index_nospec(reg->filter.opcode, IORING_OP_LAST)];

	/* same size, always ok */
	ret = 0;
	if (reg->filter.pdu_size == def->filter_pdu_size)
		;
	/* size differs, fail in strict mode */
	else if (reg->filter.flags & IO_URING_BPF_FILTER_SZ_STRICT)
		ret = -EMSGSIZE;
	/* userspace filter is bigger, always disallow */
	else if (reg->filter.pdu_size > def->filter_pdu_size)
		ret = -EMSGSIZE;

	/* copy back kernel filter size */
	reg->filter.pdu_size = def->filter_pdu_size;
	if (copy_to_user(&arg->filter, &reg->filter, sizeof(reg->filter)))
		return -EFAULT;

	return ret;
}

int io_register_bpf_filter(struct io_restriction *res,
			   struct io_uring_bpf __user *arg)
{
	struct io_bpf_filters *filters, *old_filters = NULL;
	struct io_bpf_filter *filter, *old_filter;
	struct io_uring_bpf reg;
	struct bpf_prog *prog;
	struct sock_fprog fprog;
	int ret;

	ret = io_bpf_filter_import(&reg, arg);
	if (ret)
		return ret;

	fprog.len = reg.filter.filter_len;
	fprog.filter = u64_to_user_ptr(reg.filter.filter_ptr);

	ret = bpf_prog_create_from_user(&prog, &fprog,
					io_uring_check_cbpf_filter, false);
	if (ret)
		return ret;

	/*
	 * No existing filters, allocate set.
	 */
	filters = res->bpf_filters;
	if (!filters) {
		filters = io_new_bpf_filters();
		if (IS_ERR(filters)) {
			ret = PTR_ERR(filters);
			goto err_prog;
		}
	} else if (res->bpf_filters_cow) {
		filters = io_bpf_filter_cow(res);
		if (IS_ERR(filters)) {
			ret = PTR_ERR(filters);
			goto err_prog;
		}
		/*
		 * Stash old filters, we'll put them once we know we'll
		 * succeed. Until then, res->bpf_filters is left untouched.
		 */
		old_filters = res->bpf_filters;
	}

	filter = kzalloc_obj(*filter, GFP_KERNEL_ACCOUNT);
	if (!filter) {
		ret = -ENOMEM;
		goto err;
	}
	refcount_set(&filter->refs, 1);
	filter->prog = prog;

	/*
	 * Success - install the new filter set now. If we did COW, put
	 * the old filters as we're replacing them.
	 */
	if (old_filters) {
		__io_put_bpf_filters(old_filters);
		res->bpf_filters_cow = false;
	}
	res->bpf_filters = filters;

	/*
	 * Insert filter - if the current opcode already has a filter
	 * attached, add to the set.
	 */
	rcu_read_lock();
	spin_lock_bh(&filters->lock);
	old_filter = rcu_dereference(filters->filters[reg.filter.opcode]);
	if (old_filter)
		filter->next = old_filter;
	rcu_assign_pointer(filters->filters[reg.filter.opcode], filter);

	/*
	 * If IO_URING_BPF_FILTER_DENY_REST is set, fill any unregistered
	 * opcode with the dummy filter. That will cause them to be denied.
	 */
	if (reg.filter.flags & IO_URING_BPF_FILTER_DENY_REST) {
		for (int i = 0; i < IORING_OP_LAST; i++) {
			if (i == reg.filter.opcode)
				continue;
			old_filter = rcu_dereference(filters->filters[i]);
			if (old_filter)
				continue;
			rcu_assign_pointer(filters->filters[i], &dummy_filter);
		}
	}

	spin_unlock_bh(&filters->lock);
	rcu_read_unlock();
	return 0;
err:
	if (filters != res->bpf_filters)
		__io_put_bpf_filters(filters);
err_prog:
	bpf_prog_destroy(prog);
	return ret;
}