Release 4.16 lib/iov_iter.c

#include <linux/export.h>
#include <linux/bvec.h>
#include <linux/uio.h>
#include <linux/pagemap.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/splice.h>
#include <net/checksum.h>


#define PIPE_PARANOIA 
/* for now */


#define iterate_iovec(i, n, __v, __p, skip, STEP) {       \
        size_t left;                                    \
        size_t wanted = n;                              \
        __p = i->iov;                                   \
        __v.iov_len = min(n, __p->iov_len - skip);      \
        if (likely(__v.iov_len)) {                      \
                __v.iov_base = __p->iov_base + skip;    \
                left = (STEP);                          \
                __v.iov_len -= left;                    \
                skip += __v.iov_len;                    \
                n -= __v.iov_len;                       \
        } else {                                        \
                left = 0;                               \
        }                                               \
        while (unlikely(!left && n)) {                  \
                __p++;                                  \
                __v.iov_len = min(n, __p->iov_len);     \
                if (unlikely(!__v.iov_len))             \
                        continue;                       \
                __v.iov_base = __p->iov_base;           \
                left = (STEP);                          \
                __v.iov_len -= left;                    \
                skip = __v.iov_len;                     \
                n -= __v.iov_len;                       \
        }                                               \
        n = wanted - n;                                 \
}


#define iterate_kvec(i, n, __v, __p, skip, STEP) {       \
        size_t wanted = n;                              \
        __p = i->kvec;                                  \
        __v.iov_len = min(n, __p->iov_len - skip);      \
        if (likely(__v.iov_len)) {                      \
                __v.iov_base = __p->iov_base + skip;    \
                (void)(STEP);                           \
                skip += __v.iov_len;                    \
                n -= __v.iov_len;                       \
        }                                               \
        while (unlikely(n)) {                           \
                __p++;                                  \
                __v.iov_len = min(n, __p->iov_len);     \
                if (unlikely(!__v.iov_len))             \
                        continue;                       \
                __v.iov_base = __p->iov_base;           \
                (void)(STEP);                           \
                skip = __v.iov_len;                     \
                n -= __v.iov_len;                       \
        }                                               \
        n = wanted;                                     \
}


#define iterate_bvec(i, n, __v, __bi, skip, STEP) {       \
        struct bvec_iter __start;                       \
        __start.bi_size = n;                            \
        __start.bi_bvec_done = skip;                    \
        __start.bi_idx = 0;                             \
        for_each_bvec(__v, i->bvec, __bi, __start) {    \
                if (!__v.bv_len)                        \
                        continue;                       \
                (void)(STEP);                           \
        }                                               \
}


#define iterate_all_kinds(i, n, v, I, B, K) {                       \
        if (likely(n)) {                                        \
                size_t skip = i->iov_offset;                    \
                if (unlikely(i->type & ITER_BVEC)) {            \
                        struct bio_vec v;                       \
                        struct bvec_iter __bi;                  \
                        iterate_bvec(i, n, v, __bi, skip, (B))  \
                } else if (unlikely(i->type & ITER_KVEC)) {     \
                        const struct kvec *kvec;                \
                        struct kvec v;                          \
                        iterate_kvec(i, n, v, kvec, skip, (K))  \
                } else {                                        \
                        const struct iovec *iov;                \
                        struct iovec v;                         \
                        iterate_iovec(i, n, v, iov, skip, (I))  \
                }                                               \
        }                                                       \
}


#define iterate_and_advance(i, n, v, I, B, K) {                       \
        if (unlikely(i->count < n))                             \
                n = i->count;                                   \
        if (i->count) {                                         \
                size_t skip = i->iov_offset;                    \
                if (unlikely(i->type & ITER_BVEC)) {            \
                        const struct bio_vec *bvec = i->bvec;   \
                        struct bio_vec v;                       \
                        struct bvec_iter __bi;                  \
                        iterate_bvec(i, n, v, __bi, skip, (B))  \
                        i->bvec = __bvec_iter_bvec(i->bvec, __bi);      \
                        i->nr_segs -= i->bvec - bvec;           \
                        skip = __bi.bi_bvec_done;               \
                } else if (unlikely(i->type & ITER_KVEC)) {     \
                        const struct kvec *kvec;                \
                        struct kvec v;                          \
                        iterate_kvec(i, n, v, kvec, skip, (K))  \
                        if (skip == kvec->iov_len) {            \
                                kvec++;                         \
                                skip = 0;                       \
                        }                                       \
                        i->nr_segs -= kvec - i->kvec;           \
                        i->kvec = kvec;                         \
                } else {                                        \
                        const struct iovec *iov;                \
                        struct iovec v;                         \
                        iterate_iovec(i, n, v, iov, skip, (I))  \
                        if (skip == iov->iov_len) {             \
                                iov++;                          \
                                skip = 0;                       \
                        }                                       \
                        i->nr_segs -= iov - i->iov;             \
                        i->iov = iov;                           \
                }                                               \
                i->count -= n;                                  \
                i->iov_offset = skip;                           \
        }                                                       \
}



static int copyout(void __user *to, const void *from, size_t n)
{
	if (access_ok(VERIFY_WRITE, to, n)) {
		kasan_check_read(from, n);
		n = raw_copy_to_user(to, from, n);
	}
	return n;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Al Viro
53
100.00%
1
100.00%
Total
53
100.00%
1
100.00%



static int copyin(void *to, const void __user *from, size_t n)
{
	if (access_ok(VERIFY_READ, from, n)) {
		kasan_check_write(to, n);
		n = raw_copy_from_user(to, from, n);
	}
	return n;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Al Viro
53
100.00%
1
100.00%
Total
53
100.00%
1
100.00%



static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,
			 struct iov_iter *i)
{
	size_t skip, copy, left, wanted;
	const struct iovec *iov;
	char __user *buf;
	void *kaddr, *from;

	if (unlikely(bytes > i->count))
		bytes = i->count;

	if (unlikely(!bytes))
		return 0;

	might_fault();
	wanted = bytes;
	iov = i->iov;
	skip = i->iov_offset;
	buf = iov->iov_base + skip;
	copy = min(bytes, iov->iov_len - skip);

	if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) {
		kaddr = kmap_atomic(page);
		from = kaddr + offset;

		/* first chunk, usually the only one */
		left = copyout(buf, from, copy);
		copy -= left;
		skip += copy;
		from += copy;
		bytes -= copy;

		while (unlikely(!left && bytes)) {
			iov++;
			buf = iov->iov_base;
			copy = min(bytes, iov->iov_len);
			left = copyout(buf, from, copy);
			copy -= left;
			skip = copy;
			from += copy;
			bytes -= copy;
		}
		if (likely(!bytes)) {
			kunmap_atomic(kaddr);
			goto done;
		}
		offset = from - kaddr;
		buf += copy;
		kunmap_atomic(kaddr);
		copy = min(bytes, iov->iov_len - skip);
	}
	/* Too bad - revert to non-atomic kmap */

	kaddr = kmap(page);
	from = kaddr + offset;
	left = copyout(buf, from, copy);
	copy -= left;
	skip += copy;
	from += copy;
	bytes -= copy;
	while (unlikely(!left && bytes)) {
		iov++;
		buf = iov->iov_base;
		copy = min(bytes, iov->iov_len);
		left = copyout(buf, from, copy);
		copy -= left;
		skip = copy;
		from += copy;
		bytes -= copy;
	}
	kunmap(page);

done:
	if (skip == iov->iov_len) {
		iov++;
		skip = 0;
	}
	i->count -= wanted - bytes;
	i->nr_segs -= iov - i->iov;
	i->iov = iov;
	i->iov_offset = skip;
	return wanted - bytes;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Al Viro
416
94.76%
4
66.67%
Matthew Wilcox
18
4.10%
1
16.67%
Mikulas Patocka
5
1.14%
1
16.67%
Total
439
100.00%
6
100.00%



static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
			 struct iov_iter *i)
{
	size_t skip, copy, left, wanted;
	const struct iovec *iov;
	char __user *buf;
	void *kaddr, *to;

	if (unlikely(bytes > i->count))
		bytes = i->count;

	if (unlikely(!bytes))
		return 0;

	might_fault();
	wanted = bytes;
	iov = i->iov;
	skip = i->iov_offset;
	buf = iov->iov_base + skip;
	copy = min(bytes, iov->iov_len - skip);

	if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) {
		kaddr = kmap_atomic(page);
		to = kaddr + offset;

		/* first chunk, usually the only one */
		left = copyin(to, buf, copy);
		copy -= left;
		skip += copy;
		to += copy;
		bytes -= copy;

		while (unlikely(!left && bytes)) {
			iov++;
			buf = iov->iov_base;
			copy = min(bytes, iov->iov_len);
			left = copyin(to, buf, copy);
			copy -= left;
			skip = copy;
			to += copy;
			bytes -= copy;
		}
		if (likely(!bytes)) {
			kunmap_atomic(kaddr);
			goto done;
		}
		offset = to - kaddr;
		buf += copy;
		kunmap_atomic(kaddr);
		copy = min(bytes, iov->iov_len - skip);
	}
	/* Too bad - revert to non-atomic kmap */

	kaddr = kmap(page);
	to = kaddr + offset;
	left = copyin(to, buf, copy);
	copy -= left;
	skip += copy;
	to += copy;
	bytes -= copy;
	while (unlikely(!left && bytes)) {
		iov++;
		buf = iov->iov_base;
		copy = min(bytes, iov->iov_len);
		left = copyin(to, buf, copy);
		copy -= left;
		skip = copy;
		to += copy;
		bytes -= copy;
	}
	kunmap(page);

done:
	if (skip == iov->iov_len) {
		iov++;
		skip = 0;
	}
	i->count -= wanted - bytes;
	i->nr_segs -= iov - i->iov;
	i->iov = iov;
	i->iov_offset = skip;
	return wanted - bytes;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Matthew Wilcox
361
82.23%
1
25.00%
Al Viro
73
16.63%
2
50.00%
Mikulas Patocka
5
1.14%
1
25.00%
Total
439
100.00%
4
100.00%

#ifdef PIPE_PARANOIA


static bool sanity(const struct iov_iter *i)
{
	struct pipe_inode_info *pipe = i->pipe;
	int idx = i->idx;
	int next = pipe->curbuf + pipe->nrbufs;
	if (i->iov_offset) {
		struct pipe_buffer *p;
		if (unlikely(!pipe->nrbufs))
			goto Bad;	// pipe must be non-empty
		if (unlikely(idx != ((next - 1) & (pipe->buffers - 1))))
			goto Bad;	// must be at the last buffer...

		p = &pipe->bufs[idx];
		if (unlikely(p->offset + p->len != i->iov_offset))
			goto Bad;	// ... at the end of segment
	} else {
		if (idx != (next & (pipe->buffers - 1)))
			goto Bad;	// must be right after the last buffer
	}
	return true;
Bad:
	printk(KERN_ERR "idx = %d, offset = %zd\n", i->idx, i->iov_offset);
	printk(KERN_ERR "curbuf = %d, nrbufs = %d, buffers = %d\n",
			pipe->curbuf, pipe->nrbufs, pipe->buffers);
	for (idx = 0; idx < pipe->buffers; idx++)
		printk(KERN_ERR "[%p %p %d %d]\n",
			pipe->bufs[idx].ops,
			pipe->bufs[idx].page,
			pipe->bufs[idx].offset,
			pipe->bufs[idx].len);
	WARN_ON(1);
	return false;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Al Viro
218
87.55%
2
66.67%
Anton Altaparmakov
31
12.45%
1
33.33%
Total
249
100.00%
3
100.00%

#else

#define sanity(i) true
#endif



static inline int next_idx(int idx, struct pipe_inode_info *pipe)
{
	return (idx + 1) & (pipe->buffers - 1);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Al Viro
30
100.00%
3
100.00%
Total
30
100.00%
3
100.00%



static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,
			 struct iov_iter *i)
{
	struct pipe_inode_info *pipe = i->pipe;
	struct pipe_buffer *buf;
	size_t off;
	int idx;

	if (unlikely(bytes > i->count))
		bytes = i->count;

	if (unlikely(!bytes))
		return 0;

	if (!sanity(i))
		return 0;

	off = i->iov_offset;
	idx = i->idx;
	buf = &pipe->bufs[idx];
	if (off) {
		if (offset == off && buf->page == page) {
			/* merge with the last one */
			buf->len += bytes;
			i->iov_offset += bytes;
			goto out;
		}
		idx = next_idx(idx, pipe);
		buf = &pipe->bufs[idx];
	}
	if (idx == pipe->curbuf && pipe->nrbufs)
		return 0;
	pipe->nrbufs++;
	buf->ops = &page_cache_pipe_buf_ops;
	get_page(buf->page = page);
	buf->offset = offset;
	buf->len = bytes;
	i->iov_offset = offset + bytes;
	i->idx = idx;
out:
	i->count -= bytes;
	return bytes;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Al Viro
231
100.00%
2
100.00%
Total
231
100.00%
2
100.00%

/*
 * Fault in one or more iovecs of the given iov_iter, to a maximum length of
 * bytes.  For each iovec, fault in each page that constitutes the iovec.
 *
 * Return 0 on success, or non-zero if the memory could not be accessed (i.e.
 * because it is an invalid address).
 */


int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)
{
	size_t skip = i->iov_offset;
	const struct iovec *iov;
	int err;
	struct iovec v;

	if (!(i->type & (ITER_BVEC|ITER_KVEC))) {
		iterate_iovec(i, bytes, v, iov, skip, ({
                        err = fault_in_pages_readable(v.iov_base, v.iov_len);
                        if (unlikely(err))
                        return err;
		0;}))
	}
	return 0;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Al Viro
68
98.55%
2
66.67%
Linus Torvalds
1
1.45%
1
33.33%
Total
69
100.00%
3
100.00%

EXPORT_SYMBOL(iov_iter_fault_in_readable);



void iov_iter_init(struct iov_iter *i, int direction,
			const struct iovec *iov, unsigned long nr_segs,
			size_t count)
{
	/* It will get better.  Eventually... */
	if (uaccess_kernel()) {
		direction |= ITER_KVEC;
		i->type = direction;
		i->kvec = (struct kvec *)iov;
	} else {
		i->type = direction;
		i->iov = iov;
	}
	i->nr_segs = nr_segs;
	i->iov_offset = 0;
	i->count = count;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Al Viro
88
100.00%
2
100.00%
Total
88
100.00%
2
100.00%

EXPORT_SYMBOL(iov_iter_init);



static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
{
	char *from = kmap_atomic(page);
	memcpy(to, from + offset, len);
	kunmap_atomic(from);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Al Viro
46
100.00%
1
100.00%
Total
46
100.00%
1
100.00%



static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len)
{
	char *to = kmap_atomic(page);
	memcpy(to + offset, from, len);
	kunmap_atomic(to);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Al Viro
47
100.00%
1
100.00%
Total
47
100.00%
1
100.00%



static void memzero_page(struct page *page, size_t offset, size_t len)
{
	char *addr = kmap_atomic(page);
	memset(addr + offset, 0, len);
	kunmap_atomic(addr);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Al Viro
42
100.00%
1
100.00%
Total
42
100.00%
1
100.00%



static inline bool allocated(struct pipe_buffer *buf)
{
	return buf->ops == &default_pipe_buf_ops;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Al Viro
20
100.00%
1
100.00%
Total
20
100.00%
1
100.00%



static inline void data_start(const struct iov_iter *i, int *idxp, size_t *offp)
{
	size_t off = i->iov_offset;
	int idx = i->idx;
	if (off && (!allocated(&i->pipe->bufs[idx]) || off == PAGE_SIZE)) {
		idx = next_idx(idx, i->pipe);
		off = 0;
	}
	*idxp = idx;
	*offp = off;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Al Viro
86
100.00%
1
100.00%
Total
86
100.00%
1
100.00%



static size_t push_pipe(struct iov_iter *i, size_t size,
			int *idxp, size_t *offp)
{
	struct pipe_inode_info *pipe = i->pipe;
	size_t off;
	int idx;
	ssize_t left;

	if (unlikely(size > i->count))
		size = i->count;
	if (unlikely(!size))
		return 0;

	left = size;
	data_start(i, &idx, &off);
	*idxp = idx;
	*offp = off;
	if (off) {
		left -= PAGE_SIZE - off;
		if (left <= 0) {
			pipe->bufs[idx].len += size;
			return size;
		}
		pipe->bufs[idx].len = PAGE_SIZE;
		idx = next_idx(idx, pipe);
	}
	while (idx != pipe->curbuf || !pipe->nrbufs) {
		struct page *page = alloc_page(GFP_USER);
		if (!page)
			break;
		pipe->nrbufs++;
		pipe->bufs[idx].ops = &default_pipe_buf_ops;
		pipe->bufs[idx].page = page;
		pipe->bufs[idx].offset = 0;
		if (left <= PAGE_SIZE) {
			pipe->bufs[idx].len = left;
			return size;
		}
		pipe->bufs[idx].len = PAGE_SIZE;
		left -= PAGE_SIZE;
		idx = next_idx(idx, pipe);
	}
	return size - left;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Al Viro
268
100.00%
1
100.00%
Total
268
100.00%
1
100.00%



static size_t copy_pipe_to_iter(const void *addr, size_t bytes,
				struct iov_iter *i)
{
	struct pipe_inode_info *pipe = i->pipe;
	size_t n, off;
	int idx;

	if (!sanity(i))
		return 0;

	bytes = n = push_pipe(i, bytes, &idx, &off);
	if (unlikely(!n))
		return 0;
	for ( ; n; idx = next_idx(idx, pipe), off = 0) {
		size_t chunk = min_t(size_t, n, PAGE_SIZE - off);
		memcpy_to_page(pipe->bufs[idx].page, off, addr, chunk);
		i->idx = idx;
		i->iov_offset = off + chunk;
		n -= chunk;
		addr += chunk;
	}
	i->count -= bytes;
	return bytes;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Al Viro
158
100.00%
1
100.00%
Total
158
100.00%
1
100.00%



size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
{
	const char *from = addr;
	if (unlikely(i->type & ITER_PIPE))
		return copy_pipe_to_iter(addr, bytes, i);
	if (iter_is_iovec(i))
		might_fault();
	iterate_and_advance(i, bytes, v,
		copyout(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len),
		memcpy_to_page(v.bv_page, v.bv_offset,
                               (from += v.bv_len) - v.bv_len, v.bv_len),
		memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len)
	)

	return bytes;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Al Viro
73
100.00%
3
100.00%
Total
73
100.00%
3
100.00%

EXPORT_SYMBOL(_copy_to_iter);



size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
{
	char *to = addr;
	if (unlikely(i->type & ITER_PIPE)) {
		WARN_ON(1);
		return 0;
	}
	if (iter_is_iovec(i))
		might_fault();
	iterate_and_advance(i, bytes, v,
		copyin((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len),
		memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
                                 v.bv_offset, v.bv_len),
		memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
	)

	return bytes;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Al Viro
71
100.00%
3
100.00%
Total
71
100.00%
3
100.00%

EXPORT_SYMBOL(_copy_from_iter);



bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
{
	char *to = addr;
	if (unlikely(i->type & ITER_PIPE)) {
		WARN_ON(1);
		return false;
	}
	if (unlikely(i->count < bytes))
		return false;

	if (iter_is_iovec(i))
		might_fault();
	iterate_all_kinds(i, bytes, v, ({
                if (copyin((to += v.iov_len) - v.iov_len,
                                      v.iov_base, v.iov_len))
                        return false;
		0;}),
		memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
                                 v.bv_offset, v.bv_len),
		memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
	)

	iov_iter_advance(i, bytes);
	return true;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Al Viro
94
100.00%
3
100.00%
Total
94
100.00%
3
100.00%

EXPORT_SYMBOL(_copy_from_iter_full);



size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
{
	char *to = addr;
	if (unlikely(i->type & ITER_PIPE)) {
		WARN_ON(1);
		return 0;
	}
	iterate_and_advance(i, bytes, v,
		__copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len,
                                         v.iov_base, v.iov_len),
		memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
                                 v.bv_offset, v.bv_len),
		memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
	)

	return bytes;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Al Viro
61
100.00%
3
100.00%
Total
61
100.00%
3
100.00%

EXPORT_SYMBOL(_copy_from_iter_nocache);

#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE


size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
{
	char *to = addr;
	if (unlikely(i->type & ITER_PIPE)) {
		WARN_ON(1);
		return 0;
	}
	iterate_and_advance(i, bytes, v,
		__copy_from_user_flushcache((to += v.iov_len) - v.iov_len,
                                         v.iov_base, v.iov_len),
		memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page,
                                 v.bv_offset, v.bv_len),
		memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base,
                        v.iov_len)
	)

	return bytes;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
60
98.36%
1
50.00%
Linus Torvalds
1
1.64%
1
50.00%
Total
61
100.00%
2
100.00%

EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache);
#endif



bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
{
	char *to = addr;
	if (unlikely(i->type & ITER_PIPE)) {
		WARN_ON(1);
		return false;
	}
	if (unlikely(i->count < bytes))
		return false;
	iterate_all_kinds(i, bytes, v