cregit-Linux how code gets into the kernel

Release 4.11 fs/splice.c

Directory: fs
/*
 * "splice": joining two ropes together by interweaving their strands.
 *
 * This is the "extended pipe" functionality, where a pipe is used as
 * an arbitrary in-memory buffer. Think of a pipe as a small kernel
 * buffer that you can use to transfer data from one end to the other.
 *
 * The traditional unix read/write is extended with a "splice()" operation
 * that transfers data buffers to or from a pipe buffer.
 *
 * Named by Larry McVoy, original implementation from Linus, extended by
 * Jens to support splicing to files, network, direct splicing, etc and
 * fixing lots of bugs.
 *
 * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk>
 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
 *
 */
#include <linux/bvec.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/pagemap.h>
#include <linux/splice.h>
#include <linux/memcontrol.h>
#include <linux/mm_inline.h>
#include <linux/swap.h>
#include <linux/writeback.h>
#include <linux/export.h>
#include <linux/syscalls.h>
#include <linux/uio.h>
#include <linux/security.h>
#include <linux/gfp.h>
#include <linux/socket.h>
#include <linux/compat.h>
#include <linux/sched/signal.h>

#include "internal.h"

/*
 * Attempt to steal a page from a pipe buffer. This should perhaps go into
 * a vm helper function, it's already simplified quite a bit by the
 * addition of remove_mapping(). If success is returned, the caller may
 * attempt to reuse this page for another destination.
 */



static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe,
				     struct pipe_buffer *buf)
{
	struct page *page = buf->page;
	struct address_space *mapping;

	lock_page(page);

	mapping = page_mapping(page);
	if (mapping) {
		WARN_ON(!PageUptodate(page));

		/*
                 * At least for ext2 with nobh option, we need to wait on
                 * writeback completing on this page, since we'll remove it
                 * from the pagecache.  Otherwise truncate wont wait on the
                 * page, allowing the disk blocks to be reused by someone else
                 * before we actually wrote our data to them. fs corruption
                 * ensues.
                 */
		wait_on_page_writeback(page);

		if (page_has_private(page) &&
		    !try_to_release_page(page, GFP_KERNEL))
			goto out_unlock;

		/*
                 * If we succeeded in removing the mapping, set LRU flag
                 * and return good.
                 */
		if (remove_mapping(mapping, page)) {
			buf->flags |= PIPE_BUF_FLAG_LRU;
			return 0;
		}
	}

	/*
         * Raced with truncate or failed to remove page from current
         * address space, unlock and return failure.
         */
out_unlock:
	unlock_page(page);
	return 1;
}


Contributors
PersonTokensPropCommitsCommitProp
Jens Axboe11198.23%880.00%


David Howells10.88%110.00%


Nicholas Piggin10.88%110.00%


Total113100.00%10100.00%






static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
					struct pipe_buffer *buf)
{
	put_page(buf->page);
	buf->flags &= ~PIPE_BUF_FLAG_LRU;
}


Contributors
PersonTokensPropCommitsCommitProp
Jens Axboe2996.67%375.00%


Kirill A. Shutemov13.33%125.00%


Total30100.00%4100.00%



/*
 * Check whether the contents of buf is OK to access. Since the content
 * is a page cache page, IO may be in flight.
 */



static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,
				       struct pipe_buffer *buf)
{
	struct page *page = buf->page;
	int err;

	if (!PageUptodate(page)) {
		lock_page(page);

		/*
                 * Page got truncated/unhashed. This will cause a 0-byte
                 * splice, if this is the first page.
                 */
		if (!page->mapping) {
			err = -ENODATA;
			goto error;
		}

		/*
                 * Uh oh, read-error from disk.
                 */
		if (!PageUptodate(page)) {
			err = -EIO;
			goto error;
		}

		/*
                 * Page is ok afterall, we are done.
                 */
		unlock_page(page);
	}

	return 0;
error:
	unlock_page(page);
	return err;
}


Contributors
PersonTokensPropCommitsCommitProp
Jens Axboe9797.98%583.33%


Ingo Molnar22.02%116.67%


Total99100.00%6100.00%




const struct pipe_buf_operations page_cache_pipe_buf_ops = {
	.can_merge = 0,
	.confirm = page_cache_pipe_buf_confirm,
	.release = page_cache_pipe_buf_release,
	.steal = page_cache_pipe_buf_steal,
	.get = generic_pipe_buf_get,
};




static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe,
				    struct pipe_buffer *buf)
{
	if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
		return 1;

	buf->flags |= PIPE_BUF_FLAG_LRU;
	return generic_pipe_buf_steal(pipe, buf);
}


Contributors
PersonTokensPropCommitsCommitProp
Jens Axboe44100.00%4100.00%


Total44100.00%4100.00%




static const struct pipe_buf_operations user_page_pipe_buf_ops = {
	.can_merge = 0,
	.confirm = generic_pipe_buf_confirm,
	.release = page_cache_pipe_buf_release,
	.steal = user_page_pipe_buf_steal,
	.get = generic_pipe_buf_get,
};




static void wakeup_pipe_readers(struct pipe_inode_info *pipe)
{
	smp_mb();
	if (waitqueue_active(&pipe->wait))
		wake_up_interruptible(&pipe->wait);
	kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
}


Contributors
PersonTokensPropCommitsCommitProp
Namhyung Kim44100.00%1100.00%


Total44100.00%1100.00%



/**
 * splice_to_pipe - fill passed data into a pipe
 * @pipe:       pipe to fill
 * @spd:        data to fill
 *
 * Description:
 *    @spd contains a map of pages and len/offset tuples, along with
 *    the struct pipe_buf_operations associated with these pages. This
 *    function will link that data to the pipe.
 *
 */



ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
		       struct splice_pipe_desc *spd)
{
	unsigned int spd_pages = spd->nr_pages;
	int ret = 0, page_nr = 0;

	if (!spd_pages)
		return 0;

	if (unlikely(!pipe->readers)) {
		send_sig(SIGPIPE, current, 0);
		ret = -EPIPE;
		goto out;
	}

	while (pipe->nrbufs < pipe->buffers) {
		int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
		struct pipe_buffer *buf = pipe->bufs + newbuf;

		buf->page = spd->pages[page_nr];
		buf->offset = spd->partial[page_nr].offset;
		buf->len = spd->partial[page_nr].len;
		buf->private = spd->partial[page_nr].private;
		buf->ops = spd->ops;
		buf->flags = 0;

		pipe->nrbufs++;
		page_nr++;
		ret += buf->len;

		if (!--spd->nr_pages)
			break;
	}

	if (!ret)
		ret = -EAGAIN;

out:
	while (page_nr < spd_pages)
		spd->spd_release(spd, page_nr++);

	return ret;
}

Contributors
PersonTokensPropCommitsCommitProp
Jens Axboe18780.95%861.54%


Al Viro135.63%17.69%


Ingo Molnar93.90%17.69%


Linus Torvalds83.46%17.69%


Rabin Vincent83.46%17.69%


Miklos Szeredi62.60%17.69%


Total231100.00%13100.00%




EXPORT_SYMBOL_GPL(splice_to_pipe);




ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
{
	int ret;

	if (unlikely(!pipe->readers)) {
		send_sig(SIGPIPE, current, 0);
		ret = -EPIPE;
	} else if (pipe->nrbufs == pipe->buffers) {
		ret = -EAGAIN;
	} else {
		int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
		pipe->bufs[newbuf] = *buf;
		pipe->nrbufs++;
		return buf->len;
	}
	pipe_buf_release(pipe, buf);
	return ret;
}

Contributors
PersonTokensPropCommitsCommitProp
Al Viro11599.14%150.00%


Miklos Szeredi10.86%150.00%


Total116100.00%2100.00%




EXPORT_SYMBOL(add_to_pipe);




void spd_release_page(struct splice_pipe_desc *spd, unsigned int i)
{
	put_page(spd->pages[i]);
}


Contributors
PersonTokensPropCommitsCommitProp
Jens Axboe2395.83%150.00%


Kirill A. Shutemov14.17%150.00%


Total24100.00%2100.00%



/*
 * Check if we need to grow the arrays holding pages and partial page
 * descriptions.
 */



int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)
{
	unsigned int buffers = ACCESS_ONCE(pipe->buffers);

	spd->nr_pages_max = buffers;
	if (buffers <= PIPE_DEF_BUFFERS)
		return 0;

	spd->pages = kmalloc(buffers * sizeof(struct page *), GFP_KERNEL);
	spd->partial = kmalloc(buffers * sizeof(struct partial_page), GFP_KERNEL);

	if (spd->pages && spd->partial)
		return 0;

	kfree(spd->pages);
	kfree(spd->partial);
	return -ENOMEM;
}


Contributors
PersonTokensPropCommitsCommitProp
Jens Axboe9083.33%150.00%


Eric Dumazet1816.67%150.00%


Total108100.00%2100.00%






void splice_shrink_spd(struct splice_pipe_desc *spd)
{
	if (spd->nr_pages_max <= PIPE_DEF_BUFFERS)
		return;

	kfree(spd->pages);
	kfree(spd->partial);
}


Contributors
PersonTokensPropCommitsCommitProp
Jens Axboe3193.94%150.00%


Eric Dumazet26.06%150.00%


Total33100.00%2100.00%



/**
 * generic_file_splice_read - splice data from file to a pipe
 * @in:         file to splice from
 * @ppos:       position in @in
 * @pipe:       pipe to splice to
 * @len:        number of bytes to splice
 * @flags:      splice modifier flags
 *
 * Description:
 *    Will read pages from given file and fill them into a pipe. Can be
 *    used as long as it has more or less sane ->read_iter().
 *
 */



ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
				 struct pipe_inode_info *pipe, size_t len,
				 unsigned int flags)
{
	struct iov_iter to;
	struct kiocb kiocb;
	int idx, ret;

	iov_iter_pipe(&to, ITER_PIPE | READ, pipe, len);
	idx = to.idx;
	init_sync_kiocb(&kiocb, in);
	kiocb.ki_pos = *ppos;
	ret = call_read_iter(in, &kiocb, &to);
	if (ret > 0) {
		*ppos = kiocb.ki_pos;
		file_accessed(in);
	} else if (ret < 0) {
		to.idx = idx;
		to.iov_offset = 0;
		iov_iter_advance(&to, 0); /* to free what was emitted */
		/*
                 * callers of ->splice_read() expect -EAGAIN on
                 * "can't put anything in there", rather than -EFAULT.
                 */
		if (ret == -EFAULT)
			ret = -EAGAIN;
	}

	return ret;
}

Contributors
PersonTokensPropCommitsCommitProp
Al Viro9159.48%111.11%


Jens Axboe5133.33%555.56%


Miklos Szeredi106.54%222.22%


Ingo Molnar10.65%111.11%


Total153100.00%9100.00%




EXPORT_SYMBOL(generic_file_splice_read);


const struct pipe_buf_operations default_pipe_buf_ops = {
	.can_merge = 0,
	.confirm = generic_pipe_buf_confirm,
	.release = generic_pipe_buf_release,
	.steal = generic_pipe_buf_steal,
	.get = generic_pipe_buf_get,
};




static int generic_pipe_buf_nosteal(struct pipe_inode_info *pipe,
				    struct pipe_buffer *buf)
{
	return 1;
}


Contributors
PersonTokensPropCommitsCommitProp
Miklos Szeredi19100.00%1100.00%


Total19100.00%1100.00%



/* Pipe buffer operations for a socket and similar. */

const struct pipe_buf_operations nosteal_pipe_buf_ops = {
	.can_merge = 0,
	.confirm = generic_pipe_buf_confirm,
	.release = generic_pipe_buf_release,
	.steal = generic_pipe_buf_nosteal,
	.get = generic_pipe_buf_get,
};

EXPORT_SYMBOL(nosteal_pipe_buf_ops);




static ssize_t kernel_readv(struct file *file, const struct kvec *vec,
			    unsigned long vlen, loff_t offset)
{
	mm_segment_t old_fs;
	loff_t pos = offset;
	ssize_t res;

	old_fs = get_fs();
	set_fs(get_ds());
	/* The cast to a user pointer is valid due to the set_fs() */
	res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos, 0);
	set_fs(old_fs);

	return res;
}


Contributors
PersonTokensPropCommitsCommitProp
Miklos Szeredi7596.15%133.33%


Christoph Hellwig22.56%133.33%


Al Viro11.28%133.33%


Total78100.00%3100.00%






ssize_t kernel_write(struct file *file, const char *buf, size_t count,
			    loff_t pos)
{
	mm_segment_t old_fs;
	ssize_t res;

	old_fs = get_fs();
	set_fs(get_ds());
	/* The cast to a user pointer is valid due to the set_fs() */
	res = vfs_write(file, (__force const char __user *)buf, count, &pos);
	set_fs(old_fs);

	return res;
}

Contributors
PersonTokensPropCommitsCommitProp
Miklos Szeredi6798.53%266.67%


Al Viro11.47%133.33%


Total68100.00%3100.00%




EXPORT_SYMBOL(kernel_write);




static ssize_t default_file_splice_read(struct file *in, loff_t *ppos,
				 struct pipe_inode_info *pipe, size_t len,
				 unsigned int flags)
{
	struct kvec *vec, __vec[PIPE_DEF_BUFFERS];
	struct iov_iter to;
	struct page **pages;
	unsigned int nr_pages;
	size_t offset, dummy, copied = 0;
	ssize_t res;
	int i;

	if (pipe->nrbufs == pipe->buffers)
		return -EAGAIN;

	/*
         * Try to keep page boundaries matching to source pagecache ones -
         * it probably won't be much help, but...
         */
	offset = *ppos & ~PAGE_MASK;

	iov_iter_pipe(&to, ITER_PIPE | READ, pipe, len + offset);

	res = iov_iter_get_pages_alloc(&to, &pages, len + offset, &dummy);
	if (res <= 0)
		return -ENOMEM;

	BUG_ON(dummy);
	nr_pages = DIV_ROUND_UP(res, PAGE_SIZE);

	vec = __vec;
	if (nr_pages > PIPE_DEF_BUFFERS) {
		vec = kmalloc(nr_pages * sizeof(struct kvec), GFP_KERNEL);
		if (unlikely(!vec)) {
			res = -ENOMEM;
			goto out;
		}
	}

	pipe->bufs[to.idx].offset = offset;
	pipe->bufs[to.idx].len -= offset;

	for (i = 0; i < nr_pages; i++) {
		size_t this_len = min_t(size_t, len, PAGE_SIZE - offset);
		vec[i].iov_base = page_address(pages[i]) + offset;
		vec[i].iov_len = this_len;
		len -= this_len;
		offset = 0;
	}

	res = kernel_readv(in, vec, nr_pages, *ppos);
	if (res > 0) {
		copied = res;
		*ppos += res;
	}

	if (vec != __vec)
		kfree(vec);
out:
	for (i = 0; i < nr_pages; i++)
		put_page(pages[i]);
	kvfree(pages);
	iov_iter_advance(&to, copied);	/* truncates and discards */
	return res;
}


Contributors
PersonTokensPropCommitsCommitProp
Al Viro15342.15%342.86%


Miklos Szeredi14941.05%114.29%


Jens Axboe6016.53%228.57%


Kirill A. Shutemov10.28%114.29%


Total363100.00%7100.00%



/*
 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos'
 * using sendpage(). Return the number of bytes sent.
 */



static int pipe_to_sendpage(struct pipe_inode_info *pipe,
			    struct pipe_buffer *buf, struct splice_desc *sd)
{
	struct file *file = sd->u.file;
	loff_t pos = sd->pos;
	int more;

	if (!likely(file->f_op->sendpage))
		return -EINVAL;

	more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0;

	if (sd->len < sd->total_len && pipe->nrbufs > 1)
		more |= MSG_SENDPAGE_NOTLAST;

	return file->f_op->sendpage(file, buf->page, buf->offset,
				    sd->len, &pos, more);
}


Contributors
PersonTokensPropCommitsCommitProp
Jens Axboe8974.79%666.67%


Eric Dumazet1714.29%222.22%


Michał Mirosław1310.92%111.11%


Total119100.00%9100.00%






static void wakeup_pipe_writers(struct pipe_inode_info *pipe)
{
	smp_mb();
	if (waitqueue_active(&pipe->wait))
		wake_up_interruptible(&pipe->wait);
	kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
}


Contributors
PersonTokensPropCommitsCommitProp
Miklos Szeredi44100.00%1100.00%


Total44100.00%1100.00%



/**
 * splice_from_pipe_feed - feed available data from a pipe to a file
 * @pipe:       pipe to splice from
 * @sd:         information to @actor
 * @actor:      handler that splices the data
 *
 * Description:
 *    This function loops over the pipe and calls @actor to do the
 *    actual moving of a single struct pipe_buffer to the desired
 *    destination.  It returns when there's no more buffers left in
 *    the pipe or if the requested number of bytes (@sd->total_len)
 *    have been copied.  It returns a positive number (one) if the
 *    pipe needs to be filled with more data, zero if the required
 *    number of bytes have been copied and -errno on error.
 *
 *    This, together with splice_from_pipe_{begin,end,next}, may be
 *    used to implement the functionality of __splice_from_pipe() when
 *    locking is required around copying the pipe buffers to the
 *    destination.
 */



static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,
			  splice_actor *actor)
{
	int ret;

	while (pipe->nrbufs) {
		struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;

		sd->len = buf->len;
		if (sd->len > sd->total_len)
			sd->len = sd->total_len;

		ret = pipe_buf_confirm(pipe, buf);
		if (unlikely(ret)) {
			if (ret == -ENODATA)
				ret = 0;
			return ret;
		}

		ret = actor(pipe, buf, sd);
		if (ret <= 0)
			return ret;

		buf->offset += ret;
		buf->len -= ret;

		sd->num_spliced += ret;
		sd->len -= ret;
		sd->pos += ret;
		sd->total_len -= ret;

		if (!buf->len) {
			pipe_buf_release(pipe, buf);
			pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
			pipe->nrbufs--;
			if (pipe->files)
				sd->need_wakeup = true;
		}

		if (!sd->total_len)
			return 0;
	}

	return 1;
}


Contributors
PersonTokensPropCommitsCommitProp
Jens Axboe15870.54%646.15%


Miklos Szeredi3415.18%323.08%


Michał Mirosław2310.27%17.69%


Ingo Molnar73.12%17.69%


Al Viro20.89%215.38%


Total224100.00%13100.00%



/**
 * splice_from_pipe_next - wait for some data to splice from
 * @pipe:       pipe to splice from
 * @sd:         information about the splice operation
 *
 * Description:
 *    This function will wait for some data and return a positive
 *    value (one) if pipe buffers are available.  It will return zero
 *    or -errno if no more data needs to be spliced.
 */



static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)
{
	/*
         * Check for signal early to make process killable when there are
         * always buffers available
         */
	if (signal_pending(current))
		return -ERESTARTSYS;

	while (!pipe->nrbufs) {
		if (!pipe->writers)
			return 0;

		if (!pipe->waiting_writers && sd->num_spliced)
			return 0;

		if (sd->flags & SPLICE_F_NONBLOCK)
			return -EAGAIN;

		if (signal_pending(current))
			return -ERESTARTSYS;

		if (sd->need_wakeup) {
			wakeup_pipe_writers(pipe);
			sd->need_wakeup = false;
		}

		pipe_wait(pipe);
	}

	return 1;
}


Contributors
PersonTokensPropCommitsCommitProp
Miklos Szeredi4944.14%111.11%


Jens Axboe3329.73%444.44%


Jan Kara1210.81%111.11%


Linus Torvalds87.21%111.11%


Ingo Molnar87.21%111.11%


Al Viro10.90%111.11%


Total111100.00%9100.00%



/**
 * splice_from_pipe_begin - start splicing from pipe
 * @sd:         information about the splice operation
 *
 * Description:
 *    This function should be called before a loop containing
 *    splice_from_pipe_next() and splice_from_pipe_feed() to
 *    initialize the necessary fields of @sd.
 */



static void splice_from_pipe_begin(struct splice_desc *sd)
{
	sd->num_spliced = 0;
	sd->need_wakeup = false;
}


Contributors
PersonTokensPropCommitsCommitProp
Miklos Szeredi1878.26%133.33%


Jens Axboe417.39%133.33%


Al Viro14.35%133.33%


Total23100.00%3100.00%



/**
 * splice_from_pipe_end - finish splicing from pipe
 * @pipe:       pipe to splice from
 * @sd:         information about the splice operation
 *
 * Description:
 *    This function will wake up pipe writers if necessary.  It should
 *    be called after a loop containing splice_from_pipe_next() and
 *    splice_from_pipe_feed().
 */



static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)
{
	if (sd->need_wakeup)
		wakeup_pipe_writers(pipe);
}


Contributors
PersonTokensPropCommitsCommitProp
Miklos Szeredi2281.48%125.00%


Jens Axboe414.81%250.00%


Al Viro13.70%125.00%


Total27100.00%4100.00%



/**
 * __splice_from_pipe - splice data from a pipe to given actor
 * @pipe:       pipe to splice from
 * @sd:         information to @actor
 * @actor:      handler that splices the data
 *
 * Description:
 *    This function does little more than loop over the pipe and call
 *    @actor to do the actual moving of a single struct pipe_buffer to
 *    the desired destination. See pipe_to_file, pipe_to_sendpage, or
 *    pipe_to_user.
 *
 */



ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,
			   splice_actor *actor)
{
	int ret;

	splice_from_pipe_begin(sd);
	do {
		cond_resched();
		ret = splice_from_pipe_next(pipe, sd);
		if (ret > 0)
			ret = splice_from_pipe_feed(pipe, sd, actor);
	} while (ret > 0);
	splice_from_pipe_end(pipe, sd);

	return sd->num_spliced ? sd->num_spliced : ret;
}

Contributors
PersonTokensPropCommitsCommitProp
Miklos Szeredi6476.19%120.00%


Jens Axboe1416.67%240.00%


Ingo Molnar33.57%120.00%


Jan Kara33.57%120.00%


Total84100.00%5100.00%




EXPORT_SYMBOL(__splice_from_pipe);

/**
 * splice_from_pipe - splice data from a pipe to a file
 * @pipe:       pipe to splice from
 * @out:        file to splice to
 * @ppos:       position in @out
 * @len:        how many bytes to splice
 * @flags:      splice modifier flags
 * @actor:      handler that splices the data
 *
 * Description:
 *    See __splice_from_pipe. This function locks the pipe inode,
 *    otherwise it's identical to __splice_from_pipe().
 *
 */



ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
			 loff_t *ppos, size_t len, unsigned int flags,
			 splice_actor *actor)
{
	ssize_t ret;
	struct splice_desc sd = {
		.total_len = len,
		.flags = flags,
		.pos = *ppos,
		.u.file = out,
        };

	pipe_lock(pipe);
	ret = __splice_from_pipe(pipe, &sd, actor);
	pipe_unlock(pipe);

	return ret;
}


Contributors
PersonTokensPropCommitsCommitProp
Mark Fasheh5158.62%120.00%


Jens Axboe3034.48%240.00%


Miklos Szeredi66.90%240.00%


Total87100.00%5100.00%



/**
 * iter_file_splice_write - splice data from a pipe to a file
 * @pipe:       pipe info
 * @out:        file to write to
 * @ppos:       position in @out
 * @len:        number of bytes to splice
 * @flags:      splice modifier flags
 *
 * Description:
 *    Will either move or copy pages (determined by @flags options) from
 *    the given pipe inode to the given file.
 *    This one is ->write_iter-based.
 *
 */


ssize_t

iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
			  loff_t *ppos, size_t len, unsigned int flags)
{
	struct splice_desc sd = {
		.total_len = len,
		.flags = flags,
		.pos = *ppos,
		.u.file = out,
        };
	int nbufs = pipe->buffers;
	struct bio_vec *array = kcalloc(nbufs, sizeof(struct bio_vec),
					GFP_KERNEL);
	ssize_t ret;

	if (unlikely(!array))
		return -ENOMEM;

	pipe_lock(pipe);

	splice_from_pipe_begin(&sd);
	while (sd.total_len) {
		struct iov_iter from;
		size_t left;
		int n, idx;

		ret = splice_from_pipe_next(pipe, &sd);
		if (ret <= 0)
			break;

		if (unlikely(nbufs < pipe->buffers)) {
			kfree(array);
			nbufs = pipe->buffers;
			array = kcalloc(nbufs, sizeof(struct bio_vec),
					GFP_KERNEL);
			if (!array) {
				ret = -ENOMEM;
				break;
			}
		}

		/* build the vector */
		left = sd.total_len;
		for (n = 0, idx = pipe->curbuf; left && n < pipe->nrbufs; n++, idx++) {
			struct pipe_buffer *buf = pipe->bufs + idx;
			size_t this_len = buf->len;

			if (this_len > left)
				this_len = left;

			if (idx == pipe->buffers - 1)
				idx = -1;

			ret = pipe_buf_confirm(pipe, buf);
			if (unlikely(ret)) {
				if (ret == -ENODATA)
					ret = 0;
				goto done;
			}

			array[n].bv_page = buf->page;
			array[n].bv_len = this_len;
			array[n].bv_offset = buf->offset;
			left -= this_len;
		}

		iov_iter_bvec(&from, ITER_BVEC | WRITE, array, n,
			      sd.total_len - left);
		ret = vfs_iter_write(out, &from, &sd.pos);
		if (ret <= 0)
			break;

		sd.num_spliced += ret;
		sd.total_len -= ret;
		*ppos = sd.pos;

		/* dismiss the fully eaten buffers, adjust the partial one */
		while (ret) {
			struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
			if (ret >= buf->len) {
				ret -= buf->len;
				buf->len = 0;
				pipe_buf_release(pipe, buf);
				pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
				pipe->nrbufs--;
				if (pipe->files)
					sd.need_wakeup = true;
			} else {
				buf->offset += ret;
				buf->len -= ret;
				ret = 0;
			}
		}
	}
done:
	kfree(array);
	splice_from_pipe_end(pipe, &sd);

	pipe_unlock(pipe);

	if (sd.num_spliced)
		ret = sd.num_spliced;

	return ret;
}


Contributors
PersonTokensPropCommitsCommitProp
Al Viro53398.52%240.00%


Christoph Hellwig61.11%120.00%


Miklos Szeredi20.37%240.00%


Total541100.00%5100.00%




EXPORT_SYMBOL(iter_file_splice_write);




static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
			  struct splice_desc *sd)
{
	int ret;