cregit-Linux how code gets into the kernel

Release 4.11 block/bio.c

Directory: block
 * Copyright (C) 2001 Jens Axboe <>
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * GNU General Public License for more details.
 * You should have received a copy of the GNU General Public Licens
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/uio.h>
#include <linux/iocontext.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/mempool.h>
#include <linux/workqueue.h>
#include <linux/cgroup.h>

#include <trace/events/block.h>

 * Test patch to inline a certain number of bi_io_vec's inside the bio
 * itself, to shrink a bio data allocation from two mempool calls to one

#define BIO_INLINE_VECS		4

 * if you change this list, also change bvec_alloc or things will
 * break badly! cannot be bigger than what you can fit into an
 * unsigned short

#define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) }

static struct biovec_slab bvec_slabs[BVEC_POOL_NR] __read_mostly = {
	BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES),

#undef BV

 * fs_bio_set is the bio_set containing bio and iovec memory pools used by
 * IO code that does not need private memory pools.

struct bio_set *fs_bio_set;


 * Our slab pool management

struct bio_slab {
struct kmem_cache *slab;
unsigned int slab_ref;
unsigned int slab_size;
char name[8];
static DEFINE_MUTEX(bio_slab_lock);

static struct bio_slab *bio_slabs;

static unsigned int bio_slab_nr, bio_slab_max;

static struct kmem_cache *bio_find_or_create_slab(unsigned int extra_size) { unsigned int sz = sizeof(struct bio) + extra_size; struct kmem_cache *slab = NULL; struct bio_slab *bslab, *new_bio_slabs; unsigned int new_bio_slab_max; unsigned int i, entry = -1; mutex_lock(&bio_slab_lock); i = 0; while (i < bio_slab_nr) { bslab = &bio_slabs[i]; if (!bslab->slab && entry == -1) entry = i; else if (bslab->slab_size == sz) { slab = bslab->slab; bslab->slab_ref++; break; } i++; } if (slab) goto out_unlock; if (bio_slab_nr == bio_slab_max && entry == -1) { new_bio_slab_max = bio_slab_max << 1; new_bio_slabs = krealloc(bio_slabs, new_bio_slab_max * sizeof(struct bio_slab), GFP_KERNEL); if (!new_bio_slabs) goto out_unlock; bio_slab_max = new_bio_slab_max; bio_slabs = new_bio_slabs; } if (entry == -1) entry = bio_slab_nr++; bslab = &bio_slabs[entry]; snprintf(bslab->name, sizeof(bslab->name), "bio-%d", entry); slab = kmem_cache_create(bslab->name, sz, ARCH_KMALLOC_MINALIGN, SLAB_HWCACHE_ALIGN, NULL); if (!slab) goto out_unlock; bslab->slab = slab; bslab->slab_ref = 1; bslab->slab_size = sz; out_unlock: mutex_unlock(&bio_slab_lock); return slab; }


Jens Axboe24791.14%120.00%
Anna Leuschner124.43%120.00%
Alexey Khoroshilov93.32%120.00%
Thiago Farina20.74%120.00%
Mikulas Patocka10.37%120.00%

static void bio_put_slab(struct bio_set *bs) { struct bio_slab *bslab = NULL; unsigned int i; mutex_lock(&bio_slab_lock); for (i = 0; i < bio_slab_nr; i++) { if (bs->bio_slab == bio_slabs[i].slab) { bslab = &bio_slabs[i]; break; } } if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n")) goto out; WARN_ON(!bslab->slab_ref); if (--bslab->slab_ref) goto out; kmem_cache_destroy(bslab->slab); bslab->slab = NULL; out: mutex_unlock(&bio_slab_lock); }


Jens Axboe11797.50%150.00%
Andi Kleen32.50%150.00%

unsigned int bvec_nr_vecs(unsigned short idx) { return bvec_slabs[idx].nr_vecs; }


Martin K. Petersen18100.00%1100.00%

void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx) { if (!idx) return; idx--; BIO_BUG_ON(idx >= BVEC_POOL_NR); if (idx == BVEC_POOL_MAX) { mempool_free(bv, pool); } else { struct biovec_slab *bvs = bvec_slabs + idx; kmem_cache_free(bvs->slab, bv); } }


Jens Axboe5375.71%133.33%
Christoph Hellwig1318.57%133.33%
Kent Overstreet45.71%133.33%

struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx, mempool_t *pool) { struct bio_vec *bvl; /* * see comment near bvec_array define! */ switch (nr) { case 1: *idx = 0; break; case 2 ... 4: *idx = 1; break; case 5 ... 16: *idx = 2; break; case 17 ... 64: *idx = 3; break; case 65 ... 128: *idx = 4; break; case 129 ... BIO_MAX_PAGES: *idx = 5; break; default: return NULL; } /* * idx now points to the pool we want to allocate from. only the * 1-vec entry pool is mempool backed. */ if (*idx == BVEC_POOL_MAX) { fallback: bvl = mempool_alloc(pool, gfp_mask); } else { struct biovec_slab *bvs = bvec_slabs + *idx; gfp_t __gfp_mask = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_IO); /* * Make this allocation restricted and don't dump info on * allocation failures, since we'll fallback to the mempool * in case of failure. */ __gfp_mask |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; /* * Try a slab allocation. If this fails and __GFP_DIRECT_RECLAIM * is set, retry with the 1-entry mempool */ bvl = kmem_cache_alloc(bvs->slab, __gfp_mask); if (unlikely(!bvl && (gfp_mask & __GFP_DIRECT_RECLAIM))) { *idx = BVEC_POOL_MAX; goto fallback; } } (*idx)++; return bvl; }


Linus Torvalds10150.00%433.33%
Jens Axboe7939.11%325.00%
Christoph Hellwig83.96%18.33%
Dave Olien62.97%18.33%
Kent Overstreet41.98%18.33%
Mel Gorman31.49%18.33%
Al Viro10.50%18.33%

static void __bio_free(struct bio *bio) { bio_disassociate_task(bio); if (bio_integrity(bio)) bio_integrity_free(bio); }


Kent Overstreet28100.00%1100.00%

static void bio_free(struct bio *bio) { struct bio_set *bs = bio->bi_pool; void *p; __bio_free(bio); if (bs) { bvec_free(bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio)); /* * If we have front padding, adjust the bio pointer before freeing */ p = bio; p -= bs->front_pad; mempool_free(p, bs->bio_pool); } else { /* Bio was allocated by bio_kmalloc() */ kfree(bio); } }


Kent Overstreet2936.25%218.18%
Jens Axboe2227.50%218.18%
Linus Torvalds2025.00%327.27%
Peter Osterlund45.00%19.09%
H Hartley Sweeten33.75%19.09%
Christoph Hellwig11.25%19.09%
Dave Olien11.25%19.09%

void bio_init(struct bio *bio, struct bio_vec *table, unsigned short max_vecs) { memset(bio, 0, sizeof(*bio)); atomic_set(&bio->__bi_remaining, 1); atomic_set(&bio->__bi_cnt, 1); bio->bi_io_vec = table; bio->bi_max_vecs = max_vecs; }


Linus Torvalds2132.81%333.33%
Lei Ming2132.81%111.11%
Jens Axboe1320.31%444.44%
Kent Overstreet914.06%111.11%

EXPORT_SYMBOL(bio_init); /** * bio_reset - reinitialize a bio * @bio: bio to reset * * Description: * After calling bio_reset(), @bio will be in the same state as a freshly * allocated bio returned bio bio_alloc_bioset() - the only fields that are * preserved are the ones that are initialized by bio_alloc_bioset(). See * comment in struct bio. */
void bio_reset(struct bio *bio) { unsigned long flags = bio->bi_flags & (~0UL << BIO_RESET_BITS); __bio_free(bio); memset(bio, 0, BIO_RESET_BYTES); bio->bi_flags = flags; atomic_set(&bio->__bi_remaining, 1); }


Kent Overstreet5498.18%375.00%
Jens Axboe11.82%125.00%

static struct bio *__bio_chain_endio(struct bio *bio) { struct bio *parent = bio->bi_private; if (!parent->bi_error) parent->bi_error = bio->bi_error; bio_put(bio); return parent; }


Christoph Hellwig3271.11%375.00%
Kent Overstreet1328.89%125.00%

static void bio_chain_endio(struct bio *bio) { bio_endio(__bio_chain_endio(bio)); }


Christoph Hellwig1473.68%150.00%
Kent Overstreet526.32%150.00%

/** * bio_chain - chain bio completions * @bio: the target bio * @parent: the @bio's parent bio * * The caller won't have a bi_end_io called when @bio completes - instead, * @parent's bi_end_io won't be called until both @parent and @bio have * completed; the chained bio will also be freed when it completes. * * The caller must not set bi_private or bi_end_io in @bio. */
void bio_chain(struct bio *bio, struct bio *parent) { BUG_ON(bio->bi_private || bio->bi_end_io); bio->bi_private = parent; bio->bi_end_io = bio_chain_endio; bio_inc_remaining(parent); }


Kent Overstreet4297.67%150.00%
Jens Axboe12.33%150.00%

static void bio_alloc_rescue(struct work_struct *work) { struct bio_set *bs = container_of(work, struct bio_set, rescue_work); struct bio *bio; while (1) { spin_lock(&bs->rescue_lock); bio = bio_list_pop(&bs->rescue_list); spin_unlock(&bs->rescue_lock); if (!bio) break; generic_make_request(bio); } }


Kent Overstreet74100.00%1100.00%

static void punt_bios_to_rescuer(struct bio_set *bs) { struct bio_list punt, nopunt; struct bio *bio; /* * In order to guarantee forward progress we must punt only bios that * were allocated from this bio_set; otherwise, if there was a bio on * there for a stacking driver higher up in the stack, processing it * could require allocating bios from this bio_set, and doing that from * our own rescuer would be bad. * * Since bio lists are singly linked, pop them all instead of trying to * remove from the middle of the list: */ bio_list_init(&punt); bio_list_init(&nopunt); while ((bio = bio_list_pop(&current->bio_list[0]))) bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio); current->bio_list[0] = nopunt; bio_list_init(&nopunt); while ((bio = bio_list_pop(&current->bio_list[1]))) bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio); current->bio_list[1] = nopunt; spin_lock(&bs->rescue_lock); bio_list_merge(&bs->rescue_list, &punt); spin_unlock(&bs->rescue_lock); queue_work(bs->rescue_workqueue, &bs->rescue_work); }


Kent Overstreet11066.27%150.00%
Neil Brown5633.73%150.00%

/** * bio_alloc_bioset - allocate a bio for I/O * @gfp_mask: the GFP_ mask given to the slab allocator * @nr_iovecs: number of iovecs to pre-allocate * @bs: the bio_set to allocate from. * * Description: * If @bs is NULL, uses kmalloc() to allocate the bio; else the allocation is * backed by the @bs's mempool. * * When @bs is not NULL, if %__GFP_DIRECT_RECLAIM is set then bio_alloc will * always be able to allocate a bio. This is due to the mempool guarantees. * To make this work, callers must never allocate more than 1 bio at a time * from this pool. Callers that need to allocate more than 1 bio must always * submit the previously allocated bio for IO before attempting to allocate * a new one. Failure to do so can cause deadlocks under memory pressure. * * Note that when running under generic_make_request() (i.e. any block * driver), bios are not submitted until after you return - see the code in * generic_make_request() that converts recursion into iteration, to prevent * stack overflows. * * This would normally mean allocating multiple bios under * generic_make_request() would be susceptible to deadlocks, but we have * deadlock avoidance code that resubmits any blocked bios from a rescuer * thread. * * However, we do not guarantee forward progress for allocations from other * mempools. Doing multiple allocations from the same mempool under * generic_make_request() should be avoided - instead, use bio_set's front_pad * for per bio allocations. * * RETURNS: * Pointer to new bio on success, NULL on failure. */
struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs) { gfp_t saved_gfp = gfp_mask; unsigned front_pad; unsigned inline_vecs; struct bio_vec *bvl = NULL; struct bio *bio; void *p; if (!bs) { if (nr_iovecs > UIO_MAXIOV) return NULL; p = kmalloc(sizeof(struct bio) + nr_iovecs * sizeof(struct bio_vec), gfp_mask); front_pad = 0; inline_vecs = nr_iovecs; } else { /* should not use nobvec bioset for nr_iovecs > 0 */ if (WARN_ON_ONCE(!bs->bvec_pool && nr_iovecs > 0)) return NULL; /* * generic_make_request() converts recursion to iteration; this * means if we're running beneath it, any bios we allocate and * submit will not be submitted (and thus freed) until after we * return. * * This exposes us to a potential deadlock if we allocate * multiple bios from the same bio_set() while running * underneath generic_make_request(). If we were to allocate * multiple bios (say a stacking block driver that was splitting * bios), we would deadlock if we exhausted the mempool's * reserve. * * We solve this, and guarantee forward progress, with a rescuer * workqueue per bio_set. If we go to allocate and there are * bios on current->bio_list, we first try the allocation * without __GFP_DIRECT_RECLAIM; if that fails, we punt those * bios we would be blocking to the rescuer workqueue before * we retry with the original gfp_flags. */ if (current->bio_list && (!bio_list_empty(&current->bio_list[0]) || !bio_list_empty(&current->bio_list[1]))) gfp_mask &= ~__GFP_DIRECT_RECLAIM; p = mempool_alloc(bs->bio_pool, gfp_mask); if (!p && gfp_mask != saved_gfp) { punt_bios_to_rescuer(bs); gfp_mask = saved_gfp; p = mempool_alloc(bs->bio_pool, gfp_mask); } front_pad = bs->front_pad; inline_vecs = BIO_INLINE_VECS; } if (unlikely(!p)) return NULL; bio = p + front_pad; bio_init(bio, NULL, 0); if (nr_iovecs > inline_vecs) { unsigned long idx = 0; bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool); if (!bvl && gfp_mask != saved_gfp) { punt_bios_to_rescuer(bs); gfp_mask = saved_gfp; bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool); } if (unlikely(!bvl)) goto err_free; bio->bi_flags |= idx << BVEC_POOL_OFFSET; } else if (nr_iovecs) { bvl = bio->bi_inline_vecs; } bio->bi_pool = bs; bio->bi_max_vecs = nr_iovecs; bio->bi_io_vec = bvl; return bio; err_free: mempool_free(p, bs->bio_pool); return NULL; }


Kent Overstreet17147.50%417.39%
Jens Axboe3710.28%417.39%
Linus Torvalds298.06%313.04%
Ingo Molnar267.22%14.35%
Neil Brown185.00%14.35%
Jun'ichi Nomura185.00%14.35%
Andrew Morton154.17%28.70%
Christoph Hellwig123.33%14.35%
Tejun Heo113.06%14.35%
Dave Olien102.78%14.35%
Subhash Peddamallu61.67%14.35%
Lei Ming41.11%14.35%
Mel Gorman20.56%14.35%
Al Viro10.28%14.35%

void zero_fill_bio(struct bio *bio) { unsigned long flags; struct bio_vec bv; struct bvec_iter iter; bio_for_each_segment(bv, bio, iter) { char *data = bvec_kmap_irq(&bv, &flags); memset(data, 0, bv.bv_len); flush_dcache_page(bv.bv_page); bvec_kunmap_irq(data, &flags); } }


Dave Olien6490.14%150.00%
Kent Overstreet79.86%150.00%

EXPORT_SYMBOL(zero_fill_bio); /** * bio_put - release a reference to a bio * @bio: bio to release reference to * * Description: * Put a reference to a &struct bio, either one you have gotten with * bio_alloc, bio_get or bio_clone. The last put of a bio will free it. **/
void bio_put(struct bio *bio) { if (!bio_flagged(bio, BIO_REFFED)) bio_free(bio); else { BIO_BUG_ON(!atomic_read(&bio->__bi_cnt)); /* * last put frees it */ if (atomic_dec_and_test(&bio->__bi_cnt)) bio_free(bio); } }


Linus Torvalds3053.57%360.00%
Jens Axboe2137.50%120.00%
Kent Overstreet58.93%120.00%

inline int bio_phys_segments(struct request_queue *q, struct bio *bio) { if (unlikely(!bio_flagged(bio, BIO_SEG_VALID))) blk_recount_segments(q, bio); return bio->bi_phys_segments; }


Linus Torvalds3585.37%133.33%
Jens Axboe614.63%266.67%

EXPORT_SYMBOL(bio_phys_segments); /** * __bio_clone_fast - clone a bio that shares the original bio's biovec * @bio: destination bio * @bio_src: bio to clone * * Clone a &bio. Caller will own the returned bio, but not * the actual data it points to. Reference count of returned * bio will be one. * * Caller must ensure that @bio_src is not freed before @bio. */
void __bio_clone_fast(struct bio *bio, struct bio *bio_src) { BUG_ON(bio->bi_pool && BVEC_POOL_IDX(bio)); /* * most users will be overriding ->bi_bdev with a new target, * so we don't set nor calculate new physical/hw segment counts here */ bio->bi_bdev = bio_src->bi_bdev; bio_set_flag(bio, BIO_CLONED); bio->bi_opf = bio_src->bi_opf; bio->bi_iter = bio_src->bi_iter; bio->bi_io_vec = bio_src->bi_io_vec; bio_clone_blkcg_association(bio, bio_src); }


Kent Overstreet6081.08%120.00%
Paolo Valente79.46%120.00%
Jens Axboe68.11%240.00%
Christoph Hellwig11.35%120.00%

EXPORT_SYMBOL(__bio_clone_fast); /** * bio_clone_fast - clone a bio that shares the original bio's biovec * @bio: bio to clone * @gfp_mask: allocation priority * @bs: bio_set to allocate from * * Like __bio_clone_fast, only also allocates the returned bio */
struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs) { struct bio *b; b = bio_alloc_bioset(gfp_mask, 0, bs); if (!b) return NULL; __bio_clone_fast(b, bio); if (bio_integrity(bio)) { int ret; ret = bio_integrity_clone(b, bio, gfp_mask); if (ret < 0) { bio_put(b); return NULL; } } return b; }


Kent Overstreet93100.00%1100.00%

static struct bio *__bio_clone_bioset(struct bio *bio_src, gfp_t gfp_mask, struct bio_set *bs, int offset, int size) { struct bvec_iter iter; struct bio_vec bv; struct bio *bio; struct bvec_iter iter_src = bio_src->bi_iter; /* for supporting partial clone */ if (offset || size != bio_src->bi_iter.bi_size) { bio_advance_iter(bio_src, &iter_src, offset); iter_src.bi_size = size; } /* * Pre immutable biovecs, __bio_clone() used to just do a memcpy from * bio_src->bi_io_vec to bio->bi_io_vec. * * We can't do that anymore, because: * * - The point of cloning the biovec is to produce a bio with a biovec * the caller can modify: bi_idx and bi_bvec_done should be 0. * * - The original bio could've had more than BIO_MAX_PAGES biovecs; if * we tried to clone the whole thing bio_alloc_bioset() would fail. * But the clone should succeed as long as the number of biovecs we * actually need to allocate is fewer than BIO_MAX_PAGES. * * - Lastly, bi_vcnt should not be looked at or relied upon by code * that does not own the bio - reason being drivers don't use it for * iterating over the biovec anymore, so expecting it to be kept up * to date (i.e. for clones that share the parent biovec) is just * asking for trouble and would force extra work on * __bio_clone_fast() anyways. */ bio = bio_alloc_bioset(gfp_mask, __bio_segments(bio_src, &iter_src), bs); if (!bio) return NULL; bio->bi_bdev = bio_src->bi_bdev; bio->bi_opf = bio_src->bi_opf; bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector; bio->bi_iter.bi_size = bio_src->bi_iter.bi_size; switch (bio_op(bio)) { case REQ_OP_DISCARD: case REQ_OP_SECURE_ERASE: case REQ_OP_WRITE_ZEROES: break; case REQ_OP_WRITE_SAME: bio->bi_io_vec[bio->bi_vcnt++] = bio_src->bi_io_vec[0]; break; default: __bio_for_each_segment(bv, bio_src, iter, iter_src) bio->bi_io_vec[bio->bi_vcnt++] = bv; break; } if (bio_integrity(bio_src)) { int ret; ret = bio_integrity_clone(bio, bio_src, gfp_mask); if (ret < 0) { bio_put(bio); return NULL; } } bio_clone_blkcg_association(bio, bio_src); return bio; }


Kent Overstreet10641.09%318.75%
Lei Ming5420.93%16.25%
Martin K. Petersen3112.02%16.25%
Linus Torvalds2810.85%212.50%
Adrian Hunter155.81%16.25%
Paolo Valente72.71%16.25%
Li Zefan62.33%16.25%
Chaitanya Kulkarni31.16%16.25%
Michael Christie31.16%16.25%
Jens Axboe20.78%16.25%
Al Viro10.39%16.25%
Jun'ichi Nomura10.39%16.25%
Peter Osterlund10.39%16.25%

/** * bio_clone_bioset - clone a bio * @bio_src: bio to clone * @gfp_mask: allocation priority * @bs: bio_set to allocate from * * Clone bio. Caller will own the returned bio, but not the actual data it * points to. Reference count of returned bio will be one. */
struct bio *bio_clone_bioset(struct bio *bio_src, gfp_t gfp_mask, struct bio_set *bs) { return __bio_clone_bioset(bio_src, gfp_mask, bs, 0, bio_src->bi_iter.bi_size); }


Lei Ming38100.00%1100.00%

EXPORT_SYMBOL(bio_clone_bioset); /** * bio_clone_bioset_partial - clone a partial bio * @bio_src: bio to clone * @gfp_mask: allocation priority * @bs: bio_set to allocate from * @offset: cloned starting from the offset * @size: size for the cloned bio * * Clone bio. Caller will own the returned bio, but not the actual data it * points to. Reference count of returned bio will be one. */
struct bio *bio_clone_bioset_partial(struct bio *bio_src, gfp_t gfp_mask, struct bio_set *bs, int offset, int size) { return __bio_clone_bioset(bio_src, gfp_mask, bs, offset, size); }


Lei Ming40100.00%1100.00%

EXPORT_SYMBOL(bio_clone_bioset_partial); /** * bio_add_pc_page - attempt to add page to bio * @q: the target queue * @bio: destination bio * @page: page to add * @len: vec entry length * @offset: vec entry offset * * Attempt to add a page to the bio_vec maplist. This can fail for a * number of reasons, such as the bio being full or target block device * limitations. The target block device must allow bio's up to PAGE_SIZE, * so it is always possible to add a single page to an empty bio. * * This should only be used by REQ_PC bios. */
int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page *page, unsigned int len, unsigned int offset) { int retried_segments = 0; struct bio_vec *bvec; /* * cloned bio must not modify vec list */ if (unlikely(bio_flagged(bio, BIO_CLONED))) return 0; if (((bio->bi_iter.bi_size + len) >> 9) > queue_max_hw_sectors(q)) return 0; /* * For filesystems with a blocksize smaller than the pagesize * we will often be called with the same page as last time and * a consecutive offset. Optimize this special case. */ if (bio->bi_vcnt > 0) { struct bio_vec *prev = &bio->bi_io_vec[bio->bi_vcnt - 1]; if (page == prev->bv_page && offset == prev->bv_offset + prev->bv_len) { prev->bv_len += len; bio->bi_iter.bi_size += len; goto done; } /* * If the queue doesn't support SG gaps and adding this * offset would create a gap, disallow it. */ if (bvec_gap_to_prev(q, prev, offset)) return 0; } if (bio->bi_vcnt >= bio->bi_max_vecs) return 0; /* * setup the new entry, we might clear it again later if we * cannot add the page */ bvec = &bio->bi_io_vec[bio->bi_vcnt]; bvec->bv_page = page; bvec->bv_len = len; bvec->bv_offset = offset; bio->bi_vcnt++; bio->bi_phys_segments++; bio->bi_iter.bi_size += len; /* * Perform a recount if the number of segments is greater * than queue_max_segments(q). */ while (bio->bi_phys_segments > queue_max_segments(q)) { if (retried_segments) goto failed; retried_segments = 1; blk_recount_segments(q, bio); } /* If we may be able to merge these biovecs, force a recount */ if (bio->bi_vcnt > 1 && (BIOVEC_PHYS_MERGEABLE(bvec-1, bvec))) bio_clear_flag(bio, BIO_SEG_VALID); done: return len; failed: bvec->bv_page = NULL; bvec->bv_len = 0; bvec->bv_offset = 0; bio->bi_vcnt--; bio->bi_iter.bi_size -= len; blk_recount_segments(q, bio); return 0; }


Jens Axboe21666.06%753.85%
Maurizio Lombardi8927.22%17.69%
Kent Overstreet92.75%215.38%
Mike Christie82.45%17.69%
Patrick Mansfield30.92%17.69%
Keith Busch20.61%17.69%

EXPORT_SYMBOL(bio_add_pc_page); /** * bio_add_page - attempt to add page to bio * @bio: destination bio * @page: page to add * @len: vec entry length * @offset: vec entry offset * * Attempt to add a page to the bio_vec maplist. This will only fail * if either bio->bi_vcnt == bio->bi_max_vecs or it's a cloned bio. */
int bio_add_page(struct bio *bio, struct page *page, unsigned int len, unsigned int offset) { struct bio_vec *bv; /* * cloned bio must not modify vec list */ if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return 0; /* * For filesystems with a blocksize smaller than the pagesize * we will often be called with the same page as last time and * a consecutive offset. Optimize this special case. */ if (bio->bi_vcnt > 0) { bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; if (page == bv->bv_page && offset == bv->bv_offset + bv->bv_len) { bv->bv_len += len; goto done; } } if (bio->bi_vcnt >= bio->bi_max_vecs) return 0; bv = &bio->bi_io_vec[bio->bi_vcnt]; bv->bv_page = page; bv->bv_len = len; bv->bv_offset = offset