cregit-Linux how code gets into the kernel

Release 4.17 fs/block_dev.c

Directory: fs
/*
 *  linux/fs/block_dev.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *  Copyright (C) 2001  Andrea Arcangeli <andrea@suse.de> SuSE
 */

#include <linux/init.h>
#include <linux/mm.h>
#include <linux/fcntl.h>
#include <linux/slab.h>
#include <linux/kmod.h>
#include <linux/major.h>
#include <linux/device_cgroup.h>
#include <linux/highmem.h>
#include <linux/blkdev.h>
#include <linux/backing-dev.h>
#include <linux/module.h>
#include <linux/blkpg.h>
#include <linux/magic.h>
#include <linux/dax.h>
#include <linux/buffer_head.h>
#include <linux/swap.h>
#include <linux/pagevec.h>
#include <linux/writeback.h>
#include <linux/mpage.h>
#include <linux/mount.h>
#include <linux/uio.h>
#include <linux/namei.h>
#include <linux/log2.h>
#include <linux/cleancache.h>
#include <linux/dax.h>
#include <linux/badblocks.h>
#include <linux/task_io_accounting_ops.h>
#include <linux/falloc.h>
#include <linux/uaccess.h>
#include "internal.h"


struct bdev_inode {
	
struct block_device bdev;
	
struct inode vfs_inode;
};


static const struct address_space_operations def_blk_aops;


static inline struct bdev_inode *BDEV_I(struct inode *inode) { return container_of(inode, struct bdev_inode, vfs_inode); }

Contributors

PersonTokensPropCommitsCommitProp
Andrew Morton25100.00%1100.00%
Total25100.00%1100.00%


struct block_device *I_BDEV(struct inode *inode) { return &BDEV_I(inode)->bdev; }

Contributors

PersonTokensPropCommitsCommitProp
Andrew Morton21100.00%1100.00%
Total21100.00%1100.00%

EXPORT_SYMBOL(I_BDEV);
static void bdev_write_inode(struct block_device *bdev) { struct inode *inode = bdev->bd_inode; int ret; spin_lock(&inode->i_lock); while (inode->i_state & I_DIRTY) { spin_unlock(&inode->i_lock); ret = write_inode_now(inode, true); if (ret) { char name[BDEVNAME_SIZE]; pr_warn_ratelimited("VFS: Dirty inode writeback failed " "for block device %s (err=%d).\n", bdevname(bdev, name), ret); } spin_lock(&inode->i_lock); } spin_unlock(&inode->i_lock); }

Contributors

PersonTokensPropCommitsCommitProp
Vivek Goyal4241.58%116.67%
Dave Chinner3635.64%233.33%
Christoph Hellwig1716.83%116.67%
Jan Kara43.96%116.67%
Tejun Heo21.98%116.67%
Total101100.00%6100.00%

/* Kill _all_ buffers and pagecache , dirty or not.. */
void kill_bdev(struct block_device *bdev) { struct address_space *mapping = bdev->bd_inode->i_mapping; if (mapping->nrpages == 0 && mapping->nrexceptional == 0) return; invalidate_bh_lrus(); truncate_inode_pages(mapping, 0); }

Contributors

PersonTokensPropCommitsCommitProp
Al Viro1532.61%120.00%
Linus Torvalds1328.26%120.00%
Peter Zijlstra1226.09%120.00%
Johannes Weiner510.87%120.00%
Ross Zwisler12.17%120.00%
Total46100.00%5100.00%

EXPORT_SYMBOL(kill_bdev); /* Invalidate clean unused buffers and pagecache. */
void invalidate_bdev(struct block_device *bdev) { struct address_space *mapping = bdev->bd_inode->i_mapping; if (mapping->nrpages) { invalidate_bh_lrus(); lru_add_drain_all(); /* make sure all lru add caches are flushed */ invalidate_mapping_pages(mapping, 0, -1); } /* 99% of the time, we don't need to flush the cleancache on the bdev. * But, for the strange corners, lets be cautious */ cleancache_invalidate_inode(mapping); }

Contributors

PersonTokensPropCommitsCommitProp
Al Viro3975.00%125.00%
Linus Torvalds1019.23%125.00%
Andrey Ryabinin23.85%125.00%
Dan Magenheimer11.92%125.00%
Total52100.00%4100.00%

EXPORT_SYMBOL(invalidate_bdev);
int set_blocksize(struct block_device *bdev, int size) { /* Size must be a power of two, and between 512 and PAGE_SIZE */ if (size > PAGE_SIZE || size < 512 || !is_power_of_2(size)) return -EINVAL; /* Size cannot be smaller than the size supported by the device */ if (size < bdev_logical_block_size(bdev)) return -EINVAL; /* Don't change the size if it is same as current */ if (bdev->bd_block_size != size) { sync_blockdev(bdev); bdev->bd_block_size = size; bdev->bd_inode->i_blkbits = blksize_bits(size); kill_bdev(bdev); } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds6573.03%220.00%
Al Viro1213.48%440.00%
Mika Kukkonen66.74%110.00%
Vignesh Babu44.49%110.00%
Andrew Morton11.12%110.00%
Martin K. Petersen11.12%110.00%
Total89100.00%10100.00%

EXPORT_SYMBOL(set_blocksize);
int sb_set_blocksize(struct super_block *sb, int size) { if (set_blocksize(sb->s_bdev, size)) return 0; /* If we get here, we know size is power of two * and it's value is between 512 and PAGE_SIZE */ sb->s_blocksize = size; sb->s_blocksize_bits = blksize_bits(size); return sb->s_blocksize; }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds4287.50%125.00%
Coywolf Qi Hunt48.33%125.00%
Mika Kukkonen12.08%125.00%
Al Viro12.08%125.00%
Total48100.00%4100.00%

EXPORT_SYMBOL(sb_set_blocksize);
int sb_min_blocksize(struct super_block *sb, int size) { int minsize = bdev_logical_block_size(sb->s_bdev); if (size < minsize) size = minsize; return sb_set_blocksize(sb, size); }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds3995.12%133.33%
Al Viro12.44%133.33%
Martin K. Petersen12.44%133.33%
Total41100.00%3100.00%

EXPORT_SYMBOL(sb_min_blocksize);
static int blkdev_get_block(struct inode *inode, sector_t iblock, struct buffer_head *bh, int create) { bh->b_bdev = I_BDEV(inode); bh->b_blocknr = iblock; set_buffer_mapped(bh); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Andrew Morton1942.22%218.18%
Linus Torvalds1226.67%327.27%
Linus Torvalds (pre-git)920.00%545.45%
Kenneth W. Chen511.11%19.09%
Total45100.00%11100.00%


static struct inode *bdev_file_inode(struct file *file) { return file->f_mapping->host; }

Contributors

PersonTokensPropCommitsCommitProp
Dan J Williams20100.00%1100.00%
Total20100.00%1100.00%


static unsigned int dio_bio_write_op(struct kiocb *iocb) { unsigned int op = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE; /* avoid the need for a I/O completion work item */ if (iocb->ki_flags & IOCB_DSYNC) op |= REQ_FUA; return op; }

Contributors

PersonTokensPropCommitsCommitProp
Jens Axboe38100.00%1100.00%
Total38100.00%1100.00%

#define DIO_INLINE_BIO_VECS 4
static void blkdev_bio_end_io_simple(struct bio *bio) { struct task_struct *waiter = bio->bi_private; WRITE_ONCE(bio->bi_private, NULL); wake_up_process(waiter); }

Contributors

PersonTokensPropCommitsCommitProp
Christoph Hellwig34100.00%1100.00%
Total34100.00%1100.00%


static ssize_t __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter, int nr_pages) { struct file *file = iocb->ki_filp; struct block_device *bdev = I_BDEV(bdev_file_inode(file)); struct bio_vec inline_vecs[DIO_INLINE_BIO_VECS], *vecs, *bvec; loff_t pos = iocb->ki_pos; bool should_dirty = false; struct bio bio; ssize_t ret; blk_qc_t qc; int i; if ((pos | iov_iter_alignment(iter)) & (bdev_logical_block_size(bdev) - 1)) return -EINVAL; if (nr_pages <= DIO_INLINE_BIO_VECS) vecs = inline_vecs; else { vecs = kmalloc(nr_pages * sizeof(struct bio_vec), GFP_KERNEL); if (!vecs) return -ENOMEM; } bio_init(&bio, vecs, nr_pages); bio_set_dev(&bio, bdev); bio.bi_iter.bi_sector = pos >> 9; bio.bi_write_hint = iocb->ki_hint; bio.bi_private = current; bio.bi_end_io = blkdev_bio_end_io_simple; ret = bio_iov_iter_get_pages(&bio, iter); if (unlikely(ret)) return ret; ret = bio.bi_iter.bi_size; if (iov_iter_rw(iter) == READ) { bio.bi_opf = REQ_OP_READ; if (iter_is_iovec(iter)) should_dirty = true; } else { bio.bi_opf = dio_bio_write_op(iocb); task_io_account_write(ret); } qc = submit_bio(&bio); for (;;) { set_current_state(TASK_UNINTERRUPTIBLE); if (!READ_ONCE(bio.bi_private)) break; if (!(iocb->ki_flags & IOCB_HIPRI) || !blk_poll(bdev_get_queue(bdev), qc)) io_schedule(); } __set_current_state(TASK_RUNNING); bio_for_each_segment_all(bvec, &bio, i) { if (should_dirty && !PageCompound(bvec->bv_page)) set_page_dirty_lock(bvec->bv_page); put_page(bvec->bv_page); } if (vecs != inline_vecs) kfree(vecs); if (unlikely(bio.bi_status)) ret = blk_status_to_errno(bio.bi_status); bio_uninit(&bio); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Christoph Hellwig30978.03%436.36%
Jens Axboe8220.71%545.45%
Lei Ming41.01%19.09%
Damien Le Moal10.25%19.09%
Total396100.00%11100.00%

struct blkdev_dio { union { struct kiocb *iocb; struct task_struct *waiter; }; size_t size; atomic_t ref; bool multi_bio : 1; bool should_dirty : 1; bool is_sync : 1; struct bio bio; }; static struct bio_set *blkdev_dio_pool __read_mostly;
static void blkdev_bio_end_io(struct bio *bio) { struct blkdev_dio *dio = bio->bi_private; bool should_dirty = dio->should_dirty; if (dio->multi_bio && !atomic_dec_and_test(&dio->ref)) { if (bio->bi_status && !dio->bio.bi_status) dio->bio.bi_status = bio->bi_status; } else { if (!dio->is_sync) { struct kiocb *iocb = dio->iocb; ssize_t ret; if (likely(!dio->bio.bi_status)) { ret = dio->size; iocb->ki_pos += ret; } else { ret = blk_status_to_errno(dio->bio.bi_status); } dio->iocb->ki_complete(iocb, ret, 0); bio_put(&dio->bio); } else { struct task_struct *waiter = dio->waiter; WRITE_ONCE(dio->waiter, NULL); wake_up_process(waiter); } } if (should_dirty) { bio_check_pages_dirty(bio); } else { struct bio_vec *bvec; int i; bio_for_each_segment_all(bvec, bio, i) put_page(bvec->bv_page); bio_put(bio); } }

Contributors

PersonTokensPropCommitsCommitProp
Christoph Hellwig220100.00%2100.00%
Total220100.00%2100.00%


static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages) { struct file *file = iocb->ki_filp; struct inode *inode = bdev_file_inode(file); struct block_device *bdev = I_BDEV(inode); struct blk_plug plug; struct blkdev_dio *dio; struct bio *bio; bool is_read = (iov_iter_rw(iter) == READ), is_sync; loff_t pos = iocb->ki_pos; blk_qc_t qc = BLK_QC_T_NONE; int ret = 0; if ((pos | iov_iter_alignment(iter)) & (bdev_logical_block_size(bdev) - 1)) return -EINVAL; bio = bio_alloc_bioset(GFP_KERNEL, nr_pages, blkdev_dio_pool); bio_get(bio); /* extra ref for the completion handler */ dio = container_of(bio, struct blkdev_dio, bio); dio->is_sync = is_sync = is_sync_kiocb(iocb); if (dio->is_sync) dio->waiter = current; else dio->iocb = iocb; dio->size = 0; dio->multi_bio = false; dio->should_dirty = is_read && (iter->type == ITER_IOVEC); blk_start_plug(&plug); for (;;) { bio_set_dev(bio, bdev); bio->bi_iter.bi_sector = pos >> 9; bio->bi_write_hint = iocb->ki_hint; bio->bi_private = dio; bio->bi_end_io = blkdev_bio_end_io; ret = bio_iov_iter_get_pages(bio, iter); if (unlikely(ret)) { bio->bi_status = BLK_STS_IOERR; bio_endio(bio); break; } if (is_read) { bio->bi_opf = REQ_OP_READ; if (dio->should_dirty) bio_set_pages_dirty(bio); } else { bio->bi_opf = dio_bio_write_op(iocb); task_io_account_write(bio->bi_iter.bi_size); } dio->size += bio->bi_iter.bi_size; pos += bio->bi_iter.bi_size; nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES); if (!nr_pages) { qc = submit_bio(bio); break; } if (!dio->multi_bio) { dio->multi_bio = true; atomic_set(&dio->ref, 2); } else { atomic_inc(&dio->ref); } submit_bio(bio); bio = bio_alloc(GFP_KERNEL, nr_pages); } blk_finish_plug(&plug); if (!is_sync) return -EIOCBQUEUED; for (;;) { set_current_state(TASK_UNINTERRUPTIBLE); if (!READ_ONCE(dio->waiter)) break; if (!(iocb->ki_flags & IOCB_HIPRI) || !blk_poll(bdev_get_queue(bdev), qc)) io_schedule(); } __set_current_state(TASK_RUNNING); if (!ret) ret = blk_status_to_errno(dio->bio.bi_status); if (likely(!ret)) ret = dio->size; bio_put(&dio->bio); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Christoph Hellwig51497.53%770.00%
Jens Axboe122.28%220.00%
Damien Le Moal10.19%110.00%
Total527100.00%10100.00%


static ssize_t blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter) { int nr_pages; nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES + 1); if (!nr_pages) return 0; if (is_sync_kiocb(iocb) && nr_pages <= BIO_MAX_PAGES) return __blkdev_direct_IO_simple(iocb, iter, nr_pages); return __blkdev_direct_IO(iocb, iter, min(nr_pages, BIO_MAX_PAGES)); }

Contributors

PersonTokensPropCommitsCommitProp
Christoph Hellwig4966.22%216.67%
Linus Torvalds810.81%18.33%
Andrew Morton68.11%541.67%
Kenneth W. Chen56.76%216.67%
Al Viro34.05%18.33%
Jens Axboe34.05%18.33%
Total74100.00%12100.00%


static __init int blkdev_init(void) { blkdev_dio_pool = bioset_create(4, offsetof(struct blkdev_dio, bio), BIOSET_NEED_BVECS); if (!blkdev_dio_pool) return -ENOMEM; return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Christoph Hellwig3386.84%240.00%
Neil Brown25.26%120.00%
Kenneth W. Chen25.26%120.00%
Andrew Morton12.63%120.00%
Total38100.00%5100.00%

module_init(blkdev_init);
int __sync_blockdev(struct block_device *bdev, int wait) { if (!bdev) return 0; if (!wait) return filemap_flush(bdev->bd_inode->i_mapping); return filemap_write_and_wait(bdev->bd_inode->i_mapping); }

Contributors

PersonTokensPropCommitsCommitProp
Jan Kara2656.52%150.00%
Nicholas Piggin2043.48%150.00%
Total46100.00%2100.00%

/* * Write out and wait upon all the dirty data associated with a block * device via its mapping. Does not take the superblock lock. */
int sync_blockdev(struct block_device *bdev) { return __sync_blockdev(bdev, 1); }

Contributors

PersonTokensPropCommitsCommitProp
Jan Kara1583.33%150.00%
Nicholas Piggin316.67%150.00%
Total18100.00%2100.00%

EXPORT_SYMBOL(sync_blockdev); /* * Write out and wait upon all dirty data associated with this * device. Filesystem data as well as the underlying block * device. Takes the superblock lock. */
int fsync_bdev(struct block_device *bdev) { struct super_block *sb = get_super(bdev); if (sb) { int res = sync_filesystem(sb); drop_super(sb); return res; } return sync_blockdev(bdev); }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin4797.92%150.00%
Jan Kara12.08%150.00%
Total48100.00%2100.00%

EXPORT_SYMBOL(fsync_bdev); /** * freeze_bdev -- lock a filesystem and force it into a consistent state * @bdev: blockdevice to lock * * If a superblock is found on this device, we take the s_umount semaphore * on it to make sure nobody unmounts until the snapshot creation is done. * The reference counter (bd_fsfreeze_count) guarantees that only the last * unfreeze process can unfreeze the frozen filesystem actually when multiple * freeze requests arrive simultaneously. It counts up in freeze_bdev() and * count down in thaw_bdev(). When it becomes 0, thaw_bdev() will unfreeze * actually. */
struct super_block *freeze_bdev(struct block_device *bdev) { struct super_block *sb; int error = 0; mutex_lock(&bdev->bd_fsfreeze_mutex); if (++bdev->bd_fsfreeze_count > 1) { /* * We don't even need to grab a reference - the first call * to freeze_bdev grab an active reference and only the last * thaw_bdev drops it. */ sb = get_super(bdev); if (sb) drop_super(sb); mutex_unlock(&bdev->bd_fsfreeze_mutex); return sb; } sb = get_active_super(bdev); if (!sb) goto out; if (sb->s_op->freeze_super) error = sb->s_op->freeze_super(sb); else error = freeze_super(sb); if (error) { deactivate_super(sb); bdev->bd_fsfreeze_count--; mutex_unlock(&bdev->bd_fsfreeze_mutex); return ERR_PTR(error); } deactivate_super(sb); out: sync_blockdev(bdev); mutex_unlock(&bdev->bd_fsfreeze_mutex); return sb; /* thaw_bdev releases s->s_umount */ }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin11770.91%116.67%
Christoph Hellwig2112.73%233.33%
Benjamin Marzinski2012.12%116.67%
Andrey Ryabinin42.42%116.67%
Josef Bacik31.82%116.67%
Total165100.00%6100.00%

EXPORT_SYMBOL(freeze_bdev); /** * thaw_bdev -- unlock filesystem * @bdev: blockdevice to unlock * @sb: associated superblock * * Unlocks the filesystem and marks it writeable again after freeze_bdev(). */
int thaw_bdev(struct block_device *bdev, struct super_block *sb) { int error = -EINVAL; mutex_lock(&bdev->bd_fsfreeze_mutex); if (!bdev->bd_fsfreeze_count) goto out; error = 0; if (--bdev->bd_fsfreeze_count > 0) goto out; if (!sb) goto out; if (sb->s_op->thaw_super) error = sb->s_op->thaw_super(sb); else error = thaw_super(sb); if (error) bdev->bd_fsfreeze_count++; out: mutex_unlock(&bdev->bd_fsfreeze_mutex); return error; }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin7264.29%120.00%
Benjamin Marzinski2017.86%120.00%
Christoph Hellwig1412.50%120.00%
Josef Bacik54.46%120.00%
Pierre Morel10.89%120.00%
Total112100.00%5100.00%

EXPORT_SYMBOL(thaw_bdev);
static int blkdev_writepage(struct page *page, struct writeback_control *wbc) { return block_write_full_page(page, blkdev_get_block, wbc); }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds1973.08%266.67%
Andrew Morton726.92%133.33%
Total26100.00%3100.00%


static int blkdev_readpage(struct file * file, struct page * page) { return block_read_full_page(page, blkdev_get_block); }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds2083.33%250.00%
Linus Torvalds (pre-git)416.67%250.00%
Total24100.00%4100.00%


static int blkdev_readpages(struct file *file, struct address_space *mapping, struct list_head *pages, unsigned nr_pages) { return mpage_readpages(mapping, pages, nr_pages, blkdev_get_block); }

Contributors

PersonTokensPropCommitsCommitProp
Akinobu Mita36100.00%1100.00%
Total36100.00%1100.00%


static int blkdev_write_begin(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned flags, struct page **pagep, void **fsdata) { return block_write_begin(mapping, pos, len, flags, pagep, blkdev_get_block); }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin2853.85%116.67%
Linus Torvalds2140.38%350.00%
Linus Torvalds (pre-git)23.85%116.67%
Christoph Hellwig11.92%116.67%
Total52100.00%6100.00%


static int blkdev_write_end(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned copied, struct page *page, void *fsdata) { int ret; ret = block_write_end(file, mapping, pos, len, copied, page, fsdata); unlock_page(page); put_page(page); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin4666.67%125.00%
Linus Torvalds2028.99%125.00%
Linus Torvalds (pre-git)22.90%125.00%
Kirill A. Shutemov11.45%125.00%
Total69100.00%4100.00%

/* * private llseek: * for a block special file file_inode(file)->i_size is zero * so we compute the size by hand (just as in block_read/write above) */
static loff_t block_llseek(struct file *file, loff_t offset, int whence) { struct inode *bd_inode = bdev_file_inode(file); loff_t retval; inode_lock(bd_inode); retval = fixed_size_llseek(file, offset, whence, i_size_read(bd_inode)); inode_unlock(bd_inode); return retval; }

Contributors

PersonTokensPropCommitsCommitProp
Andrew Morton2237.29%440.00%
Linus Torvalds (pre-git)1932.20%110.00%
Al Viro1220.34%220.00%
Dan J Williams35.08%110.00%
Linus Torvalds35.08%220.00%
Total59100.00%10100.00%


int blkdev_fsync(struct file *filp, loff_t start, loff_t end, int datasync) { struct inode *bd_inode = bdev_file_inode(filp); struct block_device *bdev = I_BDEV(bd_inode); int error; error = file_write_and_wait_range(filp, start, end); if (error) return error; /* * There is no need to serialise calls to blkdev_issue_flush with * i_mutex and doing so causes performance issues with concurrent * O_SYNC writers to a block device. */ error = blkdev_issue_flush(bdev, GFP_KERNEL, NULL); if (error == -EOPNOTSUPP) error = 0; return error; }

Contributors

</
PersonTokensPropCommitsCommitProp
Christoph Hellwig2933.72%18.33%
Rafael J. Wysocki1719.77%18.33%
Linus Torvalds (pre-git)1315.12%325.00%
Anton Blanchard1213.95%18.33%
Josef Bacik66.98%18.33%