Release 4.7 block/elevator.c

Directory: block
/*
 *  Block device elevator/IO-scheduler.
 *
 *  Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
 *
 * 30042000 Jens Axboe <axboe@kernel.dk> :
 *
 * Split the elevator a bit so that it is possible to choose a different
 * one or even write a new "plug in". There are three pieces:
 * - elevator_fn, inserts a new request in the queue list
 * - elevator_merge_fn, decides whether a new buffer can be merged with
 *   an existing request
 * - elevator_dequeue_fn, called when a request is taken off the active list
 *
 * 20082000 Dave Jones <davej@suse.de> :
 * Removed tests for max-bomb-segments, which was breaking elvtune
 *  when run without -bN
 *
 * Jens:
 * - Rework again to work with bio instead of buffer_heads
 * - loose bi_dev comparisons, partition handling is right now
 * - completely modularize elevator setup and teardown
 *
 */
#include <linux/kernel.h>
#include <linux/fs.h>
#include <linux/blkdev.h>
#include <linux/elevator.h>
#include <linux/bio.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/compiler.h>
#include <linux/blktrace_api.h>
#include <linux/hash.h>
#include <linux/uaccess.h>
#include <linux/pm_runtime.h>
#include <linux/blk-cgroup.h>

#include <trace/events/block.h>

#include "blk.h"

static DEFINE_SPINLOCK(elv_list_lock);
static LIST_HEAD(elv_list);

/*
 * Merge hash stuff.
 */

#define rq_hash_key(rq)		(blk_rq_pos(rq) + blk_rq_sectors(rq))

/*
 * Query io scheduler to see if the current process issuing bio may be
 * merged with rq.
 */


static int elv_iosched_allow_merge(struct request *rq, struct bio *bio)
{
	struct request_queue *q = rq->q;
	struct elevator_queue *e = q->elevator;

	if (e->type->ops.elevator_allow_merge_fn)
		return e->type->ops.elevator_allow_merge_fn(q, rq, bio);

	return 1;
}

Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 57 90.48% 3 75.00%
tejun heo tejun heo 6 9.52% 1 25.00%
Total 63 100.00% 4 100.00%

/*
 * can we safely merge with this request?
 */


bool elv_rq_merge_ok(struct request *rq, struct bio *bio)
{
	if (!blk_rq_merge_ok(rq, bio))
		return 0;

	if (!elv_iosched_allow_merge(rq, bio))
		return 0;

	return 1;
}
Contributors
Person Tokens Prop Commits CommitProp
linus torvalds linus torvalds 24 54.55% 3 50.00%
jens axboe jens axboe 13 29.55% 1 16.67%
martin k. petersen martin k. petersen 4 9.09% 1 16.67%
tejun heo tejun heo 3 6.82% 1 16.67%
Total 44 100.00% 6 100.00%


EXPORT_SYMBOL(elv_rq_merge_ok);



static struct elevator_type *elevator_find(const char *name)
{
	struct elevator_type *e;

	list_for_each_entry(e, &elv_list, list) {
		if (!strcmp(e->elevator_name, name))
			return e;
	}

	return NULL;
}

Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 37 80.43% 1 25.00%
vasily tarasov vasily tarasov 4 8.70% 1 25.00%
matthias kaehlcke matthias kaehlcke 4 8.70% 1 25.00%
adrian bunk adrian bunk 1 2.17% 1 25.00%
Total 46 100.00% 4 100.00%



static void elevator_put(struct elevator_type *e)
{
	module_put(e->elevator_owner);
}

Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 18 100.00% 1 100.00%
Total 18 100.00% 1 100.00%



static struct elevator_type *elevator_get(const char *name, bool try_loading)
{
	struct elevator_type *e;

	spin_lock(&elv_list_lock);

	e = elevator_find(name);
	if (!e && try_loading) {
		spin_unlock(&elv_list_lock);
		request_module("%s-iosched", name);
		spin_lock(&elv_list_lock);
		e = elevator_find(name);
	}

	if (e && !try_module_get(e->elevator_owner))
		e = NULL;

	spin_unlock(&elv_list_lock);

	return e;
}

Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 69 73.40% 3 42.86%
tejun heo tejun heo 23 24.47% 2 28.57%
kees cook kees cook 1 1.06% 1 14.29%
wzt wzt wzt wzt 1 1.06% 1 14.29%
Total 94 100.00% 7 100.00%


static char chosen_elevator[ELV_NAME_MAX];



static int __init elevator_setup(char *str)
{
	/*
         * Be backwards-compatible with previous kernels, so users
         * won't get the wrong elevator.
         */
	strncpy(chosen_elevator, str, sizeof(chosen_elevator) - 1);
	return 1;
}

Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 17 58.62% 2 28.57%
nate diller* nate diller* 6 20.69% 1 14.29%
pre-git pre-git 2 6.90% 1 14.29%
linus torvalds linus torvalds 2 6.90% 1 14.29%
chuck ebbert chuck ebbert 1 3.45% 1 14.29%
hirofumi ogawa hirofumi ogawa 1 3.45% 1 14.29%
Total 29 100.00% 7 100.00%

__setup("elevator=", elevator_setup);

/* called during boot to load the elevator chosen by the elevator param */


void __init load_default_elevator_module(void)
{
	struct elevator_type *e;

	if (!chosen_elevator[0])
		return;

	spin_lock(&elv_list_lock);
	e = elevator_find(chosen_elevator);
	spin_unlock(&elv_list_lock);

	if (!e)
		request_module("%s-iosched", chosen_elevator);
}

Contributors
Person Tokens Prop Commits CommitProp
tejun heo tejun heo 53 100.00% 1 100.00%
Total 53 100.00% 1 100.00%


static struct kobj_type elv_ktype;



struct elevator_queue *elevator_alloc(struct request_queue *q,
				  struct elevator_type *e)
{
	struct elevator_queue *eq;

	eq = kzalloc_node(sizeof(*eq), GFP_KERNEL, q->node);
	if (unlikely(!eq))
		return NULL;

	eq->type = e;
	kobject_init(&eq->kobj, &elv_ktype);
	mutex_init(&eq->sysfs_lock);
	hash_init(eq->hash);

	return eq;
}
Contributors
Person Tokens Prop Commits CommitProp
al viro al viro 44 51.76% 1 8.33%
jens axboe jens axboe 32 37.65% 4 33.33%
greg kroah-hartman greg kroah-hartman 3 3.53% 3 25.00%
chao yu chao yu 3 3.53% 1 8.33%
joe perches joe perches 1 1.18% 1 8.33%
tejun heo tejun heo 1 1.18% 1 8.33%
sasha levin sasha levin 1 1.18% 1 8.33%
Total 85 100.00% 12 100.00%


EXPORT_SYMBOL(elevator_alloc);



static void elevator_release(struct kobject *kobj)
{
	struct elevator_queue *e;

	e = container_of(kobj, struct elevator_queue, kobj);
	elevator_put(e->type);
	kfree(e);
}

Contributors
Person Tokens Prop Commits CommitProp
al viro al viro 31 77.50% 1 33.33%
jens axboe jens axboe 8 20.00% 1 33.33%
tejun heo tejun heo 1 2.50% 1 33.33%
Total 40 100.00% 3 100.00%



int elevator_init(struct request_queue *q, char *name)
{
	struct elevator_type *e = NULL;
	int err;

	/*
         * q->sysfs_lock must be held to provide mutual exclusion between
         * elevator_switch() and here.
         */
	lockdep_assert_held(&q->sysfs_lock);

	if (unlikely(q->elevator))
		return 0;

	INIT_LIST_HEAD(&q->queue_head);
	q->last_merge = NULL;
	q->end_sector = 0;
	q->boundary_rq = NULL;

	if (name) {
		e = elevator_get(name, true);
		if (!e)
			return -EINVAL;
	}

	/*
         * Use the default elevator specified by config boot param or
         * config option.  Don't try to load modules as we could be running
         * off async and request_module() isn't allowed from async.
         */
	if (!e && *chosen_elevator) {
		e = elevator_get(chosen_elevator, false);
		if (!e)
			printk(KERN_ERR "I/O scheduler %s not found\n",
							chosen_elevator);
	}

	if (!e) {
		e = elevator_get(CONFIG_DEFAULT_IOSCHED, false);
		if (!e) {
			printk(KERN_ERR
				"Default I/O scheduler not found. " \
				"Using noop.\n");
			e = elevator_get("noop", false);
		}
	}

	err = e->ops.elevator_init_fn(q, e);
	if (err)
		elevator_put(e);
	return err;
}
Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 58 30.05% 5 31.25%
nate diller* nate diller* 47 24.35% 2 12.50%
tejun heo tejun heo 43 22.28% 4 25.00%
mike snitzer mike snitzer 12 6.22% 1 6.25%
linus torvalds linus torvalds 12 6.22% 1 6.25%
sudip mukherjee sudip mukherjee 10 5.18% 1 6.25%
tomoki sekiyama tomoki sekiyama 9 4.66% 1 6.25%
ma jianpeng ma jianpeng 2 1.04% 1 6.25%
Total 193 100.00% 16 100.00%


EXPORT_SYMBOL(elevator_init);



void elevator_exit(struct elevator_queue *e)
{
	mutex_lock(&e->sysfs_lock);
	if (e->type->ops.elevator_exit_fn)
		e->type->ops.elevator_exit_fn(e);
	mutex_unlock(&e->sysfs_lock);

	kobject_put(&e->kobj);
}
Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 32 58.18% 2 50.00%
al viro al viro 17 30.91% 1 25.00%
tejun heo tejun heo 6 10.91% 1 25.00%
Total 55 100.00% 4 100.00%


EXPORT_SYMBOL(elevator_exit);



static inline void __elv_rqhash_del(struct request *rq)
{
	hash_del(&rq->hash);
	rq->cmd_flags &= ~REQ_HASHED;
}

Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 26 96.30% 2 66.67%
sasha levin sasha levin 1 3.70% 1 33.33%
Total 27 100.00% 3 100.00%



static void elv_rqhash_del(struct request_queue *q, struct request *rq)
{
	if (ELV_ON_HASH(rq))
		__elv_rqhash_del(rq);
}

Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 28 100.00% 2 100.00%
Total 28 100.00% 2 100.00%



static void elv_rqhash_add(struct request_queue *q, struct request *rq)
{
	struct elevator_queue *e = q->elevator;

	BUG_ON(ELV_ON_HASH(rq));
	hash_add(e->hash, &rq->hash, rq_hash_key(rq));
	rq->cmd_flags |= REQ_HASHED;
}

Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 52 92.86% 4 80.00%
sasha levin sasha levin 4 7.14% 1 20.00%
Total 56 100.00% 5 100.00%



static void elv_rqhash_reposition(struct request_queue *q, struct request *rq)
{
	__elv_rqhash_del(rq);
	elv_rqhash_add(q, rq);
}

Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 28 100.00% 2 100.00%
Total 28 100.00% 2 100.00%



static struct request *elv_rqhash_find(struct request_queue *q, sector_t offset)
{
	struct elevator_queue *e = q->elevator;
	struct hlist_node *next;
	struct request *rq;

	hash_for_each_possible_safe(e->hash, rq, next, hash, offset) {
		BUG_ON(!ELV_ON_HASH(rq));

		if (unlikely(!rq_mergeable(rq))) {
			__elv_rqhash_del(rq);
			continue;
		}

		if (rq_hash_key(rq) == offset)
			return rq;
	}

	return NULL;
}

Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 87 94.57% 3 75.00%
sasha levin sasha levin 5 5.43% 1 25.00%
Total 92 100.00% 4 100.00%

/*
 * RB-tree support functions for inserting/lookup/removal of requests
 * in a sorted RB tree.
 */


void elv_rb_add(struct rb_root *root, struct request *rq)
{
	struct rb_node **p = &root->rb_node;
	struct rb_node *parent = NULL;
	struct request *__rq;

	while (*p) {
		parent = *p;
		__rq = rb_entry(parent, struct request, rb_node);

		if (blk_rq_pos(rq) < blk_rq_pos(__rq))
			p = &(*p)->rb_left;
		else if (blk_rq_pos(rq) >= blk_rq_pos(__rq))
			p = &(*p)->rb_right;
	}

	rb_link_node(&rq->rb_node, parent, p);
	rb_insert_color(&rq->rb_node, root);
}
Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 115 89.15% 1 33.33%
tejun heo tejun heo 12 9.30% 1 33.33%
jeff moyer jeff moyer 2 1.55% 1 33.33%
Total 129 100.00% 3 100.00%


EXPORT_SYMBOL(elv_rb_add);



void elv_rb_del(struct rb_root *root, struct request *rq)
{
	BUG_ON(RB_EMPTY_NODE(&rq->rb_node));
	rb_erase(&rq->rb_node, root);
	RB_CLEAR_NODE(&rq->rb_node);
}
Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 44 100.00% 1 100.00%
Total 44 100.00% 1 100.00%


EXPORT_SYMBOL(elv_rb_del);



struct request *elv_rb_find(struct rb_root *root, sector_t sector)
{
	struct rb_node *n = root->rb_node;
	struct request *rq;

	while (n) {
		rq = rb_entry(n, struct request, rb_node);

		if (sector < blk_rq_pos(rq))
			n = n->rb_left;
		else if (sector > blk_rq_pos(rq))
			n = n->rb_right;
		else
			return rq;
	}

	return NULL;
}
Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 79 92.94% 1 50.00%
tejun heo tejun heo 6 7.06% 1 50.00%
Total 85 100.00% 2 100.00%


EXPORT_SYMBOL(elv_rb_find);

/*
 * Insert rq into dispatch queue of q.  Queue lock must be held on
 * entry.  rq is sort instead into the dispatch queue. To be used by
 * specific elevators.
 */


void elv_dispatch_sort(struct request_queue *q, struct request *rq)
{
	sector_t boundary;
	struct list_head *entry;
	int stop_flags;

	if (q->last_merge == rq)
		q->last_merge = NULL;

	elv_rqhash_del(q, rq);

	q->nr_sorted--;

	boundary = q->end_sector;
	stop_flags = REQ_SOFTBARRIER | REQ_STARTED;
	list_for_each_prev(entry, &q->queue_head) {
		struct request *pos = list_entry_rq(entry);

		if ((rq->cmd_flags & REQ_DISCARD) !=
		    (pos->cmd_flags & REQ_DISCARD))
			break;
		if (rq_data_dir(rq) != rq_data_dir(pos))
			break;
		if (pos->cmd_flags & stop_flags)
			break;
		if (blk_rq_pos(rq) >= boundary) {
			if (blk_rq_pos(pos) < boundary)
				continue;
		} else {
			if (blk_rq_pos(pos) >= boundary)
				break;
		}
		if (blk_rq_pos(rq) >= blk_rq_pos(pos))
			break;
	}

	list_add(&rq->queuelist, entry);
}
Contributors
Person Tokens Prop Commits CommitProp
tejun heo tejun heo 126 70.00% 4 33.33%
jens axboe jens axboe 35 19.44% 6 50.00%
christoph hellwig christoph hellwig 12 6.67% 1 8.33%
david woodhouse david woodhouse 7 3.89% 1 8.33%
Total 180 100.00% 12 100.00%


EXPORT_SYMBOL(elv_dispatch_sort);

/*
 * Insert rq into dispatch queue of q.  Queue lock must be held on
 * entry.  rq is added to the back of the dispatch queue. To be used by
 * specific elevators.
 */


void elv_dispatch_add_tail(struct request_queue *q, struct request *rq)
{
	if (q->last_merge == rq)
		q->last_merge = NULL;

	elv_rqhash_del(q, rq);

	q->nr_sorted--;

	q->end_sector = rq_end_sector(rq);
	q->boundary_rq = rq;
	list_add_tail(&rq->queuelist, &q->queue_head);
}
Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 69 100.00% 1 100.00%
Total 69 100.00% 1 100.00%


EXPORT_SYMBOL(elv_dispatch_add_tail);



int elv_merge(struct request_queue *q, struct request **req, struct bio *bio)
{
	struct elevator_queue *e = q->elevator;
	struct request *__rq;
	int ret;

	/*
         * Levels of merges:
         *      nomerges:  No merges at all attempted
         *      noxmerges: Only simple one-hit cache try
         *      merges:    All merge tries attempted
         */
	if (blk_queue_nomerges(q) || !bio_mergeable(bio))
		return ELEVATOR_NO_MERGE;

	/*
         * First try one-hit cache.
         */
	if (q->last_merge && elv_rq_merge_ok(q->last_merge, bio)) {
		ret = blk_try_merge(q->last_merge, bio);
		if (ret != ELEVATOR_NO_MERGE) {
			*req = q->last_merge;
			return ret;
		}
	}

	if (blk_queue_noxmerges(q))
		return ELEVATOR_NO_MERGE;

	/*
         * See if our hash lookup can find a potential backmerge.
         */
	__rq = elv_rqhash_find(q, bio->bi_iter.bi_sector);
	if (__rq && elv_rq_merge_ok(__rq, bio)) {
		*req = __rq;
		return ELEVATOR_BACK_MERGE;
	}

	if (e->type->ops.elevator_merge_fn)
		return e->type->ops.elevator_merge_fn(q, req, bio);

	return ELEVATOR_NO_MERGE;
}

Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 76 43.18% 5 38.46%
tejun heo tejun heo 55 31.25% 3 23.08%
alan d. brunelle alan d. brunelle 21 11.93% 2 15.38%
linus torvalds linus torvalds 16 9.09% 1 7.69%
ming lei ming lei 6 3.41% 1 7.69%
kent overstreet kent overstreet 2 1.14% 1 7.69%
Total 176 100.00% 13 100.00%

/*
 * Attempt to do an insertion back merge. Only check for the case where
 * we can append 'rq' to an existing request, so we can throw 'rq' away
 * afterwards.
 *
 * Returns true if we merged, false otherwise
 */


static bool elv_attempt_insert_merge(struct request_queue *q,
				     struct request *rq)
{
	struct request *__rq;
	bool ret;

	if (blk_queue_nomerges(q))
		return false;

	/*
         * First try one-hit cache.
         */
	if (q->last_merge && blk_attempt_req_merge(q, q->last_merge, rq))
		return true;

	if (blk_queue_noxmerges(q))
		return false;

	ret = false;
	/*
         * See if our hash lookup can find a potential backmerge.
         */
	while (1) {
		__rq = elv_rqhash_find(q, blk_rq_pos(rq));
		if (!__rq || !blk_attempt_req_merge(q, __rq, rq))
			break;

		/* The merged request could be merged with others, try again */
		ret = true;
		rq = __rq;
	}

	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 90 77.59% 1 50.00%
shaohua li shaohua li 26 22.41% 1 50.00%
Total 116 100.00% 2 100.00%



void elv_merged_request(struct request_queue *q, struct request *rq, int type)
{
	struct elevator_queue *e = q->elevator;

	if (e->type->ops.elevator_merged_fn)
		e->type->ops.elevator_merged_fn(q, rq, type);

	if (type == ELEVATOR_BACK_MERGE)
		elv_rqhash_reposition(q, rq);

	q->last_merge = rq;
}

Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 59 83.10% 6 75.00%
tejun heo tejun heo 12 16.90% 2 25.00%
Total 71 100.00% 8 100.00%



void elv_merge_requests(struct request_queue *q, struct request *rq,
			     struct request *next)
{
	struct elevator_queue *e = q->elevator;
	const int next_sorted = next->cmd_flags & REQ_SORTED;

	if (next_sorted && e->type->ops.elevator_merge_req_fn)
		e->type->ops.elevator_merge_req_fn(q, rq, next);

	elv_rqhash_reposition(q, rq);

	if (next_sorted) {
		elv_rqhash_del(q, next);
		q->nr_sorted--;
	}

	q->last_merge = rq;
}

Contributors
Person Tokens Prop Commits CommitProp
linus torvalds linus torvalds 40 41.24% 1 10.00%
jens axboe jens axboe 40 41.24% 6 60.00%
tejun heo tejun heo 17 17.53% 3 30.00%
Total 97 100.00% 10 100.00%



void elv_bio_merged(struct request_queue *q, struct request *rq,
			struct bio *bio)
{
	struct elevator_queue *e = q->elevator;

	if (e->type->ops.elevator_bio_merged_fn)
		e->type->ops.elevator_bio_merged_fn(q, rq, bio);
}

Contributors
Person Tokens Prop Commits CommitProp
divyesh shah divyesh shah 48 88.89% 1 50.00%
tejun heo tejun heo 6 11.11% 1 50.00%
Total 54 100.00% 2 100.00%

#ifdef CONFIG_PM


static void blk_pm_requeue_request(struct request *rq)
{
	if (rq->q->dev && !(rq->cmd_flags & REQ_PM))
		rq->q->nr_pending--;
}

Contributors
Person Tokens Prop Commits CommitProp
lin ming lin ming 35 100.00% 1 100.00%
Total 35 100.00% 1 100.00%



static void blk_pm_add_request(struct request_queue *q, struct request *rq)
{
	if (q->dev && !(rq->cmd_flags & REQ_PM) && q->nr_pending++ == 0 &&
	    (q->rpm_status == RPM_SUSPENDED || q->rpm_status == RPM_SUSPENDING))
		pm_request_resume(q->dev);
}
Contributors
Person Tokens Prop Commits CommitProp
lin ming lin ming 59 100.00% 1 100.00%
Total 59 100.00% 1 100.00%

#else


static inline void blk_pm_requeue_request(struct request *rq) {}
Contributors
Person Tokens Prop Commits CommitProp
lin ming lin ming 11 100.00% 1 100.00%
Total 11 100.00% 1 100.00%



static inline void blk_pm_add_request(struct request_queue *q,
				      struct request *rq)
{
}
Contributors
Person Tokens Prop Commits CommitProp
lin ming lin ming 16 100.00% 1 100.00%
Total 16 100.00% 1 100.00%

#endif



void elv_requeue_request(struct request_queue *q, struct request *rq)
{
	/*
         * it already went through dequeue, we need to decrement the
         * in_flight count again
         */
	if (blk_account_rq(rq)) {
		q->in_flight[rq_is_sync(rq)]--;
		if (rq->cmd_flags & REQ_SORTED)
			elv_deactivate_rq(q, rq);
	}

	rq->cmd_flags &= ~REQ_STARTED;

	blk_pm_requeue_request(rq);

	__elv_add_request(q, rq, ELEVATOR_INSERT_REQUEUE);
}

Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 42 58.33% 7 58.33%
andrew morton andrew morton 13 18.06% 1 8.33%
tejun heo tejun heo 8 11.11% 2 16.67%
lin ming lin ming 5 6.94% 1 8.33%
christoph hellwig christoph hellwig 4 5.56% 1 8.33%
Total 72 100.00% 12 100.00%



void elv_drain_elevator(struct request_queue *q)
{
	static int printed;

	lockdep_assert_held(q->queue_lock);

	while (q->elevator->type->ops.elevator_dispatch_fn(q, 1))
		;
	if (q->nr_sorted && printed++ < 10) {
		printk(KERN_ERR "%s: forced dispatching is broken "
		       "(nr_sorted=%u), please report this\n",
		       q->elevator->type->elevator_name, q->nr_sorted);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
tejun heo tejun heo 69 97.18% 3 75.00%
jens axboe jens axboe 2 2.82% 1 25.00%
Total 71 100.00% 4 100.00%



void __elv_add_request(struct request_queue *q, struct request *rq, int where)
{
	trace_block_rq_insert(q, rq);

	blk_pm_add_request(q, rq);

	rq->q = q;

	if (rq->cmd_flags & REQ_SOFTBARRIER) {
		/* barriers are scheduling boundary, update end_sector */
		if (rq->cmd_type == REQ_TYPE_FS) {
			q->end_sector = rq_end_sector(rq);
			q->boundary_rq = rq;
		}
	} else if (!(rq->cmd_flags & REQ_ELVPRIV) &&
		    (where == ELEVATOR_INSERT_SORT ||
		     where == ELEVATOR_INSERT_SORT_MERGE))
		where = ELEVATOR_INSERT_BACK;

	switch (where) {
	case ELEVATOR_INSERT_REQUEUE:
	case ELEVATOR_INSERT_FRONT:
		rq->cmd_flags |= REQ_SOFTBARRIER;
		list_add(&rq->queuelist, &q->queue_head);
		break;

	case ELEVATOR_INSERT_BACK:
		rq->cmd_flags |= REQ_SOFTBARRIER;
		elv_drain_elevator(q);
		list_add_tail(&rq->queuelist, &q->queue_head);
		/*
                 * We kick the queue here for the following reasons.
                 * - The elevator might have returned NULL previously
                 *   to delay requests and returned them now.  As the
                 *   queue wasn't empty before this request, ll_rw_blk
                 *   won't run the queue on return, resulting in hang.
                 * - Usually, back inserted requests won't be merged
                 *   with anything.  There's no point in delaying queue
                 *   processing.
                 */
		__blk_run_queue(q);
		break;

	case ELEVATOR_INSERT_SORT_MERGE:
		/*
                 * If we succeed in merging this request with one in the
                 * queue already, we are done - rq has now been freed,
                 * so no need to do anything further.
                 */
		if (elv_attempt_insert_merge(q, rq))
			break;
	case ELEVATOR_INSERT_SORT:
		BUG_ON(rq->cmd_type != REQ_TYPE_FS);
		rq->cmd_flags |= REQ_SORTED;
		q->nr_sorted++;
		if (rq_mergeable(rq)) {
			elv_rqhash_add(q, rq);
			if (!q->last_merge)
				q->last_merge = rq;
		}

		/*
                 * Some ioscheds (cfq) run q->request_fn directly, so
                 * rq cannot be accessed after calling
                 * elevator_add_req_fn.
                 */
		q->elevator->type->ops.elevator_add_req_fn(q, rq);
		break;

	case ELEVATOR_INSERT_FLUSH:
		rq->cmd_flags |= REQ_SOFTBARRIER;
		blk_insert_flush(rq);
		break;
	default:
		printk(KERN_ERR "%s: bad insertion point %d\n",
		       __func__, where);
		BUG();
	}
}
Contributors
Person Tokens Prop Commits CommitProp
tejun heo tejun heo 129 46.40% 9 36.00%
jens axboe jens axboe 120 43.17% 10 40.00%
linus torvalds linus torvalds 15 5.40% 1 4.00%
lin ming lin ming 7 2.52% 1 4.00%
christoph hellwig christoph hellwig 4 1.44% 1 4.00%
harvey harrison harvey harrison 1 0.36% 1 4.00%
arnaldo carvalho de melo arnaldo carvalho de melo 1 0.36% 1 4.00%
andrew morton andrew morton 1 0.36% 1 4.00%
Total 278 100.00% 25 100.00%


EXPORT_SYMBOL(__elv_add_request);



void elv_add_request(struct request_queue *q, struct request *rq, int where)
{
	unsigned long flags;

	spin_lock_irqsave(q->queue_lock, flags);
	__elv_add_request(q, rq, where);
	spin_unlock_irqrestore(q->queue_lock, flags);
}
Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 46 93.88% 3 75.00%
linus torvalds linus torvalds 3 6.12% 1 25.00%
Total 49 100.00% 4 100.00%


EXPORT_SYMBOL(elv_add_request);



struct request *elv_latter_request(struct request_queue *q, struct request *rq)
{
	struct elevator_queue *e = q->elevator;

	if (e->type->ops.elevator_latter_req_fn)
		return e->type->ops.elevator_latter_req_fn(q, rq);
	return NULL;
}

Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 47 88.68% 6 85.71%
tejun heo tejun heo 6 11.32% 1 14.29%
Total 53 100.00% 7 100.00%



struct request *elv_former_request(struct request_queue *q, struct request *rq)
{
	struct elevator_queue *e = q->elevator;

	if (e->type->ops.elevator_former_req_fn)
		return e->type->ops.elevator_former_req_fn(q, rq);
	return NULL;
}

Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 47 88.68% 6 85.71%
tejun heo tejun heo 6 11.32% 1 14.29%
Total 53 100.00% 7 100.00%



int elv_set_request(struct request_queue *q, struct request *rq,
		    struct bio *bio, gfp_t gfp_mask)
{
	struct elevator_queue *e = q->elevator;

	if (e->type->ops.elevator_set_req_fn)
		return e->type->ops.elevator_set_req_fn(q, rq, bio, gfp_mask);
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 49 77.78% 4 57.14%
tejun heo tejun heo 13 20.63% 2 28.57%
al viro al viro 1 1.59% 1 14.29%
Total 63 100.00% 7 100.00%



void elv_put_request(struct request_queue *q, struct request *rq)
{
	struct elevator_queue *e = q->elevator;

	if (e->type->ops.elevator_put_req_fn)
		e->type->ops.elevator_put_req_fn(rq);
}

Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 39 86.67% 4 80.00%
tejun heo tejun heo 6 13.33% 1 20.00%
Total 45 100.00% 5 100.00%



int elv_may_queue(struct request_queue *q, int rw)
{
	struct elevator_queue *e = q->elevator;

	if (e->type->ops.elevator_may_queue_fn)
		return e->type->ops.elevator_may_queue_fn(q, rw);

	return ELV_MQUEUE_MAY;
}

Contributors
Person Tokens Prop Commits CommitProp
andrew morton andrew morton 36 73.47% 1 16.67%
jens axboe jens axboe 7 14.29% 4 66.67%
tejun heo tejun heo 6 12.24% 1 16.67%
Total 49 100.00% 6 100.00%



void elv_completed_request(struct request_queue *q, struct request *rq)
{
	struct elevator_queue *e = q->elevator;

	/*
         * request is released from the driver, io must be done
         */
	if (blk_account_rq(rq)) {
		q->in_flight[rq_is_sync(rq)]--;
		if ((rq->cmd_flags & REQ_SORTED) &&
		    e->type->ops.elevator_completed_req_fn)
			e->type->ops.elevator_completed_req_fn(q, rq);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
tejun heo tejun heo 33 43.42% 3 33.33%
andrew morton andrew morton 27 35.53% 2 22.22%
jens axboe jens axboe 10 13.16% 3 33.33%
christoph hellwig christoph hellwig 6 7.89% 1 11.11%
Total 76 100.00% 9 100.00%


#define to_elv(atr) container_of((atr), struct elv_fs_entry, attr)


static ssize_t

elv_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
{
	struct elv_fs_entry *entry = to_elv(attr);
	struct elevator_queue *e;
	ssize_t error;

	if (!entry->show)
		return -EIO;

	e = container_of(kobj, struct elevator_queue, kobj);
	mutex_lock(&e->sysfs_lock);
	error = e->type ? entry->show(e, page) : -ENOENT;
	mutex_unlock(&e->sysfs_lock);
	return error;
}

Contributors
Person Tokens Prop Commits CommitProp
al viro al viro 69 70.41% 1 25.00%
jens axboe jens axboe 28 28.57% 2 50.00%
tejun heo tejun heo 1 1.02% 1 25.00%
Total 98 100.00% 4 100.00%


static ssize_t

elv_attr_store(struct kobject *kobj, struct attribute *attr,
	       const char *page, size_t length)
{
	struct elv_fs_entry *entry = to_elv(attr);
	struct elevator_queue *e;
	ssize_t error;

	if (!entry->store)
		return -EIO;

	e = container_of(kobj, struct elevator_queue, kobj);
	mutex_lock(&e->sysfs_lock);
	error = e->type ? entry->store(e, page, length) : -ENOENT;
	mutex_unlock(&e->sysfs_lock);
	return error;
}

Contributors
Person Tokens Prop Commits CommitProp
al viro al viro 71 68.27% 1 25.00%
jens axboe jens axboe 32 30.77% 2 50.00%
tejun heo tejun heo 1 0.96% 1 25.00%
Total 104 100.00% 4 100.00%


static const struct sysfs_ops elv_sysfs_ops = {
	.show	= elv_attr_show,
	.store	= elv_attr_store,
};


static struct kobj_type elv_ktype = {
	.sysfs_ops	= &elv_sysfs_ops,
	.release	= elevator_release,
};



int elv_register_queue(struct request_queue *q)
{
	struct elevator_queue *e = q->elevator;
	int error;

	error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");
	if (!error) {
		struct elv_fs_entry *attr = e->type->elevator_attrs;
		if (attr) {
			while (attr->attr.name) {
				if (sysfs_create_file(&e->kobj, &attr->attr))
					break;
				attr++;
			}
		}
		kobject_uevent(&e->kobj, KOBJ_ADD);
		e->registered = 1;
		if (e->type->ops.elevator_registered_fn)
			e->type->ops.elevator_registered_fn(q);
	}
	return error;
}
Contributors
Person Tokens Prop Commits CommitProp
al viro al viro 75 55.97% 2 18.18%
jens axboe jens axboe 40 29.85% 5 45.45%
greg kroah-hartman greg kroah-hartman 10 7.46% 2 18.18%
tejun heo tejun heo 9 6.72% 2 18.18%
Total 134 100.00% 11 100.00%


EXPORT_SYMBOL(elv_register_queue);



void elv_unregister_queue(struct request_queue *q)
{
	if (q) {
		struct elevator_queue *e = q->elevator;

		kobject_uevent(&e->kobj, KOBJ_REMOVE);
		kobject_del(&e->kobj);
		e->registered = 0;
	}
}
Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 26 53.06% 4 57.14%
tejun heo tejun heo 17 34.69% 1 14.29%
al viro al viro 5 10.20% 1 14.29%
patrick mochel patrick mochel 1 2.04% 1 14.29%
Total 49 100.00% 7 100.00%


EXPORT_SYMBOL(elv_unregister_queue);



int elv_register(struct elevator_type *e)
{
	char *def = "";

	/* create icq_cache if requested */
	if (e->icq_size) {
		if (WARN_ON(e->icq_size < sizeof(struct io_cq)) ||
		    WARN_ON(e->icq_align < __alignof__(struct io_cq)))
			return -EINVAL;

		snprintf(e->icq_cache_name, sizeof(e->icq_cache_name),
			 "%s_io_cq", e->elevator_name);
		e->icq_cache = kmem_cache_create(e->icq_cache_name, e->icq_size,
						 e->icq_align, 0, NULL);
		if (!e->icq_cache)
			return -ENOMEM;
	}

	/* register, don't allow duplicate names */
	spin_lock(&elv_list_lock);
	if (elevator_find(e->elevator_name)) {
		spin_unlock(&elv_list_lock);
		if (e->icq_cache)
			kmem_cache_destroy(e->icq_cache);
		return -EBUSY;
	}
	list_add_tail(&e->list, &elv_list);
	spin_unlock(&elv_list_lock);

	/* print pretty message */
	if (!strcmp(e->elevator_name, chosen_elevator) ||
			(!*chosen_elevator &&
			 !strcmp(e->elevator_name, CONFIG_DEFAULT_IOSCHED)))
				def = " (default)";

	printk(KERN_INFO "io scheduler %s registered%s\n", e->elevator_name,
								def);
	return 0;
}
Contributors
Person Tokens Prop Commits CommitProp
tejun heo tejun heo 132 60.83% 2 25.00%
jens axboe jens axboe 52 23.96% 3 37.50%
nate diller* nate diller* 16 7.37% 1 12.50%
thibaut varene thibaut varene 16 7.37% 1 12.50%
eric sesterhenn eric sesterhenn 1 0.46% 1 12.50%
Total 217 100.00% 8 100.00%


EXPORT_SYMBOL_GPL(elv_register);



void elv_unregister(struct elevator_type *e)
{
	/* unregister */
	spin_lock(&elv_list_lock);
	list_del_init(&e->list);
	spin_unlock(&elv_list_lock);

	/*
         * Destroy icq_cache if it exists.  icq's are RCU managed.  Make
         * sure all RCU operations are complete before proceeding.
         */
	if (e->icq_cache) {
		rcu_barrier();
		kmem_cache_destroy(e->icq_cache);
		e->icq_cache = NULL;
	}
}
Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 30 53.57% 2 66.67%
tejun heo tejun heo 26 46.43% 1 33.33%
Total 56 100.00% 3 100.00%


EXPORT_SYMBOL_GPL(elv_unregister);

/*
 * switch to new_e io scheduler. be careful not to introduce deadlocks -
 * we don't free the old io scheduler, before we have allocated what we
 * need for the new one. this way we have a chance of going back to the old
 * one, if the new one fails init for some reason.
 */


static int elevator_switch(struct request_queue *q, struct elevator_type *new_e)
{
	struct elevator_queue *old = q->elevator;
	bool registered = old->registered;
	int err;

	/*
         * Turn on BYPASS and drain all requests w/ elevator private data.
         * Block layer doesn't call into a quiesced elevator - all requests
         * are directly put on the dispatch list without elevator data
         * using INSERT_BACK.  All requests have SOFTBARRIER set and no
         * merge happens either.
         */
	blk_queue_bypass_start(q);

	/* unregister and clear all auxiliary data of the old elevator */
	if (registered)
		elv_unregister_queue(q);

	spin_lock_irq(q->queue_lock);
	ioc_clear_queue(q);
	spin_unlock_irq(q->queue_lock);

	/* allocate, init and register new elevator */
	err = new_e->ops.elevator_init_fn(q, new_e);
	if (err)
		goto fail_init;

	if (registered) {
		err = elv_register_queue(q);
		if (err)
			goto fail_register;
	}

	/* done, kill the old one and finish */
	elevator_exit(old);
	blk_queue_bypass_end(q);

	blk_add_trace_msg(q, "elv switch: %s", new_e->elevator_name);

	return 0;

fail_register:
	elevator_exit(q->elevator);
fail_init:
	/* switch failed, restore and re-register old elevator */
	q->elevator = old;
	elv_register_queue(q);
	blk_queue_bypass_end(q);

	return err;
}

Contributors
Person Tokens Prop Commits CommitProp
tejun heo tejun heo 93 55.69% 5 35.71%
jens axboe jens axboe 53 31.74% 5 35.71%
alan d. brunelle alan d. brunelle 10 5.99% 1 7.14%
al viro al viro 5 2.99% 1 7.14%
nick piggin nick piggin 4 2.40% 1 7.14%
ma jianpeng ma jianpeng 2 1.20% 1 7.14%
Total 167 100.00% 14 100.00%

/*
 * Switch this queue to the given IO scheduler.
 */


static int __elevator_change(struct request_queue *q, const char *name)
{
	char elevator_name[ELV_NAME_MAX];
	struct elevator_type *e;

	if (!q->elevator)
		return -ENXIO;

	strlcpy(elevator_name, name, sizeof(elevator_name));
	e = elevator_get(strstrip(elevator_name), true);
	if (!e) {
		printk(KERN_ERR "elevator: type %s not found\n", elevator_name);
		return -EINVAL;
	}

	if (!strcmp(elevator_name, q->elevator->type->elevator_name)) {
		elevator_put(e);
		return 0;
	}

	return elevator_switch(q, e);
}

Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 89 77.39% 4 36.36%
martin k. petersen martin k. petersen 9 7.83% 1 9.09%
nate diller* nate diller* 8 6.96% 1 9.09%
tejun heo tejun heo 3 2.61% 2 18.18%
kosaki motohiro kosaki motohiro 3 2.61% 1 9.09%
tomoki sekiyama tomoki sekiyama 2 1.74% 1 9.09%
li zefan li zefan 1 0.87% 1 9.09%
Total 115 100.00% 11 100.00%



int elevator_change(struct request_queue *q, const char *name)
{
	int ret;

	/* Protect q->elevator from elevator_init() */
	mutex_lock(&q->sysfs_lock);
	ret = __elevator_change(q, name);
	mutex_unlock(&q->sysfs_lock);

	return ret;
}
Contributors
Person Tokens Prop Commits CommitProp
tomoki sekiyama tomoki sekiyama 47 100.00% 1 100.00%
Total 47 100.00% 1 100.00%


EXPORT_SYMBOL(elevator_change);



ssize_t elv_iosched_store(struct request_queue *q, const char *name,
			  size_t count)
{
	int ret;

	if (!q->elevator)
		return count;

	ret = __elevator_change(q, name);
	if (!ret)
		return count;

	printk(KERN_ERR "elevator: switch to %s failed\n", name);
	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 48 81.36% 2 50.00%
al viro al viro 10 16.95% 1 25.00%
tomoki sekiyama tomoki sekiyama 1 1.69% 1 25.00%
Total 59 100.00% 4 100.00%



ssize_t elv_iosched_show(struct request_queue *q, char *name)
{
	struct elevator_queue *e = q->elevator;
	struct elevator_type *elv;
	struct elevator_type *__e;
	int len = 0;

	if (!q->elevator || !blk_queue_stackable(q))
		return sprintf(name, "none\n");

	elv = e->type;

	spin_lock(&elv_list_lock);
	list_for_each_entry(__e, &elv_list, list) {
		if (!strcmp(elv->elevator_name, __e->elevator_name))
			len += sprintf(name+len, "[%s] ", elv->elevator_name);
		else
			len += sprintf(name+len, "%s ", __e->elevator_name);
	}
	spin_unlock(&elv_list_lock);

	len += sprintf(len+name, "\n");
	return len;
}

Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 109 74.66% 4 44.44%
martin k. petersen martin k. petersen 20 13.70% 1 11.11%
mike snitzer mike snitzer 6 4.11% 1 11.11%
matthias kaehlcke matthias kaehlcke 6 4.11% 1 11.11%
vasily tarasov vasily tarasov 4 2.74% 1 11.11%
tejun heo tejun heo 1 0.68% 1 11.11%
Total 146 100.00% 9 100.00%



struct request *elv_rb_former_request(struct request_queue *q,
				      struct request *rq)
{
	struct rb_node *rbprev = rb_prev(&rq->rb_node);

	if (rbprev)
		return rb_entry_rq(rbprev);

	return NULL;
}
Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 39 90.70% 2 66.67%
linus torvalds linus torvalds 4 9.30% 1 33.33%
Total 43 100.00% 3 100.00%


EXPORT_SYMBOL(elv_rb_former_request);



struct request *elv_rb_latter_request(struct request_queue *q,
				      struct request *rq)
{
	struct rb_node *rbnext = rb_next(&rq->rb_node);

	if (rbnext)
		return rb_entry_rq(rbnext);

	return NULL;
}
Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 41 95.35% 3 75.00%
martin schwidefsky martin schwidefsky 2 4.65% 1 25.00%
Total 43 100.00% 4 100.00%


EXPORT_SYMBOL(elv_rb_latter_request);
Overall Contributors
Person Tokens Prop Commits CommitProp
jens axboe jens axboe 2273 51.46% 41 30.83%
tejun heo tejun heo 937 21.21% 25 18.80%
al viro al viro 375 8.49% 3 2.26%
lin ming lin ming 142 3.21% 1 0.75%
linus torvalds linus torvalds 136 3.08% 5 3.76%
andrew morton andrew morton 80 1.81% 5 3.76%
nate diller* nate diller* 77 1.74% 3 2.26%
tomoki sekiyama tomoki sekiyama 59 1.34% 2 1.50%
divyesh shah divyesh shah 48 1.09% 1 0.75%
martin k. petersen martin k. petersen 33 0.75% 2 1.50%
alan d. brunelle alan d. brunelle 31 0.70% 3 2.26%
mike snitzer mike snitzer 28 0.63% 3 2.26%
christoph hellwig christoph hellwig 26 0.59% 1 0.75%
shaohua li shaohua li 26 0.59% 1 0.75%
thibaut varene thibaut varene 16 0.36% 1 0.75%
greg kroah-hartman greg kroah-hartman 13 0.29% 4 3.01%
pre-git pre-git 13 0.29% 1 0.75%
sasha levin sasha levin 11 0.25% 1 0.75%
matthias kaehlcke matthias kaehlcke 10 0.23% 1 0.75%
sudip mukherjee sudip mukherjee 10 0.23% 1 0.75%
ma jianpeng ma jianpeng 9 0.20% 1 0.75%
vasily tarasov vasily tarasov 8 0.18% 2 1.50%
david woodhouse david woodhouse 7 0.16% 1 0.75%
martin schwidefsky martin schwidefsky 6 0.14% 1 0.75%
ming lei ming lei 6 0.14% 1 0.75%
nick piggin nick piggin 4 0.09% 1 0.75%
thomas gleixner thomas gleixner 4 0.09% 1 0.75%
chao yu chao yu 3 0.07% 1 0.75%
arnaldo carvalho de melo arnaldo carvalho de melo 3 0.07% 1 0.75%
kosaki motohiro kosaki motohiro 3 0.07% 1 0.75%
li zefan li zefan 3 0.07% 2 1.50%
jeff moyer jeff moyer 2 0.05% 1 0.75%
kent overstreet kent overstreet 2 0.05% 1 0.75%
patrick mochel patrick mochel 1 0.02% 1 0.75%
uwe kleine-koenig uwe kleine-koenig 1 0.02% 1 0.75%
wzt wzt wzt wzt 1 0.02% 1 0.75%
chuck ebbert chuck ebbert 1 0.02% 1 0.75%
kees cook kees cook 1 0.02% 1 0.75%
adrian bunk adrian bunk 1 0.02% 1 0.75%
emese revfy emese revfy 1 0.02% 1 0.75%
hirofumi ogawa hirofumi ogawa 1 0.02% 1 0.75%
harvey harrison harvey harrison 1 0.02% 1 0.75%
eric sesterhenn eric sesterhenn 1 0.02% 1 0.75%
joe perches joe perches 1 0.02% 1 0.75%
wang sheng-hui wang sheng-hui 1 0.02% 1 0.75%
rafael j. wysocki rafael j. wysocki 1 0.02% 1 0.75%
Total 4417 100.00% 133 100.00%
  Directory: block

Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
	Person	Tokens	Prop	Commits	CommitProp
jens axboe	jens axboe	57	90.48%	3	75.00%
tejun heo	tejun heo	6	9.52%	1	25.00%
	Total	63	100.00%	4	100.00%
cregit-Linux how code gets into the kernel

Release 4.7 block/elevator.c

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Overall Contributors
