Release 4.12 block/bfq-cgroup.c

Directory: block
/*
 * cgroups support for the BFQ I/O scheduler.
 *
 *  This program is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU General Public License as
 *  published by the Free Software Foundation; either version 2 of the
 *  License, or (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  General Public License for more details.
 */
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/blkdev.h>
#include <linux/cgroup.h>
#include <linux/elevator.h>
#include <linux/ktime.h>
#include <linux/rbtree.h>
#include <linux/ioprio.h>
#include <linux/sbitmap.h>
#include <linux/delay.h>

#include "bfq-iosched.h"

#ifdef CONFIG_BFQ_GROUP_IOSCHED

/* bfqg stats flags */

enum bfqg_stats_flags {
	
BFQG_stats_waiting = 0,
	
BFQG_stats_idling,
	
BFQG_stats_empty,
};


#define BFQG_FLAG_FNS(name)						\
static void bfqg_stats_mark_##name(struct bfqg_stats *stats)    \
{                                                                       \
        stats->flags |= (1 << BFQG_stats_##name);                       \
}                                                                       \
static void bfqg_stats_clear_##name(struct bfqg_stats *stats)   \
{                                                                       \
        stats->flags &= ~(1 << BFQG_stats_##name);                      \
}                                                                       \
static int bfqg_stats_##name(struct bfqg_stats *stats)          \
{                                                                       \
        return (stats->flags & (1 << BFQG_stats_##name)) != 0;          \
}                                                                       \

BFQG_FLAG_FNS(waiting)
BFQG_FLAG_FNS(idling)
BFQG_FLAG_FNS(empty)

#undef BFQG_FLAG_FNS

/* This should be called with the scheduler lock held. */


static void bfqg_stats_update_group_wait_time(struct bfqg_stats *stats)
{
	unsigned long long now;

	if (!bfqg_stats_waiting(stats))
		return;

	now = sched_clock();
	if (time_after64(now, stats->start_group_wait_time))
		blkg_stat_add(&stats->group_wait_time,
			      now - stats->start_group_wait_time);
	bfqg_stats_clear_waiting(stats);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 60 100.00% 1 100.00%
Total 60 100.00% 1 100.00%

/* This should be called with the scheduler lock held. */


static void bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg,
						 struct bfq_group *curr_bfqg)
{
	struct bfqg_stats *stats = &bfqg->stats;

	if (bfqg_stats_waiting(stats))
		return;
	if (bfqg == curr_bfqg)
		return;
	stats->start_group_wait_time = sched_clock();
	bfqg_stats_mark_waiting(stats);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 53 100.00% 1 100.00%
Total 53 100.00% 1 100.00%

/* This should be called with the scheduler lock held. */


static void bfqg_stats_end_empty_time(struct bfqg_stats *stats)
{
	unsigned long long now;

	if (!bfqg_stats_empty(stats))
		return;

	now = sched_clock();
	if (time_after64(now, stats->start_empty_time))
		blkg_stat_add(&stats->empty_time,
			      now - stats->start_empty_time);
	bfqg_stats_clear_empty(stats);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 60 100.00% 1 100.00%
Total 60 100.00% 1 100.00%



void bfqg_stats_update_dequeue(struct bfq_group *bfqg)
{
	blkg_stat_add(&bfqg->stats.dequeue, 1);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 22 100.00% 1 100.00%
Total 22 100.00% 1 100.00%



void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg)
{
	struct bfqg_stats *stats = &bfqg->stats;

	if (blkg_rwstat_total(&stats->queued))
		return;

	/*
         * group is already marked empty. This can happen if bfqq got new
         * request in parent group and moved to this group while being added
         * to service tree. Just ignore the event and move on.
         */
	if (bfqg_stats_empty(stats))
		return;

	stats->start_empty_time = sched_clock();
	bfqg_stats_mark_empty(stats);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 52 100.00% 1 100.00%
Total 52 100.00% 1 100.00%



void bfqg_stats_update_idle_time(struct bfq_group *bfqg)
{
	struct bfqg_stats *stats = &bfqg->stats;

	if (bfqg_stats_idling(stats)) {
		unsigned long long now = sched_clock();

		if (time_after64(now, stats->start_idle_time))
			blkg_stat_add(&stats->idle_time,
				      now - stats->start_idle_time);
		bfqg_stats_clear_idling(stats);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 67 100.00% 1 100.00%
Total 67 100.00% 1 100.00%



void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg)
{
	struct bfqg_stats *stats = &bfqg->stats;

	stats->start_idle_time = sched_clock();
	bfqg_stats_mark_idling(stats);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 32 100.00% 1 100.00%
Total 32 100.00% 1 100.00%



void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg)
{
	struct bfqg_stats *stats = &bfqg->stats;

	blkg_stat_add(&stats->avg_queue_size_sum,
		      blkg_rwstat_total(&stats->queued));
	blkg_stat_add(&stats->avg_queue_size_samples, 1);
	bfqg_stats_update_group_wait_time(stats);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 51 100.00% 1 100.00%
Total 51 100.00% 1 100.00%

/*
 * blk-cgroup policy-related handlers
 * The following functions help in converting between blk-cgroup
 * internal structures and BFQ-specific structures.
 */



static struct bfq_group *pd_to_bfqg(struct blkg_policy_data *pd)
{
	return pd ? container_of(pd, struct bfq_group, pd) : NULL;
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 28 100.00% 1 100.00%
Total 28 100.00% 1 100.00%



struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg)
{
	return pd_to_blkg(&bfqg->pd);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 21 100.00% 1 100.00%
Total 21 100.00% 1 100.00%



static struct bfq_group *blkg_to_bfqg(struct blkcg_gq *blkg)
{
	return pd_to_bfqg(blkg_to_pd(blkg, &blkcg_policy_bfq));
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 25 100.00% 1 100.00%
Total 25 100.00% 1 100.00%

/*
 * bfq_group handlers
 * The following functions help in navigating the bfq_group hierarchy
 * by allowing to find the parent of a bfq_group or the bfq_group
 * associated to a bfq_queue.
 */



static struct bfq_group *bfqg_parent(struct bfq_group *bfqg)
{
	struct blkcg_gq *pblkg = bfqg_to_blkg(bfqg)->parent;

	return pblkg ? blkg_to_bfqg(pblkg) : NULL;
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 35 100.00% 1 100.00%
Total 35 100.00% 1 100.00%



struct bfq_group *bfqq_group(struct bfq_queue *bfqq)
{
	struct bfq_entity *group_entity = bfqq->entity.parent;

	return group_entity ? container_of(group_entity, struct bfq_group,
					   entity) :
			      bfqq->bfqd->root_group;
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 42 100.00% 1 100.00%
Total 42 100.00% 1 100.00%

/*
 * The following two functions handle get and put of a bfq_group by
 * wrapping the related blk-cgroup hooks.
 */



static void bfqg_get(struct bfq_group *bfqg)
{
	bfqg->ref++;
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 16 100.00% 2 100.00%
Total 16 100.00% 2 100.00%



void bfqg_put(struct bfq_group *bfqg)
{
	bfqg->ref--;

	if (bfqg->ref == 0)
		kfree(bfqg);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 28 100.00% 1 100.00%
Total 28 100.00% 1 100.00%



static void bfqg_and_blkg_get(struct bfq_group *bfqg)
{
	/* see comments in bfq_bic_update_cgroup for why refcounting bfqg */
	bfqg_get(bfqg);

	blkg_get(bfqg_to_blkg(bfqg));
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 25 100.00% 2 100.00%
Total 25 100.00% 2 100.00%



void bfqg_and_blkg_put(struct bfq_group *bfqg)
{
	bfqg_put(bfqg);

	blkg_put(bfqg_to_blkg(bfqg));
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 23 100.00% 2 100.00%
Total 23 100.00% 2 100.00%



void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,
			      unsigned int op)
{
	blkg_rwstat_add(&bfqg->stats.queued, op, 1);
	bfqg_stats_end_empty_time(&bfqg->stats);
	if (!(bfqq == ((struct bfq_data *)bfqg->bfqd)->in_service_queue))
		bfqg_stats_set_start_group_wait_time(bfqg, bfqq_group(bfqq));
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 71 100.00% 1 100.00%
Total 71 100.00% 1 100.00%



void bfqg_stats_update_io_remove(struct bfq_group *bfqg, unsigned int op)
{
	blkg_rwstat_add(&bfqg->stats.queued, op, -1);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 29 100.00% 1 100.00%
Total 29 100.00% 1 100.00%



void bfqg_stats_update_io_merged(struct bfq_group *bfqg, unsigned int op)
{
	blkg_rwstat_add(&bfqg->stats.merged, op, 1);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 28 100.00% 1 100.00%
Total 28 100.00% 1 100.00%



void bfqg_stats_update_completion(struct bfq_group *bfqg, uint64_t start_time,
				  uint64_t io_start_time, unsigned int op)
{
	struct bfqg_stats *stats = &bfqg->stats;
	unsigned long long now = sched_clock();

	if (time_after64(now, io_start_time))
		blkg_rwstat_add(&stats->service_time, op,
				now - io_start_time);
	if (time_after64(io_start_time, start_time))
		blkg_rwstat_add(&stats->wait_time, op,
				io_start_time - start_time);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 84 100.00% 1 100.00%
Total 84 100.00% 1 100.00%

/* @stats = 0 */


static void bfqg_stats_reset(struct bfqg_stats *stats)
{
	/* queued stats shouldn't be cleared */
	blkg_rwstat_reset(&stats->merged);
	blkg_rwstat_reset(&stats->service_time);
	blkg_rwstat_reset(&stats->wait_time);
	blkg_stat_reset(&stats->time);
	blkg_stat_reset(&stats->avg_queue_size_sum);
	blkg_stat_reset(&stats->avg_queue_size_samples);
	blkg_stat_reset(&stats->dequeue);
	blkg_stat_reset(&stats->group_wait_time);
	blkg_stat_reset(&stats->idle_time);
	blkg_stat_reset(&stats->empty_time);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 92 100.00% 1 100.00%
Total 92 100.00% 1 100.00%

/* @to += @from */


static void bfqg_stats_add_aux(struct bfqg_stats *to, struct bfqg_stats *from)
{
	if (!to || !from)
		return;

	/* queued stats shouldn't be cleared */
	blkg_rwstat_add_aux(&to->merged, &from->merged);
	blkg_rwstat_add_aux(&to->service_time, &from->service_time);
	blkg_rwstat_add_aux(&to->wait_time, &from->wait_time);
	blkg_stat_add_aux(&from->time, &from->time);
	blkg_stat_add_aux(&to->avg_queue_size_sum, &from->avg_queue_size_sum);
	blkg_stat_add_aux(&to->avg_queue_size_samples,
			  &from->avg_queue_size_samples);
	blkg_stat_add_aux(&to->dequeue, &from->dequeue);
	blkg_stat_add_aux(&to->group_wait_time, &from->group_wait_time);
	blkg_stat_add_aux(&to->idle_time, &from->idle_time);
	blkg_stat_add_aux(&to->empty_time, &from->empty_time);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 156 100.00% 1 100.00%
Total 156 100.00% 1 100.00%

/*
 * Transfer @bfqg's stats to its parent's aux counts so that the ancestors'
 * recursive stats can still account for the amount used by this bfqg after
 * it's gone.
 */


static void bfqg_stats_xfer_dead(struct bfq_group *bfqg)
{
	struct bfq_group *parent;

	if (!bfqg) /* root_group */
		return;

	parent = bfqg_parent(bfqg);

	lockdep_assert_held(bfqg_to_blkg(bfqg)->q->queue_lock);

	if (unlikely(!parent))
		return;

	bfqg_stats_add_aux(&parent->stats, &bfqg->stats);
	bfqg_stats_reset(&bfqg->stats);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 72 100.00% 1 100.00%
Total 72 100.00% 1 100.00%



void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)
{
	struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);

	entity->weight = entity->new_weight;
	entity->orig_weight = entity->new_weight;
	if (bfqq) {
		bfqq->ioprio = bfqq->new_ioprio;
		bfqq->ioprio_class = bfqq->new_ioprio_class;
		/*
                 * Make sure that bfqg and its associated blkg do not
                 * disappear before entity.
                 */
		bfqg_and_blkg_get(bfqg);
	}
	entity->parent = bfqg->my_entity; /* NULL for root group */
	entity->sched_data = &bfqg->sched_data;
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 87 100.00% 2 100.00%
Total 87 100.00% 2 100.00%



static void bfqg_stats_exit(struct bfqg_stats *stats)
{
	blkg_rwstat_exit(&stats->merged);
	blkg_rwstat_exit(&stats->service_time);
	blkg_rwstat_exit(&stats->wait_time);
	blkg_rwstat_exit(&stats->queued);
	blkg_stat_exit(&stats->time);
	blkg_stat_exit(&stats->avg_queue_size_sum);
	blkg_stat_exit(&stats->avg_queue_size_samples);
	blkg_stat_exit(&stats->dequeue);
	blkg_stat_exit(&stats->group_wait_time);
	blkg_stat_exit(&stats->idle_time);
	blkg_stat_exit(&stats->empty_time);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 99 100.00% 1 100.00%
Total 99 100.00% 1 100.00%



static int bfqg_stats_init(struct bfqg_stats *stats, gfp_t gfp)
{
	if (blkg_rwstat_init(&stats->merged, gfp) ||
	    blkg_rwstat_init(&stats->service_time, gfp) ||
	    blkg_rwstat_init(&stats->wait_time, gfp) ||
	    blkg_rwstat_init(&stats->queued, gfp) ||
	    blkg_stat_init(&stats->time, gfp) ||
	    blkg_stat_init(&stats->avg_queue_size_sum, gfp) ||
	    blkg_stat_init(&stats->avg_queue_size_samples, gfp) ||
	    blkg_stat_init(&stats->dequeue, gfp) ||
	    blkg_stat_init(&stats->group_wait_time, gfp) ||
	    blkg_stat_init(&stats->idle_time, gfp) ||
	    blkg_stat_init(&stats->empty_time, gfp)) {
		bfqg_stats_exit(stats);
		return -ENOMEM;
	}

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 140 100.00% 1 100.00%
Total 140 100.00% 1 100.00%



static struct bfq_group_data *cpd_to_bfqgd(struct blkcg_policy_data *cpd)
{
	return cpd ? container_of(cpd, struct bfq_group_data, pd) : NULL;
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 28 100.00% 1 100.00%
Total 28 100.00% 1 100.00%



static struct bfq_group_data *blkcg_to_bfqgd(struct blkcg *blkcg)
{
	return cpd_to_bfqgd(blkcg_to_cpd(blkcg, &blkcg_policy_bfq));
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 25 100.00% 1 100.00%
Total 25 100.00% 1 100.00%



struct blkcg_policy_data *bfq_cpd_alloc(gfp_t gfp)
{
	struct bfq_group_data *bgd;

	bgd = kzalloc(sizeof(*bgd), gfp);
	if (!bgd)
		return NULL;
	return &bgd->pd;
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 42 100.00% 1 100.00%
Total 42 100.00% 1 100.00%



void bfq_cpd_init(struct blkcg_policy_data *cpd)
{
	struct bfq_group_data *d = cpd_to_bfqgd(cpd);

	d->weight = cgroup_subsys_on_dfl(io_cgrp_subsys) ?
		CGROUP_WEIGHT_DFL : BFQ_WEIGHT_LEGACY_DFL;
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 33 100.00% 1 100.00%
Total 33 100.00% 1 100.00%



void bfq_cpd_free(struct blkcg_policy_data *cpd)
{
	kfree(cpd_to_bfqgd(cpd));
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 18 100.00% 1 100.00%
Total 18 100.00% 1 100.00%



struct blkg_policy_data *bfq_pd_alloc(gfp_t gfp, int node)
{
	struct bfq_group *bfqg;

	bfqg = kzalloc_node(sizeof(*bfqg), gfp, node);
	if (!bfqg)
		return NULL;

	if (bfqg_stats_init(&bfqg->stats, gfp)) {
		kfree(bfqg);
		return NULL;
	}

	/* see comments in bfq_bic_update_cgroup for why refcounting */
	bfqg_get(bfqg);
	return &bfqg->pd;
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 75 100.00% 2 100.00%
Total 75 100.00% 2 100.00%



void bfq_pd_init(struct blkg_policy_data *pd)
{
	struct blkcg_gq *blkg = pd_to_blkg(pd);
	struct bfq_group *bfqg = blkg_to_bfqg(blkg);
	struct bfq_data *bfqd = blkg->q->elevator->elevator_data;
	struct bfq_entity *entity = &bfqg->entity;
	struct bfq_group_data *d = blkcg_to_bfqgd(blkg->blkcg);

	entity->orig_weight = entity->weight = entity->new_weight = d->weight;
	entity->my_sched_data = &bfqg->sched_data;
	bfqg->my_entity = entity; /*
                                   * the root_group's will be set to NULL
                                   * in bfq_init_queue()
                                   */
	bfqg->bfqd = bfqd;
	bfqg->active_entities = 0;
	bfqg->rq_pos_tree = RB_ROOT;
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 115 100.00% 1 100.00%
Total 115 100.00% 1 100.00%



void bfq_pd_free(struct blkg_policy_data *pd)
{
	struct bfq_group *bfqg = pd_to_bfqg(pd);

	bfqg_stats_exit(&bfqg->stats);
	bfqg_put(bfqg);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 33 100.00% 2 100.00%
Total 33 100.00% 2 100.00%



void bfq_pd_reset_stats(struct blkg_policy_data *pd)
{
	struct bfq_group *bfqg = pd_to_bfqg(pd);

	bfqg_stats_reset(&bfqg->stats);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 28 100.00% 1 100.00%
Total 28 100.00% 1 100.00%



static void bfq_group_set_parent(struct bfq_group *bfqg,
					struct bfq_group *parent)
{
	struct bfq_entity *entity;

	entity = &bfqg->entity;
	entity->parent = parent->my_entity;
	entity->sched_data = &parent->sched_data;
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 45 100.00% 1 100.00%
Total 45 100.00% 1 100.00%



static struct bfq_group *bfq_lookup_bfqg(struct bfq_data *bfqd,
					 struct blkcg *blkcg)
{
	struct blkcg_gq *blkg;

	blkg = blkg_lookup(blkcg, bfqd->queue);
	if (likely(blkg))
		return blkg_to_bfqg(blkg);
	return NULL;
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 50 100.00% 1 100.00%
Total 50 100.00% 1 100.00%



struct bfq_group *bfq_find_set_group(struct bfq_data *bfqd,
				     struct blkcg *blkcg)
{
	struct bfq_group *bfqg, *parent;
	struct bfq_entity *entity;

	bfqg = bfq_lookup_bfqg(bfqd, blkcg);

	if (unlikely(!bfqg))
		return NULL;

	/*
         * Update chain of bfq_groups as we might be handling a leaf group
         * which, along with some of its relatives, has not been hooked yet
         * to the private hierarchy of BFQ.
         */
	entity = &bfqg->entity;
	for_each_entity(entity) {
		bfqg = container_of(entity, struct bfq_group, entity);
		if (bfqg != bfqd->root_group) {
			parent = bfqg_parent(bfqg);
			if (!parent)
				parent = bfqd->root_group;
			bfq_group_set_parent(bfqg, parent);
		}
	}

	return bfqg;
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 114 100.00% 1 100.00%
Total 114 100.00% 1 100.00%

/**
 * bfq_bfqq_move - migrate @bfqq to @bfqg.
 * @bfqd: queue descriptor.
 * @bfqq: the queue to move.
 * @bfqg: the group to move to.
 *
 * Move @bfqq to @bfqg, deactivating it from its old group and reactivating
 * it on the new one.  Avoid putting the entity on the old group idle tree.
 *
 * Must be called under the scheduler lock, to make sure that the blkg
 * owning @bfqg does not disappear (see comments in
 * bfq_bic_update_cgroup on guaranteeing the consistency of blkg
 * objects).
 */


void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
		   struct bfq_group *bfqg)
{
	struct bfq_entity *entity = &bfqq->entity;

	/* If bfqq is empty, then bfq_bfqq_expire also invokes
         * bfq_del_bfqq_busy, thereby removing bfqq and its entity
         * from data structures related to current group. Otherwise we
         * need to remove bfqq explicitly with bfq_deactivate_bfqq, as
         * we do below.
         */
	if (bfqq == bfqd->in_service_queue)
		bfq_bfqq_expire(bfqd, bfqd->in_service_queue,
				false, BFQQE_PREEMPTED);

	if (bfq_bfqq_busy(bfqq))
		bfq_deactivate_bfqq(bfqd, bfqq, false, false);
	else if (entity->on_st)
		bfq_put_idle_entity(bfq_entity_service_tree(entity), entity);
	bfqg_and_blkg_put(bfqq_group(bfqq));

	entity->parent = bfqg->my_entity;
	entity->sched_data = &bfqg->sched_data;
	/* pin down bfqg and its associated blkg  */
	bfqg_and_blkg_get(bfqg);

	if (bfq_bfqq_busy(bfqq)) {
		bfq_pos_tree_add_move(bfqd, bfqq);
		bfq_activate_bfqq(bfqd, bfqq);
	}

	if (!bfqd->in_service_queue && !bfqd->rq_in_driver)
		bfq_schedule_dispatch(bfqd);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 158 100.00% 2 100.00%
Total 158 100.00% 2 100.00%

/**
 * __bfq_bic_change_cgroup - move @bic to @cgroup.
 * @bfqd: the queue descriptor.
 * @bic: the bic to move.
 * @blkcg: the blk-cgroup to move to.
 *
 * Move bic to blkcg, assuming that bfqd->lock is held; which makes
 * sure that the reference to cgroup is valid across the call (see
 * comments in bfq_bic_update_cgroup on this issue)
 *
 * NOTE: an alternative approach might have been to store the current
 * cgroup in bfqq and getting a reference to it, reducing the lookup
 * time here, at the price of slightly more complex code.
 */


static struct bfq_group *__bfq_bic_change_cgroup(struct bfq_data *bfqd,
						struct bfq_io_cq *bic,
						struct blkcg *blkcg)
{
	struct bfq_queue *async_bfqq = bic_to_bfqq(bic, 0);
	struct bfq_queue *sync_bfqq = bic_to_bfqq(bic, 1);
	struct bfq_group *bfqg;
	struct bfq_entity *entity;

	bfqg = bfq_find_set_group(bfqd, blkcg);

	if (unlikely(!bfqg))
		bfqg = bfqd->root_group;

	if (async_bfqq) {
		entity = &async_bfqq->entity;

		if (entity->sched_data != &bfqg->sched_data) {
			bic_set_bfqq(bic, NULL, 0);
			bfq_log_bfqq(bfqd, async_bfqq,
				     "bic_change_group: %p %d",
				     async_bfqq, async_bfqq->ref);
			bfq_put_queue(async_bfqq);
		}
	}

	if (sync_bfqq) {
		entity = &sync_bfqq->entity;
		if (entity->sched_data != &bfqg->sched_data)
			bfq_bfqq_move(bfqd, sync_bfqq, bfqg);
	}

	return bfqg;
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 171 100.00% 1 100.00%
Total 171 100.00% 1 100.00%



void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio)
{
	struct bfq_data *bfqd = bic_to_bfqd(bic);
	struct bfq_group *bfqg = NULL;
	uint64_t serial_nr;

	rcu_read_lock();
	serial_nr = bio_blkcg(bio)->css.serial_nr;

	/*
         * Check whether blkcg has changed.  The condition may trigger
         * spuriously on a newly created cic but there's no harm.
         */
	if (unlikely(!bfqd) || likely(bic->blkcg_serial_nr == serial_nr))
		goto out;

	bfqg = __bfq_bic_change_cgroup(bfqd, bic, bio_blkcg(bio));
	/*
         * Update blkg_path for bfq_log_* functions. We cache this
         * path, and update it here, for the following
         * reasons. Operations on blkg objects in blk-cgroup are
         * protected with the request_queue lock, and not with the
         * lock that protects the instances of this scheduler
         * (bfqd->lock). This exposes BFQ to the following sort of
         * race.
         *
         * The blkg_lookup performed in bfq_get_queue, protected
         * through rcu, may happen to return the address of a copy of
         * the original blkg. If this is the case, then the
         * bfqg_and_blkg_get performed in bfq_get_queue, to pin down
         * the blkg, is useless: it does not prevent blk-cgroup code
         * from destroying both the original blkg and all objects
         * directly or indirectly referred by the copy of the
         * blkg.
         *
         * On the bright side, destroy operations on a blkg invoke, as
         * a first step, hooks of the scheduler associated with the
         * blkg. And these hooks are executed with bfqd->lock held for
         * BFQ. As a consequence, for any blkg associated with the
         * request queue this instance of the scheduler is attached
         * to, we are guaranteed that such a blkg is not destroyed, and
         * that all the pointers it contains are consistent, while we
         * are holding bfqd->lock. A blkg_lookup performed with
         * bfqd->lock held then returns a fully consistent blkg, which
         * remains consistent until this lock is held.
         *
         * Thanks to the last fact, and to the fact that: (1) bfqg has
         * been obtained through a blkg_lookup in the above
         * assignment, and (2) bfqd->lock is being held, here we can
         * safely use the policy data for the involved blkg (i.e., the
         * field bfqg->pd) to get to the blkg associated with bfqg,
         * and then we can safely use any field of blkg. After we
         * release bfqd->lock, even just getting blkg through this
         * bfqg may cause dangling references to be traversed, as
         * bfqg->pd may not exist any more.
         *
         * In view of the above facts, here we cache, in the bfqg, any
         * blkg data we may need for this bic, and for its associated
         * bfq_queue. As of now, we need to cache only the path of the
         * blkg, which is used in the bfq_log_* functions.
         *
         * Finally, note that bfqg itself needs to be protected from
         * destruction on the blkg_free of the original blkg (which
         * invokes bfq_pd_free). We use an additional private
         * refcounter for bfqg, to let it disappear only after no
         * bfq_queue refers to it any longer.
         */
	blkg_path(bfqg_to_blkg(bfqg), bfqg->blkg_path, sizeof(bfqg->blkg_path));
	bic->blkcg_serial_nr = serial_nr;
out:
	rcu_read_unlock();
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 115 100.00% 2 100.00%
Total 115 100.00% 2 100.00%

/**
 * bfq_flush_idle_tree - deactivate any entity on the idle tree of @st.
 * @st: the service tree being flushed.
 */


static void bfq_flush_idle_tree(struct bfq_service_tree *st)
{
	struct bfq_entity *entity = st->first_idle;

	for (; entity ; entity = st->first_idle)
		__bfq_deactivate_entity(entity, false);
}

Contributors
Person Tokens Prop Commits CommitProp
Paolo Valente 38 100.00% 1 100.00%
Total 38 100.00% 1 100.00%

/**
 * bfq_reparent_leaf_entity - move leaf entity to the root_group.
 * @bfqd: the device data structure with the root group.
 * @entity: the entity to move.
 */


static void bfq_reparent_leaf_entity(struct bfq_data *bfqd,
				     struct bfq_entity *
Person	Tokens	Prop	Commits	CommitProp
Paolo Valente	60	100.00%	1	100.00%
Total	60	100.00%	1	100.00%
cregit-Linux how code gets into the kernel

Release 4.12 block/bfq-cgroup.c

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors
