cregit-Linux how code gets into the kernel

Release 4.14 net/sched/sch_qfq.c

Directory: net/sched
 * net/sched/sch_qfq.c         Quick Fair Queueing Plus Scheduler.
 * Copyright (c) 2009 Fabio Checconi, Luigi Rizzo, and Paolo Valente.
 * Copyright (c) 2012 Paolo Valente.
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * version 2 as published by the Free Software Foundation.

#include <linux/module.h>
#include <linux/init.h>
#include <linux/bitops.h>
#include <linux/errno.h>
#include <linux/netdevice.h>
#include <linux/pkt_sched.h>
#include <net/sch_generic.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>

/*  Quick Fair Queueing Plus


    [1] Paolo Valente,
    "Reducing the Execution Time of Fair-Queueing Schedulers."

    Sources for QFQ:

    [2] Fabio Checconi, Luigi Rizzo, and Paolo Valente: "QFQ: Efficient
    Packet Scheduling with Tight Bandwidth Distribution Guarantees."

    See also:


  QFQ+ divides classes into aggregates of at most MAX_AGG_CLASSES
  classes. Each aggregate is timestamped with a virtual start time S
  and a virtual finish time F, and scheduled according to its
  timestamps. S and F are computed as a function of a system virtual
  time function V. The classes within each aggregate are instead
  scheduled with DRR.

  To speed up operations, QFQ+ divides also aggregates into a limited
  number of groups. Which group a class belongs to depends on the
  ratio between the maximum packet length for the class and the weight
  of the class. Groups have their own S and F. In the end, QFQ+
  schedules groups, then aggregates within groups, then classes within
  aggregates. See [1] and [2] for a full description.

  Virtual time computations.

  S, F and V are all computed in fixed point arithmetic with
  FRAC_BITS decimal bits.

  QFQ_MAX_INDEX is the maximum index allowed for a group. We need
        one bit per index.
  QFQ_MAX_WSHIFT is the maximum power of two supported as a weight.

  The layout of the bits is as below:

                   [ MTU_SHIFT ][      FRAC_BITS    ]
                   [ MAX_INDEX    ][ MIN_SLOT_SHIFT ]
                                 ^.__grp->index = 0

  where MIN_SLOT_SHIFT is derived by difference from the others.

  The max group index corresponds to Lmax/w_min, where
  Lmax=1<<MTU_SHIFT, w_min = 1 .
  From this, and knowing how many groups (MAX_INDEX) we want,
  we can derive the shift corresponding to each group.

  Because we often need to compute
        F = S + len/w_i  and V = V + len/wsum
  instead of storing w_i store the value
        inv_w = (1<<FRAC_BITS)/w_i
  so we can do F = S + len * inv_w * wsum.
  We use W_TOT in the formulas so we can easily move between
  static and adaptive weight sum.

  The per-scheduler-instance data contain all the data structures
  for the scheduler: bitmaps and bucket lists.


 * Maximum number of consecutive slots occupied by backlogged classes
 * inside a group.

#define QFQ_MAX_SLOTS	32

 * Shifts used for aggregate<->group mapping.  We allow class weights that are
 * in the range [1, 2^MAX_WSHIFT], and we try to map each aggregate i to the
 * group with the smallest index that can support the L_i / r_i configured
 * for the classes in the aggregate.
 * grp->index is the index of the group; and grp->slot_shift
 * is the shift for the corresponding (scaled) sigma_i.

#define QFQ_MAX_INDEX		24

#define QFQ_MAX_WSHIFT		10

/* see qfq_slot_insert */


#define FRAC_BITS		30	
/* fixed point arithmetic */

#define ONE_FP			(1UL << FRAC_BITS)

#define QFQ_MTU_SHIFT		16	
/* to support TSO/GSO */

#define QFQ_MIN_LMAX		512	
/* see qfq_slot_insert */

/* max num classes per aggregate allowed */

 * Possible group states.  These values are used as indexes for the bitmaps
 * array of struct qfq_queue.

enum qfq_state { ER, IR, EB, IB, QFQ_MAX_STATE };

struct qfq_group;

struct qfq_aggregate;

struct qfq_class {
struct Qdisc_class_common common;

unsigned int filter_cnt;

struct gnet_stats_basic_packed bstats;
struct gnet_stats_queue qstats;
struct net_rate_estimator __rcu *rate_est;
struct Qdisc *qdisc;
struct list_head alist;		/* Link for active-classes list. */
struct qfq_aggregate *agg;	/* Parent aggregate. */
int deficit;			/* DRR deficit counter. */

struct qfq_aggregate {
struct hlist_node next;	/* Link for the slot list. */

u64 S, F;		/* flow timestamps (exact) */

	/* group we belong to. In principle we would need the index,
         * which is log_2(lmax/weight), but we never reference it
         * directly, only the group.
struct qfq_group *grp;

	/* these are copied from the flowset. */
u32	class_weight; /* Weight of each class in this aggregate. */
	/* Max pkt size for the classes in this aggregate, DRR quantum. */
int	lmax;

u32	inv_w;	    /* ONE_FP/(sum of weights of classes in aggr.). */
u32	budgetmax;  /* Max budget for this aggregate. */

u32	initial_budget, budget;     /* Initial and current budget. */

int		  num_classes;	/* Number of classes in this aggr. */
struct list_head  active;	/* DRR queue of active classes. */

struct hlist_node nonfull_next;	/* See nonfull_aggs in qfq_sched. */

struct qfq_group {

u64 S, F;			/* group timestamps (approx). */
unsigned int slot_shift;	/* Slot shift. */
unsigned int index;		/* Group index. */
unsigned int front;		/* Index of the front slot. */
unsigned long full_slots;	/* non-empty slots */

	/* Array of RR lists of active aggregates. */
struct hlist_head slots[QFQ_MAX_SLOTS];

struct qfq_sched {
struct tcf_proto __rcu *filter_list;
struct tcf_block	*block;
struct Qdisc_class_hash clhash;


u64			oldV, V;	/* Precise virtual times. */
struct qfq_aggregate	*in_serv_agg;   /* Aggregate being served. */
u32			wsum;		/* weight sum */
u32			iwsum;		/* inverse weight sum */

unsigned long bitmaps[QFQ_MAX_STATE];	    /* Group bitmaps. */
struct qfq_group groups[QFQ_MAX_INDEX + 1]; /* The groups. */
u32 min_slot_shift;	/* Index of the group-0 bit in the bitmaps. */

u32 max_agg_classes;		/* Max number of classes per aggr. */
struct hlist_head nonfull_aggs; /* Aggs with room for more classes. */

 * Possible reasons why the timestamps of an aggregate are updated
 * enqueue: the aggregate switches from idle to active and must scheduled
 *          for service
 * requeue: the aggregate finishes its budget, so it stops being served and
 *          must be rescheduled for service

enum update_reason {enqueue, requeue};

static struct qfq_class *qfq_find_class(struct Qdisc *sch, u32 classid) { struct qfq_sched *q = qdisc_priv(sch); struct Qdisc_class_common *clc; clc = qdisc_class_find(&q->clhash, classid); if (clc == NULL) return NULL; return container_of(clc, struct qfq_class, common); }


Stephen Hemminger63100.00%1100.00%

static void qfq_purge_queue(struct qfq_class *cl) { unsigned int len = cl->qdisc->q.qlen; unsigned int backlog = cl->qdisc->qstats.backlog; qdisc_reset(cl->qdisc); qdisc_tree_reduce_backlog(cl->qdisc, len, backlog); }


Stephen Hemminger3871.70%150.00%
Américo Wang1528.30%150.00%

static const struct nla_policy qfq_policy[TCA_QFQ_MAX + 1] = { [TCA_QFQ_WEIGHT] = { .type = NLA_U32 }, [TCA_QFQ_LMAX] = { .type = NLA_U32 }, }; /* * Calculate a flow index, given its weight and maximum packet length. * index = log_2(maxlen/weight) but we need to apply the scaling. * This is used only once at flow creation. */
static int qfq_calc_index(u32 inv_w, unsigned int maxlen, u32 min_slot_shift) { u64 slot_size = (u64)maxlen * inv_w; unsigned long size_map; int index = 0; size_map = slot_size >> min_slot_shift; if (!size_map) goto out; index = __fls(size_map) + 1; /* basically a log_2 */ index -= !(slot_size - (1ULL << (index + min_slot_shift - 1))); if (index < 0) index = 0; out: pr_debug("qfq calc_index: W = %lu, L = %u, I = %d\n", (unsigned long) ONE_FP/inv_w, maxlen, index); return index; }


Stephen Hemminger10595.45%150.00%
Paolo Valente54.55%150.00%

static void qfq_deactivate_agg(struct qfq_sched *, struct qfq_aggregate *); static void qfq_activate_agg(struct qfq_sched *, struct qfq_aggregate *, enum update_reason);
static void qfq_init_agg(struct qfq_sched *q, struct qfq_aggregate *agg, u32 lmax, u32 weight) { INIT_LIST_HEAD(&agg->active); hlist_add_head(&agg->nonfull_next, &q->nonfull_aggs); agg->lmax = lmax; agg->class_weight = weight; }


Paolo Valente55100.00%2100.00%

static struct qfq_aggregate *qfq_find_agg(struct qfq_sched *q, u32 lmax, u32 weight) { struct qfq_aggregate *agg; hlist_for_each_entry(agg, &q->nonfull_aggs, nonfull_next) if (agg->lmax == lmax && agg->class_weight == weight) return agg; return NULL; }


Paolo Valente52100.00%2100.00%

/* Update aggregate as a function of the new number of classes. */
static void qfq_update_agg(struct qfq_sched *q, struct qfq_aggregate *agg, int new_num_classes) { u32 new_agg_weight; if (new_num_classes == q->max_agg_classes) hlist_del_init(&agg->nonfull_next); if (agg->num_classes > new_num_classes && new_num_classes == q->max_agg_classes - 1) /* agg no more full */ hlist_add_head(&agg->nonfull_next, &q->nonfull_aggs); /* The next assignment may let * agg->initial_budget > agg->budgetmax * hold, we will take it into account in charge_actual_service(). */ agg->budgetmax = new_num_classes * agg->lmax; new_agg_weight = agg->class_weight * new_num_classes; agg->inv_w = ONE_FP/new_agg_weight; if (agg->grp == NULL) { int i = qfq_calc_index(agg->inv_w, agg->budgetmax, q->min_slot_shift); agg->grp = &q->groups[i]; } q->wsum += (int) agg->class_weight * (new_num_classes - agg->num_classes); q->iwsum = ONE_FP / q->wsum; agg->num_classes = new_num_classes; }


Paolo Valente170100.00%3100.00%

/* Add class to aggregate. */
static void qfq_add_to_agg(struct qfq_sched *q, struct qfq_aggregate *agg, struct qfq_class *cl) { cl->agg = agg; qfq_update_agg(q, agg, agg->num_classes+1); if (cl->qdisc->q.qlen > 0) { /* adding an active class */ list_add_tail(&cl->alist, &agg->active); if (list_first_entry(&agg->active, struct qfq_class, alist) == cl && q->in_serv_agg != agg) /* agg was inactive */ qfq_activate_agg(q, agg, enqueue); /* schedule agg */ } }


Paolo Valente102100.00%1100.00%

static struct qfq_aggregate *qfq_choose_next_agg(struct qfq_sched *);
static void qfq_destroy_agg(struct qfq_sched *q, struct qfq_aggregate *agg) { hlist_del_init(&agg->nonfull_next); q->wsum -= agg->class_weight; if (q->wsum != 0) q->iwsum = ONE_FP / q->wsum; if (q->in_serv_agg == agg) q->in_serv_agg = qfq_choose_next_agg(q); kfree(agg); }


Paolo Valente72100.00%2100.00%

/* Deschedule class from within its parent aggregate. */
static void qfq_deactivate_class(struct qfq_sched *q, struct qfq_class *cl) { struct qfq_aggregate *agg = cl->agg; list_del(&cl->alist); /* remove from RR queue of the aggregate */ if (list_empty(&agg->active)) /* agg is now inactive */ qfq_deactivate_agg(q, agg); }


Paolo Valente52100.00%1100.00%

/* Remove class from its parent aggregate. */
static void qfq_rm_from_agg(struct qfq_sched *q, struct qfq_class *cl) { struct qfq_aggregate *agg = cl->agg; cl->agg = NULL; if (agg->num_classes == 1) { /* agg being emptied, destroy it */ qfq_destroy_agg(q, agg); return; } qfq_update_agg(q, agg, agg->num_classes-1); }


Paolo Valente63100.00%2100.00%

/* Deschedule class and remove it from its parent aggregate. */
static void qfq_deact_rm_from_agg(struct qfq_sched *q, struct qfq_class *cl) { if (cl->qdisc->q.qlen > 0) /* class is active */ qfq_deactivate_class(q, cl); qfq_rm_from_agg(q, cl); }


Paolo Valente43100.00%2100.00%

/* Move class to a new aggregate, matching the new class weight and/or lmax */
static int qfq_change_agg(struct Qdisc *sch, struct qfq_class *cl, u32 weight, u32 lmax) { struct qfq_sched *q = qdisc_priv(sch); struct qfq_aggregate *new_agg = qfq_find_agg(q, lmax, weight); if (new_agg == NULL) { /* create new aggregate */ new_agg = kzalloc(sizeof(*new_agg), GFP_ATOMIC); if (new_agg == NULL) return -ENOBUFS; qfq_init_agg(q, new_agg, lmax, weight); } qfq_deact_rm_from_agg(q, cl); qfq_add_to_agg(q, new_agg, cl); return 0; }


Paolo Valente108100.00%2100.00%

static int qfq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, struct nlattr **tca, unsigned long *arg) { struct qfq_sched *q = qdisc_priv(sch); struct qfq_class *cl = (struct qfq_class *)*arg; bool existing = false; struct nlattr *tb[TCA_QFQ_MAX + 1]; struct qfq_aggregate *new_agg = NULL; u32 weight, lmax, inv_w; int err; int delta_w; if (tca[TCA_OPTIONS] == NULL) { pr_notice("qfq: no options\n"); return -EINVAL; } err = nla_parse_nested(tb, TCA_QFQ_MAX, tca[TCA_OPTIONS], qfq_policy, NULL); if (err < 0) return err; if (tb[TCA_QFQ_WEIGHT]) { weight = nla_get_u32(tb[TCA_QFQ_WEIGHT]); if (!weight || weight > (1UL << QFQ_MAX_WSHIFT)) { pr_notice("qfq: invalid weight %u\n", weight); return -EINVAL; } } else weight = 1; if (tb[TCA_QFQ_LMAX]) { lmax = nla_get_u32(tb[TCA_QFQ_LMAX]); if (lmax < QFQ_MIN_LMAX || lmax > (1UL << QFQ_MTU_SHIFT)) { pr_notice("qfq: invalid max length %u\n", lmax); return -EINVAL; } } else lmax = psched_mtu(qdisc_dev(sch)); inv_w = ONE_FP / weight; weight = ONE_FP / inv_w; if (cl != NULL && lmax == cl->agg->lmax && weight == cl->agg->class_weight) return 0; /* nothing to change */ delta_w = weight - (cl ? cl->agg->class_weight : 0); if (q->wsum + delta_w > QFQ_MAX_WSUM) { pr_notice("qfq: total weight out of range (%d + %u)\n", delta_w, q->wsum); return -EINVAL; } if (cl != NULL) { /* modify existing class */ if (tca[TCA_RATE]) { err = gen_replace_estimator(&cl->bstats, NULL, &cl->rate_est, NULL, qdisc_root_sleeping_running(sch), tca[TCA_RATE]); if (err) return err; } existing = true; goto set_change_agg; } /* create and init new class */ cl = kzalloc(sizeof(struct qfq_class), GFP_KERNEL); if (cl == NULL) return -ENOBUFS; cl->common.classid = classid; cl->deficit = lmax; cl->qdisc = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops, classid); if (cl->qdisc == NULL) cl->qdisc = &noop_qdisc; if (tca[TCA_RATE]) { err = gen_new_estimator(&cl->bstats, NULL, &cl->rate_est, NULL, qdisc_root_sleeping_running(sch), tca[TCA_RATE]); if (err) goto destroy_class; } if (cl->qdisc != &noop_qdisc) qdisc_hash_add(cl->qdisc, true); sch_tree_lock(sch); qdisc_class_hash_insert(&q->clhash, &cl->common); sch_tree_unlock(sch); qdisc_class_hash_grow(sch, &q->clhash); set_change_agg: sch_tree_lock(sch); new_agg = qfq_find_agg(q, lmax, weight); if (new_agg == NULL) { /* create new aggregate */ sch_tree_unlock(sch); new_agg = kzalloc(sizeof(*new_agg), GFP_KERNEL); if (new_agg == NULL) { err = -ENOBUFS; gen_kill_estimator(&cl->rate_est); goto destroy_class; } sch_tree_lock(sch); qfq_init_agg(q, new_agg, lmax, weight); } if (existing) qfq_deact_rm_from_agg(q, cl); qfq_add_to_agg(q, new_agg, cl); sch_tree_unlock(sch); *arg = (unsigned long)cl; return 0; destroy_class: qdisc_destroy(cl->qdisc); kfree(cl); return err; }


Stephen Hemminger39357.29%112.50%
Paolo Valente26037.90%225.00%
Jiri Kosina182.62%112.50%
Eric Dumazet91.31%225.00%
John Fastabend40.58%112.50%
Johannes Berg20.29%112.50%

static void qfq_destroy_class(struct Qdisc *sch, struct qfq_class *cl) { struct qfq_sched *q = qdisc_priv(sch); qfq_rm_from_agg(q, cl); gen_kill_estimator(&cl->rate_est); qdisc_destroy(cl->qdisc); kfree(cl); }


Stephen Hemminger4992.45%150.00%
Paolo Valente47.55%150.00%

static int qfq_delete_class(struct Qdisc *sch, unsigned long arg) { struct qfq_sched *q = qdisc_priv(sch); struct qfq_class *cl = (struct qfq_class *)arg; if (cl->filter_cnt > 0) return -EBUSY; sch_tree_lock(sch); qfq_purge_queue(cl); qdisc_class_hash_remove(&q->clhash, &cl->common); sch_tree_unlock(sch); qfq_destroy_class(sch, cl); return 0; }


Stephen Hemminger8294.25%150.00%
Américo Wang55.75%150.00%

static unsigned long qfq_search_class(struct Qdisc *sch, u32 classid) { return (unsigned long)qfq_find_class(sch, classid); }


Stephen Hemminger2385.19%150.00%
Américo Wang414.81%150.00%

static struct tcf_block *qfq_tcf_block(struct Qdisc *sch, unsigned long cl) { struct qfq_sched *q = qdisc_priv(sch); if (cl) return NULL; return q->block; }


Stephen Hemminger3692.31%150.00%
Jiri Pirko37.69%150.00%

static unsigned long qfq_bind_tcf(struct Qdisc *sch, unsigned long parent, u32 classid) { struct qfq_class *cl = qfq_find_class(sch, classid); if (cl != NULL) cl->filter_cnt++; return (unsigned long)cl; }


Stephen Hemminger49100.00%1100.00%

static void qfq_unbind_tcf(struct Qdisc *sch, unsigned long arg) { struct qfq_class *cl = (struct qfq_class *)arg; cl->filter_cnt--; }


Stephen Hemminger32100.00%1100.00%

static int qfq_graft_class(struct Qdisc *sch, unsigned long arg, struct Qdisc *new, struct Qdisc **old) { struct qfq_class *cl = (struct qfq_class *)arg; if (new == NULL) { new = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops, cl->common.classid); if (new == NULL) new = &noop_qdisc; } *old = qdisc_replace(sch, new, &cl->qdisc); return 0; }


Stephen Hemminger8692.47%150.00%
Américo Wang77.53%150.00%

static struct Qdisc *qfq_class_leaf(struct Qdisc *sch, unsigned long arg) { struct qfq_class *cl = (struct qfq_class *)arg; return cl->qdisc; }


Stephen Hemminger34100.00%1100.00%

static int qfq_dump_class(struct Qdisc *sch, unsigned long arg, struct sk_buff *skb, struct tcmsg *tcm) { struct qfq_class *cl = (struct qfq_class *)arg; struct nlattr *nest; tcm->tcm_parent = TC_H_ROOT; tcm->tcm_handle = cl->common.classid; tcm->tcm_info = cl->qdisc->handle; nest = nla_nest_start(skb, TCA_OPTIONS); if (nest == NULL) goto nla_put_failure; if (nla_put_u32(skb, TCA_QFQ_WEIGHT, cl->agg->class_weight) || nla_put_u32(skb, TCA_QFQ_LMAX, cl->agg->lmax)) goto nla_put_failure; return nla_nest_end(skb, nest); nla_put_failure: nla_nest_cancel(skb, nest); return -EMSGSIZE; }


Stephen Hemminger12489.86%133.33%
David S. Miller96.52%133.33%
Paolo Valente53.62%133.33%

static int qfq_dump_class_stats(struct Qdisc *sch, unsigned long arg, struct gnet_dump *d) { struct qfq_class *cl = (struct qfq_class *)arg; struct tc_qfq_stats xstats; memset(&xstats, 0, sizeof(xstats)); xstats.weight = cl->agg->class_weight; xstats.lmax = cl->agg->lmax; if (gnet_stats_copy_basic(qdisc_root_sleeping_running(sch), d, NULL, &cl->bstats) < 0 || gnet_stats_copy_rate_est(d, &cl->rate_est) < 0 || gnet_stats_copy_queue(d, NULL, &cl->qdisc->qstats, cl->qdisc->q.qlen) < 0) return -1; return gnet_stats_copy_app(d, &xstats, sizeof(xstats)); }


Stephen Hemminger12284.72%116.67%
John Fastabend128.33%350.00%
Eric Dumazet53.47%116.67%
Paolo Valente53.47%116.67%

static void qfq_walk(struct Qdisc *sch, struct qdisc_walker *arg) { struct qfq_sched *q = qdisc_priv(sch); struct qfq_class *cl; unsigned int i; if (arg->stop) return; for (i = 0; i < q->clhash.hashsize; i++) { hlist_for_each_entry(cl, &q->clhash.hash[i], common.hnode) { if (arg->count < arg->skip) { arg->count++; continue; } if (arg->fn(sch, (unsigned long)cl, arg) < 0) { arg->stop = 1; return; } arg->count++; } } }


Stephen Hemminger122100.00%1100.00%

static struct qfq_class *qfq_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr) { struct qfq_sched *q = qdisc_priv(sch); struct qfq_class *cl; struct tcf_result res; struct tcf_proto *fl; int result; if (TC_H_MAJ(skb->priority ^ sch->handle) == 0) { pr_debug("qfq_classify: found %d\n", skb->priority); cl = qfq_find_class(sch, skb->priority)