cregit-Linux how code gets into the kernel

Release 4.12 block/blk-mq-tag.c

Directory: block
/*
 * Tag allocation using scalable bitmaps. Uses active queue tracking to support
 * fairer distribution of tags between multiple submitters when a shared tag map
 * is used.
 *
 * Copyright (C) 2013-2014 Jens Axboe
 */
#include <linux/kernel.h>
#include <linux/module.h>

#include <linux/blk-mq.h>
#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-tag.h"


bool blk_mq_has_free_tags(struct blk_mq_tags *tags) { if (!tags) return true; return sbitmap_any_bit_clear(&tags->bitmap_tags.sb); }

Contributors

PersonTokensPropCommitsCommitProp
Jens Axboe2689.66%266.67%
Omar Sandoval310.34%133.33%
Total29100.00%3100.00%

/* * If a previously inactive queue goes active, bump the active user count. */
bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx) { if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state) && !test_and_set_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state)) atomic_inc(&hctx->tags->active_queues); return true; }

Contributors

PersonTokensPropCommitsCommitProp
Jens Axboe47100.00%1100.00%
Total47100.00%1100.00%

/* * Wakeup all potentially sleeping on tags */
void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool include_reserve) { sbitmap_queue_wake_all(&tags->bitmap_tags); if (include_reserve) sbitmap_queue_wake_all(&tags->breserved_tags); }

Contributors

PersonTokensPropCommitsCommitProp
Jens Axboe2987.88%375.00%
Omar Sandoval412.12%125.00%
Total33100.00%4100.00%

/* * If a previously busy queue goes inactive, potential waiters could now * be allowed to queue. Wake them up and check. */
void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx) { struct blk_mq_tags *tags = hctx->tags; if (!test_and_clear_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state)) return; atomic_dec(&tags->active_queues); blk_mq_tag_wakeup_all(tags, false); }

Contributors

PersonTokensPropCommitsCommitProp
Jens Axboe48100.00%2100.00%
Total48100.00%2100.00%

/* * For shared tag users, we track the number of currently active users * and attempt to provide a fair share of the tag depth for each of them. */
static inline bool hctx_may_queue(struct blk_mq_hw_ctx *hctx, struct sbitmap_queue *bt) { unsigned int depth, users; if (!hctx || !(hctx->flags & BLK_MQ_F_TAG_SHARED)) return true; if (!test_bit(BLK_MQ_S_TAG_ACTIVE, &hctx->state)) return true; /* * Don't try dividing an ant */ if (bt->sb.depth == 1) return true; users = atomic_read(&hctx->tags->active_queues); if (!users) return true; /* * Allow at least some tags */ depth = max((bt->sb.depth + users - 1) / users, 4U); return atomic_read(&hctx->nr_active) < depth; }

Contributors

PersonTokensPropCommitsCommitProp
Jens Axboe11895.93%150.00%
Omar Sandoval54.07%150.00%
Total123100.00%2100.00%


static int __blk_mq_get_tag(struct blk_mq_alloc_data *data, struct sbitmap_queue *bt) { if (!(data->flags & BLK_MQ_REQ_INTERNAL) && !hctx_may_queue(data->hctx, bt)) return -1; if (data->shallow_depth) return __sbitmap_queue_get_shallow(bt, data->shallow_depth); else return __sbitmap_queue_get(bt); }

Contributors

PersonTokensPropCommitsCommitProp
Jens Axboe4062.50%666.67%
Omar Sandoval2437.50%333.33%
Total64100.00%9100.00%


unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) { struct blk_mq_tags *tags = blk_mq_tags_from_data(data); struct sbitmap_queue *bt; struct sbq_wait_state *ws; DEFINE_WAIT(wait); unsigned int tag_offset; bool drop_ctx; int tag; if (data->flags & BLK_MQ_REQ_RESERVED) { if (unlikely(!tags->nr_reserved_tags)) { WARN_ON_ONCE(1); return BLK_MQ_TAG_FAIL; } bt = &tags->breserved_tags; tag_offset = 0; } else { bt = &tags->bitmap_tags; tag_offset = tags->nr_reserved_tags; } tag = __blk_mq_get_tag(data, bt); if (tag != -1) goto found_tag; if (data->flags & BLK_MQ_REQ_NOWAIT) return BLK_MQ_TAG_FAIL; ws = bt_wait_ptr(bt, data->hctx); drop_ctx = data->ctx == NULL; do { prepare_to_wait(&ws->wait, &wait, TASK_UNINTERRUPTIBLE); tag = __blk_mq_get_tag(data, bt); if (tag != -1) break; /* * We're out of tags on this hardware queue, kick any * pending IO submits before going to sleep waiting for * some to complete. */ blk_mq_run_hw_queue(data->hctx, false); /* * Retry tag allocation after running the hardware queue, * as running the queue may also have found completions. */ tag = __blk_mq_get_tag(data, bt); if (tag != -1) break; if (data->ctx) blk_mq_put_ctx(data->ctx); io_schedule(); data->ctx = blk_mq_get_ctx(data->q); data->hctx = blk_mq_map_queue(data->q, data->ctx->cpu); tags = blk_mq_tags_from_data(data); if (data->flags & BLK_MQ_REQ_RESERVED) bt = &tags->breserved_tags; else bt = &tags->bitmap_tags; finish_wait(&ws->wait, &wait); ws = bt_wait_ptr(bt, data->hctx); } while (1); if (drop_ctx && data->ctx) blk_mq_put_ctx(data->ctx); finish_wait(&ws->wait, &wait); found_tag: return tag + tag_offset; }

Contributors

PersonTokensPropCommitsCommitProp
Jens Axboe26375.79%753.85%
Lei Ming5917.00%17.69%
Omar Sandoval72.02%17.69%
Bart Van Assche72.02%17.69%
Christoph Hellwig61.73%17.69%
Shaohua Li41.15%17.69%
Linus Torvalds10.29%17.69%
Total347100.00%13100.00%


void blk_mq_put_tag(struct blk_mq_hw_ctx *hctx, struct blk_mq_tags *tags, struct blk_mq_ctx *ctx, unsigned int tag) { if (!blk_mq_tag_is_reserved(tags, tag)) { const int real_tag = tag - tags->nr_reserved_tags; BUG_ON(real_tag >= tags->nr_tags); sbitmap_queue_clear(&tags->bitmap_tags, real_tag, ctx->cpu); } else { BUG_ON(tag >= tags->nr_reserved_tags); sbitmap_queue_clear(&tags->breserved_tags, tag, ctx->cpu); } }

Contributors

PersonTokensPropCommitsCommitProp
Jens Axboe7376.84%555.56%
Omar Sandoval1414.74%222.22%
Sagi Grimberg66.32%111.11%
Shaohua Li22.11%111.11%
Total95100.00%9100.00%

struct bt_iter_data { struct blk_mq_hw_ctx *hctx; busy_iter_fn *fn; void *data; bool reserved; };
static bool bt_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) { struct bt_iter_data *iter_data = data; struct blk_mq_hw_ctx *hctx = iter_data->hctx; struct blk_mq_tags *tags = hctx->tags; bool reserved = iter_data->reserved; struct request *rq; if (!reserved) bitnr += tags->nr_reserved_tags; rq = tags->rqs[bitnr]; if (rq->q == hctx->queue) iter_data->fn(hctx, rq, iter_data->data, reserved); return true; }

Contributors

PersonTokensPropCommitsCommitProp
Omar Sandoval5250.00%125.00%
Christoph Hellwig2423.08%125.00%
Jens Axboe2423.08%125.00%
Ming Lei43.85%125.00%
Total104100.00%4100.00%


static void bt_for_each(struct blk_mq_hw_ctx *hctx, struct sbitmap_queue *bt, busy_iter_fn *fn, void *data, bool reserved) { struct bt_iter_data iter_data = { .hctx = hctx, .fn = fn, .data = data, .reserved = reserved, }; sbitmap_for_each_set(&bt->sb, bt_iter, &iter_data); }

Contributors

PersonTokensPropCommitsCommitProp
Omar Sandoval4162.12%150.00%
Keith Busch2537.88%150.00%
Total66100.00%2100.00%

struct bt_tags_iter_data { struct blk_mq_tags *tags; busy_tag_iter_fn *fn; void *data; bool reserved; };
static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) { struct bt_tags_iter_data *iter_data = data; struct blk_mq_tags *tags = iter_data->tags; bool reserved = iter_data->reserved; struct request *rq; if (!reserved) bitnr += tags->nr_reserved_tags; rq = tags->rqs[bitnr]; iter_data->fn(rq, iter_data->data, reserved); return true; }

Contributors

PersonTokensPropCommitsCommitProp
Omar Sandoval5768.67%133.33%
Keith Busch2226.51%133.33%
Ming Lei44.82%133.33%
Total83100.00%3100.00%


static void bt_tags_for_each(struct blk_mq_tags *tags, struct sbitmap_queue *bt, busy_tag_iter_fn *fn, void *data, bool reserved) { struct bt_tags_iter_data iter_data = { .tags = tags, .fn = fn, .data = data, .reserved = reserved, }; if (tags->rqs) sbitmap_for_each_set(&bt->sb, bt_tags_iter, &iter_data); }

Contributors

PersonTokensPropCommitsCommitProp
Omar Sandoval5880.56%133.33%
Keith Busch1318.06%133.33%
Sagi Grimberg11.39%133.33%
Total72100.00%3100.00%


static void blk_mq_all_tag_busy_iter(struct blk_mq_tags *tags, busy_tag_iter_fn *fn, void *priv) { if (tags->nr_reserved_tags) bt_tags_for_each(tags, &tags->breserved_tags, fn, priv, true); bt_tags_for_each(tags, &tags->bitmap_tags, fn, priv, false); }

Contributors

PersonTokensPropCommitsCommitProp
Keith Busch4070.18%150.00%
Omar Sandoval1729.82%150.00%
Total57100.00%2100.00%


void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset, busy_tag_iter_fn *fn, void *priv) { int i; for (i = 0; i < tagset->nr_hw_queues; i++) { if (tagset->tags && tagset->tags[i]) blk_mq_all_tag_busy_iter(tagset->tags[i], fn, priv); } }

Contributors

PersonTokensPropCommitsCommitProp
Sagi Grimberg65100.00%1100.00%
Total65100.00%1100.00%

EXPORT_SYMBOL(blk_mq_tagset_busy_iter);
int blk_mq_reinit_tagset(struct blk_mq_tag_set *set) { int i, j, ret = 0; if (!set->ops->reinit_request) goto out; for (i = 0; i < set->nr_hw_queues; i++) { struct blk_mq_tags *tags = set->tags[i]; if (!tags) continue; for (j = 0; j < tags->nr_tags; j++) { if (!tags->static_rqs[j]) continue; ret = set->ops->reinit_request(set->driver_data, tags->static_rqs[j]); if (ret) goto out; } } out: return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Sagi Grimberg9273.02%228.57%
Omar Sandoval2015.87%114.29%
Jens Axboe1411.11%457.14%
Total126100.00%7100.00%

EXPORT_SYMBOL_GPL(blk_mq_reinit_tagset);
void blk_mq_queue_tag_busy_iter(struct request_queue *q, busy_iter_fn *fn, void *priv) { struct blk_mq_hw_ctx *hctx; int i; queue_for_each_hw_ctx(q, hctx, i) { struct blk_mq_tags *tags = hctx->tags; /* * If not software queues are currently mapped to this * hardware queue, there's nothing to check */ if (!blk_mq_hw_queue_mapped(hctx)) continue; if (tags->nr_reserved_tags) bt_for_each(hctx, &tags->breserved_tags, fn, priv, true); bt_for_each(hctx, &tags->bitmap_tags, fn, priv, false); } }

Contributors

PersonTokensPropCommitsCommitProp
Omar Sandoval7176.34%120.00%
Jens Axboe1617.20%240.00%
Alexander Gordeev55.38%120.00%
Tony Battersby11.08%120.00%
Total93100.00%5100.00%


static int bt_alloc(struct sbitmap_queue *bt, unsigned int depth, bool round_robin, int node) { return sbitmap_queue_init_node(bt, depth, -1, round_robin, GFP_KERNEL, node); }

Contributors

PersonTokensPropCommitsCommitProp
Omar Sandoval3489.47%250.00%
Jens Axboe410.53%250.00%
Total38100.00%4100.00%


static struct blk_mq_tags *blk_mq_init_bitmap_tags(struct blk_mq_tags *tags, int node, int alloc_policy) { unsigned int depth = tags->nr_tags - tags->nr_reserved_tags; bool round_robin = alloc_policy == BLK_TAG_ALLOC_RR; if (bt_alloc(&tags->bitmap_tags, depth, round_robin, node)) goto free_tags; if (bt_alloc(&tags->breserved_tags, tags->nr_reserved_tags, round_robin, node)) goto free_bitmap_tags; return tags; free_bitmap_tags: sbitmap_queue_free(&tags->bitmap_tags); free_tags: kfree(tags); return NULL; }

Contributors

PersonTokensPropCommitsCommitProp
Jens Axboe8180.20%240.00%
Omar Sandoval1615.84%240.00%
Shaohua Li43.96%120.00%
Total101100.00%5100.00%


struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags, unsigned int reserved_tags, int node, int alloc_policy) { struct blk_mq_tags *tags; if (total_tags > BLK_MQ_TAG_MAX) { pr_err("blk-mq: tag depth too large\n"); return NULL; } tags = kzalloc_node(sizeof(*tags), GFP_KERNEL, node); if (!tags) return NULL; tags->nr_tags = total_tags; tags->nr_reserved_tags = reserved_tags; return blk_mq_init_bitmap_tags(tags, node, alloc_policy); }

Contributors

PersonTokensPropCommitsCommitProp
Jens Axboe8294.25%266.67%
Shaohua Li55.75%133.33%
Total87100.00%3100.00%


void blk_mq_free_tags(struct blk_mq_tags *tags) { sbitmap_queue_free(&tags->bitmap_tags); sbitmap_queue_free(&tags->breserved_tags); kfree(tags); }

Contributors

PersonTokensPropCommitsCommitProp
Jens Axboe2993.55%266.67%
Omar Sandoval26.45%133.33%
Total31100.00%3100.00%


int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx, struct blk_mq_tags **tagsptr, unsigned int tdepth, bool can_grow) { struct blk_mq_tags *tags = *tagsptr; if (tdepth <= tags->nr_reserved_tags) return -EINVAL; tdepth -= tags->nr_reserved_tags; /* * If we are allowed to grow beyond the original size, allocate * a new set of tags before freeing the old one. */ if (tdepth > tags->nr_tags) { struct blk_mq_tag_set *set = hctx->queue->tag_set; struct blk_mq_tags *new; bool ret; if (!can_grow) return -EINVAL; /* * We need some sort of upper limit, set it high enough that * no valid use cases should require more. */ if (tdepth > 16 * BLKDEV_MAX_RQ) return -EINVAL; new = blk_mq_alloc_rq_map(set, hctx->queue_num, tdepth, 0); if (!new) return -ENOMEM; ret = blk_mq_alloc_rqs(set, new, hctx->queue_num, tdepth); if (ret) { blk_mq_free_rq_map(new); return -ENOMEM; } blk_mq_free_rqs(set, *tagsptr, hctx->queue_num); blk_mq_free_rq_map(*tagsptr); *tagsptr = new; } else { /* * Don't need (or can't) update reserved tags here, they * remain static and should never need resizing. */ sbitmap_queue_resize(&tags->bitmap_tags, tdepth); } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Jens Axboe195100.00%3100.00%
Total195100.00%3100.00%

/** * blk_mq_unique_tag() - return a tag that is unique queue-wide * @rq: request for which to compute a unique tag * * The tag field in struct request is unique per hardware queue but not over * all hardware queues. Hence this function that returns a tag with the * hardware context index in the upper bits and the per hardware queue tag in * the lower bits. * * Note: When called for a request that is queued on a non-multiqueue request * queue, the hardware context index is set to zero. */
u32 blk_mq_unique_tag(struct request *rq) { struct request_queue *q = rq->q; struct blk_mq_hw_ctx *hctx; int hwq = 0; if (q->mq_ops) { hctx = blk_mq_map_queue(q, rq->mq_ctx->cpu); hwq = hctx->queue_num; } return (hwq << BLK_MQ_UNIQUE_TAG_BITS) | (rq->tag & BLK_MQ_UNIQUE_TAG_MASK); }

Contributors

PersonTokensPropCommitsCommitProp
Bart Van Assche7098.59%150.00%
Christoph Hellwig11.41%150.00%
Total71100.00%2100.00%

EXPORT_SYMBOL(blk_mq_unique_tag);

Overall Contributors

PersonTokensPropCommitsCommitProp
Jens Axboe111554.23%1541.67%
Omar Sandoval46722.71%411.11%
Sagi Grimberg1698.22%513.89%
Keith Busch1004.86%12.78%
Bart Van Assche834.04%25.56%
Lei Ming592.87%12.78%
Christoph Hellwig331.61%38.33%
Shaohua Li150.73%12.78%
Ming Lei80.39%12.78%
Alexander Gordeev50.24%12.78%
Tony Battersby10.05%12.78%
Linus Torvalds10.05%12.78%
Total2056100.00%36100.00%
Directory: block
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with cregit.