Author | Tokens | Token Proportion | Commits | Commit Proportion |
---|---|---|---|---|
Damien Le Moal | 1986 | 80.24% | 6 | 15.38% |
Christoph Hellwig | 267 | 10.79% | 9 | 23.08% |
Mike Snitzer | 106 | 4.28% | 10 | 25.64% |
Alan Cox | 64 | 2.59% | 1 | 2.56% |
Mikulas Patocka | 26 | 1.05% | 4 | 10.26% |
Denis Semakin | 8 | 0.32% | 1 | 2.56% |
Hannes Reinecke | 5 | 0.20% | 1 | 2.56% |
Andrew Morton | 4 | 0.16% | 1 | 2.56% |
Alasdair G. Kergon | 3 | 0.12% | 1 | 2.56% |
Bart Van Assche | 2 | 0.08% | 1 | 2.56% |
Jun'ichi Nomura | 1 | 0.04% | 1 | 2.56% |
Kiyoshi Ueda | 1 | 0.04% | 1 | 2.56% |
Milan Broz | 1 | 0.04% | 1 | 2.56% |
Heinz Mauelshagen | 1 | 0.04% | 1 | 2.56% |
Total | 2475 | 39 |
// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2021 Western Digital Corporation or its affiliates. */ #include <linux/blkdev.h> #include <linux/mm.h> #include <linux/sched/mm.h> #include <linux/slab.h> #include "dm-core.h" #define DM_MSG_PREFIX "zone" #define DM_ZONE_INVALID_WP_OFST UINT_MAX /* * For internal zone reports bypassing the top BIO submission path. */ static int dm_blk_do_report_zones(struct mapped_device *md, struct dm_table *t, sector_t sector, unsigned int nr_zones, report_zones_cb cb, void *data) { struct gendisk *disk = md->disk; int ret; struct dm_report_zones_args args = { .next_sector = sector, .orig_data = data, .orig_cb = cb, }; do { struct dm_target *tgt; tgt = dm_table_find_target(t, args.next_sector); if (WARN_ON_ONCE(!tgt->type->report_zones)) return -EIO; args.tgt = tgt; ret = tgt->type->report_zones(tgt, &args, nr_zones - args.zone_idx); if (ret < 0) return ret; } while (args.zone_idx < nr_zones && args.next_sector < get_capacity(disk)); return args.zone_idx; } /* * User facing dm device block device report zone operation. This calls the * report_zones operation for each target of a device table. This operation is * generally implemented by targets using dm_report_zones(). */ int dm_blk_report_zones(struct gendisk *disk, sector_t sector, unsigned int nr_zones, report_zones_cb cb, void *data) { struct mapped_device *md = disk->private_data; struct dm_table *map; int srcu_idx, ret; if (dm_suspended_md(md)) return -EAGAIN; map = dm_get_live_table(md, &srcu_idx); if (!map) return -EIO; ret = dm_blk_do_report_zones(md, map, sector, nr_zones, cb, data); dm_put_live_table(md, srcu_idx); return ret; } static int dm_report_zones_cb(struct blk_zone *zone, unsigned int idx, void *data) { struct dm_report_zones_args *args = data; sector_t sector_diff = args->tgt->begin - args->start; /* * Ignore zones beyond the target range. */ if (zone->start >= args->start + args->tgt->len) return 0; /* * Remap the start sector and write pointer position of the zone * to match its position in the target range. */ zone->start += sector_diff; if (zone->type != BLK_ZONE_TYPE_CONVENTIONAL) { if (zone->cond == BLK_ZONE_COND_FULL) zone->wp = zone->start + zone->len; else if (zone->cond == BLK_ZONE_COND_EMPTY) zone->wp = zone->start; else zone->wp += sector_diff; } args->next_sector = zone->start + zone->len; return args->orig_cb(zone, args->zone_idx++, args->orig_data); } /* * Helper for drivers of zoned targets to implement struct target_type * report_zones operation. */ int dm_report_zones(struct block_device *bdev, sector_t start, sector_t sector, struct dm_report_zones_args *args, unsigned int nr_zones) { /* * Set the target mapping start sector first so that * dm_report_zones_cb() can correctly remap zone information. */ args->start = start; return blkdev_report_zones(bdev, sector, nr_zones, dm_report_zones_cb, args); } EXPORT_SYMBOL_GPL(dm_report_zones); bool dm_is_zone_write(struct mapped_device *md, struct bio *bio) { struct request_queue *q = md->queue; if (!blk_queue_is_zoned(q)) return false; switch (bio_op(bio)) { case REQ_OP_WRITE_ZEROES: case REQ_OP_WRITE: return !op_is_flush(bio->bi_opf) && bio_sectors(bio); default: return false; } } void dm_cleanup_zoned_dev(struct mapped_device *md) { if (md->disk) { kfree(md->disk->conv_zones_bitmap); md->disk->conv_zones_bitmap = NULL; kfree(md->disk->seq_zones_wlock); md->disk->seq_zones_wlock = NULL; } kvfree(md->zwp_offset); md->zwp_offset = NULL; md->nr_zones = 0; } static unsigned int dm_get_zone_wp_offset(struct blk_zone *zone) { switch (zone->cond) { case BLK_ZONE_COND_IMP_OPEN: case BLK_ZONE_COND_EXP_OPEN: case BLK_ZONE_COND_CLOSED: return zone->wp - zone->start; case BLK_ZONE_COND_FULL: return zone->len; case BLK_ZONE_COND_EMPTY: case BLK_ZONE_COND_NOT_WP: case BLK_ZONE_COND_OFFLINE: case BLK_ZONE_COND_READONLY: default: /* * Conventional, offline and read-only zones do not have a valid * write pointer. Use 0 as for an empty zone. */ return 0; } } static int dm_zone_revalidate_cb(struct blk_zone *zone, unsigned int idx, void *data) { struct mapped_device *md = data; struct gendisk *disk = md->disk; switch (zone->type) { case BLK_ZONE_TYPE_CONVENTIONAL: if (!disk->conv_zones_bitmap) { disk->conv_zones_bitmap = kcalloc(BITS_TO_LONGS(disk->nr_zones), sizeof(unsigned long), GFP_NOIO); if (!disk->conv_zones_bitmap) return -ENOMEM; } set_bit(idx, disk->conv_zones_bitmap); break; case BLK_ZONE_TYPE_SEQWRITE_REQ: case BLK_ZONE_TYPE_SEQWRITE_PREF: if (!disk->seq_zones_wlock) { disk->seq_zones_wlock = kcalloc(BITS_TO_LONGS(disk->nr_zones), sizeof(unsigned long), GFP_NOIO); if (!disk->seq_zones_wlock) return -ENOMEM; } if (!md->zwp_offset) { md->zwp_offset = kvcalloc(disk->nr_zones, sizeof(unsigned int), GFP_KERNEL); if (!md->zwp_offset) return -ENOMEM; } md->zwp_offset[idx] = dm_get_zone_wp_offset(zone); break; default: DMERR("Invalid zone type 0x%x at sectors %llu", (int)zone->type, zone->start); return -ENODEV; } return 0; } /* * Revalidate the zones of a mapped device to initialize resource necessary * for zone append emulation. Note that we cannot simply use the block layer * blk_revalidate_disk_zones() function here as the mapped device is suspended * (this is called from __bind() context). */ static int dm_revalidate_zones(struct mapped_device *md, struct dm_table *t) { struct gendisk *disk = md->disk; unsigned int noio_flag; int ret; /* * Check if something changed. If yes, cleanup the current resources * and reallocate everything. */ if (!disk->nr_zones || disk->nr_zones != md->nr_zones) dm_cleanup_zoned_dev(md); if (md->nr_zones) return 0; /* * Scan all zones to initialize everything. Ensure that all vmalloc * operations in this context are done as if GFP_NOIO was specified. */ noio_flag = memalloc_noio_save(); ret = dm_blk_do_report_zones(md, t, 0, disk->nr_zones, dm_zone_revalidate_cb, md); memalloc_noio_restore(noio_flag); if (ret < 0) goto err; if (ret != disk->nr_zones) { ret = -EIO; goto err; } md->nr_zones = disk->nr_zones; return 0; err: DMERR("Revalidate zones failed %d", ret); dm_cleanup_zoned_dev(md); return ret; } static int device_not_zone_append_capable(struct dm_target *ti, struct dm_dev *dev, sector_t start, sector_t len, void *data) { return !bdev_is_zoned(dev->bdev); } static bool dm_table_supports_zone_append(struct dm_table *t) { for (unsigned int i = 0; i < t->num_targets; i++) { struct dm_target *ti = dm_table_get_target(t, i); if (ti->emulate_zone_append) return false; if (!ti->type->iterate_devices || ti->type->iterate_devices(ti, device_not_zone_append_capable, NULL)) return false; } return true; } int dm_set_zones_restrictions(struct dm_table *t, struct request_queue *q) { struct mapped_device *md = t->md; /* * For a zoned target, the number of zones should be updated for the * correct value to be exposed in sysfs queue/nr_zones. */ WARN_ON_ONCE(queue_is_mq(q)); md->disk->nr_zones = bdev_nr_zones(md->disk->part0); /* Check if zone append is natively supported */ if (dm_table_supports_zone_append(t)) { clear_bit(DMF_EMULATE_ZONE_APPEND, &md->flags); dm_cleanup_zoned_dev(md); return 0; } /* * Mark the mapped device as needing zone append emulation and * initialize the emulation resources once the capacity is set. */ set_bit(DMF_EMULATE_ZONE_APPEND, &md->flags); if (!get_capacity(md->disk)) return 0; return dm_revalidate_zones(md, t); } static int dm_update_zone_wp_offset_cb(struct blk_zone *zone, unsigned int idx, void *data) { unsigned int *wp_offset = data; *wp_offset = dm_get_zone_wp_offset(zone); return 0; } static int dm_update_zone_wp_offset(struct mapped_device *md, unsigned int zno, unsigned int *wp_ofst) { sector_t sector = zno * bdev_zone_sectors(md->disk->part0); unsigned int noio_flag; struct dm_table *t; int srcu_idx, ret; t = dm_get_live_table(md, &srcu_idx); if (!t) return -EIO; /* * Ensure that all memory allocations in this context are done as if * GFP_NOIO was specified. */ noio_flag = memalloc_noio_save(); ret = dm_blk_do_report_zones(md, t, sector, 1, dm_update_zone_wp_offset_cb, wp_ofst); memalloc_noio_restore(noio_flag); dm_put_live_table(md, srcu_idx); if (ret != 1) return -EIO; return 0; } struct orig_bio_details { enum req_op op; unsigned int nr_sectors; }; /* * First phase of BIO mapping for targets with zone append emulation: * check all BIO that change a zone writer pointer and change zone * append operations into regular write operations. */ static bool dm_zone_map_bio_begin(struct mapped_device *md, unsigned int zno, struct bio *clone) { sector_t zsectors = bdev_zone_sectors(md->disk->part0); unsigned int zwp_offset = READ_ONCE(md->zwp_offset[zno]); /* * If the target zone is in an error state, recover by inspecting the * zone to get its current write pointer position. Note that since the * target zone is already locked, a BIO issuing context should never * see the zone write in the DM_ZONE_UPDATING_WP_OFST state. */ if (zwp_offset == DM_ZONE_INVALID_WP_OFST) { if (dm_update_zone_wp_offset(md, zno, &zwp_offset)) return false; WRITE_ONCE(md->zwp_offset[zno], zwp_offset); } switch (bio_op(clone)) { case REQ_OP_ZONE_RESET: case REQ_OP_ZONE_FINISH: return true; case REQ_OP_WRITE_ZEROES: case REQ_OP_WRITE: /* Writes must be aligned to the zone write pointer */ if ((clone->bi_iter.bi_sector & (zsectors - 1)) != zwp_offset) return false; break; case REQ_OP_ZONE_APPEND: /* * Change zone append operations into a non-mergeable regular * writes directed at the current write pointer position of the * target zone. */ clone->bi_opf = REQ_OP_WRITE | REQ_NOMERGE | (clone->bi_opf & (~REQ_OP_MASK)); clone->bi_iter.bi_sector += zwp_offset; break; default: DMWARN_LIMIT("Invalid BIO operation"); return false; } /* Cannot write to a full zone */ if (zwp_offset >= zsectors) return false; return true; } /* * Second phase of BIO mapping for targets with zone append emulation: * update the zone write pointer offset array to account for the additional * data written to a zone. Note that at this point, the remapped clone BIO * may already have completed, so we do not touch it. */ static blk_status_t dm_zone_map_bio_end(struct mapped_device *md, unsigned int zno, struct orig_bio_details *orig_bio_details, unsigned int nr_sectors) { unsigned int zwp_offset = READ_ONCE(md->zwp_offset[zno]); /* The clone BIO may already have been completed and failed */ if (zwp_offset == DM_ZONE_INVALID_WP_OFST) return BLK_STS_IOERR; /* Update the zone wp offset */ switch (orig_bio_details->op) { case REQ_OP_ZONE_RESET: WRITE_ONCE(md->zwp_offset[zno], 0); return BLK_STS_OK; case REQ_OP_ZONE_FINISH: WRITE_ONCE(md->zwp_offset[zno], bdev_zone_sectors(md->disk->part0)); return BLK_STS_OK; case REQ_OP_WRITE_ZEROES: case REQ_OP_WRITE: WRITE_ONCE(md->zwp_offset[zno], zwp_offset + nr_sectors); return BLK_STS_OK; case REQ_OP_ZONE_APPEND: /* * Check that the target did not truncate the write operation * emulating a zone append. */ if (nr_sectors != orig_bio_details->nr_sectors) { DMWARN_LIMIT("Truncated write for zone append"); return BLK_STS_IOERR; } WRITE_ONCE(md->zwp_offset[zno], zwp_offset + nr_sectors); return BLK_STS_OK; default: DMWARN_LIMIT("Invalid BIO operation"); return BLK_STS_IOERR; } } static inline void dm_zone_lock(struct gendisk *disk, unsigned int zno, struct bio *clone) { if (WARN_ON_ONCE(bio_flagged(clone, BIO_ZONE_WRITE_LOCKED))) return; wait_on_bit_lock_io(disk->seq_zones_wlock, zno, TASK_UNINTERRUPTIBLE); bio_set_flag(clone, BIO_ZONE_WRITE_LOCKED); } static inline void dm_zone_unlock(struct gendisk *disk, unsigned int zno, struct bio *clone) { if (!bio_flagged(clone, BIO_ZONE_WRITE_LOCKED)) return; WARN_ON_ONCE(!test_bit(zno, disk->seq_zones_wlock)); clear_bit_unlock(zno, disk->seq_zones_wlock); smp_mb__after_atomic(); wake_up_bit(disk->seq_zones_wlock, zno); bio_clear_flag(clone, BIO_ZONE_WRITE_LOCKED); } static bool dm_need_zone_wp_tracking(struct bio *bio) { /* * Special processing is not needed for operations that do not need the * zone write lock, that is, all operations that target conventional * zones and all operations that do not modify directly a sequential * zone write pointer. */ if (op_is_flush(bio->bi_opf) && !bio_sectors(bio)) return false; switch (bio_op(bio)) { case REQ_OP_WRITE_ZEROES: case REQ_OP_WRITE: case REQ_OP_ZONE_RESET: case REQ_OP_ZONE_FINISH: case REQ_OP_ZONE_APPEND: return bio_zone_is_seq(bio); default: return false; } } /* * Special IO mapping for targets needing zone append emulation. */ int dm_zone_map_bio(struct dm_target_io *tio) { struct dm_io *io = tio->io; struct dm_target *ti = tio->ti; struct mapped_device *md = io->md; struct bio *clone = &tio->clone; struct orig_bio_details orig_bio_details; unsigned int zno; blk_status_t sts; int r; /* * IOs that do not change a zone write pointer do not need * any additional special processing. */ if (!dm_need_zone_wp_tracking(clone)) return ti->type->map(ti, clone); /* Lock the target zone */ zno = bio_zone_no(clone); dm_zone_lock(md->disk, zno, clone); orig_bio_details.nr_sectors = bio_sectors(clone); orig_bio_details.op = bio_op(clone); /* * Check that the bio and the target zone write pointer offset are * both valid, and if the bio is a zone append, remap it to a write. */ if (!dm_zone_map_bio_begin(md, zno, clone)) { dm_zone_unlock(md->disk, zno, clone); return DM_MAPIO_KILL; } /* Let the target do its work */ r = ti->type->map(ti, clone); switch (r) { case DM_MAPIO_SUBMITTED: /* * The target submitted the clone BIO. The target zone will * be unlocked on completion of the clone. */ sts = dm_zone_map_bio_end(md, zno, &orig_bio_details, *tio->len_ptr); break; case DM_MAPIO_REMAPPED: /* * The target only remapped the clone BIO. In case of error, * unlock the target zone here as the clone will not be * submitted. */ sts = dm_zone_map_bio_end(md, zno, &orig_bio_details, *tio->len_ptr); if (sts != BLK_STS_OK) dm_zone_unlock(md->disk, zno, clone); break; case DM_MAPIO_REQUEUE: case DM_MAPIO_KILL: default: dm_zone_unlock(md->disk, zno, clone); sts = BLK_STS_IOERR; break; } if (sts != BLK_STS_OK) return DM_MAPIO_KILL; return r; } /* * IO completion callback called from clone_endio(). */ void dm_zone_endio(struct dm_io *io, struct bio *clone) { struct mapped_device *md = io->md; struct gendisk *disk = md->disk; struct bio *orig_bio = io->orig_bio; unsigned int zwp_offset; unsigned int zno; /* * For targets that do not emulate zone append, we only need to * handle native zone-append bios. */ if (!dm_emulate_zone_append(md)) { /* * Get the offset within the zone of the written sector * and add that to the original bio sector position. */ if (clone->bi_status == BLK_STS_OK && bio_op(clone) == REQ_OP_ZONE_APPEND) { sector_t mask = (sector_t)bdev_zone_sectors(disk->part0) - 1; orig_bio->bi_iter.bi_sector += clone->bi_iter.bi_sector & mask; } return; } /* * For targets that do emulate zone append, if the clone BIO does not * own the target zone write lock, we have nothing to do. */ if (!bio_flagged(clone, BIO_ZONE_WRITE_LOCKED)) return; zno = bio_zone_no(orig_bio); if (clone->bi_status != BLK_STS_OK) { /* * BIOs that modify a zone write pointer may leave the zone * in an unknown state in case of failure (e.g. the write * pointer was only partially advanced). In this case, set * the target zone write pointer as invalid unless it is * already being updated. */ WRITE_ONCE(md->zwp_offset[zno], DM_ZONE_INVALID_WP_OFST); } else if (bio_op(orig_bio) == REQ_OP_ZONE_APPEND) { /* * Get the written sector for zone append operation that were * emulated using regular write operations. */ zwp_offset = READ_ONCE(md->zwp_offset[zno]); if (WARN_ON_ONCE(zwp_offset < bio_sectors(orig_bio))) WRITE_ONCE(md->zwp_offset[zno], DM_ZONE_INVALID_WP_OFST); else orig_bio->bi_iter.bi_sector += zwp_offset - bio_sectors(orig_bio); } dm_zone_unlock(disk, zno, clone); }
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1