| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| Matthew Sakai | 1329 | 99.11% | 5 | 71.43% |
| Mike Snitzer | 12 | 0.89% | 2 | 28.57% |
| Total | 1341 | 7 |
/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright 2023 Red Hat */ #ifndef VDO_SLAB_DEPOT_H #define VDO_SLAB_DEPOT_H #include <linux/atomic.h> #include <linux/dm-kcopyd.h> #include <linux/list.h> #include "numeric.h" #include "admin-state.h" #include "completion.h" #include "data-vio.h" #include "encodings.h" #include "physical-zone.h" #include "priority-table.h" #include "recovery-journal.h" #include "statistics.h" #include "types.h" #include "vio.h" #include "wait-queue.h" /* * A slab_depot is responsible for managing all of the slabs and block allocators of a VDO. It has * a single array of slabs in order to eliminate the need for additional math in order to compute * which physical zone a PBN is in. It also has a block_allocator per zone. * * Each physical zone has a single dedicated queue and thread for performing all updates to the * slabs assigned to that zone. The concurrency guarantees of this single-threaded model allow the * code to omit more fine-grained locking for the various slab structures. Each physical zone * maintains a separate copy of the slab summary to remove the need for explicit locking on that * structure as well. * * Load operations must be performed on the admin thread. Normal operations, such as allocations * and reference count updates, must be performed on the appropriate physical zone thread. Requests * from the recovery journal to commit slab journal tail blocks must be scheduled from the recovery * journal thread to run on the appropriate physical zone thread. Save operations must be launched * from the same admin thread as the original load operation. */ enum { /* The number of vios in the vio pool is proportional to the throughput of the VDO. */ BLOCK_ALLOCATOR_VIO_POOL_SIZE = 128, }; /* * Represents the possible status of a block. */ enum reference_status { RS_FREE, /* this block is free */ RS_SINGLE, /* this block is singly-referenced */ RS_SHARED, /* this block is shared */ RS_PROVISIONAL /* this block is provisionally allocated */ }; struct vdo_slab; struct journal_lock { u16 count; sequence_number_t recovery_start; }; struct slab_journal { /* A waiter object for getting a VIO pool entry */ struct vdo_waiter resource_waiter; /* A waiter object for updating the slab summary */ struct vdo_waiter slab_summary_waiter; /* A waiter object for getting a vio with which to flush */ struct vdo_waiter flush_waiter; /* The queue of VIOs waiting to make an entry */ struct vdo_wait_queue entry_waiters; /* The parent slab reference of this journal */ struct vdo_slab *slab; /* Whether a tail block commit is pending */ bool waiting_to_commit; /* Whether the journal is updating the slab summary */ bool updating_slab_summary; /* Whether the journal is adding entries from the entry_waiters queue */ bool adding_entries; /* Whether a partial write is in progress */ bool partial_write_in_progress; /* The oldest block in the journal on disk */ sequence_number_t head; /* The oldest block in the journal which may not be reaped */ sequence_number_t unreapable; /* The end of the half-open interval of the active journal */ sequence_number_t tail; /* The next journal block to be committed */ sequence_number_t next_commit; /* The tail sequence number that is written in the slab summary */ sequence_number_t summarized; /* The tail sequence number that was last summarized in slab summary */ sequence_number_t last_summarized; /* The sequence number of the recovery journal lock */ sequence_number_t recovery_lock; /* * The number of entries which fit in a single block. Can't use the constant because unit * tests change this number. */ journal_entry_count_t entries_per_block; /* * The number of full entries which fit in a single block. Can't use the constant because * unit tests change this number. */ journal_entry_count_t full_entries_per_block; /* The recovery journal of the VDO (slab journal holds locks on it) */ struct recovery_journal *recovery_journal; /* The statistics shared by all slab journals in our physical zone */ struct slab_journal_statistics *events; /* A list of the VIO pool entries for outstanding journal block writes */ struct list_head uncommitted_blocks; /* * The current tail block header state. This will be packed into the block just before it * is written. */ struct slab_journal_block_header tail_header; /* A pointer to a block-sized buffer holding the packed block data */ struct packed_slab_journal_block *block; /* The number of blocks in the on-disk journal */ block_count_t size; /* The number of blocks at which to start pushing reference blocks */ block_count_t flushing_threshold; /* The number of blocks at which all reference blocks should be writing */ block_count_t flushing_deadline; /* The number of blocks at which to wait for reference blocks to write */ block_count_t blocking_threshold; /* The number of blocks at which to scrub the slab before coming online */ block_count_t scrubbing_threshold; /* This list entry is for block_allocator to keep a queue of dirty journals */ struct list_head dirty_entry; /* The lock for the oldest unreaped block of the journal */ struct journal_lock *reap_lock; /* The locks for each on disk block */ struct journal_lock *locks; }; /* * Reference_block structure * * Blocks are used as a proxy, permitting saves of partial refcounts. */ struct reference_block { /* This block waits on the ref_counts to tell it to write */ struct vdo_waiter waiter; /* The slab to which this reference_block belongs */ struct vdo_slab *slab; /* The number of references in this block that represent allocations */ block_size_t allocated_count; /* The slab journal block on which this block must hold a lock */ sequence_number_t slab_journal_lock; /* The slab journal block which should be released when this block is committed */ sequence_number_t slab_journal_lock_to_release; /* The point up to which each sector is accurate on disk */ struct journal_point commit_points[VDO_SECTORS_PER_BLOCK]; /* Whether this block has been modified since it was written to disk */ bool is_dirty; /* Whether this block is currently writing */ bool is_writing; }; /* The search_cursor represents the saved position of a free block search. */ struct search_cursor { /* The reference block containing the current search index */ struct reference_block *block; /* The position at which to start searching for the next free counter */ slab_block_number index; /* The position just past the last valid counter in the current block */ slab_block_number end_index; /* A pointer to the first reference block in the slab */ struct reference_block *first_block; /* A pointer to the last reference block in the slab */ struct reference_block *last_block; }; enum slab_rebuild_status { VDO_SLAB_REBUILT, VDO_SLAB_REPLAYING, VDO_SLAB_REQUIRES_SCRUBBING, VDO_SLAB_REQUIRES_HIGH_PRIORITY_SCRUBBING, VDO_SLAB_REBUILDING, }; /* * This is the type declaration for the vdo_slab type. A vdo_slab currently consists of a run of * 2^23 data blocks, but that will soon change to dedicate a small number of those blocks for * metadata storage for the reference counts and slab journal for the slab. * * A reference count is maintained for each physical block number. The vast majority of blocks have * a very small reference count (usually 0 or 1). For references less than or equal to MAXIMUM_REFS * (254) the reference count is stored in counters[pbn]. */ struct vdo_slab { /* A list entry to queue this slab in a block_allocator list */ struct list_head allocq_entry; /* The struct block_allocator that owns this slab */ struct block_allocator *allocator; /* The journal for this slab */ struct slab_journal journal; /* The slab number of this slab */ slab_count_t slab_number; /* The offset in the allocator partition of the first block in this slab */ physical_block_number_t start; /* The offset of the first block past the end of this slab */ physical_block_number_t end; /* The starting translated PBN of the slab journal */ physical_block_number_t journal_origin; /* The starting translated PBN of the reference counts */ physical_block_number_t ref_counts_origin; /* The administrative state of the slab */ struct admin_state state; /* The status of the slab */ enum slab_rebuild_status status; /* Whether the slab was ever queued for scrubbing */ bool was_queued_for_scrubbing; /* The priority at which this slab has been queued for allocation */ u8 priority; /* Fields beyond this point are the reference counts for the data blocks in this slab. */ /* The size of the counters array */ u32 block_count; /* The number of free blocks */ u32 free_blocks; /* The array of reference counts */ vdo_refcount_t *counters; /* use vdo_allocate() to align data ptr */ /* The saved block pointer and array indexes for the free block search */ struct search_cursor search_cursor; /* A list of the dirty blocks waiting to be written out */ struct vdo_wait_queue dirty_blocks; /* The number of blocks which are currently writing */ size_t active_count; /* A waiter object for updating the slab summary */ struct vdo_waiter summary_waiter; /* The latest slab journal for which there has been a reference count update */ struct journal_point slab_journal_point; /* The number of reference count blocks */ u32 reference_block_count; /* reference count block array */ struct reference_block *reference_blocks; }; enum block_allocator_drain_step { VDO_DRAIN_ALLOCATOR_START, VDO_DRAIN_ALLOCATOR_STEP_SCRUBBER, VDO_DRAIN_ALLOCATOR_STEP_SLABS, VDO_DRAIN_ALLOCATOR_STEP_SUMMARY, VDO_DRAIN_ALLOCATOR_STEP_FINISHED, }; struct slab_scrubber { /* The queue of slabs to scrub first */ struct list_head high_priority_slabs; /* The queue of slabs to scrub once there are no high_priority_slabs */ struct list_head slabs; /* The queue of VIOs waiting for a slab to be scrubbed */ struct vdo_wait_queue waiters; /* * The number of slabs that are unrecovered or being scrubbed. This field is modified by * the physical zone thread, but is queried by other threads. */ slab_count_t slab_count; /* The administrative state of the scrubber */ struct admin_state admin_state; /* Whether to only scrub high-priority slabs */ bool high_priority_only; /* The slab currently being scrubbed */ struct vdo_slab *slab; /* The vio for loading slab journal blocks */ struct vio vio; }; /* A sub-structure for applying actions in parallel to all an allocator's slabs. */ struct slab_actor { /* The number of slabs performing a slab action */ slab_count_t slab_action_count; /* The method to call when a slab action has been completed by all slabs */ vdo_action_fn callback; }; /* A slab_iterator is a structure for iterating over a set of slabs. */ struct slab_iterator { struct vdo_slab **slabs; struct vdo_slab *next; slab_count_t end; slab_count_t stride; }; /* * The slab_summary provides hints during load and recovery about the state of the slabs in order * to avoid the need to read the slab journals in their entirety before a VDO can come online. * * The information in the summary for each slab includes the rough number of free blocks (which is * used to prioritize scrubbing), the cleanliness of a slab (so that clean slabs containing free * space will be used on restart), and the location of the tail block of the slab's journal. * * The slab_summary has its own partition at the end of the volume which is sized to allow for a * complete copy of the summary for each of up to 16 physical zones. * * During resize, the slab_summary moves its backing partition and is saved once moved; the * slab_summary is not permitted to overwrite the previous recovery journal space. * * The slab_summary does not have its own version information, but relies on the VDO volume version * number. */ /* * A slab status is a very small structure for use in determining the ordering of slabs in the * scrubbing process. */ struct slab_status { slab_count_t slab_number; bool is_clean; u8 emptiness; }; struct slab_summary_block { /* The block_allocator to which this block belongs */ struct block_allocator *allocator; /* The index of this block in its zone's summary */ block_count_t index; /* Whether this block has a write outstanding */ bool writing; /* Ring of updates waiting on the outstanding write */ struct vdo_wait_queue current_update_waiters; /* Ring of updates waiting on the next write */ struct vdo_wait_queue next_update_waiters; /* The active slab_summary_entry array for this block */ struct slab_summary_entry *entries; /* The vio used to write this block */ struct vio vio; /* The packed entries, one block long, backing the vio */ char *outgoing_entries; }; /* * The statistics for all the slab summary zones owned by this slab summary. These fields are all * mutated only by their physical zone threads, but are read by other threads when gathering * statistics for the entire depot. */ struct atomic_slab_summary_statistics { /* Number of blocks written */ atomic64_t blocks_written; }; struct block_allocator { struct vdo_completion completion; /* The slab depot for this allocator */ struct slab_depot *depot; /* The nonce of the VDO */ nonce_t nonce; /* The physical zone number of this allocator */ zone_count_t zone_number; /* The thread ID for this allocator's physical zone */ thread_id_t thread_id; /* The number of slabs in this allocator */ slab_count_t slab_count; /* The number of the last slab owned by this allocator */ slab_count_t last_slab; /* The reduced priority level used to preserve unopened slabs */ unsigned int unopened_slab_priority; /* The state of this allocator */ struct admin_state state; /* The actor for applying an action to all slabs */ struct slab_actor slab_actor; /* The slab from which blocks are currently being allocated */ struct vdo_slab *open_slab; /* A priority queue containing all slabs available for allocation */ struct priority_table *prioritized_slabs; /* The slab scrubber */ struct slab_scrubber scrubber; /* What phase of the close operation the allocator is to perform */ enum block_allocator_drain_step drain_step; /* * These statistics are all mutated only by the physical zone thread, but are read by other * threads when gathering statistics for the entire depot. */ /* * The count of allocated blocks in this zone. Not in block_allocator_statistics for * historical reasons. */ u64 allocated_blocks; /* Statistics for this block allocator */ struct block_allocator_statistics statistics; /* Cumulative statistics for the slab journals in this zone */ struct slab_journal_statistics slab_journal_statistics; /* Cumulative statistics for the reference counters in this zone */ struct ref_counts_statistics ref_counts_statistics; /* * This is the head of a queue of slab journals which have entries in their tail blocks * which have not yet started to commit. When the recovery journal is under space pressure, * slab journals which have uncommitted entries holding a lock on the recovery journal head * are forced to commit their blocks early. This list is kept in order, with the tail * containing the slab journal holding the most recent recovery journal lock. */ struct list_head dirty_slab_journals; /* The vio pool for reading and writing block allocator metadata */ struct vio_pool *vio_pool; /* The dm_kcopyd client for erasing slab journals */ struct dm_kcopyd_client *eraser; /* Iterator over the slabs to be erased */ struct slab_iterator slabs_to_erase; /* The portion of the slab summary managed by this allocator */ /* The state of the slab summary */ struct admin_state summary_state; /* The number of outstanding summary writes */ block_count_t summary_write_count; /* The array (owned by the blocks) of all entries */ struct slab_summary_entry *summary_entries; /* The array of slab_summary_blocks */ struct slab_summary_block *summary_blocks; }; enum slab_depot_load_type { VDO_SLAB_DEPOT_NORMAL_LOAD, VDO_SLAB_DEPOT_RECOVERY_LOAD, VDO_SLAB_DEPOT_REBUILD_LOAD }; struct slab_depot { zone_count_t zone_count; zone_count_t old_zone_count; struct vdo *vdo; struct slab_config slab_config; struct action_manager *action_manager; physical_block_number_t first_block; physical_block_number_t last_block; physical_block_number_t origin; /* slab_size == (1 << slab_size_shift) */ unsigned int slab_size_shift; /* Determines how slabs should be queued during load */ enum slab_depot_load_type load_type; /* The state for notifying slab journals to release recovery journal */ sequence_number_t active_release_request; sequence_number_t new_release_request; /* State variables for scrubbing complete handling */ atomic_t zones_to_scrub; /* Array of pointers to individually allocated slabs */ struct vdo_slab **slabs; /* The number of slabs currently allocated and stored in 'slabs' */ slab_count_t slab_count; /* Array of pointers to a larger set of slabs (used during resize) */ struct vdo_slab **new_slabs; /* The number of slabs currently allocated and stored in 'new_slabs' */ slab_count_t new_slab_count; /* The size that 'new_slabs' was allocated for */ block_count_t new_size; /* The last block before resize, for rollback */ physical_block_number_t old_last_block; /* The last block after resize, for resize */ physical_block_number_t new_last_block; /* The statistics for the slab summary */ struct atomic_slab_summary_statistics summary_statistics; /* The start of the slab summary partition */ physical_block_number_t summary_origin; /* The number of bits to shift to get a 7-bit fullness hint */ unsigned int hint_shift; /* The slab summary entries for all of the zones the partition can hold */ struct slab_summary_entry *summary_entries; /* The block allocators for this depot */ struct block_allocator allocators[]; }; struct reference_updater; bool __must_check vdo_attempt_replay_into_slab(struct vdo_slab *slab, physical_block_number_t pbn, enum journal_operation operation, bool increment, struct journal_point *recovery_point, struct vdo_completion *parent); int __must_check vdo_adjust_reference_count_for_rebuild(struct slab_depot *depot, physical_block_number_t pbn, enum journal_operation operation); static inline struct block_allocator *vdo_as_block_allocator(struct vdo_completion *completion) { vdo_assert_completion_type(completion, VDO_BLOCK_ALLOCATOR_COMPLETION); return container_of(completion, struct block_allocator, completion); } int __must_check vdo_acquire_provisional_reference(struct vdo_slab *slab, physical_block_number_t pbn, struct pbn_lock *lock); int __must_check vdo_allocate_block(struct block_allocator *allocator, physical_block_number_t *block_number_ptr); int vdo_enqueue_clean_slab_waiter(struct block_allocator *allocator, struct vdo_waiter *waiter); void vdo_modify_reference_count(struct vdo_completion *completion, struct reference_updater *updater); int __must_check vdo_release_block_reference(struct block_allocator *allocator, physical_block_number_t pbn); void vdo_notify_slab_journals_are_recovered(struct vdo_completion *completion); void vdo_dump_block_allocator(const struct block_allocator *allocator); int __must_check vdo_decode_slab_depot(struct slab_depot_state_2_0 state, struct vdo *vdo, struct partition *summary_partition, struct slab_depot **depot_ptr); void vdo_free_slab_depot(struct slab_depot *depot); struct slab_depot_state_2_0 __must_check vdo_record_slab_depot(const struct slab_depot *depot); int __must_check vdo_allocate_reference_counters(struct slab_depot *depot); struct vdo_slab * __must_check vdo_get_slab(const struct slab_depot *depot, physical_block_number_t pbn); u8 __must_check vdo_get_increment_limit(struct slab_depot *depot, physical_block_number_t pbn); bool __must_check vdo_is_physical_data_block(const struct slab_depot *depot, physical_block_number_t pbn); block_count_t __must_check vdo_get_slab_depot_allocated_blocks(const struct slab_depot *depot); block_count_t __must_check vdo_get_slab_depot_data_blocks(const struct slab_depot *depot); void vdo_get_slab_depot_statistics(const struct slab_depot *depot, struct vdo_statistics *stats); void vdo_load_slab_depot(struct slab_depot *depot, const struct admin_state_code *operation, struct vdo_completion *parent, void *context); void vdo_prepare_slab_depot_to_allocate(struct slab_depot *depot, enum slab_depot_load_type load_type, struct vdo_completion *parent); void vdo_update_slab_depot_size(struct slab_depot *depot); int __must_check vdo_prepare_to_grow_slab_depot(struct slab_depot *depot, const struct partition *partition); void vdo_use_new_slabs(struct slab_depot *depot, struct vdo_completion *parent); void vdo_abandon_new_slabs(struct slab_depot *depot); void vdo_drain_slab_depot(struct slab_depot *depot, const struct admin_state_code *operation, struct vdo_completion *parent); void vdo_resume_slab_depot(struct slab_depot *depot, struct vdo_completion *parent); void vdo_commit_oldest_slab_journal_tail_blocks(struct slab_depot *depot, sequence_number_t recovery_block_number); void vdo_scrub_all_unrecovered_slabs(struct slab_depot *depot, struct vdo_completion *parent); void vdo_dump_slab_depot(const struct slab_depot *depot); #endif /* VDO_SLAB_DEPOT_H */
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1