| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| Matthew Sakai | 19213 | 98.89% | 4 | 22.22% |
| Mike Snitzer | 186 | 0.96% | 10 | 55.56% |
| Kuan-Wei Chiu | 22 | 0.11% | 2 | 11.11% |
| Susan LeGendre-McGhee | 5 | 0.03% | 1 | 5.56% |
| Jiapeng Chong | 2 | 0.01% | 1 | 5.56% |
| Total | 19428 | 18 |
// SPDX-License-Identifier: GPL-2.0-only /* * Copyright 2023 Red Hat */ #include "slab-depot.h" #include <linux/atomic.h> #include <linux/bio.h> #include <linux/err.h> #include <linux/log2.h> #include <linux/min_heap.h> #include <linux/minmax.h> #include "logger.h" #include "memory-alloc.h" #include "numeric.h" #include "permassert.h" #include "string-utils.h" #include "action-manager.h" #include "admin-state.h" #include "completion.h" #include "constants.h" #include "data-vio.h" #include "encodings.h" #include "io-submitter.h" #include "physical-zone.h" #include "priority-table.h" #include "recovery-journal.h" #include "repair.h" #include "status-codes.h" #include "types.h" #include "vdo.h" #include "vio.h" #include "wait-queue.h" static const u64 BYTES_PER_WORD = sizeof(u64); static const bool NORMAL_OPERATION = true; /** * get_lock() - Get the lock object for a slab journal block by sequence number. * @journal: vdo_slab journal to retrieve from. * @sequence_number: Sequence number of the block. * * Return: The lock object for the given sequence number. */ static inline struct journal_lock * __must_check get_lock(struct slab_journal *journal, sequence_number_t sequence_number) { return &journal->locks[sequence_number % journal->size]; } static bool is_slab_open(struct vdo_slab *slab) { return (!vdo_is_state_quiescing(&slab->state) && !vdo_is_state_quiescent(&slab->state)); } /** * must_make_entries_to_flush() - Check whether there are entry waiters which should delay a flush. * @journal: The journal to check. * * Return: true if there are no entry waiters, or if the slab is unrecovered. */ static inline bool __must_check must_make_entries_to_flush(struct slab_journal *journal) { return ((journal->slab->status != VDO_SLAB_REBUILDING) && vdo_waitq_has_waiters(&journal->entry_waiters)); } /** * is_reaping() - Check whether a reap is currently in progress. * @journal: The journal which may be reaping. * * Return: true if the journal is reaping. */ static inline bool __must_check is_reaping(struct slab_journal *journal) { return (journal->head != journal->unreapable); } /** * initialize_tail_block() - Initialize tail block as a new block. * @journal: The journal whose tail block is being initialized. */ static void initialize_tail_block(struct slab_journal *journal) { struct slab_journal_block_header *header = &journal->tail_header; header->sequence_number = journal->tail; header->entry_count = 0; header->has_block_map_increments = false; } /** * initialize_journal_state() - Set all journal fields appropriately to start journaling. * @journal: The journal to be reset, based on its tail sequence number. */ static void initialize_journal_state(struct slab_journal *journal) { journal->unreapable = journal->head; journal->reap_lock = get_lock(journal, journal->unreapable); journal->next_commit = journal->tail; journal->summarized = journal->last_summarized = journal->tail; initialize_tail_block(journal); } /** * block_is_full() - Check whether a journal block is full. * @journal: The slab journal for the block. * * Return: true if the tail block is full. */ static bool __must_check block_is_full(struct slab_journal *journal) { journal_entry_count_t count = journal->tail_header.entry_count; return (journal->tail_header.has_block_map_increments ? (journal->full_entries_per_block == count) : (journal->entries_per_block == count)); } static void add_entries(struct slab_journal *journal); static void update_tail_block_location(struct slab_journal *journal); static void release_journal_locks(struct vdo_waiter *waiter, void *context); /** * is_slab_journal_blank() - Check whether a slab's journal is blank. * * A slab journal is blank if it has never had any entries recorded in it. * * Return: true if the slab's journal has never been modified. */ static bool is_slab_journal_blank(const struct vdo_slab *slab) { return ((slab->journal.tail == 1) && (slab->journal.tail_header.entry_count == 0)); } /** * mark_slab_journal_dirty() - Put a slab journal on the dirty ring of its allocator in the correct * order. * @journal: The journal to be marked dirty. * @lock: The recovery journal lock held by the slab journal. */ static void mark_slab_journal_dirty(struct slab_journal *journal, sequence_number_t lock) { struct slab_journal *dirty_journal; struct list_head *dirty_list = &journal->slab->allocator->dirty_slab_journals; VDO_ASSERT_LOG_ONLY(journal->recovery_lock == 0, "slab journal was clean"); journal->recovery_lock = lock; list_for_each_entry_reverse(dirty_journal, dirty_list, dirty_entry) { if (dirty_journal->recovery_lock <= journal->recovery_lock) break; } list_move_tail(&journal->dirty_entry, dirty_journal->dirty_entry.next); } static void mark_slab_journal_clean(struct slab_journal *journal) { journal->recovery_lock = 0; list_del_init(&journal->dirty_entry); } static void check_if_slab_drained(struct vdo_slab *slab) { bool read_only; struct slab_journal *journal = &slab->journal; const struct admin_state_code *code; if (!vdo_is_state_draining(&slab->state) || must_make_entries_to_flush(journal) || is_reaping(journal) || journal->waiting_to_commit || !list_empty(&journal->uncommitted_blocks) || journal->updating_slab_summary || (slab->active_count > 0)) return; /* When not suspending or recovering, the slab must be clean. */ code = vdo_get_admin_state_code(&slab->state); read_only = vdo_is_read_only(slab->allocator->depot->vdo); if (!read_only && vdo_waitq_has_waiters(&slab->dirty_blocks) && (code != VDO_ADMIN_STATE_SUSPENDING) && (code != VDO_ADMIN_STATE_RECOVERING)) return; vdo_finish_draining_with_result(&slab->state, (read_only ? VDO_READ_ONLY : VDO_SUCCESS)); } /* FULLNESS HINT COMPUTATION */ /** * compute_fullness_hint() - Translate a slab's free block count into a 'fullness hint' that can be * stored in a slab_summary_entry's 7 bits that are dedicated to its free * count. * @depot: The depot whose summary being updated. * @free_blocks: The number of free blocks. * * Note: the number of free blocks must be strictly less than 2^23 blocks, even though * theoretically slabs could contain precisely 2^23 blocks; there is an assumption that at least * one block is used by metadata. This assumption is necessary; otherwise, the fullness hint might * overflow. The fullness hint formula is roughly (fullness >> 16) & 0x7f, but (2^23 >> 16) & 0x7f * is 0, which would make it impossible to distinguish completely full from completely empty. * * Return: A fullness hint, which can be stored in 7 bits. */ static u8 __must_check compute_fullness_hint(struct slab_depot *depot, block_count_t free_blocks) { block_count_t hint; VDO_ASSERT_LOG_ONLY((free_blocks < (1 << 23)), "free blocks must be less than 2^23"); if (free_blocks == 0) return 0; hint = free_blocks >> depot->hint_shift; return ((hint == 0) ? 1 : hint); } /** * check_summary_drain_complete() - Check whether an allocators summary has finished draining. */ static void check_summary_drain_complete(struct block_allocator *allocator) { if (!vdo_is_state_draining(&allocator->summary_state) || (allocator->summary_write_count > 0)) return; vdo_finish_operation(&allocator->summary_state, (vdo_is_read_only(allocator->depot->vdo) ? VDO_READ_ONLY : VDO_SUCCESS)); } /** * notify_summary_waiters() - Wake all the waiters in a given queue. * @allocator: The block allocator summary which owns the queue. * @queue: The queue to notify. */ static void notify_summary_waiters(struct block_allocator *allocator, struct vdo_wait_queue *queue) { int result = (vdo_is_read_only(allocator->depot->vdo) ? VDO_READ_ONLY : VDO_SUCCESS); vdo_waitq_notify_all_waiters(queue, NULL, &result); } static void launch_write(struct slab_summary_block *summary_block); /** * finish_updating_slab_summary_block() - Finish processing a block which attempted to write, * whether or not the attempt succeeded. * @block: The block. */ static void finish_updating_slab_summary_block(struct slab_summary_block *block) { notify_summary_waiters(block->allocator, &block->current_update_waiters); block->writing = false; block->allocator->summary_write_count--; if (vdo_waitq_has_waiters(&block->next_update_waiters)) launch_write(block); else check_summary_drain_complete(block->allocator); } /** * finish_update() - This is the callback for a successful summary block write. * @completion: The write vio. */ static void finish_update(struct vdo_completion *completion) { struct slab_summary_block *block = container_of(as_vio(completion), struct slab_summary_block, vio); atomic64_inc(&block->allocator->depot->summary_statistics.blocks_written); finish_updating_slab_summary_block(block); } /** * handle_write_error() - Handle an error writing a slab summary block. * @completion: The write VIO. */ static void handle_write_error(struct vdo_completion *completion) { struct slab_summary_block *block = container_of(as_vio(completion), struct slab_summary_block, vio); vio_record_metadata_io_error(as_vio(completion)); vdo_enter_read_only_mode(completion->vdo, completion->result); finish_updating_slab_summary_block(block); } static void write_slab_summary_endio(struct bio *bio) { struct vio *vio = bio->bi_private; struct slab_summary_block *block = container_of(vio, struct slab_summary_block, vio); continue_vio_after_io(vio, finish_update, block->allocator->thread_id); } /** * launch_write() - Write a slab summary block unless it is currently out for writing. * @block: The block that needs to be committed. */ static void launch_write(struct slab_summary_block *block) { struct block_allocator *allocator = block->allocator; struct slab_depot *depot = allocator->depot; physical_block_number_t pbn; if (block->writing) return; allocator->summary_write_count++; vdo_waitq_transfer_all_waiters(&block->next_update_waiters, &block->current_update_waiters); block->writing = true; if (vdo_is_read_only(depot->vdo)) { finish_updating_slab_summary_block(block); return; } memcpy(block->outgoing_entries, block->entries, VDO_BLOCK_SIZE); /* * Flush before writing to ensure that the slab journal tail blocks and reference updates * covered by this summary update are stable. Otherwise, a subsequent recovery could * encounter a slab summary update that refers to a slab journal tail block that has not * actually been written. In such cases, the slab journal referenced will be treated as * empty, causing any data within the slab which predates the existing recovery journal * entries to be lost. */ pbn = (depot->summary_origin + (VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE * allocator->zone_number) + block->index); vdo_submit_metadata_vio(&block->vio, pbn, write_slab_summary_endio, handle_write_error, REQ_OP_WRITE | REQ_PREFLUSH); } /** * update_slab_summary_entry() - Update the entry for a slab. * @slab: The slab whose entry is to be updated * @waiter: The waiter that is updating the summary. * @tail_block_offset: The offset of the slab journal's tail block. * @load_ref_counts: Whether the reference counts must be loaded from disk on the vdo load. * @is_clean: Whether the slab is clean. * @free_blocks: The number of free blocks. */ static void update_slab_summary_entry(struct vdo_slab *slab, struct vdo_waiter *waiter, tail_block_offset_t tail_block_offset, bool load_ref_counts, bool is_clean, block_count_t free_blocks) { u8 index = slab->slab_number / VDO_SLAB_SUMMARY_ENTRIES_PER_BLOCK; struct block_allocator *allocator = slab->allocator; struct slab_summary_block *block = &allocator->summary_blocks[index]; int result; struct slab_summary_entry *entry; if (vdo_is_read_only(block->vio.completion.vdo)) { result = VDO_READ_ONLY; waiter->callback(waiter, &result); return; } if (vdo_is_state_draining(&allocator->summary_state) || vdo_is_state_quiescent(&allocator->summary_state)) { result = VDO_INVALID_ADMIN_STATE; waiter->callback(waiter, &result); return; } entry = &allocator->summary_entries[slab->slab_number]; *entry = (struct slab_summary_entry) { .tail_block_offset = tail_block_offset, .load_ref_counts = (entry->load_ref_counts || load_ref_counts), .is_dirty = !is_clean, .fullness_hint = compute_fullness_hint(allocator->depot, free_blocks), }; vdo_waitq_enqueue_waiter(&block->next_update_waiters, waiter); launch_write(block); } /** * finish_reaping() - Actually advance the head of the journal now that any necessary flushes are * complete. * @journal: The journal to be reaped. */ static void finish_reaping(struct slab_journal *journal) { journal->head = journal->unreapable; add_entries(journal); check_if_slab_drained(journal->slab); } static void reap_slab_journal(struct slab_journal *journal); /** * complete_reaping() - Finish reaping now that we have flushed the lower layer and then try * reaping again in case we deferred reaping due to an outstanding vio. * @completion: The flush vio. */ static void complete_reaping(struct vdo_completion *completion) { struct slab_journal *journal = completion->parent; return_vio_to_pool(journal->slab->allocator->vio_pool, vio_as_pooled_vio(as_vio(vdo_forget(completion)))); finish_reaping(journal); reap_slab_journal(journal); } /** * handle_flush_error() - Handle an error flushing the lower layer. * @completion: The flush vio. */ static void handle_flush_error(struct vdo_completion *completion) { vio_record_metadata_io_error(as_vio(completion)); vdo_enter_read_only_mode(completion->vdo, completion->result); complete_reaping(completion); } static void flush_endio(struct bio *bio) { struct vio *vio = bio->bi_private; struct slab_journal *journal = vio->completion.parent; continue_vio_after_io(vio, complete_reaping, journal->slab->allocator->thread_id); } /** * flush_for_reaping() - A waiter callback for getting a vio with which to flush the lower layer * prior to reaping. * @waiter: The journal as a flush waiter. * @context: The newly acquired flush vio. */ static void flush_for_reaping(struct vdo_waiter *waiter, void *context) { struct slab_journal *journal = container_of(waiter, struct slab_journal, flush_waiter); struct pooled_vio *pooled = context; struct vio *vio = &pooled->vio; vio->completion.parent = journal; vdo_submit_flush_vio(vio, flush_endio, handle_flush_error); } /** * reap_slab_journal() - Conduct a reap on a slab journal to reclaim unreferenced blocks. * @journal: The slab journal. */ static void reap_slab_journal(struct slab_journal *journal) { bool reaped = false; if (is_reaping(journal)) { /* We already have a reap in progress so wait for it to finish. */ return; } if ((journal->slab->status != VDO_SLAB_REBUILT) || !vdo_is_state_normal(&journal->slab->state) || vdo_is_read_only(journal->slab->allocator->depot->vdo)) { /* * We must not reap in the first two cases, and there's no point in read-only mode. */ return; } /* * Start reclaiming blocks only when the journal head has no references. Then stop when a * block is referenced or reap reaches the most recently written block, referenced by the * slab summary, which has the sequence number just before the tail. */ while ((journal->unreapable < journal->tail) && (journal->reap_lock->count == 0)) { reaped = true; journal->unreapable++; journal->reap_lock++; if (journal->reap_lock == &journal->locks[journal->size]) journal->reap_lock = &journal->locks[0]; } if (!reaped) return; /* * It is never safe to reap a slab journal block without first issuing a flush, regardless * of whether a user flush has been received or not. In the absence of the flush, the * reference block write which released the locks allowing the slab journal to reap may not * be persisted. Although slab summary writes will eventually issue flushes, multiple slab * journal block writes can be issued while previous slab summary updates have not yet been * made. Even though those slab journal block writes will be ignored if the slab summary * update is not persisted, they may still overwrite the to-be-reaped slab journal block * resulting in a loss of reference count updates. */ journal->flush_waiter.callback = flush_for_reaping; acquire_vio_from_pool(journal->slab->allocator->vio_pool, &journal->flush_waiter); } /** * adjust_slab_journal_block_reference() - Adjust the reference count for a slab journal block. * @journal: The slab journal. * @sequence_number: The journal sequence number of the referenced block. * @adjustment: Amount to adjust the reference counter. * * Note that when the adjustment is negative, the slab journal will be reaped. */ static void adjust_slab_journal_block_reference(struct slab_journal *journal, sequence_number_t sequence_number, int adjustment) { struct journal_lock *lock; if (sequence_number == 0) return; if (journal->slab->status == VDO_SLAB_REPLAYING) { /* Locks should not be used during offline replay. */ return; } VDO_ASSERT_LOG_ONLY((adjustment != 0), "adjustment must be non-zero"); lock = get_lock(journal, sequence_number); if (adjustment < 0) { VDO_ASSERT_LOG_ONLY((-adjustment <= lock->count), "adjustment %d of lock count %u for slab journal block %llu must not underflow", adjustment, lock->count, (unsigned long long) sequence_number); } lock->count += adjustment; if (lock->count == 0) reap_slab_journal(journal); } /** * release_journal_locks() - Callback invoked after a slab summary update completes. * @waiter: The slab summary waiter that has just been notified. * @context: The result code of the update. * * Registered in the constructor on behalf of update_tail_block_location(). * * Implements waiter_callback_fn. */ static void release_journal_locks(struct vdo_waiter *waiter, void *context) { sequence_number_t first, i; struct slab_journal *journal = container_of(waiter, struct slab_journal, slab_summary_waiter); int result = *((int *) context); if (result != VDO_SUCCESS) { if (result != VDO_READ_ONLY) { /* * Don't bother logging what might be lots of errors if we are already in * read-only mode. */ vdo_log_error_strerror(result, "failed slab summary update %llu", (unsigned long long) journal->summarized); } journal->updating_slab_summary = false; vdo_enter_read_only_mode(journal->slab->allocator->depot->vdo, result); check_if_slab_drained(journal->slab); return; } if (journal->partial_write_in_progress && (journal->summarized == journal->tail)) { journal->partial_write_in_progress = false; add_entries(journal); } first = journal->last_summarized; journal->last_summarized = journal->summarized; for (i = journal->summarized - 1; i >= first; i--) { /* * Release the lock the summarized block held on the recovery journal. (During * replay, recovery_start will always be 0.) */ if (journal->recovery_journal != NULL) { zone_count_t zone_number = journal->slab->allocator->zone_number; struct journal_lock *lock = get_lock(journal, i); vdo_release_recovery_journal_block_reference(journal->recovery_journal, lock->recovery_start, VDO_ZONE_TYPE_PHYSICAL, zone_number); } /* * Release our own lock against reaping for blocks that are committed. (This * function will not change locks during replay.) */ adjust_slab_journal_block_reference(journal, i, -1); } journal->updating_slab_summary = false; reap_slab_journal(journal); /* Check if the slab summary needs to be updated again. */ update_tail_block_location(journal); } /** * update_tail_block_location() - Update the tail block location in the slab summary, if necessary. * @journal: The slab journal that is updating its tail block location. */ static void update_tail_block_location(struct slab_journal *journal) { block_count_t free_block_count; struct vdo_slab *slab = journal->slab; if (journal->updating_slab_summary || vdo_is_read_only(journal->slab->allocator->depot->vdo) || (journal->last_summarized >= journal->next_commit)) { check_if_slab_drained(slab); return; } if (slab->status != VDO_SLAB_REBUILT) { u8 hint = slab->allocator->summary_entries[slab->slab_number].fullness_hint; free_block_count = ((block_count_t) hint) << slab->allocator->depot->hint_shift; } else { free_block_count = slab->free_blocks; } journal->summarized = journal->next_commit; journal->updating_slab_summary = true; /* * Update slab summary as dirty. * vdo_slab journal can only reap past sequence number 1 when all the ref counts for this * slab have been written to the layer. Therefore, indicate that the ref counts must be * loaded when the journal head has reaped past sequence number 1. */ update_slab_summary_entry(slab, &journal->slab_summary_waiter, journal->summarized % journal->size, (journal->head > 1), false, free_block_count); } /** * reopen_slab_journal() - Reopen a slab's journal by emptying it and then adding pending entries. */ static void reopen_slab_journal(struct vdo_slab *slab) { struct slab_journal *journal = &slab->journal; sequence_number_t block; VDO_ASSERT_LOG_ONLY(journal->tail_header.entry_count == 0, "vdo_slab journal's active block empty before reopening"); journal->head = journal->tail; initialize_journal_state(journal); /* Ensure no locks are spuriously held on an empty journal. */ for (block = 1; block <= journal->size; block++) { VDO_ASSERT_LOG_ONLY((get_lock(journal, block)->count == 0), "Scrubbed journal's block %llu is not locked", (unsigned long long) block); } add_entries(journal); } static sequence_number_t get_committing_sequence_number(const struct pooled_vio *vio) { const struct packed_slab_journal_block *block = (const struct packed_slab_journal_block *) vio->vio.data; return __le64_to_cpu(block->header.sequence_number); } /** * complete_write() - Handle post-commit processing. * @completion: The write vio as a completion. * * This is the callback registered by write_slab_journal_block(). */ static void complete_write(struct vdo_completion *completion) { int result = completion->result; struct pooled_vio *pooled = vio_as_pooled_vio(as_vio(completion)); struct slab_journal *journal = completion->parent; sequence_number_t committed = get_committing_sequence_number(pooled); list_del_init(&pooled->list_entry); return_vio_to_pool(journal->slab->allocator->vio_pool, vdo_forget(pooled)); if (result != VDO_SUCCESS) { vio_record_metadata_io_error(as_vio(completion)); vdo_log_error_strerror(result, "cannot write slab journal block %llu", (unsigned long long) committed); vdo_enter_read_only_mode(journal->slab->allocator->depot->vdo, result); check_if_slab_drained(journal->slab); return; } WRITE_ONCE(journal->events->blocks_written, journal->events->blocks_written + 1); if (list_empty(&journal->uncommitted_blocks)) { /* If no blocks are outstanding, then the commit point is at the tail. */ journal->next_commit = journal->tail; } else { /* The commit point is always the beginning of the oldest incomplete block. */ pooled = container_of(journal->uncommitted_blocks.next, struct pooled_vio, list_entry); journal->next_commit = get_committing_sequence_number(pooled); } update_tail_block_location(journal); } static void write_slab_journal_endio(struct bio *bio) { struct vio *vio = bio->bi_private; struct slab_journal *journal = vio->completion.parent; continue_vio_after_io(vio, complete_write, journal->slab->allocator->thread_id); } /** * write_slab_journal_block() - Write a slab journal block. * @waiter: The vio pool waiter which was just notified. * @context: The vio pool entry for the write. * * Callback from acquire_vio_from_pool() registered in commit_tail(). */ static void write_slab_journal_block(struct vdo_waiter *waiter, void *context) { struct pooled_vio *pooled = context; struct vio *vio = &pooled->vio; struct slab_journal *journal = container_of(waiter, struct slab_journal, resource_waiter); struct slab_journal_block_header *header = &journal->tail_header; int unused_entries = journal->entries_per_block - header->entry_count; physical_block_number_t block_number; const struct admin_state_code *operation; header->head = journal->head; list_add_tail(&pooled->list_entry, &journal->uncommitted_blocks); vdo_pack_slab_journal_block_header(header, &journal->block->header); /* Copy the tail block into the vio. */ memcpy(pooled->vio.data, journal->block, VDO_BLOCK_SIZE); VDO_ASSERT_LOG_ONLY(unused_entries >= 0, "vdo_slab journal block is not overfull"); if (unused_entries > 0) { /* * Release the per-entry locks for any unused entries in the block we are about to * write. */ adjust_slab_journal_block_reference(journal, header->sequence_number, -unused_entries); journal->partial_write_in_progress = !block_is_full(journal); } block_number = journal->slab->journal_origin + (header->sequence_number % journal->size); vio->completion.parent = journal; /* * This block won't be read in recovery until the slab summary is updated to refer to it. * The slab summary update does a flush which is sufficient to protect us from corruption * due to out of order slab journal, reference block, or block map writes. */ vdo_submit_metadata_vio(vdo_forget(vio), block_number, write_slab_journal_endio, complete_write, REQ_OP_WRITE); /* Since the write is submitted, the tail block structure can be reused. */ journal->tail++; initialize_tail_block(journal); journal->waiting_to_commit = false; operation = vdo_get_admin_state_code(&journal->slab->state); if (operation == VDO_ADMIN_STATE_WAITING_FOR_RECOVERY) { vdo_finish_operation(&journal->slab->state, (vdo_is_read_only(journal->slab->allocator->depot->vdo) ? VDO_READ_ONLY : VDO_SUCCESS)); return; } add_entries(journal); } /** * commit_tail() - Commit the tail block of the slab journal. * @journal: The journal whose tail block should be committed. */ static void commit_tail(struct slab_journal *journal) { if ((journal->tail_header.entry_count == 0) && must_make_entries_to_flush(journal)) { /* * There are no entries at the moment, but there are some waiters, so defer * initiating the flush until those entries are ready to write. */ return; } if (vdo_is_read_only(journal->slab->allocator->depot->vdo) || journal->waiting_to_commit || (journal->tail_header.entry_count == 0)) { /* * There is nothing to do since the tail block is empty, or writing, or the journal * is in read-only mode. */ return; } /* * Since we are about to commit the tail block, this journal no longer needs to be on the * ring of journals which the recovery journal might ask to commit. */ mark_slab_journal_clean(journal); journal->waiting_to_commit = true; journal->resource_waiter.callback = write_slab_journal_block; acquire_vio_from_pool(journal->slab->allocator->vio_pool, &journal->resource_waiter); } /** * encode_slab_journal_entry() - Encode a slab journal entry. * @tail_header: The unpacked header for the block. * @payload: The journal block payload to hold the entry. * @sbn: The slab block number of the entry to encode. * @operation: The type of the entry. * @increment: True if this is an increment. * * Exposed for unit tests. */ static void encode_slab_journal_entry(struct slab_journal_block_header *tail_header, slab_journal_payload *payload, slab_block_number sbn, enum journal_operation operation, bool increment) { journal_entry_count_t entry_number = tail_header->entry_count++; if (operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING) { if (!tail_header->has_block_map_increments) { memset(payload->full_entries.entry_types, 0, VDO_SLAB_JOURNAL_ENTRY_TYPES_SIZE); tail_header->has_block_map_increments = true; } payload->full_entries.entry_types[entry_number / 8] |= ((u8)1 << (entry_number % 8)); } vdo_pack_slab_journal_entry(&payload->entries[entry_number], sbn, increment); } /** * expand_journal_point() - Convert a recovery journal journal_point which refers to both an * increment and a decrement to a single point which refers to one or the * other. * @recovery_point: The journal point to convert. * @increment: Whether the current entry is an increment. * * Return: The expanded journal point * * Because each data_vio has but a single recovery journal point, but may need to make both * increment and decrement entries in the same slab journal. In order to distinguish the two * entries, the entry count of the expanded journal point is twice the actual recovery journal * entry count for increments, and one more than that for decrements. */ static struct journal_point expand_journal_point(struct journal_point recovery_point, bool increment) { recovery_point.entry_count *= 2; if (!increment) recovery_point.entry_count++; return recovery_point; } /** * add_entry() - Actually add an entry to the slab journal, potentially firing off a write if a * block becomes full. * @journal: The slab journal to append to. * @pbn: The pbn being adjusted. * @operation: The type of entry to make. * @increment: True if this is an increment. * @recovery_point: The expanded recovery point. * * This function is synchronous. */ static void add_entry(struct slab_journal *journal, physical_block_number_t pbn, enum journal_operation operation, bool increment, struct journal_point recovery_point) { struct packed_slab_journal_block *block = journal->block; int result; result = VDO_ASSERT(vdo_before_journal_point(&journal->tail_header.recovery_point, &recovery_point), "recovery journal point is monotonically increasing, recovery point: %llu.%u, block recovery point: %llu.%u", (unsigned long long) recovery_point.sequence_number, recovery_point.entry_count, (unsigned long long) journal->tail_header.recovery_point.sequence_number, journal->tail_header.recovery_point.entry_count); if (result != VDO_SUCCESS) { vdo_enter_read_only_mode(journal->slab->allocator->depot->vdo, result); return; } if (operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING) { result = VDO_ASSERT((journal->tail_header.entry_count < journal->full_entries_per_block), "block has room for full entries"); if (result != VDO_SUCCESS) { vdo_enter_read_only_mode(journal->slab->allocator->depot->vdo, result); return; } } encode_slab_journal_entry(&journal->tail_header, &block->payload, pbn - journal->slab->start, operation, increment); journal->tail_header.recovery_point = recovery_point; if (block_is_full(journal)) commit_tail(journal); } static inline block_count_t journal_length(const struct slab_journal *journal) { return journal->tail - journal->head; } /** * vdo_attempt_replay_into_slab() - Replay a recovery journal entry into a slab's journal. * @slab: The slab to play into. * @pbn: The PBN for the entry. * @operation: The type of entry to add. * @increment: True if this entry is an increment. * @recovery_point: The recovery journal point corresponding to this entry. * @parent: The completion to notify when there is space to add the entry if the entry could not be * added immediately. * * Return: true if the entry was added immediately. */ bool vdo_attempt_replay_into_slab(struct vdo_slab *slab, physical_block_number_t pbn, enum journal_operation operation, bool increment, struct journal_point *recovery_point, struct vdo_completion *parent) { struct slab_journal *journal = &slab->journal; struct slab_journal_block_header *header = &journal->tail_header; struct journal_point expanded = expand_journal_point(*recovery_point, increment); /* Only accept entries after the current recovery point. */ if (!vdo_before_journal_point(&journal->tail_header.recovery_point, &expanded)) return true; if ((header->entry_count >= journal->full_entries_per_block) && (header->has_block_map_increments || (operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING))) { /* * The tail block does not have room for the entry we are attempting to add so * commit the tail block now. */ commit_tail(journal); } if (journal->waiting_to_commit) { vdo_start_operation_with_waiter(&journal->slab->state, VDO_ADMIN_STATE_WAITING_FOR_RECOVERY, parent, NULL); return false; } if (journal_length(journal) >= journal->size) { /* * We must have reaped the current head before the crash, since the blocked * threshold keeps us from having more entries than fit in a slab journal; hence we * can just advance the head (and unreapable block), as needed. */ journal->head++; journal->unreapable++; } if (journal->slab->status == VDO_SLAB_REBUILT) journal->slab->status = VDO_SLAB_REPLAYING; add_entry(journal, pbn, operation, increment, expanded); return true; } /** * requires_reaping() - Check whether the journal must be reaped before adding new entries. * @journal: The journal to check. * * Return: true if the journal must be reaped. */ static bool requires_reaping(const struct slab_journal *journal) { return (journal_length(journal) >= journal->blocking_threshold); } /** finish_summary_update() - A waiter callback that resets the writing state of a slab. */ static void finish_summary_update(struct vdo_waiter *waiter, void *context) { struct vdo_slab *slab = container_of(waiter, struct vdo_slab, summary_waiter); int result = *((int *) context); slab->active_count--; if ((result != VDO_SUCCESS) && (result != VDO_READ_ONLY)) { vdo_log_error_strerror(result, "failed to update slab summary"); vdo_enter_read_only_mode(slab->allocator->depot->vdo, result); } check_if_slab_drained(slab); } static void write_reference_block(struct vdo_waiter *waiter, void *context); /** * launch_reference_block_write() - Launch the write of a dirty reference block by first acquiring * a VIO for it from the pool. * @waiter: The waiter of the block which is starting to write. * @context: The parent slab of the block. * * This can be asynchronous since the writer will have to wait if all VIOs in the pool are * currently in use. */ static void launch_reference_block_write(struct vdo_waiter *waiter, void *context) { struct vdo_slab *slab = context; if (vdo_is_read_only(slab->allocator->depot->vdo)) return; slab->active_count++; container_of(waiter, struct reference_block, waiter)->is_writing = true; waiter->callback = write_reference_block; acquire_vio_from_pool(slab->allocator->vio_pool, waiter); } static void save_dirty_reference_blocks(struct vdo_slab *slab) { vdo_waitq_notify_all_waiters(&slab->dirty_blocks, launch_reference_block_write, slab); check_if_slab_drained(slab); } /** * finish_reference_block_write() - After a reference block has written, clean it, release its * locks, and return its VIO to the pool. * @completion: The VIO that just finished writing. */ static void finish_reference_block_write(struct vdo_completion *completion) { struct vio *vio = as_vio(completion); struct pooled_vio *pooled = vio_as_pooled_vio(vio); struct reference_block *block = completion->parent; struct vdo_slab *slab = block->slab; tail_block_offset_t offset; slab->active_count--; /* Release the slab journal lock. */ adjust_slab_journal_block_reference(&slab->journal, block->slab_journal_lock_to_release, -1); return_vio_to_pool(slab->allocator->vio_pool, pooled); /* * We can't clear the is_writing flag earlier as releasing the slab journal lock may cause * us to be dirtied again, but we don't want to double enqueue. */ block->is_writing = false; if (vdo_is_read_only(completion->vdo)) { check_if_slab_drained(slab); return; } /* Re-queue the block if it was re-dirtied while it was writing. */ if (block->is_dirty) { vdo_waitq_enqueue_waiter(&block->slab->dirty_blocks, &block->waiter); if (vdo_is_state_draining(&slab->state)) { /* We must be saving, and this block will otherwise not be relaunched. */ save_dirty_reference_blocks(slab); } return; } /* * Mark the slab as clean in the slab summary if there are no dirty or writing blocks * and no summary update in progress. */ if ((slab->active_count > 0) || vdo_waitq_has_waiters(&slab->dirty_blocks)) { check_if_slab_drained(slab); return; } offset = slab->allocator->summary_entries[slab->slab_number].tail_block_offset; slab->active_count++; slab->summary_waiter.callback = finish_summary_update; update_slab_summary_entry(slab, &slab->summary_waiter, offset, true, true, slab->free_blocks); } /** * get_reference_counters_for_block() - Find the reference counters for a given block. * @block: The reference_block in question. * * Return: A pointer to the reference counters for this block. */ static vdo_refcount_t * __must_check get_reference_counters_for_block(struct reference_block *block) { size_t block_index = block - block->slab->reference_blocks; return &block->slab->counters[block_index * COUNTS_PER_BLOCK]; } /** * pack_reference_block() - Copy data from a reference block to a buffer ready to be written out. * @block: The block to copy. * @buffer: The char buffer to fill with the packed block. */ static void pack_reference_block(struct reference_block *block, void *buffer) { struct packed_reference_block *packed = buffer; vdo_refcount_t *counters = get_reference_counters_for_block(block); sector_count_t i; struct packed_journal_point commit_point; vdo_pack_journal_point(&block->slab->slab_journal_point, &commit_point); for (i = 0; i < VDO_SECTORS_PER_BLOCK; i++) { packed->sectors[i].commit_point = commit_point; memcpy(packed->sectors[i].counts, counters + (i * COUNTS_PER_SECTOR), (sizeof(vdo_refcount_t) * COUNTS_PER_SECTOR)); } } static void write_reference_block_endio(struct bio *bio) { struct vio *vio = bio->bi_private; struct reference_block *block = vio->completion.parent; thread_id_t thread_id = block->slab->allocator->thread_id; continue_vio_after_io(vio, finish_reference_block_write, thread_id); } /** * handle_io_error() - Handle an I/O error reading or writing a reference count block. * @completion: The VIO doing the I/O as a completion. */ static void handle_io_error(struct vdo_completion *completion) { int result = completion->result; struct vio *vio = as_vio(completion); struct vdo_slab *slab = ((struct reference_block *) completion->parent)->slab; vio_record_metadata_io_error(vio); return_vio_to_pool(slab->allocator->vio_pool, vio_as_pooled_vio(vio)); slab->active_count--; vdo_enter_read_only_mode(slab->allocator->depot->vdo, result); check_if_slab_drained(slab); } /** * write_reference_block() - After a dirty block waiter has gotten a VIO from the VIO pool, copy * its counters and associated data into the VIO, and launch the write. * @waiter: The waiter of the dirty block. * @context: The VIO returned by the pool. */ static void write_reference_block(struct vdo_waiter *waiter, void *context) { size_t block_offset; physical_block_number_t pbn; struct pooled_vio *pooled = context; struct vdo_completion *completion = &pooled->vio.completion; struct reference_block *block = container_of(waiter, struct reference_block, waiter); pack_reference_block(block, pooled->vio.data); block_offset = (block - block->slab->reference_blocks); pbn = (block->slab->ref_counts_origin + block_offset); block->slab_journal_lock_to_release = block->slab_journal_lock; completion->parent = block; /* * Mark the block as clean, since we won't be committing any updates that happen after this * moment. As long as VIO order is preserved, two VIOs updating this block at once will not * cause complications. */ block->is_dirty = false; /* * Flush before writing to ensure that the recovery journal and slab journal entries which * cover this reference update are stable. This prevents data corruption that can be caused * by out of order writes. */ WRITE_ONCE(block->slab->allocator->ref_counts_statistics.blocks_written, block->slab->allocator->ref_counts_statistics.blocks_written + 1); completion->callback_thread_id = ((struct block_allocator *) pooled->context)->thread_id; vdo_submit_metadata_vio(&pooled->vio, pbn, write_reference_block_endio, handle_io_error, REQ_OP_WRITE | REQ_PREFLUSH); } static void reclaim_journal_space(struct slab_journal *journal) { block_count_t length = journal_length(journal); struct vdo_slab *slab = journal->slab; block_count_t write_count = vdo_waitq_num_waiters(&slab->dirty_blocks); block_count_t written; if ((length < journal->flushing_threshold) || (write_count == 0)) return; /* The slab journal is over the first threshold, schedule some reference block writes. */ WRITE_ONCE(journal->events->flush_count, journal->events->flush_count + 1); if (length < journal->flushing_deadline) { /* Schedule more writes the closer to the deadline we get. */ write_count /= journal->flushing_deadline - length + 1; write_count = max_t(block_count_t, write_count, 1); } for (written = 0; written < write_count; written++) { vdo_waitq_notify_next_waiter(&slab->dirty_blocks, launch_reference_block_write, slab); } } /** * reference_count_to_status() - Convert a reference count to a reference status. * @count: The count to convert. * * Return: The appropriate reference status. */ static enum reference_status __must_check reference_count_to_status(vdo_refcount_t count) { if (count == EMPTY_REFERENCE_COUNT) return RS_FREE; else if (count == 1) return RS_SINGLE; else if (count == PROVISIONAL_REFERENCE_COUNT) return RS_PROVISIONAL; else return RS_SHARED; } /** * dirty_block() - Mark a reference count block as dirty, potentially adding it to the dirty queue * if it wasn't already dirty. * @block: The reference block to mark as dirty. */ static void dirty_block(struct reference_block *block) { if (block->is_dirty) return; block->is_dirty = true; if (!block->is_writing) vdo_waitq_enqueue_waiter(&block->slab->dirty_blocks, &block->waiter); } /** * get_reference_block() - Get the reference block that covers the given block index. */ static struct reference_block * __must_check get_reference_block(struct vdo_slab *slab, slab_block_number index) { return &slab->reference_blocks[index / COUNTS_PER_BLOCK]; } /** * slab_block_number_from_pbn() - Determine the index within the slab of a particular physical * block number. * @slab: The slab. * @physical_block_number: The physical block number. * @slab_block_number_ptr: A pointer to the slab block number. * * Return: VDO_SUCCESS or an error code. */ static int __must_check slab_block_number_from_pbn(struct vdo_slab *slab, physical_block_number_t pbn, slab_block_number *slab_block_number_ptr) { u64 slab_block_number; if (pbn < slab->start) return VDO_OUT_OF_RANGE; slab_block_number = pbn - slab->start; if (slab_block_number >= slab->allocator->depot->slab_config.data_blocks) return VDO_OUT_OF_RANGE; *slab_block_number_ptr = slab_block_number; return VDO_SUCCESS; } /** * get_reference_counter() - Get the reference counter that covers the given physical block number. * @slab: The slab to query. * @pbn: The physical block number. * @counter_ptr: A pointer to the reference counter. */ static int __must_check get_reference_counter(struct vdo_slab *slab, physical_block_number_t pbn, vdo_refcount_t **counter_ptr) { slab_block_number index; int result = slab_block_number_from_pbn(slab, pbn, &index); if (result != VDO_SUCCESS) return result; *counter_ptr = &slab->counters[index]; return VDO_SUCCESS; } static unsigned int calculate_slab_priority(struct vdo_slab *slab) { block_count_t free_blocks = slab->free_blocks; unsigned int unopened_slab_priority = slab->allocator->unopened_slab_priority; unsigned int priority; /* * Wholly full slabs must be the only ones with lowest priority, 0. * * Slabs that have never been opened (empty, newly initialized, and never been written to) * have lower priority than previously opened slabs that have a significant number of free * blocks. This ranking causes VDO to avoid writing physical blocks for the first time * unless there are very few free blocks that have been previously written to. * * Since VDO doesn't discard blocks currently, reusing previously written blocks makes VDO * a better client of any underlying storage that is thinly-provisioned (though discarding * would be better). * * For all other slabs, the priority is derived from the logarithm of the number of free * blocks. Slabs with the same order of magnitude of free blocks have the same priority. * With 2^23 blocks, the priority will range from 1 to 25. The reserved * unopened_slab_priority divides the range and is skipped by the logarithmic mapping. */ if (free_blocks == 0) return 0; if (is_slab_journal_blank(slab)) return unopened_slab_priority; priority = (1 + ilog2(free_blocks)); return ((priority < unopened_slab_priority) ? priority : priority + 1); } /* * Slabs are essentially prioritized by an approximation of the number of free blocks in the slab * so slabs with lots of free blocks will be opened for allocation before slabs that have few free * blocks. */ static void prioritize_slab(struct vdo_slab *slab) { VDO_ASSERT_LOG_ONLY(list_empty(&slab->allocq_entry), "a slab must not already be on a ring when prioritizing"); slab->priority = calculate_slab_priority(slab); vdo_priority_table_enqueue(slab->allocator->prioritized_slabs, slab->priority, &slab->allocq_entry); } /** * adjust_free_block_count() - Adjust the free block count and (if needed) reprioritize the slab. * @incremented: true if the free block count went up. */ static void adjust_free_block_count(struct vdo_slab *slab, bool incremented) { struct block_allocator *allocator = slab->allocator; WRITE_ONCE(allocator->allocated_blocks, allocator->allocated_blocks + (incremented ? -1 : 1)); /* The open slab doesn't need to be reprioritized until it is closed. */ if (slab == allocator->open_slab) return; /* Don't bother adjusting the priority table if unneeded. */ if (slab->priority == calculate_slab_priority(slab)) return; /* * Reprioritize the slab to reflect the new free block count by removing it from the table * and re-enqueuing it with the new priority. */ vdo_priority_table_remove(allocator->prioritized_slabs, &slab->allocq_entry); prioritize_slab(slab); } /** * increment_for_data() - Increment the reference count for a data block. * @slab: The slab which owns the block. * @block: The reference block which contains the block being updated. * @block_number: The block to update. * @old_status: The reference status of the data block before this increment. * @lock: The pbn_lock associated with this increment (may be NULL). * @counter_ptr: A pointer to the count for the data block (in, out). * @adjust_block_count: Whether to update the allocator's free block count. * * Return: VDO_SUCCESS or an error. */ static int increment_for_data(struct vdo_slab *slab, struct reference_block *block, slab_block_number block_number, enum reference_status old_status, struct pbn_lock *lock, vdo_refcount_t *counter_ptr, bool adjust_block_count) { switch (old_status) { case RS_FREE: *counter_ptr = 1; block->allocated_count++; slab->free_blocks--; if (adjust_block_count) adjust_free_block_count(slab, false); break; case RS_PROVISIONAL: *counter_ptr = 1; break; default: /* Single or shared */ if (*counter_ptr >= MAXIMUM_REFERENCE_COUNT) { return vdo_log_error_strerror(VDO_REF_COUNT_INVALID, "Incrementing a block already having 254 references (slab %u, offset %u)", slab->slab_number, block_number); } (*counter_ptr)++; } if (lock != NULL) vdo_unassign_pbn_lock_provisional_reference(lock); return VDO_SUCCESS; } /** * decrement_for_data() - Decrement the reference count for a data block. * @slab: The slab which owns the block. * @block: The reference block which contains the block being updated. * @block_number: The block to update. * @old_status: The reference status of the data block before this decrement. * @updater: The reference updater doing this operation in case we need to look up the pbn lock. * @lock: The pbn_lock associated with the block being decremented (may be NULL). * @counter_ptr: A pointer to the count for the data block (in, out). * @adjust_block_count: Whether to update the allocator's free block count. * * Return: VDO_SUCCESS or an error. */ static int decrement_for_data(struct vdo_slab *slab, struct reference_block *block, slab_block_number block_number, enum reference_status old_status, struct reference_updater *updater, vdo_refcount_t *counter_ptr, bool adjust_block_count) { switch (old_status) { case RS_FREE: return vdo_log_error_strerror(VDO_REF_COUNT_INVALID, "Decrementing free block at offset %u in slab %u", block_number, slab->slab_number); case RS_PROVISIONAL: case RS_SINGLE: if (updater->zpbn.zone != NULL) { struct pbn_lock *lock = vdo_get_physical_zone_pbn_lock(updater->zpbn.zone, updater->zpbn.pbn); if (lock != NULL) { /* * There is a read lock on this block, so the block must not become * unreferenced. */ *counter_ptr = PROVISIONAL_REFERENCE_COUNT; vdo_assign_pbn_lock_provisional_reference(lock); break; } } *counter_ptr = EMPTY_REFERENCE_COUNT; block->allocated_count--; slab->free_blocks++; if (adjust_block_count) adjust_free_block_count(slab, true); break; default: /* Shared */ (*counter_ptr)--; } return VDO_SUCCESS; } /** * increment_for_block_map() - Increment the reference count for a block map page. * @slab: The slab which owns the block. * @block: The reference block which contains the block being updated. * @block_number: The block to update. * @old_status: The reference status of the block before this increment. * @lock: The pbn_lock associated with this increment (may be NULL). * @normal_operation: Whether we are in normal operation vs. recovery or rebuild. * @counter_ptr: A pointer to the count for the block (in, out). * @adjust_block_count: Whether to update the allocator's free block count. * * All block map increments should be from provisional to MAXIMUM_REFERENCE_COUNT. Since block map * blocks never dedupe they should never be adjusted from any other state. The adjustment always * results in MAXIMUM_REFERENCE_COUNT as this value is used to prevent dedupe against block map * blocks. * * Return: VDO_SUCCESS or an error. */ static int increment_for_block_map(struct vdo_slab *slab, struct reference_block *block, slab_block_number block_number, enum reference_status old_status, struct pbn_lock *lock, bool normal_operation, vdo_refcount_t *counter_ptr, bool adjust_block_count) { switch (old_status) { case RS_FREE: if (normal_operation) { return vdo_log_error_strerror(VDO_REF_COUNT_INVALID, "Incrementing unallocated block map block (slab %u, offset %u)", slab->slab_number, block_number); } *counter_ptr = MAXIMUM_REFERENCE_COUNT; block->allocated_count++; slab->free_blocks--; if (adjust_block_count) adjust_free_block_count(slab, false); return VDO_SUCCESS; case RS_PROVISIONAL: if (!normal_operation) return vdo_log_error_strerror(VDO_REF_COUNT_INVALID, "Block map block had provisional reference during replay (slab %u, offset %u)", slab->slab_number, block_number); *counter_ptr = MAXIMUM_REFERENCE_COUNT; if (lock != NULL) vdo_unassign_pbn_lock_provisional_reference(lock); return VDO_SUCCESS; default: return vdo_log_error_strerror(VDO_REF_COUNT_INVALID, "Incrementing a block map block which is already referenced %u times (slab %u, offset %u)", *counter_ptr, slab->slab_number, block_number); } } static bool __must_check is_valid_journal_point(const struct journal_point *point) { return ((point != NULL) && (point->sequence_number > 0)); } /** * update_reference_count() - Update the reference count of a block. * @slab: The slab which owns the block. * @block: The reference block which contains the block being updated. * @block_number: The block to update. * @slab_journal_point: The slab journal point at which this update is journaled. * @updater: The reference updater. * @normal_operation: Whether we are in normal operation vs. recovery or rebuild. * @adjust_block_count: Whether to update the slab's free block count. * @provisional_decrement_ptr: A pointer which will be set to true if this update was a decrement * of a provisional reference. * * Return: VDO_SUCCESS or an error. */ static int update_reference_count(struct vdo_slab *slab, struct reference_block *block, slab_block_number block_number, const struct journal_point *slab_journal_point, struct reference_updater *updater, bool normal_operation, bool adjust_block_count, bool *provisional_decrement_ptr) { vdo_refcount_t *counter_ptr = &slab->counters[block_number]; enum reference_status old_status = reference_count_to_status(*counter_ptr); int result; if (!updater->increment) { result = decrement_for_data(slab, block, block_number, old_status, updater, counter_ptr, adjust_block_count); if ((result == VDO_SUCCESS) && (old_status == RS_PROVISIONAL)) { if (provisional_decrement_ptr != NULL) *provisional_decrement_ptr = true; return VDO_SUCCESS; } } else if (updater->operation == VDO_JOURNAL_DATA_REMAPPING) { result = increment_for_data(slab, block, block_number, old_status, updater->lock, counter_ptr, adjust_block_count); } else { result = increment_for_block_map(slab, block, block_number, old_status, updater->lock, normal_operation, counter_ptr, adjust_block_count); } if (result != VDO_SUCCESS) return result; if (is_valid_journal_point(slab_journal_point)) slab->slab_journal_point = *slab_journal_point; return VDO_SUCCESS; } static int __must_check adjust_reference_count(struct vdo_slab *slab, struct reference_updater *updater, const struct journal_point *slab_journal_point) { slab_block_number block_number; int result; struct reference_block *block; bool provisional_decrement = false; if (!is_slab_open(slab)) return VDO_INVALID_ADMIN_STATE; result = slab_block_number_from_pbn(slab, updater->zpbn.pbn, &block_number); if (result != VDO_SUCCESS) return result; block = get_reference_block(slab, block_number); result = update_reference_count(slab, block, block_number, slab_journal_point, updater, NORMAL_OPERATION, true, &provisional_decrement); if ((result != VDO_SUCCESS) || provisional_decrement) return result; if (block->is_dirty && (block->slab_journal_lock > 0)) { sequence_number_t entry_lock = slab_journal_point->sequence_number; /* * This block is already dirty and a slab journal entry has been made for it since * the last time it was clean. We must release the per-entry slab journal lock for * the entry associated with the update we are now doing. */ result = VDO_ASSERT(is_valid_journal_point(slab_journal_point), "Reference count adjustments need slab journal points."); if (result != VDO_SUCCESS) return result; adjust_slab_journal_block_reference(&slab->journal, entry_lock, -1); return VDO_SUCCESS; } /* * This may be the first time we are applying an update for which there is a slab journal * entry to this block since the block was cleaned. Therefore, we convert the per-entry * slab journal lock to an uncommitted reference block lock, if there is a per-entry lock. */ if (is_valid_journal_point(slab_journal_point)) block->slab_journal_lock = slab_journal_point->sequence_number; else block->slab_journal_lock = 0; dirty_block(block); return VDO_SUCCESS; } /** * add_entry_from_waiter() - Add an entry to the slab journal. * @waiter: The vio which should make an entry now. * @context: The slab journal to make an entry in. * * This callback is invoked by add_entries() once it has determined that we are ready to make * another entry in the slab journal. Implements waiter_callback_fn. */ static void add_entry_from_waiter(struct vdo_waiter *waiter, void *context) { int result; struct reference_updater *updater = container_of(waiter, struct reference_updater, waiter); struct data_vio *data_vio = data_vio_from_reference_updater(updater); struct slab_journal *journal = context; struct slab_journal_block_header *header = &journal->tail_header; struct journal_point slab_journal_point = { .sequence_number = header->sequence_number, .entry_count = header->entry_count, }; sequence_number_t recovery_block = data_vio->recovery_journal_point.sequence_number; if (header->entry_count == 0) { /* * This is the first entry in the current tail block, so get a lock on the recovery * journal which we will hold until this tail block is committed. */ get_lock(journal, header->sequence_number)->recovery_start = recovery_block; if (journal->recovery_journal != NULL) { zone_count_t zone_number = journal->slab->allocator->zone_number; vdo_acquire_recovery_journal_block_reference(journal->recovery_journal, recovery_block, VDO_ZONE_TYPE_PHYSICAL, zone_number); } mark_slab_journal_dirty(journal, recovery_block); reclaim_journal_space(journal); } add_entry(journal, updater->zpbn.pbn, updater->operation, updater->increment, expand_journal_point(data_vio->recovery_journal_point, updater->increment)); if (journal->slab->status != VDO_SLAB_REBUILT) { /* * If the slab is unrecovered, scrubbing will take care of the count since the * update is now recorded in the journal. */ adjust_slab_journal_block_reference(journal, slab_journal_point.sequence_number, -1); result = VDO_SUCCESS; } else { /* Now that an entry has been made in the slab journal, update the counter. */ result = adjust_reference_count(journal->slab, updater, &slab_journal_point); } if (updater->increment) continue_data_vio_with_error(data_vio, result); else vdo_continue_completion(&data_vio->decrement_completion, result); } /** * is_next_entry_a_block_map_increment() - Check whether the next entry to be made is a block map * increment. * @journal: The journal. * * Return: true if the first entry waiter's operation is a block map increment. */ static inline bool is_next_entry_a_block_map_increment(struct slab_journal *journal) { struct vdo_waiter *waiter = vdo_waitq_get_first_waiter(&journal->entry_waiters); struct reference_updater *updater = container_of(waiter, struct reference_updater, waiter); return (updater->operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING); } /** * add_entries() - Add as many entries as possible from the queue of vios waiting to make entries. * @journal: The journal to which entries may be added. * * By processing the queue in order, we ensure that slab journal entries are made in the same order * as recovery journal entries for the same increment or decrement. */ static void add_entries(struct slab_journal *journal) { if (journal->adding_entries) { /* Protect against re-entrancy. */ return; } journal->adding_entries = true; while (vdo_waitq_has_waiters(&journal->entry_waiters)) { struct slab_journal_block_header *header = &journal->tail_header; if (journal->partial_write_in_progress || (journal->slab->status == VDO_SLAB_REBUILDING)) { /* * Don't add entries while rebuilding or while a partial write is * outstanding, as it could result in reference count corruption. */ break; } if (journal->waiting_to_commit) { /* * If we are waiting for resources to write the tail block, and the tail * block is full, we can't make another entry. */ WRITE_ONCE(journal->events->tail_busy_count, journal->events->tail_busy_count + 1); break; } else if (is_next_entry_a_block_map_increment(journal) && (header->entry_count >= journal->full_entries_per_block)) { /* * The tail block does not have room for a block map increment, so commit * it now. */ commit_tail(journal); if (journal->waiting_to_commit) { WRITE_ONCE(journal->events->tail_busy_count, journal->events->tail_busy_count + 1); break; } } /* If the slab is over the blocking threshold, make the vio wait. */ if (requires_reaping(journal)) { WRITE_ONCE(journal->events->blocked_count, journal->events->blocked_count + 1); save_dirty_reference_blocks(journal->slab); break; } if (header->entry_count == 0) { struct journal_lock *lock = get_lock(journal, header->sequence_number); /* * Check if the on disk slab journal is full. Because of the blocking and * scrubbing thresholds, this should never happen. */ if (lock->count > 0) { VDO_ASSERT_LOG_ONLY((journal->head + journal->size) == journal->tail, "New block has locks, but journal is not full"); /* * The blocking threshold must let the journal fill up if the new * block has locks; if the blocking threshold is smaller than the * journal size, the new block cannot possibly have locks already. */ VDO_ASSERT_LOG_ONLY((journal->blocking_threshold >= journal->size), "New block can have locks already iff blocking threshold is at the end of the journal"); WRITE_ONCE(journal->events->disk_full_count, journal->events->disk_full_count + 1); save_dirty_reference_blocks(journal->slab); break; } /* * Don't allow the new block to be reaped until all of the reference count * blocks are written and the journal block has been fully committed as * well. */ lock->count = journal->entries_per_block + 1; if (header->sequence_number == 1) { struct vdo_slab *slab = journal->slab; block_count_t i; /* * This is the first entry in this slab journal, ever. Dirty all of * the reference count blocks. Each will acquire a lock on the tail * block so that the journal won't be reaped until the reference * counts are initialized. The lock acquisition must be done by the * ref_counts since here we don't know how many reference blocks * the ref_counts has. */ for (i = 0; i < slab->reference_block_count; i++) { slab->reference_blocks[i].slab_journal_lock = 1; dirty_block(&slab->reference_blocks[i]); } adjust_slab_journal_block_reference(journal, 1, slab->reference_block_count); } } vdo_waitq_notify_next_waiter(&journal->entry_waiters, add_entry_from_waiter, journal); } journal->adding_entries = false; /* If there are no waiters, and we are flushing or saving, commit the tail block. */ if (vdo_is_state_draining(&journal->slab->state) && !vdo_is_state_suspending(&journal->slab->state) && !vdo_waitq_has_waiters(&journal->entry_waiters)) commit_tail(journal); } /** * reset_search_cursor() - Reset the free block search back to the first reference counter in the * first reference block of a slab. */ static void reset_search_cursor(struct vdo_slab *slab) { struct search_cursor *cursor = &slab->search_cursor; cursor->block = cursor->first_block; cursor->index = 0; /* Unit tests have slabs with only one reference block (and it's a runt). */ cursor->end_index = min_t(u32, COUNTS_PER_BLOCK, slab->block_count); } /** * advance_search_cursor() - Advance the search cursor to the start of the next reference block in * a slab, * * Wraps around to the first reference block if the current block is the last reference block. * * Return: true unless the cursor was at the last reference block. */ static bool advance_search_cursor(struct vdo_slab *slab) { struct search_cursor *cursor = &slab->search_cursor; /* * If we just finished searching the last reference block, then wrap back around to the * start of the array. */ if (cursor->block == cursor->last_block) { reset_search_cursor(slab); return false; } /* We're not already at the end, so advance to cursor to the next block. */ cursor->block++; cursor->index = cursor->end_index; if (cursor->block == cursor->last_block) { /* The last reference block will usually be a runt. */ cursor->end_index = slab->block_count; } else { cursor->end_index += COUNTS_PER_BLOCK; } return true; } /** * vdo_adjust_reference_count_for_rebuild() - Adjust the reference count of a block during rebuild. * * Return: VDO_SUCCESS or an error. */ int vdo_adjust_reference_count_for_rebuild(struct slab_depot *depot, physical_block_number_t pbn, enum journal_operation operation) { int result; slab_block_number block_number; struct reference_block *block; struct vdo_slab *slab = vdo_get_slab(depot, pbn); struct reference_updater updater = { .operation = operation, .increment = true, }; result = slab_block_number_from_pbn(slab, pbn, &block_number); if (result != VDO_SUCCESS) return result; block = get_reference_block(slab, block_number); result = update_reference_count(slab, block, block_number, NULL, &updater, !NORMAL_OPERATION, false, NULL); if (result != VDO_SUCCESS) return result; dirty_block(block); return VDO_SUCCESS; } /** * replay_reference_count_change() - Replay the reference count adjustment from a slab journal * entry into the reference count for a block. * @slab: The slab. * @entry_point: The slab journal point for the entry. * @entry: The slab journal entry being replayed. * * The adjustment will be ignored if it was already recorded in the reference count. * * Return: VDO_SUCCESS or an error code. */ static int replay_reference_count_change(struct vdo_slab *slab, const struct journal_point *entry_point, struct slab_journal_entry entry) { int result; struct reference_block *block = get_reference_block(slab, entry.sbn); sector_count_t sector = (entry.sbn % COUNTS_PER_BLOCK) / COUNTS_PER_SECTOR; struct reference_updater updater = { .operation = entry.operation, .increment = entry.increment, }; if (!vdo_before_journal_point(&block->commit_points[sector], entry_point)) { /* This entry is already reflected in the existing counts, so do nothing. */ return VDO_SUCCESS; } /* This entry is not yet counted in the reference counts. */ result = update_reference_count(slab, block, entry.sbn, entry_point, &updater, !NORMAL_OPERATION, false, NULL); if (result != VDO_SUCCESS) return result; dirty_block(block); return VDO_SUCCESS; } /** * find_zero_byte_in_word() - Find the array index of the first zero byte in word-sized range of * reference counters. * @word_ptr: A pointer to the eight counter bytes to check. * @start_index: The array index corresponding to word_ptr[0]. * @fail_index: The array index to return if no zero byte is found. * * The search does no bounds checking; the function relies on the array being sufficiently padded. * * Return: The array index of the first zero byte in the word, or the value passed as fail_index if * no zero byte was found. */ static inline slab_block_number find_zero_byte_in_word(const u8 *word_ptr, slab_block_number start_index, slab_block_number fail_index) { u64 word = get_unaligned_le64(word_ptr); /* This looks like a loop, but GCC will unroll the eight iterations for us. */ unsigned int offset; for (offset = 0; offset < BYTES_PER_WORD; offset++) { /* Assumes little-endian byte order, which we have on X86. */ if ((word & 0xFF) == 0) return (start_index + offset); word >>= 8; } return fail_index; } /** * find_free_block() - Find the first block with a reference count of zero in the specified * range of reference counter indexes. * @slab: The slab counters to scan. * @index_ptr: A pointer to hold the array index of the free block. * * Exposed for unit testing. * * Return: true if a free block was found in the specified range. */ static bool find_free_block(const struct vdo_slab *slab, slab_block_number *index_ptr) { slab_block_number zero_index; slab_block_number next_index = slab->search_cursor.index; slab_block_number end_index = slab->search_cursor.end_index; u8 *next_counter = &slab->counters[next_index]; u8 *end_counter = &slab->counters[end_index]; /* * Search every byte of the first unaligned word. (Array is padded so reading past end is * safe.) */ zero_index = find_zero_byte_in_word(next_counter, next_index, end_index); if (zero_index < end_index) { *index_ptr = zero_index; return true; } /* * On architectures where unaligned word access is expensive, this would be a good place to * advance to an alignment boundary. */ next_index += BYTES_PER_WORD; next_counter += BYTES_PER_WORD; /* * Now we're word-aligned; check an word at a time until we find a word containing a zero. * (Array is padded so reading past end is safe.) */ while (next_counter < end_counter) { /* * The following code is currently an exact copy of the code preceding the loop, * but if you try to merge them by using a do loop, it runs slower because a jump * instruction gets added at the start of the iteration. */ zero_index = find_zero_byte_in_word(next_counter, next_index, end_index); if (zero_index < end_index) { *index_ptr = zero_index; return true; } next_index += BYTES_PER_WORD; next_counter += BYTES_PER_WORD; } return false; } /** * search_current_reference_block() - Search the reference block currently saved in the search * cursor for a reference count of zero, starting at the saved * counter index. * @slab: The slab to search. * @free_index_ptr: A pointer to receive the array index of the zero reference count. * * Return: true if an unreferenced counter was found. */ static bool search_current_reference_block(const struct vdo_slab *slab, slab_block_number *free_index_ptr) { /* Don't bother searching if the current block is known to be full. */ return ((slab->search_cursor.block->allocated_count < COUNTS_PER_BLOCK) && find_free_block(slab, free_index_ptr)); } /** * search_reference_blocks() - Search each reference block for a reference count of zero. * @slab: The slab to search. * @free_index_ptr: A pointer to receive the array index of the zero reference count. * * Searches each reference block for a reference count of zero, starting at the reference block and * counter index saved in the search cursor and searching up to the end of the last reference * block. The search does not wrap. * * Return: true if an unreferenced counter was found. */ static bool search_reference_blocks(struct vdo_slab *slab, slab_block_number *free_index_ptr) { /* Start searching at the saved search position in the current block. */ if (search_current_reference_block(slab, free_index_ptr)) return true; /* Search each reference block up to the end of the slab. */ while (advance_search_cursor(slab)) { if (search_current_reference_block(slab, free_index_ptr)) return true; } return false; } /** * make_provisional_reference() - Do the bookkeeping for making a provisional reference. */ static void make_provisional_reference(struct vdo_slab *slab, slab_block_number block_number) { struct reference_block *block = get_reference_block(slab, block_number); /* * Make the initial transition from an unreferenced block to a * provisionally allocated block. */ slab->counters[block_number] = PROVISIONAL_REFERENCE_COUNT; /* Account for the allocation. */ block->allocated_count++; slab->free_blocks--; } /** * dirty_all_reference_blocks() - Mark all reference count blocks in a slab as dirty. */ static void dirty_all_reference_blocks(struct vdo_slab *slab) { block_count_t i; for (i = 0; i < slab->reference_block_count; i++) dirty_block(&slab->reference_blocks[i]); } /** * clear_provisional_references() - Clear the provisional reference counts from a reference block. * @block: The block to clear. */ static void clear_provisional_references(struct reference_block *block) { vdo_refcount_t *counters = get_reference_counters_for_block(block); block_count_t j; for (j = 0; j < COUNTS_PER_BLOCK; j++) { if (counters[j] == PROVISIONAL_REFERENCE_COUNT) { counters[j] = EMPTY_REFERENCE_COUNT; block->allocated_count--; } } } static inline bool journal_points_equal(struct journal_point first, struct journal_point second) { return ((first.sequence_number == second.sequence_number) && (first.entry_count == second.entry_count)); } /** * unpack_reference_block() - Unpack reference counts blocks into the internal memory structure. * @packed: The written reference block to be unpacked. * @block: The internal reference block to be loaded. */ static void unpack_reference_block(struct packed_reference_block *packed, struct reference_block *block) { block_count_t index; sector_count_t i; struct vdo_slab *slab = block->slab; vdo_refcount_t *counters = get_reference_counters_for_block(block); for (i = 0; i < VDO_SECTORS_PER_BLOCK; i++) { struct packed_reference_sector *sector = &packed->sectors[i]; vdo_unpack_journal_point(§or->commit_point, &block->commit_points[i]); memcpy(counters + (i * COUNTS_PER_SECTOR), sector->counts, (sizeof(vdo_refcount_t) * COUNTS_PER_SECTOR)); /* The slab_journal_point must be the latest point found in any sector. */ if (vdo_before_journal_point(&slab->slab_journal_point, &block->commit_points[i])) slab->slab_journal_point = block->commit_points[i]; if ((i > 0) && !journal_points_equal(block->commit_points[0], block->commit_points[i])) { size_t block_index = block - block->slab->reference_blocks; vdo_log_warning("Torn write detected in sector %u of reference block %zu of slab %u", i, block_index, block->slab->slab_number); } } block->allocated_count = 0; for (index = 0; index < COUNTS_PER_BLOCK; index++) { if (counters[index] != EMPTY_REFERENCE_COUNT) block->allocated_count++; } } /** * finish_reference_block_load() - After a reference block has been read, unpack it. * @completion: The VIO that just finished reading. */ static void finish_reference_block_load(struct vdo_completion *completion) { struct vio *vio = as_vio(completion); struct pooled_vio *pooled = vio_as_pooled_vio(vio); struct reference_block *block = completion->parent; struct vdo_slab *slab = block->slab; unpack_reference_block((struct packed_reference_block *) vio->data, block); return_vio_to_pool(slab->allocator->vio_pool, pooled); slab->active_count--; clear_provisional_references(block); slab->free_blocks -= block->allocated_count; check_if_slab_drained(slab); } static void load_reference_block_endio(struct bio *bio) { struct vio *vio = bio->bi_private; struct reference_block *block = vio->completion.parent; continue_vio_after_io(vio, finish_reference_block_load, block->slab->allocator->thread_id); } /** * load_reference_block() - After a block waiter has gotten a VIO from the VIO pool, load the * block. * @waiter: The waiter of the block to load. * @context: The VIO returned by the pool. */ static void load_reference_block(struct vdo_waiter *waiter, void *context) { struct pooled_vio *pooled = context; struct vio *vio = &pooled->vio; struct reference_block *block = container_of(waiter, struct reference_block, waiter); size_t block_offset = (block - block->slab->reference_blocks); vio->completion.parent = block; vdo_submit_metadata_vio(vio, block->slab->ref_counts_origin + block_offset, load_reference_block_endio, handle_io_error, REQ_OP_READ); } /** * load_reference_blocks() - Load a slab's reference blocks from the underlying storage into a * pre-allocated reference counter. */ static void load_reference_blocks(struct vdo_slab *slab) { block_count_t i; slab->free_blocks = slab->block_count; slab->active_count = slab->reference_block_count; for (i = 0; i < slab->reference_block_count; i++) { struct vdo_waiter *waiter = &slab->reference_blocks[i].waiter; waiter->callback = load_reference_block; acquire_vio_from_pool(slab->allocator->vio_pool, waiter); } } /** * drain_slab() - Drain all reference count I/O. * * Depending upon the type of drain being performed (as recorded in the ref_count's vdo_slab), the * reference blocks may be loaded from disk or dirty reference blocks may be written out. */ static void drain_slab(struct vdo_slab *slab) { bool save; bool load; const struct admin_state_code *state = vdo_get_admin_state_code(&slab->state); if (state == VDO_ADMIN_STATE_SUSPENDING) return; if ((state != VDO_ADMIN_STATE_REBUILDING) && (state != VDO_ADMIN_STATE_SAVE_FOR_SCRUBBING)) commit_tail(&slab->journal); if ((state == VDO_ADMIN_STATE_RECOVERING) || (slab->counters == NULL)) return; save = false; load = slab->allocator->summary_entries[slab->slab_number].load_ref_counts; if (state == VDO_ADMIN_STATE_SCRUBBING) { if (load) { load_reference_blocks(slab); return; } } else if (state == VDO_ADMIN_STATE_SAVE_FOR_SCRUBBING) { if (!load) { /* These reference counts were never written, so mark them all dirty. */ dirty_all_reference_blocks(slab); } save = true; } else if (state == VDO_ADMIN_STATE_REBUILDING) { /* * Write out the counters if the slab has written them before, or it has any * non-zero reference counts, or there are any slab journal blocks. */ block_count_t data_blocks = slab->allocator->depot->slab_config.data_blocks; if (load || (slab->free_blocks != data_blocks) || !is_slab_journal_blank(slab)) { dirty_all_reference_blocks(slab); save = true; } } else if (state == VDO_ADMIN_STATE_SAVING) { save = (slab->status == VDO_SLAB_REBUILT); } else { vdo_finish_draining_with_result(&slab->state, VDO_SUCCESS); return; } if (save) save_dirty_reference_blocks(slab); } static int allocate_slab_counters(struct vdo_slab *slab) { int result; size_t index, bytes; result = VDO_ASSERT(slab->reference_blocks == NULL, "vdo_slab %u doesn't allocate refcounts twice", slab->slab_number); if (result != VDO_SUCCESS) return result; result = vdo_allocate(slab->reference_block_count, struct reference_block, __func__, &slab->reference_blocks); if (result != VDO_SUCCESS) return result; /* * Allocate such that the runt slab has a full-length memory array, plus a little padding * so we can word-search even at the very end. */ bytes = (slab->reference_block_count * COUNTS_PER_BLOCK) + (2 * BYTES_PER_WORD); result = vdo_allocate(bytes, vdo_refcount_t, "ref counts array", &slab->counters); if (result != VDO_SUCCESS) { vdo_free(vdo_forget(slab->reference_blocks)); return result; } slab->search_cursor.first_block = slab->reference_blocks; slab->search_cursor.last_block = &slab->reference_blocks[slab->reference_block_count - 1]; reset_search_cursor(slab); for (index = 0; index < slab->reference_block_count; index++) { slab->reference_blocks[index] = (struct reference_block) { .slab = slab, }; } return VDO_SUCCESS; } static int allocate_counters_if_clean(struct vdo_slab *slab) { if (vdo_is_state_clean_load(&slab->state)) return allocate_slab_counters(slab); return VDO_SUCCESS; } static void finish_loading_journal(struct vdo_completion *completion) { struct vio *vio = as_vio(completion); struct slab_journal *journal = completion->parent; struct vdo_slab *slab = journal->slab; struct packed_slab_journal_block *block = (struct packed_slab_journal_block *) vio->data; struct slab_journal_block_header header; vdo_unpack_slab_journal_block_header(&block->header, &header); /* FIXME: should it be an error if the following conditional fails? */ if ((header.metadata_type == VDO_METADATA_SLAB_JOURNAL) && (header.nonce == slab->allocator->nonce)) { journal->tail = header.sequence_number + 1; /* * If the slab is clean, this implies the slab journal is empty, so advance the * head appropriately. */ journal->head = (slab->allocator->summary_entries[slab->slab_number].is_dirty ? header.head : journal->tail); journal->tail_header = header; initialize_journal_state(journal); } return_vio_to_pool(slab->allocator->vio_pool, vio_as_pooled_vio(vio)); vdo_finish_loading_with_result(&slab->state, allocate_counters_if_clean(slab)); } static void read_slab_journal_tail_endio(struct bio *bio) { struct vio *vio = bio->bi_private; struct slab_journal *journal = vio->completion.parent; continue_vio_after_io(vio, finish_loading_journal, journal->slab->allocator->thread_id); } static void handle_load_error(struct vdo_completion *completion) { int result = completion->result; struct slab_journal *journal = completion->parent; struct vio *vio = as_vio(completion); vio_record_metadata_io_error(vio); return_vio_to_pool(journal->slab->allocator->vio_pool, vio_as_pooled_vio(vio)); vdo_finish_loading_with_result(&journal->slab->state, result); } /** * read_slab_journal_tail() - Read the slab journal tail block by using a vio acquired from the vio * pool. * @waiter: The vio pool waiter which has just been notified. * @context: The vio pool entry given to the waiter. * * This is the success callback from acquire_vio_from_pool() when loading a slab journal. */ static void read_slab_journal_tail(struct vdo_waiter *waiter, void *context) { struct slab_journal *journal = container_of(waiter, struct slab_journal, resource_waiter); struct vdo_slab *slab = journal->slab; struct pooled_vio *pooled = context; struct vio *vio = &pooled->vio; tail_block_offset_t last_commit_point = slab->allocator->summary_entries[slab->slab_number].tail_block_offset; /* * Slab summary keeps the commit point offset, so the tail block is the block before that. * Calculation supports small journals in unit tests. */ tail_block_offset_t tail_block = ((last_commit_point == 0) ? (tail_block_offset_t)(journal->size - 1) : (last_commit_point - 1)); vio->completion.parent = journal; vio->completion.callback_thread_id = slab->allocator->thread_id; vdo_submit_metadata_vio(vio, slab->journal_origin + tail_block, read_slab_journal_tail_endio, handle_load_error, REQ_OP_READ); } /** * load_slab_journal() - Load a slab's journal by reading the journal's tail. */ static void load_slab_journal(struct vdo_slab *slab) { struct slab_journal *journal = &slab->journal; tail_block_offset_t last_commit_point; last_commit_point = slab->allocator->summary_entries[slab->slab_number].tail_block_offset; if ((last_commit_point == 0) && !slab->allocator->summary_entries[slab->slab_number].load_ref_counts) { /* * This slab claims that it has a tail block at (journal->size - 1), but a head of * 1. This is impossible, due to the scrubbing threshold, on a real system, so * don't bother reading the (bogus) data off disk. */ VDO_ASSERT_LOG_ONLY(((journal->size < 16) || (journal->scrubbing_threshold < (journal->size - 1))), "Scrubbing threshold protects against reads of unwritten slab journal blocks"); vdo_finish_loading_with_result(&slab->state, allocate_counters_if_clean(slab)); return; } journal->resource_waiter.callback = read_slab_journal_tail; acquire_vio_from_pool(slab->allocator->vio_pool, &journal->resource_waiter); } static void register_slab_for_scrubbing(struct vdo_slab *slab, bool high_priority) { struct slab_scrubber *scrubber = &slab->allocator->scrubber; VDO_ASSERT_LOG_ONLY((slab->status != VDO_SLAB_REBUILT), "slab to be scrubbed is unrecovered"); if (slab->status != VDO_SLAB_REQUIRES_SCRUBBING) return; list_del_init(&slab->allocq_entry); if (!slab->was_queued_for_scrubbing) { WRITE_ONCE(scrubber->slab_count, scrubber->slab_count + 1); slab->was_queued_for_scrubbing = true; } if (high_priority) { slab->status = VDO_SLAB_REQUIRES_HIGH_PRIORITY_SCRUBBING; list_add_tail(&slab->allocq_entry, &scrubber->high_priority_slabs); return; } list_add_tail(&slab->allocq_entry, &scrubber->slabs); } /* Queue a slab for allocation or scrubbing. */ static void queue_slab(struct vdo_slab *slab) { struct block_allocator *allocator = slab->allocator; block_count_t free_blocks; int result; VDO_ASSERT_LOG_ONLY(list_empty(&slab->allocq_entry), "a requeued slab must not already be on a ring"); if (vdo_is_read_only(allocator->depot->vdo)) return; free_blocks = slab->free_blocks; result = VDO_ASSERT((free_blocks <= allocator->depot->slab_config.data_blocks), "rebuilt slab %u must have a valid free block count (has %llu, expected maximum %llu)", slab->slab_number, (unsigned long long) free_blocks, (unsigned long long) allocator->depot->slab_config.data_blocks); if (result != VDO_SUCCESS) { vdo_enter_read_only_mode(allocator->depot->vdo, result); return; } if (slab->status != VDO_SLAB_REBUILT) { register_slab_for_scrubbing(slab, false); return; } if (!vdo_is_state_resuming(&slab->state)) { /* * If the slab is resuming, we've already accounted for it here, so don't do it * again. * FIXME: under what situation would the slab be resuming here? */ WRITE_ONCE(allocator->allocated_blocks, allocator->allocated_blocks - free_blocks); if (!is_slab_journal_blank(slab)) { WRITE_ONCE(allocator->statistics.slabs_opened, allocator->statistics.slabs_opened + 1); } } if (allocator->depot->vdo->suspend_type == VDO_ADMIN_STATE_SAVING) reopen_slab_journal(slab); prioritize_slab(slab); } /** * initiate_slab_action() - Initiate a slab action. * * Implements vdo_admin_initiator_fn. */ static void initiate_slab_action(struct admin_state *state) { struct vdo_slab *slab = container_of(state, struct vdo_slab, state); if (vdo_is_state_draining(state)) { const struct admin_state_code *operation = vdo_get_admin_state_code(state); if (operation == VDO_ADMIN_STATE_SCRUBBING) slab->status = VDO_SLAB_REBUILDING; drain_slab(slab); check_if_slab_drained(slab); return; } if (vdo_is_state_loading(state)) { load_slab_journal(slab); return; } if (vdo_is_state_resuming(state)) { queue_slab(slab); vdo_finish_resuming(state); return; } vdo_finish_operation(state, VDO_INVALID_ADMIN_STATE); } /** * get_next_slab() - Get the next slab to scrub. * @scrubber: The slab scrubber. * * Return: The next slab to scrub or NULL if there are none. */ static struct vdo_slab *get_next_slab(struct slab_scrubber *scrubber) { struct vdo_slab *slab; slab = list_first_entry_or_null(&scrubber->high_priority_slabs, struct vdo_slab, allocq_entry); if (slab != NULL) return slab; return list_first_entry_or_null(&scrubber->slabs, struct vdo_slab, allocq_entry); } /** * has_slabs_to_scrub() - Check whether a scrubber has slabs to scrub. * @scrubber: The scrubber to check. * * Return: true if the scrubber has slabs to scrub. */ static inline bool __must_check has_slabs_to_scrub(struct slab_scrubber *scrubber) { return (get_next_slab(scrubber) != NULL); } /** * uninitialize_scrubber_vio() - Clean up the slab_scrubber's vio. * @scrubber: The scrubber. */ static void uninitialize_scrubber_vio(struct slab_scrubber *scrubber) { vdo_free(vdo_forget(scrubber->vio.data)); free_vio_components(&scrubber->vio); } /** * finish_scrubbing() - Stop scrubbing, either because there are no more slabs to scrub or because * there's been an error. * @scrubber: The scrubber. */ static void finish_scrubbing(struct slab_scrubber *scrubber, int result) { bool notify = vdo_waitq_has_waiters(&scrubber->waiters); bool done = !has_slabs_to_scrub(scrubber); struct block_allocator *allocator = container_of(scrubber, struct block_allocator, scrubber); if (done) uninitialize_scrubber_vio(scrubber); if (scrubber->high_priority_only) { scrubber->high_priority_only = false; vdo_fail_completion(vdo_forget(scrubber->vio.completion.parent), result); } else if (done && (atomic_add_return(-1, &allocator->depot->zones_to_scrub) == 0)) { /* All of our slabs were scrubbed, and we're the last allocator to finish. */ enum vdo_state prior_state = atomic_cmpxchg(&allocator->depot->vdo->state, VDO_RECOVERING, VDO_DIRTY); /* * To be safe, even if the CAS failed, ensure anything that follows is ordered with * respect to whatever state change did happen. */ smp_mb__after_atomic(); /* * We must check the VDO state here and not the depot's read_only_notifier since * the compare-swap-above could have failed due to a read-only entry which our own * thread does not yet know about. */ if (prior_state == VDO_DIRTY) vdo_log_info("VDO commencing normal operation"); else if (prior_state == VDO_RECOVERING) vdo_log_info("Exiting recovery mode"); } /* * Note that the scrubber has stopped, and inform anyone who might be waiting for that to * happen. */ if (!vdo_finish_draining(&scrubber->admin_state)) WRITE_ONCE(scrubber->admin_state.current_state, VDO_ADMIN_STATE_SUSPENDED); /* * We can't notify waiters until after we've finished draining or they'll just requeue. * Fortunately if there were waiters, we can't have been freed yet. */ if (notify) vdo_waitq_notify_all_waiters(&scrubber->waiters, NULL, NULL); } static void scrub_next_slab(struct slab_scrubber *scrubber); /** * slab_scrubbed() - Notify the scrubber that a slab has been scrubbed. * @completion: The slab rebuild completion. * * This callback is registered in apply_journal_entries(). */ static void slab_scrubbed(struct vdo_completion *completion) { struct slab_scrubber *scrubber = container_of(as_vio(completion), struct slab_scrubber, vio); struct vdo_slab *slab = scrubber->slab; slab->status = VDO_SLAB_REBUILT; queue_slab(slab); reopen_slab_journal(slab); WRITE_ONCE(scrubber->slab_count, scrubber->slab_count - 1); scrub_next_slab(scrubber); } /** * abort_scrubbing() - Abort scrubbing due to an error. * @scrubber: The slab scrubber. * @result: The error. */ static void abort_scrubbing(struct slab_scrubber *scrubber, int result) { vdo_enter_read_only_mode(scrubber->vio.completion.vdo, result); finish_scrubbing(scrubber, result); } /** * handle_scrubber_error() - Handle errors while rebuilding a slab. * @completion: The slab rebuild completion. */ static void handle_scrubber_error(struct vdo_completion *completion) { struct vio *vio = as_vio(completion); vio_record_metadata_io_error(vio); abort_scrubbing(container_of(vio, struct slab_scrubber, vio), completion->result); } /** * apply_block_entries() - Apply all the entries in a block to the reference counts. * @block: A block with entries to apply. * @entry_count: The number of entries to apply. * @block_number: The sequence number of the block. * @slab: The slab to apply the entries to. * * Return: VDO_SUCCESS or an error code. */ static int apply_block_entries(struct packed_slab_journal_block *block, journal_entry_count_t entry_count, sequence_number_t block_number, struct vdo_slab *slab) { struct journal_point entry_point = { .sequence_number = block_number, .entry_count = 0, }; int result; slab_block_number max_sbn = slab->end - slab->start; while (entry_point.entry_count < entry_count) { struct slab_journal_entry entry = vdo_decode_slab_journal_entry(block, entry_point.entry_count); if (entry.sbn > max_sbn) { /* This entry is out of bounds. */ return vdo_log_error_strerror(VDO_CORRUPT_JOURNAL, "vdo_slab journal entry (%llu, %u) had invalid offset %u in slab (size %u blocks)", (unsigned long long) block_number, entry_point.entry_count, entry.sbn, max_sbn); } result = replay_reference_count_change(slab, &entry_point, entry); if (result != VDO_SUCCESS) { vdo_log_error_strerror(result, "vdo_slab journal entry (%llu, %u) (%s of offset %u) could not be applied in slab %u", (unsigned long long) block_number, entry_point.entry_count, vdo_get_journal_operation_name(entry.operation), entry.sbn, slab->slab_number); return result; } entry_point.entry_count++; } return VDO_SUCCESS; } /** * apply_journal_entries() - Find the relevant vio of the slab journal and apply all valid entries. * @completion: The metadata read vio completion. * * This is a callback registered in start_scrubbing(). */ static void apply_journal_entries(struct vdo_completion *completion) { int result; struct slab_scrubber *scrubber = container_of(as_vio(completion), struct slab_scrubber, vio); struct vdo_slab *slab = scrubber->slab; struct slab_journal *journal = &slab->journal; /* Find the boundaries of the useful part of the journal. */ sequence_number_t tail = journal->tail; tail_block_offset_t end_index = (tail - 1) % journal->size; char *end_data = scrubber->vio.data + (end_index * VDO_BLOCK_SIZE); struct packed_slab_journal_block *end_block = (struct packed_slab_journal_block *) end_data; sequence_number_t head = __le64_to_cpu(end_block->header.head); tail_block_offset_t head_index = head % journal->size; block_count_t index = head_index; struct journal_point ref_counts_point = slab->slab_journal_point; struct journal_point last_entry_applied = ref_counts_point; sequence_number_t sequence; for (sequence = head; sequence < tail; sequence++) { char *block_data = scrubber->vio.data + (index * VDO_BLOCK_SIZE); struct packed_slab_journal_block *block = (struct packed_slab_journal_block *) block_data; struct slab_journal_block_header header; vdo_unpack_slab_journal_block_header(&block->header, &header); if ((header.nonce != slab->allocator->nonce) || (header.metadata_type != VDO_METADATA_SLAB_JOURNAL) || (header.sequence_number != sequence) || (header.entry_count > journal->entries_per_block) || (header.has_block_map_increments && (header.entry_count > journal->full_entries_per_block))) { /* The block is not what we expect it to be. */ vdo_log_error("vdo_slab journal block for slab %u was invalid", slab->slab_number); abort_scrubbing(scrubber, VDO_CORRUPT_JOURNAL); return; } result = apply_block_entries(block, header.entry_count, sequence, slab); if (result != VDO_SUCCESS) { abort_scrubbing(scrubber, result); return; } last_entry_applied.sequence_number = sequence; last_entry_applied.entry_count = header.entry_count - 1; index++; if (index == journal->size) index = 0; } /* * At the end of rebuild, the reference counters should be accurate to the end of the * journal we just applied. */ result = VDO_ASSERT(!vdo_before_journal_point(&last_entry_applied, &ref_counts_point), "Refcounts are not more accurate than the slab journal"); if (result != VDO_SUCCESS) { abort_scrubbing(scrubber, result); return; } /* Save out the rebuilt reference blocks. */ vdo_prepare_completion(completion, slab_scrubbed, handle_scrubber_error, slab->allocator->thread_id, completion->parent); vdo_start_operation_with_waiter(&slab->state, VDO_ADMIN_STATE_SAVE_FOR_SCRUBBING, completion, initiate_slab_action); } static void read_slab_journal_endio(struct bio *bio) { struct vio *vio = bio->bi_private; struct slab_scrubber *scrubber = container_of(vio, struct slab_scrubber, vio); continue_vio_after_io(bio->bi_private, apply_journal_entries, scrubber->slab->allocator->thread_id); } /** * start_scrubbing() - Read the current slab's journal from disk now that it has been flushed. * @completion: The scrubber's vio completion. * * This callback is registered in scrub_next_slab(). */ static void start_scrubbing(struct vdo_completion *completion) { struct slab_scrubber *scrubber = container_of(as_vio(completion), struct slab_scrubber, vio); struct vdo_slab *slab = scrubber->slab; if (!slab->allocator->summary_entries[slab->slab_number].is_dirty) { slab_scrubbed(completion); return; } vdo_submit_metadata_vio(&scrubber->vio, slab->journal_origin, read_slab_journal_endio, handle_scrubber_error, REQ_OP_READ); } /** * scrub_next_slab() - Scrub the next slab if there is one. * @scrubber: The scrubber. */ static void scrub_next_slab(struct slab_scrubber *scrubber) { struct vdo_completion *completion = &scrubber->vio.completion; struct vdo_slab *slab; /* * Note: this notify call is always safe only because scrubbing can only be started when * the VDO is quiescent. */ vdo_waitq_notify_all_waiters(&scrubber->waiters, NULL, NULL); if (vdo_is_read_only(completion->vdo)) { finish_scrubbing(scrubber, VDO_READ_ONLY); return; } slab = get_next_slab(scrubber); if ((slab == NULL) || (scrubber->high_priority_only && list_empty(&scrubber->high_priority_slabs))) { finish_scrubbing(scrubber, VDO_SUCCESS); return; } if (vdo_finish_draining(&scrubber->admin_state)) return; list_del_init(&slab->allocq_entry); scrubber->slab = slab; vdo_prepare_completion(completion, start_scrubbing, handle_scrubber_error, slab->allocator->thread_id, completion->parent); vdo_start_operation_with_waiter(&slab->state, VDO_ADMIN_STATE_SCRUBBING, completion, initiate_slab_action); } /** * scrub_slabs() - Scrub all of an allocator's slabs that are eligible for scrubbing. * @allocator: The block_allocator to scrub. * @parent: The completion to notify when scrubbing is done, implies high_priority, may be NULL. */ static void scrub_slabs(struct block_allocator *allocator, struct vdo_completion *parent) { struct slab_scrubber *scrubber = &allocator->scrubber; scrubber->vio.completion.parent = parent; scrubber->high_priority_only = (parent != NULL); if (!has_slabs_to_scrub(scrubber)) { finish_scrubbing(scrubber, VDO_SUCCESS); return; } if (scrubber->high_priority_only && vdo_is_priority_table_empty(allocator->prioritized_slabs) && list_empty(&scrubber->high_priority_slabs)) register_slab_for_scrubbing(get_next_slab(scrubber), true); vdo_resume_if_quiescent(&scrubber->admin_state); scrub_next_slab(scrubber); } static inline void assert_on_allocator_thread(thread_id_t thread_id, const char *function_name) { VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == thread_id), "%s called on correct thread", function_name); } static void register_slab_with_allocator(struct block_allocator *allocator, struct vdo_slab *slab) { allocator->slab_count++; allocator->last_slab = slab->slab_number; } /** * get_depot_slab_iterator() - Return a slab_iterator over the slabs in a slab_depot. * @depot: The depot over which to iterate. * @start: The number of the slab to start iterating from. * @end: The number of the last slab which may be returned. * @stride: The difference in slab number between successive slabs. * * Iteration always occurs from higher to lower numbered slabs. * * Return: An initialized iterator structure. */ static struct slab_iterator get_depot_slab_iterator(struct slab_depot *depot, slab_count_t start, slab_count_t end, slab_count_t stride) { struct vdo_slab **slabs = depot->slabs; return (struct slab_iterator) { .slabs = slabs, .next = (((slabs == NULL) || (start < end)) ? NULL : slabs[start]), .end = end, .stride = stride, }; } static struct slab_iterator get_slab_iterator(const struct block_allocator *allocator) { return get_depot_slab_iterator(allocator->depot, allocator->last_slab, allocator->zone_number, allocator->depot->zone_count); } /** * next_slab() - Get the next slab from a slab_iterator and advance the iterator * @iterator: The slab_iterator. * * Return: The next slab or NULL if the iterator is exhausted. */ static struct vdo_slab *next_slab(struct slab_iterator *iterator) { struct vdo_slab *slab = iterator->next; if ((slab == NULL) || (slab->slab_number < iterator->end + iterator->stride)) iterator->next = NULL; else iterator->next = iterator->slabs[slab->slab_number - iterator->stride]; return slab; } /** * abort_waiter() - Abort vios waiting to make journal entries when read-only. * * This callback is invoked on all vios waiting to make slab journal entries after the VDO has gone * into read-only mode. Implements waiter_callback_fn. */ static void abort_waiter(struct vdo_waiter *waiter, void *context __always_unused) { struct reference_updater *updater = container_of(waiter, struct reference_updater, waiter); struct data_vio *data_vio = data_vio_from_reference_updater(updater); if (updater->increment) { continue_data_vio_with_error(data_vio, VDO_READ_ONLY); return; } vdo_continue_completion(&data_vio->decrement_completion, VDO_READ_ONLY); } /* Implements vdo_read_only_notification_fn. */ static void notify_block_allocator_of_read_only_mode(void *listener, struct vdo_completion *parent) { struct block_allocator *allocator = listener; struct slab_iterator iterator; assert_on_allocator_thread(allocator->thread_id, __func__); iterator = get_slab_iterator(allocator); while (iterator.next != NULL) { struct vdo_slab *slab = next_slab(&iterator); vdo_waitq_notify_all_waiters(&slab->journal.entry_waiters, abort_waiter, &slab->journal); check_if_slab_drained(slab); } vdo_finish_completion(parent); } /** * vdo_acquire_provisional_reference() - Acquire a provisional reference on behalf of a PBN lock if * the block it locks is unreferenced. * @slab: The slab which contains the block. * @pbn: The physical block to reference. * @lock: The lock. * * Return: VDO_SUCCESS or an error. */ int vdo_acquire_provisional_reference(struct vdo_slab *slab, physical_block_number_t pbn, struct pbn_lock *lock) { slab_block_number block_number; int result; if (vdo_pbn_lock_has_provisional_reference(lock)) return VDO_SUCCESS; if (!is_slab_open(slab)) return VDO_INVALID_ADMIN_STATE; result = slab_block_number_from_pbn(slab, pbn, &block_number); if (result != VDO_SUCCESS) return result; if (slab->counters[block_number] == EMPTY_REFERENCE_COUNT) { make_provisional_reference(slab, block_number); if (lock != NULL) vdo_assign_pbn_lock_provisional_reference(lock); } if (vdo_pbn_lock_has_provisional_reference(lock)) adjust_free_block_count(slab, false); return VDO_SUCCESS; } static int __must_check allocate_slab_block(struct vdo_slab *slab, physical_block_number_t *block_number_ptr) { slab_block_number free_index; if (!is_slab_open(slab)) return VDO_INVALID_ADMIN_STATE; if (!search_reference_blocks(slab, &free_index)) return VDO_NO_SPACE; VDO_ASSERT_LOG_ONLY((slab->counters[free_index] == EMPTY_REFERENCE_COUNT), "free block must have ref count of zero"); make_provisional_reference(slab, free_index); adjust_free_block_count(slab, false); /* * Update the search hint so the next search will start at the array index just past the * free block we just found. */ slab->search_cursor.index = (free_index + 1); *block_number_ptr = slab->start + free_index; return VDO_SUCCESS; } /** * open_slab() - Prepare a slab to be allocated from. * @slab: The slab. */ static void open_slab(struct vdo_slab *slab) { reset_search_cursor(slab); if (is_slab_journal_blank(slab)) { WRITE_ONCE(slab->allocator->statistics.slabs_opened, slab->allocator->statistics.slabs_opened + 1); dirty_all_reference_blocks(slab); } else { WRITE_ONCE(slab->allocator->statistics.slabs_reopened, slab->allocator->statistics.slabs_reopened + 1); } slab->allocator->open_slab = slab; } /* * The block allocated will have a provisional reference and the reference must be either confirmed * with a subsequent increment or vacated with a subsequent decrement via * vdo_release_block_reference(). */ int vdo_allocate_block(struct block_allocator *allocator, physical_block_number_t *block_number_ptr) { int result; if (allocator->open_slab != NULL) { /* Try to allocate the next block in the currently open slab. */ result = allocate_slab_block(allocator->open_slab, block_number_ptr); if ((result == VDO_SUCCESS) || (result != VDO_NO_SPACE)) return result; /* Put the exhausted open slab back into the priority table. */ prioritize_slab(allocator->open_slab); } /* Remove the highest priority slab from the priority table and make it the open slab. */ open_slab(list_entry(vdo_priority_table_dequeue(allocator->prioritized_slabs), struct vdo_slab, allocq_entry)); /* * Try allocating again. If we're out of space immediately after opening a slab, then every * slab must be fully allocated. */ return allocate_slab_block(allocator->open_slab, block_number_ptr); } /** * vdo_enqueue_clean_slab_waiter() - Wait for a clean slab. * @allocator: The block_allocator on which to wait. * @waiter: The waiter. * * Return: VDO_SUCCESS if the waiter was queued, VDO_NO_SPACE if there are no slabs to scrub, and * some other error otherwise. */ int vdo_enqueue_clean_slab_waiter(struct block_allocator *allocator, struct vdo_waiter *waiter) { if (vdo_is_read_only(allocator->depot->vdo)) return VDO_READ_ONLY; if (vdo_is_state_quiescent(&allocator->scrubber.admin_state)) return VDO_NO_SPACE; vdo_waitq_enqueue_waiter(&allocator->scrubber.waiters, waiter); return VDO_SUCCESS; } /** * vdo_modify_reference_count() - Modify the reference count of a block by first making a slab * journal entry and then updating the reference counter. * * @data_vio: The data_vio for which to add the entry. * @updater: Which of the data_vio's reference updaters is being submitted. */ void vdo_modify_reference_count(struct vdo_completion *completion, struct reference_updater *updater) { struct vdo_slab *slab = vdo_get_slab(completion->vdo->depot, updater->zpbn.pbn); if (!is_slab_open(slab)) { vdo_continue_completion(completion, VDO_INVALID_ADMIN_STATE); return; } if (vdo_is_read_only(completion->vdo)) { vdo_continue_completion(completion, VDO_READ_ONLY); return; } vdo_waitq_enqueue_waiter(&slab->journal.entry_waiters, &updater->waiter); if ((slab->status != VDO_SLAB_REBUILT) && requires_reaping(&slab->journal)) register_slab_for_scrubbing(slab, true); add_entries(&slab->journal); } /* Release an unused provisional reference. */ int vdo_release_block_reference(struct block_allocator *allocator, physical_block_number_t pbn) { struct reference_updater updater; if (pbn == VDO_ZERO_BLOCK) return VDO_SUCCESS; updater = (struct reference_updater) { .operation = VDO_JOURNAL_DATA_REMAPPING, .increment = false, .zpbn = { .pbn = pbn, }, }; return adjust_reference_count(vdo_get_slab(allocator->depot, pbn), &updater, NULL); } /* * This is a min_heap callback function orders slab_status structures using the 'is_clean' field as * the primary key and the 'emptiness' field as the secondary key. * * Slabs need to be pushed onto the rings in the same order they are to be popped off. Popping * should always get the most empty first, so pushing should be from most empty to least empty. * Thus, the ordering is reversed from the usual sense since min_heap returns smaller elements * before larger ones. */ static bool slab_status_is_less_than(const void *item1, const void *item2, void __always_unused *args) { const struct slab_status *info1 = item1; const struct slab_status *info2 = item2; if (info1->is_clean != info2->is_clean) return info1->is_clean; if (info1->emptiness != info2->emptiness) return info1->emptiness > info2->emptiness; return info1->slab_number < info2->slab_number; } static void swap_slab_statuses(void *item1, void *item2, void __always_unused *args) { struct slab_status *info1 = item1; struct slab_status *info2 = item2; swap(*info1, *info2); } static const struct min_heap_callbacks slab_status_min_heap = { .less = slab_status_is_less_than, .swp = swap_slab_statuses, }; /* Inform the slab actor that a action has finished on some slab; used by apply_to_slabs(). */ static void slab_action_callback(struct vdo_completion *completion) { struct block_allocator *allocator = vdo_as_block_allocator(completion); struct slab_actor *actor = &allocator->slab_actor; if (--actor->slab_action_count == 0) { actor->callback(completion); return; } vdo_reset_completion(completion); } /* Preserve the error from part of an action and continue. */ static void handle_operation_error(struct vdo_completion *completion) { struct block_allocator *allocator = vdo_as_block_allocator(completion); if (allocator->state.waiter != NULL) vdo_set_completion_result(allocator->state.waiter, completion->result); completion->callback(completion); } /* Perform an action on each of an allocator's slabs in parallel. */ static void apply_to_slabs(struct block_allocator *allocator, vdo_action_fn callback) { struct slab_iterator iterator; vdo_prepare_completion(&allocator->completion, slab_action_callback, handle_operation_error, allocator->thread_id, NULL); allocator->completion.requeue = false; /* * Since we are going to dequeue all of the slabs, the open slab will become invalid, so * clear it. */ allocator->open_slab = NULL; /* Ensure that we don't finish before we're done starting. */ allocator->slab_actor = (struct slab_actor) { .slab_action_count = 1, .callback = callback, }; iterator = get_slab_iterator(allocator); while (iterator.next != NULL) { const struct admin_state_code *operation = vdo_get_admin_state_code(&allocator->state); struct vdo_slab *slab = next_slab(&iterator); list_del_init(&slab->allocq_entry); allocator->slab_actor.slab_action_count++; vdo_start_operation_with_waiter(&slab->state, operation, &allocator->completion, initiate_slab_action); } slab_action_callback(&allocator->completion); } static void finish_loading_allocator(struct vdo_completion *completion) { struct block_allocator *allocator = vdo_as_block_allocator(completion); const struct admin_state_code *operation = vdo_get_admin_state_code(&allocator->state); if (allocator->eraser != NULL) dm_kcopyd_client_destroy(vdo_forget(allocator->eraser)); if (operation == VDO_ADMIN_STATE_LOADING_FOR_RECOVERY) { void *context = vdo_get_current_action_context(allocator->depot->action_manager); vdo_replay_into_slab_journals(allocator, context); return; } vdo_finish_loading(&allocator->state); } static void erase_next_slab_journal(struct block_allocator *allocator); static void copy_callback(int read_err, unsigned long write_err, void *context) { struct block_allocator *allocator = context; int result = (((read_err == 0) && (write_err == 0)) ? VDO_SUCCESS : -EIO); if (result != VDO_SUCCESS) { vdo_fail_completion(&allocator->completion, result); return; } erase_next_slab_journal(allocator); } /* erase_next_slab_journal() - Erase the next slab journal. */ static void erase_next_slab_journal(struct block_allocator *allocator) { struct vdo_slab *slab; physical_block_number_t pbn; struct dm_io_region regions[1]; struct slab_depot *depot = allocator->depot; block_count_t blocks = depot->slab_config.slab_journal_blocks; if (allocator->slabs_to_erase.next == NULL) { vdo_finish_completion(&allocator->completion); return; } slab = next_slab(&allocator->slabs_to_erase); pbn = slab->journal_origin - depot->vdo->geometry.bio_offset; regions[0] = (struct dm_io_region) { .bdev = vdo_get_backing_device(depot->vdo), .sector = pbn * VDO_SECTORS_PER_BLOCK, .count = blocks * VDO_SECTORS_PER_BLOCK, }; dm_kcopyd_zero(allocator->eraser, 1, regions, 0, copy_callback, allocator); } /* Implements vdo_admin_initiator_fn. */ static void initiate_load(struct admin_state *state) { struct block_allocator *allocator = container_of(state, struct block_allocator, state); const struct admin_state_code *operation = vdo_get_admin_state_code(state); if (operation == VDO_ADMIN_STATE_LOADING_FOR_REBUILD) { /* * Must requeue because the kcopyd client cannot be freed in the same stack frame * as the kcopyd callback, lest it deadlock. */ vdo_prepare_completion_for_requeue(&allocator->completion, finish_loading_allocator, handle_operation_error, allocator->thread_id, NULL); allocator->eraser = dm_kcopyd_client_create(NULL); if (IS_ERR(allocator->eraser)) { vdo_fail_completion(&allocator->completion, PTR_ERR(allocator->eraser)); allocator->eraser = NULL; return; } allocator->slabs_to_erase = get_slab_iterator(allocator); erase_next_slab_journal(allocator); return; } apply_to_slabs(allocator, finish_loading_allocator); } /** * vdo_notify_slab_journals_are_recovered() - Inform a block allocator that its slab journals have * been recovered from the recovery journal. * @completion The allocator completion */ void vdo_notify_slab_journals_are_recovered(struct vdo_completion *completion) { struct block_allocator *allocator = vdo_as_block_allocator(completion); vdo_finish_loading_with_result(&allocator->state, completion->result); } static int get_slab_statuses(struct block_allocator *allocator, struct slab_status **statuses_ptr) { int result; struct slab_status *statuses; struct slab_iterator iterator = get_slab_iterator(allocator); result = vdo_allocate(allocator->slab_count, struct slab_status, __func__, &statuses); if (result != VDO_SUCCESS) return result; *statuses_ptr = statuses; while (iterator.next != NULL) { slab_count_t slab_number = next_slab(&iterator)->slab_number; *statuses++ = (struct slab_status) { .slab_number = slab_number, .is_clean = !allocator->summary_entries[slab_number].is_dirty, .emptiness = allocator->summary_entries[slab_number].fullness_hint, }; } return VDO_SUCCESS; } /* Prepare slabs for allocation or scrubbing. */ static int __must_check vdo_prepare_slabs_for_allocation(struct block_allocator *allocator) { struct slab_status current_slab_status; DEFINE_MIN_HEAP(struct slab_status, heap) heap; int result; struct slab_status *slab_statuses; struct slab_depot *depot = allocator->depot; WRITE_ONCE(allocator->allocated_blocks, allocator->slab_count * depot->slab_config.data_blocks); result = get_slab_statuses(allocator, &slab_statuses); if (result != VDO_SUCCESS) return result; /* Sort the slabs by cleanliness, then by emptiness hint. */ heap = (struct heap) { .data = slab_statuses, .nr = allocator->slab_count, .size = allocator->slab_count, }; min_heapify_all(&heap, &slab_status_min_heap, NULL); while (heap.nr > 0) { bool high_priority; struct vdo_slab *slab; struct slab_journal *journal; current_slab_status = slab_statuses[0]; min_heap_pop(&heap, &slab_status_min_heap, NULL); slab = depot->slabs[current_slab_status.slab_number]; if ((depot->load_type == VDO_SLAB_DEPOT_REBUILD_LOAD) || (!allocator->summary_entries[slab->slab_number].load_ref_counts && current_slab_status.is_clean)) { queue_slab(slab); continue; } slab->status = VDO_SLAB_REQUIRES_SCRUBBING; journal = &slab->journal; high_priority = ((current_slab_status.is_clean && (depot->load_type == VDO_SLAB_DEPOT_NORMAL_LOAD)) || (journal_length(journal) >= journal->scrubbing_threshold)); register_slab_for_scrubbing(slab, high_priority); } vdo_free(slab_statuses); return VDO_SUCCESS; } static const char *status_to_string(enum slab_rebuild_status status) { switch (status) { case VDO_SLAB_REBUILT: return "REBUILT"; case VDO_SLAB_REQUIRES_SCRUBBING: return "SCRUBBING"; case VDO_SLAB_REQUIRES_HIGH_PRIORITY_SCRUBBING: return "PRIORITY_SCRUBBING"; case VDO_SLAB_REBUILDING: return "REBUILDING"; case VDO_SLAB_REPLAYING: return "REPLAYING"; default: return "UNKNOWN"; } } void vdo_dump_block_allocator(const struct block_allocator *allocator) { unsigned int pause_counter = 0; struct slab_iterator iterator = get_slab_iterator(allocator); const struct slab_scrubber *scrubber = &allocator->scrubber; vdo_log_info("block_allocator zone %u", allocator->zone_number); while (iterator.next != NULL) { struct vdo_slab *slab = next_slab(&iterator); struct slab_journal *journal = &slab->journal; if (slab->reference_blocks != NULL) { /* Terse because there are a lot of slabs to dump and syslog is lossy. */ vdo_log_info("slab %u: P%u, %llu free", slab->slab_number, slab->priority, (unsigned long long) slab->free_blocks); } else { vdo_log_info("slab %u: status %s", slab->slab_number, status_to_string(slab->status)); } vdo_log_info(" slab journal: entry_waiters=%zu waiting_to_commit=%s updating_slab_summary=%s head=%llu unreapable=%llu tail=%llu next_commit=%llu summarized=%llu last_summarized=%llu recovery_lock=%llu dirty=%s", vdo_waitq_num_waiters(&journal->entry_waiters), vdo_bool_to_string(journal->waiting_to_commit), vdo_bool_to_string(journal->updating_slab_summary), (unsigned long long) journal->head, (unsigned long long) journal->unreapable, (unsigned long long) journal->tail, (unsigned long long) journal->next_commit, (unsigned long long) journal->summarized, (unsigned long long) journal->last_summarized, (unsigned long long) journal->recovery_lock, vdo_bool_to_string(journal->recovery_lock != 0)); /* * Given the frequency with which the locks are just a tiny bit off, it might be * worth dumping all the locks, but that might be too much logging. */ if (slab->counters != NULL) { /* Terse because there are a lot of slabs to dump and syslog is lossy. */ vdo_log_info(" slab: free=%u/%u blocks=%u dirty=%zu active=%zu journal@(%llu,%u)", slab->free_blocks, slab->block_count, slab->reference_block_count, vdo_waitq_num_waiters(&slab->dirty_blocks), slab->active_count, (unsigned long long) slab->slab_journal_point.sequence_number, slab->slab_journal_point.entry_count); } else { vdo_log_info(" no counters"); } /* * Wait for a while after each batch of 32 slabs dumped, an arbitrary number, * allowing the kernel log a chance to be flushed instead of being overrun. */ if (pause_counter++ == 31) { pause_counter = 0; vdo_pause_for_logger(); } } vdo_log_info("slab_scrubber slab_count %u waiters %zu %s%s", READ_ONCE(scrubber->slab_count), vdo_waitq_num_waiters(&scrubber->waiters), vdo_get_admin_state_code(&scrubber->admin_state)->name, scrubber->high_priority_only ? ", high_priority_only " : ""); } static void free_slab(struct vdo_slab *slab) { if (slab == NULL) return; list_del(&slab->allocq_entry); vdo_free(vdo_forget(slab->journal.block)); vdo_free(vdo_forget(slab->journal.locks)); vdo_free(vdo_forget(slab->counters)); vdo_free(vdo_forget(slab->reference_blocks)); vdo_free(slab); } static int initialize_slab_journal(struct vdo_slab *slab) { struct slab_journal *journal = &slab->journal; const struct slab_config *slab_config = &slab->allocator->depot->slab_config; int result; result = vdo_allocate(slab_config->slab_journal_blocks, struct journal_lock, __func__, &journal->locks); if (result != VDO_SUCCESS) return result; result = vdo_allocate(VDO_BLOCK_SIZE, char, "struct packed_slab_journal_block", (char **) &journal->block); if (result != VDO_SUCCESS) return result; journal->slab = slab; journal->size = slab_config->slab_journal_blocks; journal->flushing_threshold = slab_config->slab_journal_flushing_threshold; journal->blocking_threshold = slab_config->slab_journal_blocking_threshold; journal->scrubbing_threshold = slab_config->slab_journal_scrubbing_threshold; journal->entries_per_block = VDO_SLAB_JOURNAL_ENTRIES_PER_BLOCK; journal->full_entries_per_block = VDO_SLAB_JOURNAL_FULL_ENTRIES_PER_BLOCK; journal->events = &slab->allocator->slab_journal_statistics; journal->recovery_journal = slab->allocator->depot->vdo->recovery_journal; journal->tail = 1; journal->head = 1; journal->flushing_deadline = journal->flushing_threshold; /* * Set there to be some time between the deadline and the blocking threshold, so that * hopefully all are done before blocking. */ if ((journal->blocking_threshold - journal->flushing_threshold) > 5) journal->flushing_deadline = journal->blocking_threshold - 5; journal->slab_summary_waiter.callback = release_journal_locks; INIT_LIST_HEAD(&journal->dirty_entry); INIT_LIST_HEAD(&journal->uncommitted_blocks); journal->tail_header.nonce = slab->allocator->nonce; journal->tail_header.metadata_type = VDO_METADATA_SLAB_JOURNAL; initialize_journal_state(journal); return VDO_SUCCESS; } /** * make_slab() - Construct a new, empty slab. * @slab_origin: The physical block number within the block allocator partition of the first block * in the slab. * @allocator: The block allocator to which the slab belongs. * @slab_number: The slab number of the slab. * @is_new: true if this slab is being allocated as part of a resize. * @slab_ptr: A pointer to receive the new slab. * * Return: VDO_SUCCESS or an error code. */ static int __must_check make_slab(physical_block_number_t slab_origin, struct block_allocator *allocator, slab_count_t slab_number, bool is_new, struct vdo_slab **slab_ptr) { const struct slab_config *slab_config = &allocator->depot->slab_config; struct vdo_slab *slab; int result; result = vdo_allocate(1, struct vdo_slab, __func__, &slab); if (result != VDO_SUCCESS) return result; *slab = (struct vdo_slab) { .allocator = allocator, .start = slab_origin, .end = slab_origin + slab_config->slab_blocks, .slab_number = slab_number, .ref_counts_origin = slab_origin + slab_config->data_blocks, .journal_origin = vdo_get_slab_journal_start_block(slab_config, slab_origin), .block_count = slab_config->data_blocks, .free_blocks = slab_config->data_blocks, .reference_block_count = vdo_get_saved_reference_count_size(slab_config->data_blocks), }; INIT_LIST_HEAD(&slab->allocq_entry); result = initialize_slab_journal(slab); if (result != VDO_SUCCESS) { free_slab(slab); return result; } if (is_new) { vdo_set_admin_state_code(&slab->state, VDO_ADMIN_STATE_NEW); result = allocate_slab_counters(slab); if (result != VDO_SUCCESS) { free_slab(slab); return result; } } else { vdo_set_admin_state_code(&slab->state, VDO_ADMIN_STATE_NORMAL_OPERATION); } *slab_ptr = slab; return VDO_SUCCESS; } /** * allocate_slabs() - Allocate a new slab pointer array. * @depot: The depot. * @slab_count: The number of slabs the depot should have in the new array. * * Any existing slab pointers will be copied into the new array, and slabs will be allocated as * needed. The newly allocated slabs will not be distributed for use by the block allocators. * * Return: VDO_SUCCESS or an error code. */ static int allocate_slabs(struct slab_depot *depot, slab_count_t slab_count) { block_count_t slab_size; bool resizing = false; physical_block_number_t slab_origin; int result; result = vdo_allocate(slab_count, struct vdo_slab *, "slab pointer array", &depot->new_slabs); if (result != VDO_SUCCESS) return result; if (depot->slabs != NULL) { memcpy(depot->new_slabs, depot->slabs, depot->slab_count * sizeof(struct vdo_slab *)); resizing = true; } slab_size = depot->slab_config.slab_blocks; slab_origin = depot->first_block + (depot->slab_count * slab_size); for (depot->new_slab_count = depot->slab_count; depot->new_slab_count < slab_count; depot->new_slab_count++, slab_origin += slab_size) { struct block_allocator *allocator = &depot->allocators[depot->new_slab_count % depot->zone_count]; struct vdo_slab **slab_ptr = &depot->new_slabs[depot->new_slab_count]; result = make_slab(slab_origin, allocator, depot->new_slab_count, resizing, slab_ptr); if (result != VDO_SUCCESS) return result; } return VDO_SUCCESS; } /** * vdo_abandon_new_slabs() - Abandon any new slabs in this depot, freeing them as needed. * @depot: The depot. */ void vdo_abandon_new_slabs(struct slab_depot *depot) { slab_count_t i; if (depot->new_slabs == NULL) return; for (i = depot->slab_count; i < depot->new_slab_count; i++) free_slab(vdo_forget(depot->new_slabs[i])); depot->new_slab_count = 0; depot->new_size = 0; vdo_free(vdo_forget(depot->new_slabs)); } /** * get_allocator_thread_id() - Get the ID of the thread on which a given allocator operates. * * Implements vdo_zone_thread_getter_fn. */ static thread_id_t get_allocator_thread_id(void *context, zone_count_t zone_number) { return ((struct slab_depot *) context)->allocators[zone_number].thread_id; } /** * release_recovery_journal_lock() - Request the slab journal to release the recovery journal lock * it may hold on a specified recovery journal block. * @journal: The slab journal. * @recovery_lock: The sequence number of the recovery journal block whose locks should be * released. * * Return: true if the journal does hold a lock on the specified block (which it will release). */ static bool __must_check release_recovery_journal_lock(struct slab_journal *journal, sequence_number_t recovery_lock) { if (recovery_lock > journal->recovery_lock) { VDO_ASSERT_LOG_ONLY((recovery_lock < journal->recovery_lock), "slab journal recovery lock is not older than the recovery journal head"); return false; } if ((recovery_lock < journal->recovery_lock) || vdo_is_read_only(journal->slab->allocator->depot->vdo)) return false; /* All locks are held by the block which is in progress; write it. */ commit_tail(journal); return true; } /* * Request a commit of all dirty tail blocks which are locking the recovery journal block the depot * is seeking to release. * * Implements vdo_zone_action_fn. */ static void release_tail_block_locks(void *context, zone_count_t zone_number, struct vdo_completion *parent) { struct slab_journal *journal, *tmp; struct slab_depot *depot = context; struct list_head *list = &depot->allocators[zone_number].dirty_slab_journals; list_for_each_entry_safe(journal, tmp, list, dirty_entry) { if (!release_recovery_journal_lock(journal, depot->active_release_request)) break; } vdo_finish_completion(parent); } /** * prepare_for_tail_block_commit() - Prepare to commit oldest tail blocks. * * Implements vdo_action_preamble_fn. */ static void prepare_for_tail_block_commit(void *context, struct vdo_completion *parent) { struct slab_depot *depot = context; depot->active_release_request = depot->new_release_request; vdo_finish_completion(parent); } /** * schedule_tail_block_commit() - Schedule a tail block commit if necessary. * * This method should not be called directly. Rather, call vdo_schedule_default_action() on the * depot's action manager. * * Implements vdo_action_scheduler_fn. */ static bool schedule_tail_block_commit(void *context) { struct slab_depot *depot = context; if (depot->new_release_request == depot->active_release_request) return false; return vdo_schedule_action(depot->action_manager, prepare_for_tail_block_commit, release_tail_block_locks, NULL, NULL); } /** * initialize_slab_scrubber() - Initialize an allocator's slab scrubber. * @allocator: The allocator being initialized * * Return: VDO_SUCCESS or an error. */ static int initialize_slab_scrubber(struct block_allocator *allocator) { struct slab_scrubber *scrubber = &allocator->scrubber; block_count_t slab_journal_size = allocator->depot->slab_config.slab_journal_blocks; char *journal_data; int result; result = vdo_allocate(VDO_BLOCK_SIZE * slab_journal_size, char, __func__, &journal_data); if (result != VDO_SUCCESS) return result; result = allocate_vio_components(allocator->completion.vdo, VIO_TYPE_SLAB_JOURNAL, VIO_PRIORITY_METADATA, allocator, slab_journal_size, journal_data, &scrubber->vio); if (result != VDO_SUCCESS) { vdo_free(journal_data); return result; } INIT_LIST_HEAD(&scrubber->high_priority_slabs); INIT_LIST_HEAD(&scrubber->slabs); vdo_set_admin_state_code(&scrubber->admin_state, VDO_ADMIN_STATE_SUSPENDED); return VDO_SUCCESS; } /** * initialize_slab_summary_block() - Initialize a slab_summary_block. * @allocator: The allocator which owns the block. * @index: The index of this block in its zone's summary. * * Return: VDO_SUCCESS or an error. */ static int __must_check initialize_slab_summary_block(struct block_allocator *allocator, block_count_t index) { struct slab_summary_block *block = &allocator->summary_blocks[index]; int result; result = vdo_allocate(VDO_BLOCK_SIZE, char, __func__, &block->outgoing_entries); if (result != VDO_SUCCESS) return result; result = allocate_vio_components(allocator->depot->vdo, VIO_TYPE_SLAB_SUMMARY, VIO_PRIORITY_METADATA, NULL, 1, block->outgoing_entries, &block->vio); if (result != VDO_SUCCESS) return result; block->allocator = allocator; block->entries = &allocator->summary_entries[VDO_SLAB_SUMMARY_ENTRIES_PER_BLOCK * index]; block->index = index; return VDO_SUCCESS; } static int __must_check initialize_block_allocator(struct slab_depot *depot, zone_count_t zone) { int result; block_count_t i; struct block_allocator *allocator = &depot->allocators[zone]; struct vdo *vdo = depot->vdo; block_count_t max_free_blocks = depot->slab_config.data_blocks; unsigned int max_priority = (2 + ilog2(max_free_blocks)); *allocator = (struct block_allocator) { .depot = depot, .zone_number = zone, .thread_id = vdo->thread_config.physical_threads[zone], .nonce = vdo->states.vdo.nonce, }; INIT_LIST_HEAD(&allocator->dirty_slab_journals); vdo_set_admin_state_code(&allocator->state, VDO_ADMIN_STATE_NORMAL_OPERATION); result = vdo_register_read_only_listener(vdo, allocator, notify_block_allocator_of_read_only_mode, allocator->thread_id); if (result != VDO_SUCCESS) return result; vdo_initialize_completion(&allocator->completion, vdo, VDO_BLOCK_ALLOCATOR_COMPLETION); result = make_vio_pool(vdo, BLOCK_ALLOCATOR_VIO_POOL_SIZE, allocator->thread_id, VIO_TYPE_SLAB_JOURNAL, VIO_PRIORITY_METADATA, allocator, &allocator->vio_pool); if (result != VDO_SUCCESS) return result; result = initialize_slab_scrubber(allocator); if (result != VDO_SUCCESS) return result; result = vdo_make_priority_table(max_priority, &allocator->prioritized_slabs); if (result != VDO_SUCCESS) return result; result = vdo_allocate(VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE, struct slab_summary_block, __func__, &allocator->summary_blocks); if (result != VDO_SUCCESS) return result; vdo_set_admin_state_code(&allocator->summary_state, VDO_ADMIN_STATE_NORMAL_OPERATION); allocator->summary_entries = depot->summary_entries + (MAX_VDO_SLABS * zone); /* Initialize each summary block. */ for (i = 0; i < VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE; i++) { result = initialize_slab_summary_block(allocator, i); if (result != VDO_SUCCESS) return result; } /* * Performing well atop thin provisioned storage requires either that VDO discards freed * blocks, or that the block allocator try to use slabs that already have allocated blocks * in preference to slabs that have never been opened. For reasons we have not been able to * fully understand, some SSD machines have been have been very sensitive (50% reduction in * test throughput) to very slight differences in the timing and locality of block * allocation. Assigning a low priority to unopened slabs (max_priority/2, say) would be * ideal for the story, but anything less than a very high threshold (max_priority - 1) * hurts on these machines. * * This sets the free block threshold for preferring to open an unopened slab to the binary * floor of 3/4ths the total number of data blocks in a slab, which will generally evaluate * to about half the slab size. */ allocator->unopened_slab_priority = (1 + ilog2((max_free_blocks * 3) / 4)); return VDO_SUCCESS; } static int allocate_components(struct slab_depot *depot, struct partition *summary_partition) { int result; zone_count_t zone; slab_count_t slab_count; u8 hint; u32 i; const struct thread_config *thread_config = &depot->vdo->thread_config; result = vdo_make_action_manager(depot->zone_count, get_allocator_thread_id, thread_config->journal_thread, depot, schedule_tail_block_commit, depot->vdo, &depot->action_manager); if (result != VDO_SUCCESS) return result; depot->origin = depot->first_block; /* block size must be a multiple of entry size */ BUILD_BUG_ON((VDO_BLOCK_SIZE % sizeof(struct slab_summary_entry)) != 0); depot->summary_origin = summary_partition->offset; depot->hint_shift = vdo_get_slab_summary_hint_shift(depot->slab_size_shift); result = vdo_allocate(MAXIMUM_VDO_SLAB_SUMMARY_ENTRIES, struct slab_summary_entry, __func__, &depot->summary_entries); if (result != VDO_SUCCESS) return result; /* Initialize all the entries. */ hint = compute_fullness_hint(depot, depot->slab_config.data_blocks); for (i = 0; i < MAXIMUM_VDO_SLAB_SUMMARY_ENTRIES; i++) { /* * This default tail block offset must be reflected in * slabJournal.c::read_slab_journal_tail(). */ depot->summary_entries[i] = (struct slab_summary_entry) { .tail_block_offset = 0, .fullness_hint = hint, .load_ref_counts = false, .is_dirty = false, }; } slab_count = vdo_compute_slab_count(depot->first_block, depot->last_block, depot->slab_size_shift); if (thread_config->physical_zone_count > slab_count) { return vdo_log_error_strerror(VDO_BAD_CONFIGURATION, "%u physical zones exceeds slab count %u", thread_config->physical_zone_count, slab_count); } /* Initialize the block allocators. */ for (zone = 0; zone < depot->zone_count; zone++) { result = initialize_block_allocator(depot, zone); if (result != VDO_SUCCESS) return result; } /* Allocate slabs. */ result = allocate_slabs(depot, slab_count); if (result != VDO_SUCCESS) return result; /* Use the new slabs. */ for (i = depot->slab_count; i < depot->new_slab_count; i++) { struct vdo_slab *slab = depot->new_slabs[i]; register_slab_with_allocator(slab->allocator, slab); WRITE_ONCE(depot->slab_count, depot->slab_count + 1); } depot->slabs = depot->new_slabs; depot->new_slabs = NULL; depot->new_slab_count = 0; return VDO_SUCCESS; } /** * vdo_decode_slab_depot() - Make a slab depot and configure it with the state read from the super * block. * @state: The slab depot state from the super block. * @vdo: The VDO which will own the depot. * @summary_partition: The partition which holds the slab summary. * @depot_ptr: A pointer to hold the depot. * * Return: A success or error code. */ int vdo_decode_slab_depot(struct slab_depot_state_2_0 state, struct vdo *vdo, struct partition *summary_partition, struct slab_depot **depot_ptr) { unsigned int slab_size_shift; struct slab_depot *depot; int result; /* * Calculate the bit shift for efficiently mapping block numbers to slabs. Using a shift * requires that the slab size be a power of two. */ block_count_t slab_size = state.slab_config.slab_blocks; if (!is_power_of_2(slab_size)) { return vdo_log_error_strerror(UDS_INVALID_ARGUMENT, "slab size must be a power of two"); } slab_size_shift = ilog2(slab_size); result = vdo_allocate_extended(struct slab_depot, vdo->thread_config.physical_zone_count, struct block_allocator, __func__, &depot); if (result != VDO_SUCCESS) return result; depot->vdo = vdo; depot->old_zone_count = state.zone_count; depot->zone_count = vdo->thread_config.physical_zone_count; depot->slab_config = state.slab_config; depot->first_block = state.first_block; depot->last_block = state.last_block; depot->slab_size_shift = slab_size_shift; result = allocate_components(depot, summary_partition); if (result != VDO_SUCCESS) { vdo_free_slab_depot(depot); return result; } *depot_ptr = depot; return VDO_SUCCESS; } static void uninitialize_allocator_summary(struct block_allocator *allocator) { block_count_t i; if (allocator->summary_blocks == NULL) return; for (i = 0; i < VDO_SLAB_SUMMARY_BLOCKS_PER_ZONE; i++) { free_vio_components(&allocator->summary_blocks[i].vio); vdo_free(vdo_forget(allocator->summary_blocks[i].outgoing_entries)); } vdo_free(vdo_forget(allocator->summary_blocks)); } /** * vdo_free_slab_depot() - Destroy a slab depot. * @depot: The depot to destroy. */ void vdo_free_slab_depot(struct slab_depot *depot) { zone_count_t zone = 0; if (depot == NULL) return; vdo_abandon_new_slabs(depot); for (zone = 0; zone < depot->zone_count; zone++) { struct block_allocator *allocator = &depot->allocators[zone]; if (allocator->eraser != NULL) dm_kcopyd_client_destroy(vdo_forget(allocator->eraser)); uninitialize_allocator_summary(allocator); uninitialize_scrubber_vio(&allocator->scrubber); free_vio_pool(vdo_forget(allocator->vio_pool)); vdo_free_priority_table(vdo_forget(allocator->prioritized_slabs)); } if (depot->slabs != NULL) { slab_count_t i; for (i = 0; i < depot->slab_count; i++) free_slab(vdo_forget(depot->slabs[i])); } vdo_free(vdo_forget(depot->slabs)); vdo_free(vdo_forget(depot->action_manager)); vdo_free(vdo_forget(depot->summary_entries)); vdo_free(depot); } /** * vdo_record_slab_depot() - Record the state of a slab depot for encoding into the super block. * @depot: The depot to encode. * * Return: The depot state. */ struct slab_depot_state_2_0 vdo_record_slab_depot(const struct slab_depot *depot) { /* * If this depot is currently using 0 zones, it must have been synchronously loaded by a * tool and is now being saved. We did not load and combine the slab summary, so we still * need to do that next time we load with the old zone count rather than 0. */ struct slab_depot_state_2_0 state; zone_count_t zones_to_record = depot->zone_count; if (depot->zone_count == 0) zones_to_record = depot->old_zone_count; state = (struct slab_depot_state_2_0) { .slab_config = depot->slab_config, .first_block = depot->first_block, .last_block = depot->last_block, .zone_count = zones_to_record, }; return state; } /** * vdo_allocate_reference_counters() - Allocate the reference counters for all slabs in the depot. * * Context: This method may be called only before entering normal operation from the load thread. * * Return: VDO_SUCCESS or an error. */ int vdo_allocate_reference_counters(struct slab_depot *depot) { struct slab_iterator iterator = get_depot_slab_iterator(depot, depot->slab_count - 1, 0, 1); while (iterator.next != NULL) { int result = allocate_slab_counters(next_slab(&iterator)); if (result != VDO_SUCCESS) return result; } return VDO_SUCCESS; } /** * get_slab_number() - Get the number of the slab that contains a specified block. * @depot: The slab depot. * @pbn: The physical block number. * @slab_number_ptr: A pointer to hold the slab number. * * Return: VDO_SUCCESS or an error. */ static int __must_check get_slab_number(const struct slab_depot *depot, physical_block_number_t pbn, slab_count_t *slab_number_ptr) { slab_count_t slab_number; if (pbn < depot->first_block) return VDO_OUT_OF_RANGE; slab_number = (pbn - depot->first_block) >> depot->slab_size_shift; if (slab_number >= depot->slab_count) return VDO_OUT_OF_RANGE; *slab_number_ptr = slab_number; return VDO_SUCCESS; } /** * vdo_get_slab() - Get the slab object for the slab that contains a specified block. * @depot: The slab depot. * @pbn: The physical block number. * * Will put the VDO in read-only mode if the PBN is not a valid data block nor the zero block. * * Return: The slab containing the block, or NULL if the block number is the zero block or * otherwise out of range. */ struct vdo_slab *vdo_get_slab(const struct slab_depot *depot, physical_block_number_t pbn) { slab_count_t slab_number; int result; if (pbn == VDO_ZERO_BLOCK) return NULL; result = get_slab_number(depot, pbn, &slab_number); if (result != VDO_SUCCESS) { vdo_enter_read_only_mode(depot->vdo, result); return NULL; } return depot->slabs[slab_number]; } /** * vdo_get_increment_limit() - Determine how many new references a block can acquire. * @depot: The slab depot. * @pbn: The physical block number that is being queried. * * Context: This method must be called from the physical zone thread of the PBN. * * Return: The number of available references. */ u8 vdo_get_increment_limit(struct slab_depot *depot, physical_block_number_t pbn) { struct vdo_slab *slab = vdo_get_slab(depot, pbn); vdo_refcount_t *counter_ptr = NULL; int result; if ((slab == NULL) || (slab->status != VDO_SLAB_REBUILT)) return 0; result = get_reference_counter(slab, pbn, &counter_ptr); if (result != VDO_SUCCESS) return 0; if (*counter_ptr == PROVISIONAL_REFERENCE_COUNT) return (MAXIMUM_REFERENCE_COUNT - 1); return (MAXIMUM_REFERENCE_COUNT - *counter_ptr); } /** * vdo_is_physical_data_block() - Determine whether the given PBN refers to a data block. * @depot: The depot. * @pbn: The physical block number to ask about. * * Return: True if the PBN corresponds to a data block. */ bool vdo_is_physical_data_block(const struct slab_depot *depot, physical_block_number_t pbn) { slab_count_t slab_number; slab_block_number sbn; return ((pbn == VDO_ZERO_BLOCK) || ((get_slab_number(depot, pbn, &slab_number) == VDO_SUCCESS) && (slab_block_number_from_pbn(depot->slabs[slab_number], pbn, &sbn) == VDO_SUCCESS))); } /** * vdo_get_slab_depot_allocated_blocks() - Get the total number of data blocks allocated across all * the slabs in the depot. * @depot: The slab depot. * * This is the total number of blocks with a non-zero reference count. * * Context: This may be called from any thread. * * Return: The total number of blocks with a non-zero reference count. */ block_count_t vdo_get_slab_depot_allocated_blocks(const struct slab_depot *depot) { block_count_t total = 0; zone_count_t zone; for (zone = 0; zone < depot->zone_count; zone++) { /* The allocators are responsible for thread safety. */ total += READ_ONCE(depot->allocators[zone].allocated_blocks); } return total; } /** * vdo_get_slab_depot_data_blocks() - Get the total number of data blocks in all the slabs in the * depot. * @depot: The slab depot. * * Context: This may be called from any thread. * * Return: The total number of data blocks in all slabs. */ block_count_t vdo_get_slab_depot_data_blocks(const struct slab_depot *depot) { return (READ_ONCE(depot->slab_count) * depot->slab_config.data_blocks); } /** * finish_combining_zones() - Clean up after saving out the combined slab summary. * @completion: The vio which was used to write the summary data. */ static void finish_combining_zones(struct vdo_completion *completion) { int result = completion->result; struct vdo_completion *parent = completion->parent; free_vio(as_vio(vdo_forget(completion))); vdo_fail_completion(parent, result); } static void handle_combining_error(struct vdo_completion *completion) { vio_record_metadata_io_error(as_vio(completion)); finish_combining_zones(completion); } static void write_summary_endio(struct bio *bio) { struct vio *vio = bio->bi_private; struct vdo *vdo = vio->completion.vdo; continue_vio_after_io(vio, finish_combining_zones, vdo->thread_config.admin_thread); } /** * combine_summaries() - Treating the current entries buffer as the on-disk value of all zones, * update every zone to the correct values for every slab. * @depot: The depot whose summary entries should be combined. */ static void combine_summaries(struct slab_depot *depot) { /* * Combine all the old summary data into the portion of the buffer corresponding to the * first zone. */ zone_count_t zone = 0; struct slab_summary_entry *entries = depot->summary_entries; if (depot->old_zone_count > 1) { slab_count_t entry_number; for (entry_number = 0; entry_number < MAX_VDO_SLABS; entry_number++) { if (zone != 0) { memcpy(entries + entry_number, entries + (zone * MAX_VDO_SLABS) + entry_number, sizeof(struct slab_summary_entry)); } zone++; if (zone == depot->old_zone_count) zone = 0; } } /* Copy the combined data to each zones's region of the buffer. */ for (zone = 1; zone < MAX_VDO_PHYSICAL_ZONES; zone++) { memcpy(entries + (zone * MAX_VDO_SLABS), entries, MAX_VDO_SLABS * sizeof(struct slab_summary_entry)); } } /** * finish_loading_summary() - Finish loading slab summary data. * @completion: The vio which was used to read the summary data. * * Combines the slab summary data from all the previously written zones and copies the combined * summary to each partition's data region. Then writes the combined summary back out to disk. This * callback is registered in load_summary_endio(). */ static void finish_loading_summary(struct vdo_completion *completion) { struct slab_depot *depot = completion->vdo->depot; /* Combine the summary from each zone so each zone is correct for all slabs. */ combine_summaries(depot); /* Write the combined summary back out. */ vdo_submit_metadata_vio(as_vio(completion), depot->summary_origin, write_summary_endio, handle_combining_error, REQ_OP_WRITE); } static void load_summary_endio(struct bio *bio) { struct vio *vio = bio->bi_private; struct vdo *vdo = vio->completion.vdo; continue_vio_after_io(vio, finish_loading_summary, vdo->thread_config.admin_thread); } /** * load_slab_summary() - The preamble of a load operation. * * Implements vdo_action_preamble_fn. */ static void load_slab_summary(void *context, struct vdo_completion *parent) { int result; struct vio *vio; struct slab_depot *depot = context; const struct admin_state_code *operation = vdo_get_current_manager_operation(depot->action_manager); result = create_multi_block_metadata_vio(depot->vdo, VIO_TYPE_SLAB_SUMMARY, VIO_PRIORITY_METADATA, parent, VDO_SLAB_SUMMARY_BLOCKS, (char *) depot->summary_entries, &vio); if (result != VDO_SUCCESS) { vdo_fail_completion(parent, result); return; } if ((operation == VDO_ADMIN_STATE_FORMATTING) || (operation == VDO_ADMIN_STATE_LOADING_FOR_REBUILD)) { finish_loading_summary(&vio->completion); return; } vdo_submit_metadata_vio(vio, depot->summary_origin, load_summary_endio, handle_combining_error, REQ_OP_READ); } /* Implements vdo_zone_action_fn. */ static void load_allocator(void *context, zone_count_t zone_number, struct vdo_completion *parent) { struct slab_depot *depot = context; vdo_start_loading(&depot->allocators[zone_number].state, vdo_get_current_manager_operation(depot->action_manager), parent, initiate_load); } /** * vdo_load_slab_depot() - Asynchronously load any slab depot state that isn't included in the * super_block component. * @depot: The depot to load. * @operation: The type of load to perform. * @parent: The completion to notify when the load is complete. * @context: Additional context for the load operation; may be NULL. * * This method may be called only before entering normal operation from the load thread. */ void vdo_load_slab_depot(struct slab_depot *depot, const struct admin_state_code *operation, struct vdo_completion *parent, void *context) { if (!vdo_assert_load_operation(operation, parent)) return; vdo_schedule_operation_with_context(depot->action_manager, operation, load_slab_summary, load_allocator, NULL, context, parent); } /* Implements vdo_zone_action_fn. */ static void prepare_to_allocate(void *context, zone_count_t zone_number, struct vdo_completion *parent) { struct slab_depot *depot = context; struct block_allocator *allocator = &depot->allocators[zone_number]; int result; result = vdo_prepare_slabs_for_allocation(allocator); if (result != VDO_SUCCESS) { vdo_fail_completion(parent, result); return; } scrub_slabs(allocator, parent); } /** * vdo_prepare_slab_depot_to_allocate() - Prepare the slab depot to come online and start * allocating blocks. * @depot: The depot to prepare. * @load_type: The load type. * @parent: The completion to notify when the operation is complete. * * This method may be called only before entering normal operation from the load thread. It must be * called before allocation may proceed. */ void vdo_prepare_slab_depot_to_allocate(struct slab_depot *depot, enum slab_depot_load_type load_type, struct vdo_completion *parent) { depot->load_type = load_type; atomic_set(&depot->zones_to_scrub, depot->zone_count); vdo_schedule_action(depot->action_manager, NULL, prepare_to_allocate, NULL, parent); } /** * vdo_update_slab_depot_size() - Update the slab depot to reflect its new size in memory. * @depot: The depot to update. * * This size is saved to disk as part of the super block. */ void vdo_update_slab_depot_size(struct slab_depot *depot) { depot->last_block = depot->new_last_block; } /** * vdo_prepare_to_grow_slab_depot() - Allocate new memory needed for a resize of a slab depot to * the given size. * @depot: The depot to prepare to resize. * @partition: The new depot partition * * Return: VDO_SUCCESS or an error. */ int vdo_prepare_to_grow_slab_depot(struct slab_depot *depot, const struct partition *partition) { struct slab_depot_state_2_0 new_state; int result; slab_count_t new_slab_count; if ((partition->count >> depot->slab_size_shift) <= depot->slab_count) return VDO_INCREMENT_TOO_SMALL; /* Generate the depot configuration for the new block count. */ VDO_ASSERT_LOG_ONLY(depot->first_block == partition->offset, "New slab depot partition doesn't change origin"); result = vdo_configure_slab_depot(partition, depot->slab_config, depot->zone_count, &new_state); if (result != VDO_SUCCESS) return result; new_slab_count = vdo_compute_slab_count(depot->first_block, new_state.last_block, depot->slab_size_shift); if (new_slab_count <= depot->slab_count) return vdo_log_error_strerror(VDO_INCREMENT_TOO_SMALL, "Depot can only grow"); if (new_slab_count == depot->new_slab_count) { /* Check it out, we've already got all the new slabs allocated! */ return VDO_SUCCESS; } vdo_abandon_new_slabs(depot); result = allocate_slabs(depot, new_slab_count); if (result != VDO_SUCCESS) { vdo_abandon_new_slabs(depot); return result; } depot->new_size = partition->count; depot->old_last_block = depot->last_block; depot->new_last_block = new_state.last_block; return VDO_SUCCESS; } /** * finish_registration() - Finish registering new slabs now that all of the allocators have * received their new slabs. * * Implements vdo_action_conclusion_fn. */ static int finish_registration(void *context) { struct slab_depot *depot = context; WRITE_ONCE(depot->slab_count, depot->new_slab_count); vdo_free(depot->slabs); depot->slabs = depot->new_slabs; depot->new_slabs = NULL; depot->new_slab_count = 0; return VDO_SUCCESS; } /* Implements vdo_zone_action_fn. */ static void register_new_slabs(void *context, zone_count_t zone_number, struct vdo_completion *parent) { struct slab_depot *depot = context; struct block_allocator *allocator = &depot->allocators[zone_number]; slab_count_t i; for (i = depot->slab_count; i < depot->new_slab_count; i++) { struct vdo_slab *slab = depot->new_slabs[i]; if (slab->allocator == allocator) register_slab_with_allocator(allocator, slab); } vdo_finish_completion(parent); } /** * vdo_use_new_slabs() - Use the new slabs allocated for resize. * @depot: The depot. * @parent: The object to notify when complete. */ void vdo_use_new_slabs(struct slab_depot *depot, struct vdo_completion *parent) { VDO_ASSERT_LOG_ONLY(depot->new_slabs != NULL, "Must have new slabs to use"); vdo_schedule_operation(depot->action_manager, VDO_ADMIN_STATE_SUSPENDED_OPERATION, NULL, register_new_slabs, finish_registration, parent); } /** * stop_scrubbing() - Tell the scrubber to stop scrubbing after it finishes the slab it is * currently working on. * @scrubber: The scrubber to stop. * @parent: The completion to notify when scrubbing has stopped. */ static void stop_scrubbing(struct block_allocator *allocator) { struct slab_scrubber *scrubber = &allocator->scrubber; if (vdo_is_state_quiescent(&scrubber->admin_state)) { vdo_finish_completion(&allocator->completion); } else { vdo_start_draining(&scrubber->admin_state, VDO_ADMIN_STATE_SUSPENDING, &allocator->completion, NULL); } } /* Implements vdo_admin_initiator_fn. */ static void initiate_summary_drain(struct admin_state *state) { check_summary_drain_complete(container_of(state, struct block_allocator, summary_state)); } static void do_drain_step(struct vdo_completion *completion) { struct block_allocator *allocator = vdo_as_block_allocator(completion); vdo_prepare_completion_for_requeue(&allocator->completion, do_drain_step, handle_operation_error, allocator->thread_id, NULL); switch (++allocator->drain_step) { case VDO_DRAIN_ALLOCATOR_STEP_SCRUBBER: stop_scrubbing(allocator); return; case VDO_DRAIN_ALLOCATOR_STEP_SLABS: apply_to_slabs(allocator, do_drain_step); return; case VDO_DRAIN_ALLOCATOR_STEP_SUMMARY: vdo_start_draining(&allocator->summary_state, vdo_get_admin_state_code(&allocator->state), completion, initiate_summary_drain); return; case VDO_DRAIN_ALLOCATOR_STEP_FINISHED: VDO_ASSERT_LOG_ONLY(!is_vio_pool_busy(allocator->vio_pool), "vio pool not busy"); vdo_finish_draining_with_result(&allocator->state, completion->result); return; default: vdo_finish_draining_with_result(&allocator->state, UDS_BAD_STATE); } } /* Implements vdo_admin_initiator_fn. */ static void initiate_drain(struct admin_state *state) { struct block_allocator *allocator = container_of(state, struct block_allocator, state); allocator->drain_step = VDO_DRAIN_ALLOCATOR_START; do_drain_step(&allocator->completion); } /* * Drain all allocator I/O. Depending upon the type of drain, some or all dirty metadata may be * written to disk. The type of drain will be determined from the state of the allocator's depot. * * Implements vdo_zone_action_fn. */ static void drain_allocator(void *context, zone_count_t zone_number, struct vdo_completion *parent) { struct slab_depot *depot = context; vdo_start_draining(&depot->allocators[zone_number].state, vdo_get_current_manager_operation(depot->action_manager), parent, initiate_drain); } /** * vdo_drain_slab_depot() - Drain all slab depot I/O. * @depot: The depot to drain. * @operation: The drain operation (flush, rebuild, suspend, or save). * @parent: The completion to finish when the drain is complete. * * If saving, or flushing, all dirty depot metadata will be written out. If saving or suspending, * the depot will be left in a suspended state. */ void vdo_drain_slab_depot(struct slab_depot *depot, const struct admin_state_code *operation, struct vdo_completion *parent) { vdo_schedule_operation(depot->action_manager, operation, NULL, drain_allocator, NULL, parent); } /** * resume_scrubbing() - Tell the scrubber to resume scrubbing if it has been stopped. * @allocator: The allocator being resumed. */ static void resume_scrubbing(struct block_allocator *allocator) { int result; struct slab_scrubber *scrubber = &allocator->scrubber; if (!has_slabs_to_scrub(scrubber)) { vdo_finish_completion(&allocator->completion); return; } result = vdo_resume_if_quiescent(&scrubber->admin_state); if (result != VDO_SUCCESS) { vdo_fail_completion(&allocator->completion, result); return; } scrub_next_slab(scrubber); vdo_finish_completion(&allocator->completion); } static void do_resume_step(struct vdo_completion *completion) { struct block_allocator *allocator = vdo_as_block_allocator(completion); vdo_prepare_completion_for_requeue(&allocator->completion, do_resume_step, handle_operation_error, allocator->thread_id, NULL); switch (--allocator->drain_step) { case VDO_DRAIN_ALLOCATOR_STEP_SUMMARY: vdo_fail_completion(completion, vdo_resume_if_quiescent(&allocator->summary_state)); return; case VDO_DRAIN_ALLOCATOR_STEP_SLABS: apply_to_slabs(allocator, do_resume_step); return; case VDO_DRAIN_ALLOCATOR_STEP_SCRUBBER: resume_scrubbing(allocator); return; case VDO_DRAIN_ALLOCATOR_START: vdo_finish_resuming_with_result(&allocator->state, completion->result); return; default: vdo_finish_resuming_with_result(&allocator->state, UDS_BAD_STATE); } } /* Implements vdo_admin_initiator_fn. */ static void initiate_resume(struct admin_state *state) { struct block_allocator *allocator = container_of(state, struct block_allocator, state); allocator->drain_step = VDO_DRAIN_ALLOCATOR_STEP_FINISHED; do_resume_step(&allocator->completion); } /* Implements vdo_zone_action_fn. */ static void resume_allocator(void *context, zone_count_t zone_number, struct vdo_completion *parent) { struct slab_depot *depot = context; vdo_start_resuming(&depot->allocators[zone_number].state, vdo_get_current_manager_operation(depot->action_manager), parent, initiate_resume); } /** * vdo_resume_slab_depot() - Resume a suspended slab depot. * @depot: The depot to resume. * @parent: The completion to finish when the depot has resumed. */ void vdo_resume_slab_depot(struct slab_depot *depot, struct vdo_completion *parent) { if (vdo_is_read_only(depot->vdo)) { vdo_continue_completion(parent, VDO_READ_ONLY); return; } vdo_schedule_operation(depot->action_manager, VDO_ADMIN_STATE_RESUMING, NULL, resume_allocator, NULL, parent); } /** * vdo_commit_oldest_slab_journal_tail_blocks() - Commit all dirty tail blocks which are locking a * given recovery journal block. * @depot: The depot. * @recovery_block_number: The sequence number of the recovery journal block whose locks should be * released. * * Context: This method must be called from the journal zone thread. */ void vdo_commit_oldest_slab_journal_tail_blocks(struct slab_depot *depot, sequence_number_t recovery_block_number) { if (depot == NULL) return; depot->new_release_request = recovery_block_number; vdo_schedule_default_action(depot->action_manager); } /* Implements vdo_zone_action_fn. */ static void scrub_all_unrecovered_slabs(void *context, zone_count_t zone_number, struct vdo_completion *parent) { struct slab_depot *depot = context; scrub_slabs(&depot->allocators[zone_number], NULL); vdo_launch_completion(parent); } /** * vdo_scrub_all_unrecovered_slabs() - Scrub all unrecovered slabs. * @depot: The depot to scrub. * @parent: The object to notify when scrubbing has been launched for all zones. */ void vdo_scrub_all_unrecovered_slabs(struct slab_depot *depot, struct vdo_completion *parent) { vdo_schedule_action(depot->action_manager, NULL, scrub_all_unrecovered_slabs, NULL, parent); } /** * get_block_allocator_statistics() - Get the total of the statistics from all the block allocators * in the depot. * @depot: The slab depot. * * Return: The statistics from all block allocators in the depot. */ static struct block_allocator_statistics __must_check get_block_allocator_statistics(const struct slab_depot *depot) { struct block_allocator_statistics totals; zone_count_t zone; memset(&totals, 0, sizeof(totals)); for (zone = 0; zone < depot->zone_count; zone++) { const struct block_allocator *allocator = &depot->allocators[zone]; const struct block_allocator_statistics *stats = &allocator->statistics; totals.slab_count += allocator->slab_count; totals.slabs_opened += READ_ONCE(stats->slabs_opened); totals.slabs_reopened += READ_ONCE(stats->slabs_reopened); } return totals; } /** * get_ref_counts_statistics() - Get the cumulative ref_counts statistics for the depot. * @depot: The slab depot. * * Return: The cumulative statistics for all ref_counts in the depot. */ static struct ref_counts_statistics __must_check get_ref_counts_statistics(const struct slab_depot *depot) { struct ref_counts_statistics totals; zone_count_t zone; memset(&totals, 0, sizeof(totals)); for (zone = 0; zone < depot->zone_count; zone++) { totals.blocks_written += READ_ONCE(depot->allocators[zone].ref_counts_statistics.blocks_written); } return totals; } /** * get_slab_journal_statistics() - Get the aggregated slab journal statistics for the depot. * @depot: The slab depot. * * Return: The aggregated statistics for all slab journals in the depot. */ static struct slab_journal_statistics __must_check get_slab_journal_statistics(const struct slab_depot *depot) { struct slab_journal_statistics totals; zone_count_t zone; memset(&totals, 0, sizeof(totals)); for (zone = 0; zone < depot->zone_count; zone++) { const struct slab_journal_statistics *stats = &depot->allocators[zone].slab_journal_statistics; totals.disk_full_count += READ_ONCE(stats->disk_full_count); totals.flush_count += READ_ONCE(stats->flush_count); totals.blocked_count += READ_ONCE(stats->blocked_count); totals.blocks_written += READ_ONCE(stats->blocks_written); totals.tail_busy_count += READ_ONCE(stats->tail_busy_count); } return totals; } /** * vdo_get_slab_depot_statistics() - Get all the vdo_statistics fields that are properties of the * slab depot. * @depot: The slab depot. * @stats: The vdo statistics structure to partially fill. */ void vdo_get_slab_depot_statistics(const struct slab_depot *depot, struct vdo_statistics *stats) { slab_count_t slab_count = READ_ONCE(depot->slab_count); slab_count_t unrecovered = 0; zone_count_t zone; for (zone = 0; zone < depot->zone_count; zone++) { /* The allocators are responsible for thread safety. */ unrecovered += READ_ONCE(depot->allocators[zone].scrubber.slab_count); } stats->recovery_percentage = (slab_count - unrecovered) * 100 / slab_count; stats->allocator = get_block_allocator_statistics(depot); stats->ref_counts = get_ref_counts_statistics(depot); stats->slab_journal = get_slab_journal_statistics(depot); stats->slab_summary = (struct slab_summary_statistics) { .blocks_written = atomic64_read(&depot->summary_statistics.blocks_written), }; } /** * vdo_dump_slab_depot() - Dump the slab depot, in a thread-unsafe fashion. * @depot: The slab depot. */ void vdo_dump_slab_depot(const struct slab_depot *depot) { vdo_log_info("vdo slab depot"); vdo_log_info(" zone_count=%u old_zone_count=%u slabCount=%u active_release_request=%llu new_release_request=%llu", (unsigned int) depot->zone_count, (unsigned int) depot->old_zone_count, READ_ONCE(depot->slab_count), (unsigned long long) depot->active_release_request, (unsigned long long) depot->new_release_request); }
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1