cregit-Linux how code gets into the kernel

Release 4.14 drivers/lightnvm/pblk-gc.c

Directory: drivers/lightnvm
/*
 * Copyright (C) 2016 CNEX Labs
 * Initial release: Javier Gonzalez <javier@cnexlabs.com>
 *                  Matias Bjorling <matias@cnexlabs.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License version
 * 2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * pblk-gc.c - pblk's garbage collector
 */

#include "pblk.h"
#include <linux/delay.h>


static void pblk_gc_free_gc_rq(struct pblk_gc_rq *gc_rq) { vfree(gc_rq->data); kfree(gc_rq); }

Contributors

PersonTokensPropCommitsCommitProp
Javier González23100.00%2100.00%
Total23100.00%2100.00%


static int pblk_gc_write(struct pblk *pblk) { struct pblk_gc *gc = &pblk->gc; struct pblk_gc_rq *gc_rq, *tgc_rq; LIST_HEAD(w_list); spin_lock(&gc->w_lock); if (list_empty(&gc->w_list)) { spin_unlock(&gc->w_lock); return 1; } list_cut_position(&w_list, &gc->w_list, gc->w_list.prev); gc->w_entries = 0; spin_unlock(&gc->w_lock); list_for_each_entry_safe(gc_rq, tgc_rq, &w_list, list) { pblk_write_gc_to_cache(pblk, gc_rq->data, gc_rq->lba_list, gc_rq->nr_secs, gc_rq->secs_to_gc, gc_rq->line, PBLK_IOTYPE_GC); list_del(&gc_rq->list); kref_put(&gc_rq->line->ref, pblk_line_put); pblk_gc_free_gc_rq(gc_rq); } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Javier González163100.00%2100.00%
Total163100.00%2100.00%


static void pblk_gc_writer_kick(struct pblk_gc *gc) { wake_up_process(gc->gc_writer_ts); }

Contributors

PersonTokensPropCommitsCommitProp
Javier González18100.00%1100.00%
Total18100.00%1100.00%

/* * Responsible for managing all memory related to a gc request. Also in case of * failure */
static int pblk_gc_move_valid_secs(struct pblk *pblk, struct pblk_gc_rq *gc_rq) { struct nvm_tgt_dev *dev = pblk->dev; struct nvm_geo *geo = &dev->geo; struct pblk_gc *gc = &pblk->gc; struct pblk_line *line = gc_rq->line; void *data; unsigned int secs_to_gc; int ret = 0; data = vmalloc(gc_rq->nr_secs * geo->sec_size); if (!data) { ret = -ENOMEM; goto out; } /* Read from GC victim block */ if (pblk_submit_read_gc(pblk, gc_rq->lba_list, data, gc_rq->nr_secs, &secs_to_gc, line)) { ret = -EFAULT; goto free_data; } if (!secs_to_gc) goto free_rq; gc_rq->data = data; gc_rq->secs_to_gc = secs_to_gc; retry: spin_lock(&gc->w_lock); if (gc->w_entries >= PBLK_GC_W_QD) { spin_unlock(&gc->w_lock); pblk_gc_writer_kick(&pblk->gc); usleep_range(128, 256); goto retry; } gc->w_entries++; list_add_tail(&gc_rq->list, &gc->w_list); spin_unlock(&gc->w_lock); pblk_gc_writer_kick(&pblk->gc); return 0; free_rq: kfree(gc_rq); free_data: vfree(data); out: kref_put(&line->ref, pblk_line_put); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Javier González260100.00%3100.00%
Total260100.00%3100.00%


static void pblk_put_line_back(struct pblk *pblk, struct pblk_line *line) { struct pblk_line_mgmt *l_mg = &pblk->l_mg; struct list_head *move_list; spin_lock(&line->lock); WARN_ON(line->state != PBLK_LINESTATE_GC); line->state = PBLK_LINESTATE_CLOSED; move_list = pblk_line_gc_list(pblk, line); spin_unlock(&line->lock); if (move_list) { spin_lock(&l_mg->gc_lock); list_add_tail(&line->list, move_list); spin_unlock(&l_mg->gc_lock); } }

Contributors

PersonTokensPropCommitsCommitProp
Javier González103100.00%1100.00%
Total103100.00%1100.00%


static void pblk_gc_line_ws(struct work_struct *work) { struct pblk_line_ws *line_rq_ws = container_of(work, struct pblk_line_ws, ws); struct pblk *pblk = line_rq_ws->pblk; struct pblk_gc *gc = &pblk->gc; struct pblk_line *line = line_rq_ws->line; struct pblk_gc_rq *gc_rq = line_rq_ws->priv; up(&gc->gc_sem); if (pblk_gc_move_valid_secs(pblk, gc_rq)) { pr_err("pblk: could not GC all sectors: line:%d (%d/%d)\n", line->id, *line->vsc, gc_rq->nr_secs); } mempool_free(line_rq_ws, pblk->line_ws_pool); }

Contributors

PersonTokensPropCommitsCommitProp
Javier González109100.00%3100.00%
Total109100.00%3100.00%


static void pblk_gc_line_prepare_ws(struct work_struct *work) { struct pblk_line_ws *line_ws = container_of(work, struct pblk_line_ws, ws); struct pblk *pblk = line_ws->pblk; struct pblk_line *line = line_ws->line; struct pblk_line_mgmt *l_mg = &pblk->l_mg; struct pblk_line_meta *lm = &pblk->lm; struct pblk_gc *gc = &pblk->gc; struct line_emeta *emeta_buf; struct pblk_line_ws *line_rq_ws; struct pblk_gc_rq *gc_rq; __le64 *lba_list; int sec_left, nr_secs, bit; int ret; emeta_buf = pblk_malloc(lm->emeta_len[0], l_mg->emeta_alloc_type, GFP_KERNEL); if (!emeta_buf) { pr_err("pblk: cannot use GC emeta\n"); return; } ret = pblk_line_read_emeta(pblk, line, emeta_buf); if (ret) { pr_err("pblk: line %d read emeta failed (%d)\n", line->id, ret); goto fail_free_emeta; } /* If this read fails, it means that emeta is corrupted. For now, leave * the line untouched. TODO: Implement a recovery routine that scans and * moves all sectors on the line. */ lba_list = pblk_recov_get_lba_list(pblk, emeta_buf); if (!lba_list) { pr_err("pblk: could not interpret emeta (line %d)\n", line->id); goto fail_free_emeta; } sec_left = pblk_line_vsc(line); if (sec_left < 0) { pr_err("pblk: corrupted GC line (%d)\n", line->id); goto fail_free_emeta; } bit = -1; next_rq: gc_rq = kmalloc(sizeof(struct pblk_gc_rq), GFP_KERNEL); if (!gc_rq) goto fail_free_emeta; nr_secs = 0; do { bit = find_next_zero_bit(line->invalid_bitmap, lm->sec_per_line, bit + 1); if (bit > line->emeta_ssec) break; gc_rq->lba_list[nr_secs++] = le64_to_cpu(lba_list[bit]); } while (nr_secs < pblk->max_write_pgs); if (unlikely(!nr_secs)) { kfree(gc_rq); goto out; } gc_rq->nr_secs = nr_secs; gc_rq->line = line; line_rq_ws = mempool_alloc(pblk->line_ws_pool, GFP_KERNEL); if (!line_rq_ws) goto fail_free_gc_rq; line_rq_ws->pblk = pblk; line_rq_ws->line = line; line_rq_ws->priv = gc_rq; down(&gc->gc_sem); kref_get(&line->ref); INIT_WORK(&line_rq_ws->ws, pblk_gc_line_ws); queue_work(gc->gc_line_reader_wq, &line_rq_ws->ws); sec_left -= nr_secs; if (sec_left > 0) goto next_rq; out: pblk_mfree(emeta_buf, l_mg->emeta_alloc_type); mempool_free(line_ws, pblk->line_ws_pool); kref_put(&line->ref, pblk_line_put); atomic_dec(&gc->inflight_gc); return; fail_free_gc_rq: kfree(gc_rq); fail_free_emeta: pblk_mfree(emeta_buf, l_mg->emeta_alloc_type); pblk_put_line_back(pblk, line); kref_put(&line->ref, pblk_line_put); mempool_free(line_ws, pblk->line_ws_pool); atomic_dec(&gc->inflight_gc); pr_err("pblk: Failed to GC line %d\n", line->id); }

Contributors

PersonTokensPropCommitsCommitProp
Javier González525100.00%3100.00%
Total525100.00%3100.00%


static int pblk_gc_line(struct pblk *pblk, struct pblk_line *line) { struct pblk_gc *gc = &pblk->gc; struct pblk_line_ws *line_ws; pr_debug("pblk: line '%d' being reclaimed for GC\n", line->id); line_ws = mempool_alloc(pblk->line_ws_pool, GFP_KERNEL); if (!line_ws) return -ENOMEM; line_ws->pblk = pblk; line_ws->line = line; INIT_WORK(&line_ws->ws, pblk_gc_line_prepare_ws); queue_work(gc->gc_reader_wq, &line_ws->ws); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Javier González97100.00%3100.00%
Total97100.00%3100.00%


static int pblk_gc_read(struct pblk *pblk) { struct pblk_gc *gc = &pblk->gc; struct pblk_line *line; spin_lock(&gc->r_lock); if (list_empty(&gc->r_list)) { spin_unlock(&gc->r_lock); return 1; } line = list_first_entry(&gc->r_list, struct pblk_line, list); list_del(&line->list); spin_unlock(&gc->r_lock); pblk_gc_kick(pblk); if (pblk_gc_line(pblk, line)) pr_err("pblk: failed to GC line %d\n", line->id); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Javier González114100.00%2100.00%
Total114100.00%2100.00%


static void pblk_gc_reader_kick(struct pblk_gc *gc) { wake_up_process(gc->gc_reader_ts); }

Contributors

PersonTokensPropCommitsCommitProp
Javier González18100.00%2100.00%
Total18100.00%2100.00%


static struct pblk_line *pblk_gc_get_victim_line(struct pblk *pblk, struct list_head *group_list) { struct pblk_line *line, *victim; int line_vsc, victim_vsc; victim = list_first_entry(group_list, struct pblk_line, list); list_for_each_entry(line, group_list, list) { line_vsc = le32_to_cpu(*line->vsc); victim_vsc = le32_to_cpu(*victim->vsc); if (line_vsc < victim_vsc) victim = line; } return victim; }

Contributors

PersonTokensPropCommitsCommitProp
Javier González86100.00%2100.00%
Total86100.00%2100.00%


static bool pblk_gc_should_run(struct pblk_gc *gc, struct pblk_rl *rl) { unsigned int nr_blocks_free, nr_blocks_need; nr_blocks_need = pblk_rl_high_thrs(rl); nr_blocks_free = pblk_rl_nr_free_blks(rl); /* This is not critical, no need to take lock here */ return ((gc->gc_active) && (nr_blocks_need > nr_blocks_free)); }

Contributors

PersonTokensPropCommitsCommitProp
Javier González52100.00%1100.00%
Total52100.00%1100.00%

/* * Lines with no valid sectors will be returned to the free list immediately. If * GC is activated - either because the free block count is under the determined * threshold, or because it is being forced from user space - only lines with a * high count of invalid sectors will be recycled. */
static void pblk_gc_run(struct pblk *pblk) { struct pblk_line_mgmt *l_mg = &pblk->l_mg; struct pblk_gc *gc = &pblk->gc; struct pblk_line *line; struct list_head *group_list; bool run_gc; int inflight_gc, gc_group = 0, prev_group = 0; do { spin_lock(&l_mg->gc_lock); if (list_empty(&l_mg->gc_full_list)) { spin_unlock(&l_mg->gc_lock); break; } line = list_first_entry(&l_mg->gc_full_list, struct pblk_line, list); spin_lock(&line->lock); WARN_ON(line->state != PBLK_LINESTATE_CLOSED); line->state = PBLK_LINESTATE_GC; spin_unlock(&line->lock); list_del(&line->list); spin_unlock(&l_mg->gc_lock); kref_put(&line->ref, pblk_line_put); } while (1); run_gc = pblk_gc_should_run(&pblk->gc, &pblk->rl); if (!run_gc || (atomic_read(&gc->inflight_gc) >= PBLK_GC_L_QD)) return; next_gc_group: group_list = l_mg->gc_lists[gc_group++]; do { spin_lock(&l_mg->gc_lock); if (list_empty(group_list)) { spin_unlock(&l_mg->gc_lock); break; } line = pblk_gc_get_victim_line(pblk, group_list); spin_lock(&line->lock); WARN_ON(line->state != PBLK_LINESTATE_CLOSED); line->state = PBLK_LINESTATE_GC; spin_unlock(&line->lock); list_del(&line->list); spin_unlock(&l_mg->gc_lock); spin_lock(&gc->r_lock); list_add_tail(&line->list, &gc->r_list); spin_unlock(&gc->r_lock); inflight_gc = atomic_inc_return(&gc->inflight_gc); pblk_gc_reader_kick(gc); prev_group = 1; /* No need to queue up more GC lines than we can handle */ run_gc = pblk_gc_should_run(&pblk->gc, &pblk->rl); if (!run_gc || inflight_gc >= PBLK_GC_L_QD) break; } while (1); if (!prev_group && pblk->rl.rb_state > gc_group && gc_group < PBLK_GC_NR_LISTS) goto next_gc_group; }

Contributors

PersonTokensPropCommitsCommitProp
Javier González393100.00%3100.00%
Total393100.00%3100.00%


void pblk_gc_kick(struct pblk *pblk) { struct pblk_gc *gc = &pblk->gc; wake_up_process(gc->gc_ts); pblk_gc_writer_kick(gc); pblk_gc_reader_kick(gc); mod_timer(&gc->gc_timer, jiffies + msecs_to_jiffies(GC_TIME_MSECS)); }

Contributors

PersonTokensPropCommitsCommitProp
Javier González52100.00%2100.00%
Total52100.00%2100.00%


static void pblk_gc_timer(unsigned long data) { struct pblk *pblk = (struct pblk *)data; pblk_gc_kick(pblk); }

Contributors

PersonTokensPropCommitsCommitProp
Javier González27100.00%1100.00%
Total27100.00%1100.00%


static int pblk_gc_ts(void *data) { struct pblk *pblk = data; while (!kthread_should_stop()) { pblk_gc_run(pblk); set_current_state(TASK_INTERRUPTIBLE); io_schedule(); } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Javier González41100.00%1100.00%
Total41100.00%1100.00%


static int pblk_gc_writer_ts(void *data) { struct pblk *pblk = data; while (!kthread_should_stop()) { if (!pblk_gc_write(pblk)) continue; set_current_state(TASK_INTERRUPTIBLE); io_schedule(); } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Javier González45100.00%1100.00%
Total45100.00%1100.00%


static int pblk_gc_reader_ts(void *data) { struct pblk *pblk = data; while (!kthread_should_stop()) { if (!pblk_gc_read(pblk)) continue; set_current_state(TASK_INTERRUPTIBLE); io_schedule(); } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Javier González45100.00%2100.00%
Total45100.00%2100.00%


static void pblk_gc_start(struct pblk *pblk) { pblk->gc.gc_active = 1; pr_debug("pblk: gc start\n"); }

Contributors

PersonTokensPropCommitsCommitProp
Javier González24100.00%2100.00%
Total24100.00%2100.00%


void pblk_gc_should_start(struct pblk *pblk) { struct pblk_gc *gc = &pblk->gc; if (gc->gc_enabled && !gc->gc_active) pblk_gc_start(pblk); pblk_gc_kick(pblk); }

Contributors

PersonTokensPropCommitsCommitProp
Javier González41100.00%2100.00%
Total41100.00%2100.00%

/* * If flush_wq == 1 then no lock should be held by the caller since * flush_workqueue can sleep */
static void pblk_gc_stop(struct pblk *pblk, int flush_wq) { pblk->gc.gc_active = 0; pr_debug("pblk: gc stop\n"); }

Contributors

PersonTokensPropCommitsCommitProp
Javier González27100.00%1100.00%
Total27100.00%1100.00%


void pblk_gc_should_stop(struct pblk *pblk) { struct pblk_gc *gc = &pblk->gc; if (gc->gc_active && !gc->gc_forced) pblk_gc_stop(pblk, 0); }

Contributors

PersonTokensPropCommitsCommitProp
Javier González38100.00%1100.00%
Total38100.00%1100.00%


void pblk_gc_sysfs_state_show(struct pblk *pblk, int *gc_enabled, int *gc_active) { struct pblk_gc *gc = &pblk->gc; spin_lock(&gc->lock); *gc_enabled = gc->gc_enabled; *gc_active = gc->gc_active; spin_unlock(&gc->lock); }

Contributors

PersonTokensPropCommitsCommitProp
Javier González58100.00%1100.00%
Total58100.00%1100.00%


int pblk_gc_sysfs_force(struct pblk *pblk, int force) { struct pblk_gc *gc = &pblk->gc; if (force < 0 || force > 1) return -EINVAL; spin_lock(&gc->lock); gc->gc_forced = force; if (force) gc->gc_enabled = 1; else gc->gc_enabled = 0; spin_unlock(&gc->lock); pblk_gc_should_start(pblk); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Javier González84100.00%2100.00%
Total84100.00%2100.00%


int pblk_gc_init(struct pblk *pblk) { struct pblk_gc *gc = &pblk->gc; int ret; gc->gc_ts = kthread_create(pblk_gc_ts, pblk, "pblk-gc-ts"); if (IS_ERR(gc->gc_ts)) { pr_err("pblk: could not allocate GC main kthread\n"); return PTR_ERR(gc->gc_ts); } gc->gc_writer_ts = kthread_create(pblk_gc_writer_ts, pblk, "pblk-gc-writer-ts"); if (IS_ERR(gc->gc_writer_ts)) { pr_err("pblk: could not allocate GC writer kthread\n"); ret = PTR_ERR(gc->gc_writer_ts); goto fail_free_main_kthread; } gc->gc_reader_ts = kthread_create(pblk_gc_reader_ts, pblk, "pblk-gc-reader-ts"); if (IS_ERR(gc->gc_reader_ts)) { pr_err("pblk: could not allocate GC reader kthread\n"); ret = PTR_ERR(gc->gc_reader_ts); goto fail_free_writer_kthread; } setup_timer(&gc->gc_timer, pblk_gc_timer, (unsigned long)pblk); mod_timer(&gc->gc_timer, jiffies + msecs_to_jiffies(GC_TIME_MSECS)); gc->gc_active = 0; gc->gc_forced = 0; gc->gc_enabled = 1; gc->w_entries = 0; atomic_set(&gc->inflight_gc, 0); /* Workqueue that reads valid sectors from a line and submit them to the * GC writer to be recycled. */ gc->gc_line_reader_wq = alloc_workqueue("pblk-gc-line-reader-wq", WQ_MEM_RECLAIM | WQ_UNBOUND, PBLK_GC_MAX_READERS); if (!gc->gc_line_reader_wq) { pr_err("pblk: could not allocate GC line reader workqueue\n"); ret = -ENOMEM; goto fail_free_reader_kthread; } /* Workqueue that prepare lines for GC */ gc->gc_reader_wq = alloc_workqueue("pblk-gc-line_wq", WQ_MEM_RECLAIM | WQ_UNBOUND, 1); if (!gc->gc_reader_wq) { pr_err("pblk: could not allocate GC reader workqueue\n"); ret = -ENOMEM; goto fail_free_reader_line_wq; } spin_lock_init(&gc->lock); spin_lock_init(&gc->w_lock); spin_lock_init(&gc->r_lock); sema_init(&gc->gc_sem, 128); INIT_LIST_HEAD(&gc->w_list); INIT_LIST_HEAD(&gc->r_list); return 0; fail_free_reader_line_wq: destroy_workqueue(gc->gc_line_reader_wq); fail_free_reader_kthread: kthread_stop(gc->gc_reader_ts); fail_free_writer_kthread: kthread_stop(gc->gc_writer_ts); fail_free_main_kthread: kthread_stop(gc->gc_ts); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Javier González37198.93%266.67%
Dan Carpenter41.07%133.33%
Total375100.00%3100.00%


void pblk_gc_exit(struct pblk *pblk) { struct pblk_gc *gc = &pblk->gc; flush_workqueue(gc->gc_reader_wq); flush_workqueue(gc->gc_line_reader_wq); del_timer(&gc->gc_timer); pblk_gc_stop(pblk, 1); if (gc->gc_ts) kthread_stop(gc->gc_ts); if (gc->gc_reader_wq) destroy_workqueue(gc->gc_reader_wq); if (gc->gc_line_reader_wq) destroy_workqueue(gc->gc_line_reader_wq); if (gc->gc_writer_ts) kthread_stop(gc->gc_writer_ts); if (gc->gc_reader_ts) kthread_stop(gc->gc_reader_ts); }

Contributors

PersonTokensPropCommitsCommitProp
Javier González114100.00%2100.00%
Total114100.00%2100.00%


Overall Contributors

PersonTokensPropCommitsCommitProp
Javier González293899.86%685.71%
Dan Carpenter40.14%114.29%
Total2942100.00%7100.00%
Directory: drivers/lightnvm
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with cregit.