| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| Paul E. McKenney | 3044 | 50.11% | 27 | 39.13% |
| Davidlohr Bueso A | 1593 | 26.22% | 15 | 21.74% |
| John Stultz | 437 | 7.19% | 4 | 5.80% |
| Chris Wilson | 298 | 4.91% | 1 | 1.45% |
| Waiman Long | 227 | 3.74% | 3 | 4.35% |
| Zqiang | 219 | 3.60% | 1 | 1.45% |
| Joel A Fernandes | 105 | 1.73% | 2 | 2.90% |
| Hou Tao | 80 | 1.32% | 2 | 2.90% |
| Kees Cook | 19 | 0.31% | 2 | 2.90% |
| Yang Shi | 16 | 0.26% | 1 | 1.45% |
| Dietmar Eggemann | 11 | 0.18% | 1 | 1.45% |
| Ingo Molnar | 6 | 0.10% | 2 | 2.90% |
| Jeff Johnson | 5 | 0.08% | 1 | 1.45% |
| Chuhong Yuan | 4 | 0.07% | 1 | 1.45% |
| Zou Wei | 4 | 0.07% | 1 | 1.45% |
| Dongsheng Yang | 2 | 0.03% | 1 | 1.45% |
| Li Zhijian | 2 | 0.03% | 1 | 1.45% |
| Thomas Hellstrom | 1 | 0.02% | 1 | 1.45% |
| Alexey Kodanev | 1 | 0.02% | 1 | 1.45% |
| Wei Yongjun | 1 | 0.02% | 1 | 1.45% |
| Total | 6075 | 69 |
// SPDX-License-Identifier: GPL-2.0+ /* * Module-based torture test facility for locking * * Copyright (C) IBM Corporation, 2014 * * Authors: Paul E. McKenney <paulmck@linux.ibm.com> * Davidlohr Bueso <dave@stgolabs.net> * Based on kernel/rcu/torture.c. */ #define pr_fmt(fmt) fmt #include <linux/kernel.h> #include <linux/module.h> #include <linux/kthread.h> #include <linux/sched/rt.h> #include <linux/spinlock.h> #include <linux/mutex.h> #include <linux/rwsem.h> #include <linux/smp.h> #include <linux/interrupt.h> #include <linux/sched.h> #include <uapi/linux/sched/types.h> #include <linux/rtmutex.h> #include <linux/atomic.h> #include <linux/moduleparam.h> #include <linux/delay.h> #include <linux/slab.h> #include <linux/torture.h> #include <linux/reboot.h> MODULE_DESCRIPTION("torture test facility for locking"); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>"); torture_param(int, acq_writer_lim, 0, "Write_acquisition time limit (jiffies)."); torture_param(int, call_rcu_chains, 0, "Self-propagate call_rcu() chains during test (0=disable)."); torture_param(int, long_hold, 100, "Do occasional long hold of lock (ms), 0=disable"); torture_param(int, nested_locks, 0, "Number of nested locks (max = 8)"); torture_param(int, nreaders_stress, -1, "Number of read-locking stress-test threads"); torture_param(int, nwriters_stress, -1, "Number of write-locking stress-test threads"); torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (s), 0=disable"); torture_param(int, rt_boost, 2, "Do periodic rt-boost. 0=Disable, 1=Only for rt_mutex, 2=For all lock types."); torture_param(int, rt_boost_factor, 50, "A factor determining how often rt-boost happens."); torture_param(int, shuffle_interval, 3, "Number of jiffies between shuffles, 0=disable"); torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable."); torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s"); torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable"); torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); torture_param(int, writer_fifo, 0, "Run writers at sched_set_fifo() priority"); /* Going much higher trips "BUG: MAX_LOCKDEP_CHAIN_HLOCKS too low!" errors */ #define MAX_NESTED_LOCKS 8 static char *torture_type = IS_ENABLED(CONFIG_PREEMPT_RT) ? "raw_spin_lock" : "spin_lock"; module_param(torture_type, charp, 0444); MODULE_PARM_DESC(torture_type, "Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)"); static cpumask_var_t bind_readers; // Bind the readers to the specified set of CPUs. static cpumask_var_t bind_writers; // Bind the writers to the specified set of CPUs. // Parse a cpumask kernel parameter. If there are more users later on, // this might need to got to a more central location. static int param_set_cpumask(const char *val, const struct kernel_param *kp) { cpumask_var_t *cm_bind = kp->arg; int ret; char *s; if (!alloc_cpumask_var(cm_bind, GFP_KERNEL)) { s = "Out of memory"; ret = -ENOMEM; goto out_err; } ret = cpulist_parse(val, *cm_bind); if (!ret) return ret; s = "Bad CPU range"; out_err: pr_warn("%s: %s, all CPUs set\n", kp->name, s); cpumask_setall(*cm_bind); return ret; } // Output a cpumask kernel parameter. static int param_get_cpumask(char *buffer, const struct kernel_param *kp) { cpumask_var_t *cm_bind = kp->arg; return sprintf(buffer, "%*pbl", cpumask_pr_args(*cm_bind)); } static bool cpumask_nonempty(cpumask_var_t mask) { return cpumask_available(mask) && !cpumask_empty(mask); } static const struct kernel_param_ops lt_bind_ops = { .set = param_set_cpumask, .get = param_get_cpumask, }; module_param_cb(bind_readers, <_bind_ops, &bind_readers, 0644); module_param_cb(bind_writers, <_bind_ops, &bind_writers, 0644); long torture_sched_setaffinity(pid_t pid, const struct cpumask *in_mask); static struct task_struct *stats_task; static struct task_struct **writer_tasks; static struct task_struct **reader_tasks; static bool lock_is_write_held; static atomic_t lock_is_read_held; static unsigned long last_lock_release; struct lock_stress_stats { long n_lock_fail; long n_lock_acquired; }; struct call_rcu_chain { struct rcu_head crc_rh; bool crc_stop; }; struct call_rcu_chain *call_rcu_chain_list; /* Forward reference. */ static void lock_torture_cleanup(void); /* * Operations vector for selecting different types of tests. */ struct lock_torture_ops { void (*init)(void); void (*exit)(void); int (*nested_lock)(int tid, u32 lockset); int (*writelock)(int tid); void (*write_delay)(struct torture_random_state *trsp); void (*task_boost)(struct torture_random_state *trsp); void (*writeunlock)(int tid); void (*nested_unlock)(int tid, u32 lockset); int (*readlock)(int tid); void (*read_delay)(struct torture_random_state *trsp); void (*readunlock)(int tid); unsigned long flags; /* for irq spinlocks */ const char *name; }; struct lock_torture_cxt { int nrealwriters_stress; int nrealreaders_stress; bool debug_lock; bool init_called; atomic_t n_lock_torture_errors; struct lock_torture_ops *cur_ops; struct lock_stress_stats *lwsa; /* writer statistics */ struct lock_stress_stats *lrsa; /* reader statistics */ }; static struct lock_torture_cxt cxt = { 0, 0, false, false, ATOMIC_INIT(0), NULL, NULL}; /* * Definitions for lock torture testing. */ static int torture_lock_busted_write_lock(int tid __maybe_unused) { return 0; /* BUGGY, do not use in real life!!! */ } static void torture_lock_busted_write_delay(struct torture_random_state *trsp) { /* We want a long delay occasionally to force massive contention. */ if (long_hold && !(torture_random(trsp) % (cxt.nrealwriters_stress * 2000 * long_hold))) mdelay(long_hold); if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) torture_preempt_schedule(); /* Allow test to be preempted. */ } static void torture_lock_busted_write_unlock(int tid __maybe_unused) { /* BUGGY, do not use in real life!!! */ } static void __torture_rt_boost(struct torture_random_state *trsp) { const unsigned int factor = rt_boost_factor; if (!rt_task(current)) { /* * Boost priority once every rt_boost_factor operations. When * the task tries to take the lock, the rtmutex it will account * for the new priority, and do any corresponding pi-dance. */ if (trsp && !(torture_random(trsp) % (cxt.nrealwriters_stress * factor))) { sched_set_fifo(current); } else /* common case, do nothing */ return; } else { /* * The task will remain boosted for another 10 * rt_boost_factor * operations, then restored back to its original prio, and so * forth. * * When @trsp is nil, we want to force-reset the task for * stopping the kthread. */ if (!trsp || !(torture_random(trsp) % (cxt.nrealwriters_stress * factor * 2))) { sched_set_normal(current, 0); } else /* common case, do nothing */ return; } } static void torture_rt_boost(struct torture_random_state *trsp) { if (rt_boost != 2) return; __torture_rt_boost(trsp); } static struct lock_torture_ops lock_busted_ops = { .writelock = torture_lock_busted_write_lock, .write_delay = torture_lock_busted_write_delay, .task_boost = torture_rt_boost, .writeunlock = torture_lock_busted_write_unlock, .readlock = NULL, .read_delay = NULL, .readunlock = NULL, .name = "lock_busted" }; static DEFINE_SPINLOCK(torture_spinlock); static int torture_spin_lock_write_lock(int tid __maybe_unused) __acquires(torture_spinlock) { spin_lock(&torture_spinlock); return 0; } static void torture_spin_lock_write_delay(struct torture_random_state *trsp) { const unsigned long shortdelay_us = 2; unsigned long j; /* We want a short delay mostly to emulate likely code, and * we want a long delay occasionally to force massive contention. */ if (long_hold && !(torture_random(trsp) % (cxt.nrealwriters_stress * 2000 * long_hold))) { j = jiffies; mdelay(long_hold); pr_alert("%s: delay = %lu jiffies.\n", __func__, jiffies - j); } if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 200 * shortdelay_us))) udelay(shortdelay_us); if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) torture_preempt_schedule(); /* Allow test to be preempted. */ } static void torture_spin_lock_write_unlock(int tid __maybe_unused) __releases(torture_spinlock) { spin_unlock(&torture_spinlock); } static struct lock_torture_ops spin_lock_ops = { .writelock = torture_spin_lock_write_lock, .write_delay = torture_spin_lock_write_delay, .task_boost = torture_rt_boost, .writeunlock = torture_spin_lock_write_unlock, .readlock = NULL, .read_delay = NULL, .readunlock = NULL, .name = "spin_lock" }; static int torture_spin_lock_write_lock_irq(int tid __maybe_unused) __acquires(torture_spinlock) { unsigned long flags; spin_lock_irqsave(&torture_spinlock, flags); cxt.cur_ops->flags = flags; return 0; } static void torture_lock_spin_write_unlock_irq(int tid __maybe_unused) __releases(torture_spinlock) { spin_unlock_irqrestore(&torture_spinlock, cxt.cur_ops->flags); } static struct lock_torture_ops spin_lock_irq_ops = { .writelock = torture_spin_lock_write_lock_irq, .write_delay = torture_spin_lock_write_delay, .task_boost = torture_rt_boost, .writeunlock = torture_lock_spin_write_unlock_irq, .readlock = NULL, .read_delay = NULL, .readunlock = NULL, .name = "spin_lock_irq" }; static DEFINE_RAW_SPINLOCK(torture_raw_spinlock); static int torture_raw_spin_lock_write_lock(int tid __maybe_unused) __acquires(torture_raw_spinlock) { raw_spin_lock(&torture_raw_spinlock); return 0; } static void torture_raw_spin_lock_write_unlock(int tid __maybe_unused) __releases(torture_raw_spinlock) { raw_spin_unlock(&torture_raw_spinlock); } static struct lock_torture_ops raw_spin_lock_ops = { .writelock = torture_raw_spin_lock_write_lock, .write_delay = torture_spin_lock_write_delay, .task_boost = torture_rt_boost, .writeunlock = torture_raw_spin_lock_write_unlock, .readlock = NULL, .read_delay = NULL, .readunlock = NULL, .name = "raw_spin_lock" }; static int torture_raw_spin_lock_write_lock_irq(int tid __maybe_unused) __acquires(torture_raw_spinlock) { unsigned long flags; raw_spin_lock_irqsave(&torture_raw_spinlock, flags); cxt.cur_ops->flags = flags; return 0; } static void torture_raw_spin_lock_write_unlock_irq(int tid __maybe_unused) __releases(torture_raw_spinlock) { raw_spin_unlock_irqrestore(&torture_raw_spinlock, cxt.cur_ops->flags); } static struct lock_torture_ops raw_spin_lock_irq_ops = { .writelock = torture_raw_spin_lock_write_lock_irq, .write_delay = torture_spin_lock_write_delay, .task_boost = torture_rt_boost, .writeunlock = torture_raw_spin_lock_write_unlock_irq, .readlock = NULL, .read_delay = NULL, .readunlock = NULL, .name = "raw_spin_lock_irq" }; static DEFINE_RWLOCK(torture_rwlock); static int torture_rwlock_write_lock(int tid __maybe_unused) __acquires(torture_rwlock) { write_lock(&torture_rwlock); return 0; } static void torture_rwlock_write_delay(struct torture_random_state *trsp) { const unsigned long shortdelay_us = 2; /* We want a short delay mostly to emulate likely code, and * we want a long delay occasionally to force massive contention. */ if (long_hold && !(torture_random(trsp) % (cxt.nrealwriters_stress * 2000 * long_hold))) mdelay(long_hold); else udelay(shortdelay_us); } static void torture_rwlock_write_unlock(int tid __maybe_unused) __releases(torture_rwlock) { write_unlock(&torture_rwlock); } static int torture_rwlock_read_lock(int tid __maybe_unused) __acquires(torture_rwlock) { read_lock(&torture_rwlock); return 0; } static void torture_rwlock_read_delay(struct torture_random_state *trsp) { const unsigned long shortdelay_us = 10; /* We want a short delay mostly to emulate likely code, and * we want a long delay occasionally to force massive contention. */ if (long_hold && !(torture_random(trsp) % (cxt.nrealreaders_stress * 2000 * long_hold))) mdelay(long_hold); else udelay(shortdelay_us); } static void torture_rwlock_read_unlock(int tid __maybe_unused) __releases(torture_rwlock) { read_unlock(&torture_rwlock); } static struct lock_torture_ops rw_lock_ops = { .writelock = torture_rwlock_write_lock, .write_delay = torture_rwlock_write_delay, .task_boost = torture_rt_boost, .writeunlock = torture_rwlock_write_unlock, .readlock = torture_rwlock_read_lock, .read_delay = torture_rwlock_read_delay, .readunlock = torture_rwlock_read_unlock, .name = "rw_lock" }; static int torture_rwlock_write_lock_irq(int tid __maybe_unused) __acquires(torture_rwlock) { unsigned long flags; write_lock_irqsave(&torture_rwlock, flags); cxt.cur_ops->flags = flags; return 0; } static void torture_rwlock_write_unlock_irq(int tid __maybe_unused) __releases(torture_rwlock) { write_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags); } static int torture_rwlock_read_lock_irq(int tid __maybe_unused) __acquires(torture_rwlock) { unsigned long flags; read_lock_irqsave(&torture_rwlock, flags); cxt.cur_ops->flags = flags; return 0; } static void torture_rwlock_read_unlock_irq(int tid __maybe_unused) __releases(torture_rwlock) { read_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags); } static struct lock_torture_ops rw_lock_irq_ops = { .writelock = torture_rwlock_write_lock_irq, .write_delay = torture_rwlock_write_delay, .task_boost = torture_rt_boost, .writeunlock = torture_rwlock_write_unlock_irq, .readlock = torture_rwlock_read_lock_irq, .read_delay = torture_rwlock_read_delay, .readunlock = torture_rwlock_read_unlock_irq, .name = "rw_lock_irq" }; static DEFINE_MUTEX(torture_mutex); static struct mutex torture_nested_mutexes[MAX_NESTED_LOCKS]; static struct lock_class_key nested_mutex_keys[MAX_NESTED_LOCKS]; static void torture_mutex_init(void) { int i; for (i = 0; i < MAX_NESTED_LOCKS; i++) __mutex_init(&torture_nested_mutexes[i], __func__, &nested_mutex_keys[i]); } static int torture_mutex_nested_lock(int tid __maybe_unused, u32 lockset) { int i; for (i = 0; i < nested_locks; i++) if (lockset & (1 << i)) mutex_lock(&torture_nested_mutexes[i]); return 0; } static int torture_mutex_lock(int tid __maybe_unused) __acquires(torture_mutex) { mutex_lock(&torture_mutex); return 0; } static void torture_mutex_delay(struct torture_random_state *trsp) { /* We want a long delay occasionally to force massive contention. */ if (long_hold && !(torture_random(trsp) % (cxt.nrealwriters_stress * 2000 * long_hold))) mdelay(long_hold * 5); if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) torture_preempt_schedule(); /* Allow test to be preempted. */ } static void torture_mutex_unlock(int tid __maybe_unused) __releases(torture_mutex) { mutex_unlock(&torture_mutex); } static void torture_mutex_nested_unlock(int tid __maybe_unused, u32 lockset) { int i; for (i = nested_locks - 1; i >= 0; i--) if (lockset & (1 << i)) mutex_unlock(&torture_nested_mutexes[i]); } static struct lock_torture_ops mutex_lock_ops = { .init = torture_mutex_init, .nested_lock = torture_mutex_nested_lock, .writelock = torture_mutex_lock, .write_delay = torture_mutex_delay, .task_boost = torture_rt_boost, .writeunlock = torture_mutex_unlock, .nested_unlock = torture_mutex_nested_unlock, .readlock = NULL, .read_delay = NULL, .readunlock = NULL, .name = "mutex_lock" }; #include <linux/ww_mutex.h> /* * The torture ww_mutexes should belong to the same lock class as * torture_ww_class to avoid lockdep problem. The ww_mutex_init() * function is called for initialization to ensure that. */ static DEFINE_WD_CLASS(torture_ww_class); static struct ww_mutex torture_ww_mutex_0, torture_ww_mutex_1, torture_ww_mutex_2; static struct ww_acquire_ctx *ww_acquire_ctxs; static void torture_ww_mutex_init(void) { ww_mutex_init(&torture_ww_mutex_0, &torture_ww_class); ww_mutex_init(&torture_ww_mutex_1, &torture_ww_class); ww_mutex_init(&torture_ww_mutex_2, &torture_ww_class); ww_acquire_ctxs = kmalloc_array(cxt.nrealwriters_stress, sizeof(*ww_acquire_ctxs), GFP_KERNEL); if (!ww_acquire_ctxs) VERBOSE_TOROUT_STRING("ww_acquire_ctx: Out of memory"); } static void torture_ww_mutex_exit(void) { kfree(ww_acquire_ctxs); } static int torture_ww_mutex_lock(int tid) __acquires(torture_ww_mutex_0) __acquires(torture_ww_mutex_1) __acquires(torture_ww_mutex_2) { LIST_HEAD(list); struct reorder_lock { struct list_head link; struct ww_mutex *lock; } locks[3], *ll, *ln; struct ww_acquire_ctx *ctx = &ww_acquire_ctxs[tid]; locks[0].lock = &torture_ww_mutex_0; list_add(&locks[0].link, &list); locks[1].lock = &torture_ww_mutex_1; list_add(&locks[1].link, &list); locks[2].lock = &torture_ww_mutex_2; list_add(&locks[2].link, &list); ww_acquire_init(ctx, &torture_ww_class); list_for_each_entry(ll, &list, link) { int err; err = ww_mutex_lock(ll->lock, ctx); if (!err) continue; ln = ll; list_for_each_entry_continue_reverse(ln, &list, link) ww_mutex_unlock(ln->lock); if (err != -EDEADLK) return err; ww_mutex_lock_slow(ll->lock, ctx); list_move(&ll->link, &list); } return 0; } static void torture_ww_mutex_unlock(int tid) __releases(torture_ww_mutex_0) __releases(torture_ww_mutex_1) __releases(torture_ww_mutex_2) { struct ww_acquire_ctx *ctx = &ww_acquire_ctxs[tid]; ww_mutex_unlock(&torture_ww_mutex_0); ww_mutex_unlock(&torture_ww_mutex_1); ww_mutex_unlock(&torture_ww_mutex_2); ww_acquire_fini(ctx); } static struct lock_torture_ops ww_mutex_lock_ops = { .init = torture_ww_mutex_init, .exit = torture_ww_mutex_exit, .writelock = torture_ww_mutex_lock, .write_delay = torture_mutex_delay, .task_boost = torture_rt_boost, .writeunlock = torture_ww_mutex_unlock, .readlock = NULL, .read_delay = NULL, .readunlock = NULL, .name = "ww_mutex_lock" }; #ifdef CONFIG_RT_MUTEXES static DEFINE_RT_MUTEX(torture_rtmutex); static struct rt_mutex torture_nested_rtmutexes[MAX_NESTED_LOCKS]; static struct lock_class_key nested_rtmutex_keys[MAX_NESTED_LOCKS]; static void torture_rtmutex_init(void) { int i; for (i = 0; i < MAX_NESTED_LOCKS; i++) __rt_mutex_init(&torture_nested_rtmutexes[i], __func__, &nested_rtmutex_keys[i]); } static int torture_rtmutex_nested_lock(int tid __maybe_unused, u32 lockset) { int i; for (i = 0; i < nested_locks; i++) if (lockset & (1 << i)) rt_mutex_lock(&torture_nested_rtmutexes[i]); return 0; } static int torture_rtmutex_lock(int tid __maybe_unused) __acquires(torture_rtmutex) { rt_mutex_lock(&torture_rtmutex); return 0; } static void torture_rtmutex_delay(struct torture_random_state *trsp) { const unsigned long shortdelay_us = 2; /* * We want a short delay mostly to emulate likely code, and * we want a long delay occasionally to force massive contention. */ if (long_hold && !(torture_random(trsp) % (cxt.nrealwriters_stress * 2000 * long_hold))) mdelay(long_hold); if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 200 * shortdelay_us))) udelay(shortdelay_us); if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) torture_preempt_schedule(); /* Allow test to be preempted. */ } static void torture_rtmutex_unlock(int tid __maybe_unused) __releases(torture_rtmutex) { rt_mutex_unlock(&torture_rtmutex); } static void torture_rt_boost_rtmutex(struct torture_random_state *trsp) { if (!rt_boost) return; __torture_rt_boost(trsp); } static void torture_rtmutex_nested_unlock(int tid __maybe_unused, u32 lockset) { int i; for (i = nested_locks - 1; i >= 0; i--) if (lockset & (1 << i)) rt_mutex_unlock(&torture_nested_rtmutexes[i]); } static struct lock_torture_ops rtmutex_lock_ops = { .init = torture_rtmutex_init, .nested_lock = torture_rtmutex_nested_lock, .writelock = torture_rtmutex_lock, .write_delay = torture_rtmutex_delay, .task_boost = torture_rt_boost_rtmutex, .writeunlock = torture_rtmutex_unlock, .nested_unlock = torture_rtmutex_nested_unlock, .readlock = NULL, .read_delay = NULL, .readunlock = NULL, .name = "rtmutex_lock" }; #endif static DECLARE_RWSEM(torture_rwsem); static int torture_rwsem_down_write(int tid __maybe_unused) __acquires(torture_rwsem) { down_write(&torture_rwsem); return 0; } static void torture_rwsem_write_delay(struct torture_random_state *trsp) { /* We want a long delay occasionally to force massive contention. */ if (long_hold && !(torture_random(trsp) % (cxt.nrealwriters_stress * 2000 * long_hold))) mdelay(long_hold * 10); if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000))) torture_preempt_schedule(); /* Allow test to be preempted. */ } static void torture_rwsem_up_write(int tid __maybe_unused) __releases(torture_rwsem) { up_write(&torture_rwsem); } static int torture_rwsem_down_read(int tid __maybe_unused) __acquires(torture_rwsem) { down_read(&torture_rwsem); return 0; } static void torture_rwsem_read_delay(struct torture_random_state *trsp) { /* We want a long delay occasionally to force massive contention. */ if (long_hold && !(torture_random(trsp) % (cxt.nrealreaders_stress * 2000 * long_hold))) mdelay(long_hold * 2); else mdelay(long_hold / 2); if (!(torture_random(trsp) % (cxt.nrealreaders_stress * 20000))) torture_preempt_schedule(); /* Allow test to be preempted. */ } static void torture_rwsem_up_read(int tid __maybe_unused) __releases(torture_rwsem) { up_read(&torture_rwsem); } static struct lock_torture_ops rwsem_lock_ops = { .writelock = torture_rwsem_down_write, .write_delay = torture_rwsem_write_delay, .task_boost = torture_rt_boost, .writeunlock = torture_rwsem_up_write, .readlock = torture_rwsem_down_read, .read_delay = torture_rwsem_read_delay, .readunlock = torture_rwsem_up_read, .name = "rwsem_lock" }; #include <linux/percpu-rwsem.h> static struct percpu_rw_semaphore pcpu_rwsem; static void torture_percpu_rwsem_init(void) { BUG_ON(percpu_init_rwsem(&pcpu_rwsem)); } static void torture_percpu_rwsem_exit(void) { percpu_free_rwsem(&pcpu_rwsem); } static int torture_percpu_rwsem_down_write(int tid __maybe_unused) __acquires(pcpu_rwsem) { percpu_down_write(&pcpu_rwsem); return 0; } static void torture_percpu_rwsem_up_write(int tid __maybe_unused) __releases(pcpu_rwsem) { percpu_up_write(&pcpu_rwsem); } static int torture_percpu_rwsem_down_read(int tid __maybe_unused) __acquires(pcpu_rwsem) { percpu_down_read(&pcpu_rwsem); return 0; } static void torture_percpu_rwsem_up_read(int tid __maybe_unused) __releases(pcpu_rwsem) { percpu_up_read(&pcpu_rwsem); } static struct lock_torture_ops percpu_rwsem_lock_ops = { .init = torture_percpu_rwsem_init, .exit = torture_percpu_rwsem_exit, .writelock = torture_percpu_rwsem_down_write, .write_delay = torture_rwsem_write_delay, .task_boost = torture_rt_boost, .writeunlock = torture_percpu_rwsem_up_write, .readlock = torture_percpu_rwsem_down_read, .read_delay = torture_rwsem_read_delay, .readunlock = torture_percpu_rwsem_up_read, .name = "percpu_rwsem_lock" }; /* * Lock torture writer kthread. Repeatedly acquires and releases * the lock, checking for duplicate acquisitions. */ static int lock_torture_writer(void *arg) { unsigned long j; unsigned long j1; u32 lockset_mask; struct lock_stress_stats *lwsp = arg; DEFINE_TORTURE_RANDOM(rand); bool skip_main_lock; int tid = lwsp - cxt.lwsa; VERBOSE_TOROUT_STRING("lock_torture_writer task started"); if (!rt_task(current)) set_user_nice(current, MAX_NICE); do { if ((torture_random(&rand) & 0xfffff) == 0) schedule_timeout_uninterruptible(1); lockset_mask = torture_random(&rand); /* * When using nested_locks, we want to occasionally * skip the main lock so we can avoid always serializing * the lock chains on that central lock. By skipping the * main lock occasionally, we can create different * contention patterns (allowing for multiple disjoint * blocked trees) */ skip_main_lock = (nested_locks && !(torture_random(&rand) % 100)); cxt.cur_ops->task_boost(&rand); if (cxt.cur_ops->nested_lock) cxt.cur_ops->nested_lock(tid, lockset_mask); if (!skip_main_lock) { if (acq_writer_lim > 0) j = jiffies; cxt.cur_ops->writelock(tid); if (WARN_ON_ONCE(lock_is_write_held)) lwsp->n_lock_fail++; lock_is_write_held = true; if (WARN_ON_ONCE(atomic_read(&lock_is_read_held))) lwsp->n_lock_fail++; /* rare, but... */ if (acq_writer_lim > 0) { j1 = jiffies; WARN_ONCE(time_after(j1, j + acq_writer_lim), "%s: Lock acquisition took %lu jiffies.\n", __func__, j1 - j); } lwsp->n_lock_acquired++; cxt.cur_ops->write_delay(&rand); lock_is_write_held = false; WRITE_ONCE(last_lock_release, jiffies); cxt.cur_ops->writeunlock(tid); } if (cxt.cur_ops->nested_unlock) cxt.cur_ops->nested_unlock(tid, lockset_mask); stutter_wait("lock_torture_writer"); } while (!torture_must_stop()); cxt.cur_ops->task_boost(NULL); /* reset prio */ torture_kthread_stopping("lock_torture_writer"); return 0; } /* * Lock torture reader kthread. Repeatedly acquires and releases * the reader lock. */ static int lock_torture_reader(void *arg) { struct lock_stress_stats *lrsp = arg; int tid = lrsp - cxt.lrsa; DEFINE_TORTURE_RANDOM(rand); VERBOSE_TOROUT_STRING("lock_torture_reader task started"); set_user_nice(current, MAX_NICE); do { if ((torture_random(&rand) & 0xfffff) == 0) schedule_timeout_uninterruptible(1); cxt.cur_ops->readlock(tid); atomic_inc(&lock_is_read_held); if (WARN_ON_ONCE(lock_is_write_held)) lrsp->n_lock_fail++; /* rare, but... */ lrsp->n_lock_acquired++; cxt.cur_ops->read_delay(&rand); atomic_dec(&lock_is_read_held); cxt.cur_ops->readunlock(tid); stutter_wait("lock_torture_reader"); } while (!torture_must_stop()); torture_kthread_stopping("lock_torture_reader"); return 0; } /* * Create an lock-torture-statistics message in the specified buffer. */ static void __torture_print_stats(char *page, struct lock_stress_stats *statp, bool write) { long cur; bool fail = false; int i, n_stress; long max = 0, min = statp ? data_race(statp[0].n_lock_acquired) : 0; long long sum = 0; n_stress = write ? cxt.nrealwriters_stress : cxt.nrealreaders_stress; for (i = 0; i < n_stress; i++) { if (data_race(statp[i].n_lock_fail)) fail = true; cur = data_race(statp[i].n_lock_acquired); sum += cur; if (max < cur) max = cur; if (min > cur) min = cur; } page += sprintf(page, "%s: Total: %lld Max/Min: %ld/%ld %s Fail: %d %s\n", write ? "Writes" : "Reads ", sum, max, min, !onoff_interval && max / 2 > min ? "???" : "", fail, fail ? "!!!" : ""); if (fail) atomic_inc(&cxt.n_lock_torture_errors); } /* * Print torture statistics. Caller must ensure that there is only one * call to this function at a given time!!! This is normally accomplished * by relying on the module system to only have one copy of the module * loaded, and then by giving the lock_torture_stats kthread full control * (or the init/cleanup functions when lock_torture_stats thread is not * running). */ static void lock_torture_stats_print(void) { int size = cxt.nrealwriters_stress * 200 + 8192; char *buf; if (cxt.cur_ops->readlock) size += cxt.nrealreaders_stress * 200 + 8192; buf = kmalloc(size, GFP_KERNEL); if (!buf) { pr_err("lock_torture_stats_print: Out of memory, need: %d", size); return; } __torture_print_stats(buf, cxt.lwsa, true); pr_alert("%s", buf); kfree(buf); if (cxt.cur_ops->readlock) { buf = kmalloc(size, GFP_KERNEL); if (!buf) { pr_err("lock_torture_stats_print: Out of memory, need: %d", size); return; } __torture_print_stats(buf, cxt.lrsa, false); pr_alert("%s", buf); kfree(buf); } } /* * Periodically prints torture statistics, if periodic statistics printing * was specified via the stat_interval module parameter. * * No need to worry about fullstop here, since this one doesn't reference * volatile state or register callbacks. */ static int lock_torture_stats(void *arg) { VERBOSE_TOROUT_STRING("lock_torture_stats task started"); do { schedule_timeout_interruptible(stat_interval * HZ); lock_torture_stats_print(); torture_shutdown_absorb("lock_torture_stats"); } while (!torture_must_stop()); torture_kthread_stopping("lock_torture_stats"); return 0; } static inline void lock_torture_print_module_parms(struct lock_torture_ops *cur_ops, const char *tag) { static cpumask_t cpumask_all; cpumask_t *rcmp = cpumask_nonempty(bind_readers) ? bind_readers : &cpumask_all; cpumask_t *wcmp = cpumask_nonempty(bind_writers) ? bind_writers : &cpumask_all; cpumask_setall(&cpumask_all); pr_alert("%s" TORTURE_FLAG "--- %s%s: acq_writer_lim=%d bind_readers=%*pbl bind_writers=%*pbl call_rcu_chains=%d long_hold=%d nested_locks=%d nreaders_stress=%d nwriters_stress=%d onoff_holdoff=%d onoff_interval=%d rt_boost=%d rt_boost_factor=%d shuffle_interval=%d shutdown_secs=%d stat_interval=%d stutter=%d verbose=%d writer_fifo=%d\n", torture_type, tag, cxt.debug_lock ? " [debug]": "", acq_writer_lim, cpumask_pr_args(rcmp), cpumask_pr_args(wcmp), call_rcu_chains, long_hold, nested_locks, cxt.nrealreaders_stress, cxt.nrealwriters_stress, onoff_holdoff, onoff_interval, rt_boost, rt_boost_factor, shuffle_interval, shutdown_secs, stat_interval, stutter, verbose, writer_fifo); } // If requested, maintain call_rcu() chains to keep a grace period always // in flight. These increase the probability of getting an RCU CPU stall // warning and associated diagnostics when a locking primitive stalls. static void call_rcu_chain_cb(struct rcu_head *rhp) { struct call_rcu_chain *crcp = container_of(rhp, struct call_rcu_chain, crc_rh); if (!smp_load_acquire(&crcp->crc_stop)) { (void)start_poll_synchronize_rcu(); // Start one grace period... call_rcu(&crcp->crc_rh, call_rcu_chain_cb); // ... and later start another. } } // Start the requested number of call_rcu() chains. static int call_rcu_chain_init(void) { int i; if (call_rcu_chains <= 0) return 0; call_rcu_chain_list = kcalloc(call_rcu_chains, sizeof(*call_rcu_chain_list), GFP_KERNEL); if (!call_rcu_chain_list) return -ENOMEM; for (i = 0; i < call_rcu_chains; i++) { call_rcu_chain_list[i].crc_stop = false; call_rcu(&call_rcu_chain_list[i].crc_rh, call_rcu_chain_cb); } return 0; } // Stop all of the call_rcu() chains. static void call_rcu_chain_cleanup(void) { int i; if (!call_rcu_chain_list) return; for (i = 0; i < call_rcu_chains; i++) smp_store_release(&call_rcu_chain_list[i].crc_stop, true); rcu_barrier(); kfree(call_rcu_chain_list); call_rcu_chain_list = NULL; } static void lock_torture_cleanup(void) { int i; if (torture_cleanup_begin()) return; /* * Indicates early cleanup, meaning that the test has not run, * such as when passing bogus args when loading the module. * However cxt->cur_ops.init() may have been invoked, so beside * perform the underlying torture-specific cleanups, cur_ops.exit() * will be invoked if needed. */ if (!cxt.lwsa && !cxt.lrsa) goto end; if (writer_tasks) { for (i = 0; i < cxt.nrealwriters_stress; i++) torture_stop_kthread(lock_torture_writer, writer_tasks[i]); kfree(writer_tasks); writer_tasks = NULL; } if (reader_tasks) { for (i = 0; i < cxt.nrealreaders_stress; i++) torture_stop_kthread(lock_torture_reader, reader_tasks[i]); kfree(reader_tasks); reader_tasks = NULL; } torture_stop_kthread(lock_torture_stats, stats_task); lock_torture_stats_print(); /* -After- the stats thread is stopped! */ if (atomic_read(&cxt.n_lock_torture_errors)) lock_torture_print_module_parms(cxt.cur_ops, "End of test: FAILURE"); else if (torture_onoff_failures()) lock_torture_print_module_parms(cxt.cur_ops, "End of test: LOCK_HOTPLUG"); else lock_torture_print_module_parms(cxt.cur_ops, "End of test: SUCCESS"); kfree(cxt.lwsa); cxt.lwsa = NULL; kfree(cxt.lrsa); cxt.lrsa = NULL; call_rcu_chain_cleanup(); end: if (cxt.init_called) { if (cxt.cur_ops->exit) cxt.cur_ops->exit(); cxt.init_called = false; } torture_cleanup_end(); } static int __init lock_torture_init(void) { int i, j; int firsterr = 0; static struct lock_torture_ops *torture_ops[] = { &lock_busted_ops, &spin_lock_ops, &spin_lock_irq_ops, &raw_spin_lock_ops, &raw_spin_lock_irq_ops, &rw_lock_ops, &rw_lock_irq_ops, &mutex_lock_ops, &ww_mutex_lock_ops, #ifdef CONFIG_RT_MUTEXES &rtmutex_lock_ops, #endif &rwsem_lock_ops, &percpu_rwsem_lock_ops, }; if (!torture_init_begin(torture_type, verbose)) return -EBUSY; /* Process args and tell the world that the torturer is on the job. */ for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { cxt.cur_ops = torture_ops[i]; if (strcmp(torture_type, cxt.cur_ops->name) == 0) break; } if (i == ARRAY_SIZE(torture_ops)) { pr_alert("lock-torture: invalid torture type: \"%s\"\n", torture_type); pr_alert("lock-torture types:"); for (i = 0; i < ARRAY_SIZE(torture_ops); i++) pr_alert(" %s", torture_ops[i]->name); pr_alert("\n"); firsterr = -EINVAL; goto unwind; } if (nwriters_stress == 0 && (!cxt.cur_ops->readlock || nreaders_stress == 0)) { pr_alert("lock-torture: must run at least one locking thread\n"); firsterr = -EINVAL; goto unwind; } if (nwriters_stress >= 0) cxt.nrealwriters_stress = nwriters_stress; else cxt.nrealwriters_stress = 2 * num_online_cpus(); if (cxt.cur_ops->init) { cxt.cur_ops->init(); cxt.init_called = true; } #ifdef CONFIG_DEBUG_MUTEXES if (str_has_prefix(torture_type, "mutex")) cxt.debug_lock = true; #endif #ifdef CONFIG_DEBUG_RT_MUTEXES if (str_has_prefix(torture_type, "rtmutex")) cxt.debug_lock = true; #endif #ifdef CONFIG_DEBUG_SPINLOCK if ((str_has_prefix(torture_type, "spin")) || (str_has_prefix(torture_type, "rw_lock"))) cxt.debug_lock = true; #endif /* Initialize the statistics so that each run gets its own numbers. */ if (nwriters_stress) { lock_is_write_held = false; cxt.lwsa = kmalloc_array(cxt.nrealwriters_stress, sizeof(*cxt.lwsa), GFP_KERNEL); if (cxt.lwsa == NULL) { VERBOSE_TOROUT_STRING("cxt.lwsa: Out of memory"); firsterr = -ENOMEM; goto unwind; } for (i = 0; i < cxt.nrealwriters_stress; i++) { cxt.lwsa[i].n_lock_fail = 0; cxt.lwsa[i].n_lock_acquired = 0; } } if (cxt.cur_ops->readlock) { if (nreaders_stress >= 0) cxt.nrealreaders_stress = nreaders_stress; else { /* * By default distribute evenly the number of * readers and writers. We still run the same number * of threads as the writer-only locks default. */ if (nwriters_stress < 0) /* user doesn't care */ cxt.nrealwriters_stress = num_online_cpus(); cxt.nrealreaders_stress = cxt.nrealwriters_stress; } if (nreaders_stress) { cxt.lrsa = kmalloc_array(cxt.nrealreaders_stress, sizeof(*cxt.lrsa), GFP_KERNEL); if (cxt.lrsa == NULL) { VERBOSE_TOROUT_STRING("cxt.lrsa: Out of memory"); firsterr = -ENOMEM; kfree(cxt.lwsa); cxt.lwsa = NULL; goto unwind; } for (i = 0; i < cxt.nrealreaders_stress; i++) { cxt.lrsa[i].n_lock_fail = 0; cxt.lrsa[i].n_lock_acquired = 0; } } } firsterr = call_rcu_chain_init(); if (torture_init_error(firsterr)) goto unwind; lock_torture_print_module_parms(cxt.cur_ops, "Start of test"); /* Prepare torture context. */ if (onoff_interval > 0) { firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval * HZ, NULL); if (torture_init_error(firsterr)) goto unwind; } if (shuffle_interval > 0) { firsterr = torture_shuffle_init(shuffle_interval); if (torture_init_error(firsterr)) goto unwind; } if (shutdown_secs > 0) { firsterr = torture_shutdown_init(shutdown_secs, lock_torture_cleanup); if (torture_init_error(firsterr)) goto unwind; } if (stutter > 0) { firsterr = torture_stutter_init(stutter, stutter); if (torture_init_error(firsterr)) goto unwind; } if (nwriters_stress) { writer_tasks = kcalloc(cxt.nrealwriters_stress, sizeof(writer_tasks[0]), GFP_KERNEL); if (writer_tasks == NULL) { TOROUT_ERRSTRING("writer_tasks: Out of memory"); firsterr = -ENOMEM; goto unwind; } } /* cap nested_locks to MAX_NESTED_LOCKS */ if (nested_locks > MAX_NESTED_LOCKS) nested_locks = MAX_NESTED_LOCKS; if (cxt.cur_ops->readlock) { reader_tasks = kcalloc(cxt.nrealreaders_stress, sizeof(reader_tasks[0]), GFP_KERNEL); if (reader_tasks == NULL) { TOROUT_ERRSTRING("reader_tasks: Out of memory"); kfree(writer_tasks); writer_tasks = NULL; firsterr = -ENOMEM; goto unwind; } } /* * Create the kthreads and start torturing (oh, those poor little locks). * * TODO: Note that we interleave writers with readers, giving writers a * slight advantage, by creating its kthread first. This can be modified * for very specific needs, or even let the user choose the policy, if * ever wanted. */ for (i = 0, j = 0; i < cxt.nrealwriters_stress || j < cxt.nrealreaders_stress; i++, j++) { if (i >= cxt.nrealwriters_stress) goto create_reader; /* Create writer. */ firsterr = torture_create_kthread_cb(lock_torture_writer, &cxt.lwsa[i], writer_tasks[i], writer_fifo ? sched_set_fifo : NULL); if (torture_init_error(firsterr)) goto unwind; if (cpumask_nonempty(bind_writers)) torture_sched_setaffinity(writer_tasks[i]->pid, bind_writers); create_reader: if (cxt.cur_ops->readlock == NULL || (j >= cxt.nrealreaders_stress)) continue; /* Create reader. */ firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j], reader_tasks[j]); if (torture_init_error(firsterr)) goto unwind; if (cpumask_nonempty(bind_readers)) torture_sched_setaffinity(reader_tasks[j]->pid, bind_readers); } if (stat_interval > 0) { firsterr = torture_create_kthread(lock_torture_stats, NULL, stats_task); if (torture_init_error(firsterr)) goto unwind; } torture_init_end(); return 0; unwind: torture_init_end(); lock_torture_cleanup(); if (shutdown_secs) { WARN_ON(!IS_MODULE(CONFIG_LOCK_TORTURE_TEST)); kernel_power_off(); } return firsterr; } module_init(lock_torture_init); module_exit(lock_torture_cleanup);
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1