cregit-Linux how code gets into the kernel

Release 4.7 kernel/softirq.c

Directory: kernel
/*
 *      linux/kernel/softirq.c
 *
 *      Copyright (C) 1992 Linus Torvalds
 *
 *      Distribute under GPLv2.
 *
 *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
 */


#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/export.h>
#include <linux/kernel_stat.h>
#include <linux/interrupt.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/notifier.h>
#include <linux/percpu.h>
#include <linux/cpu.h>
#include <linux/freezer.h>
#include <linux/kthread.h>
#include <linux/rcupdate.h>
#include <linux/ftrace.h>
#include <linux/smp.h>
#include <linux/smpboot.h>
#include <linux/tick.h>
#include <linux/irq.h>


#define CREATE_TRACE_POINTS
#include <trace/events/irq.h>

/*
   - No shared variables, all the data are CPU local.
   - If a softirq needs serialization, let it serialize itself
     by its own spinlocks.
   - Even if softirq is serialized, only local cpu is marked for
     execution. Hence, we get something sort of weak cpu binding.
     Though it is still not clear, will it result in better locality
     or will not.

   Examples:
   - NET RX softirq. It is multithreaded and does not require
     any global serialization.
   - NET TX softirq. It kicks software netdevice queues, hence
     it is logically serialized per device, but this serialization
     is invisible to common code.
   - Tasklets: serialized wrt itself.
 */

#ifndef __ARCH_IRQ_STAT

irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;

EXPORT_SYMBOL(irq_stat);
#endif


static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;

DEFINE_PER_CPU(struct task_struct *, ksoftirqd);


const char * const softirq_to_name[NR_SOFTIRQS] = {
	"HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
	"TASKLET", "SCHED", "HRTIMER", "RCU"
};

/*
 * we cannot loop indefinitely here to avoid userspace starvation,
 * but we also don't want to introduce a worst case 1/HZ latency
 * to the pending events, so lets the scheduler to balance
 * the softirq load for us.
 */

static void wakeup_softirqd(void) { /* Interrupts are disabled: no need to stop preemption */ struct task_struct *tsk = __this_cpu_read(ksoftirqd); if (tsk && tsk->state != TASK_RUNNING) wake_up_process(tsk); }

Contributors

PersonTokensPropCommitsCommitProp
linus torvaldslinus torvalds2985.29%125.00%
rusty russellrusty russell38.82%125.00%
thomas gleixnerthomas gleixner12.94%125.00%
christoph lameterchristoph lameter12.94%125.00%
Total34100.00%4100.00%

/* * preempt_count and SOFTIRQ_OFFSET usage: * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving * softirq processing. * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET) * on local_bh_disable or local_bh_enable. * This lets us distinguish between whether we are currently processing * softirq and whether we just have bh disabled. */ /* * This one is for softirq.c-internal use, * where hardirqs are disabled legitimately: */ #ifdef CONFIG_TRACE_IRQFLAGS
void __local_bh_disable_ip(unsigned long ip, unsigned int cnt) { unsigned long flags; WARN_ON_ONCE(in_irq()); raw_local_irq_save(flags); /* * The preempt tracer hooks into preempt_count_add and will break * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET * is set and before current->softirq_enabled is cleared. * We must manually increment preempt_count here and manually * call the trace_preempt_off later. */ __preempt_count_add(cnt); /* * Were softirqs turned off above: */ if (softirq_count() == (cnt & SOFTIRQ_MASK)) trace_softirqs_off(ip); raw_local_irq_restore(flags); if (preempt_count() == cnt) { #ifdef CONFIG_DEBUG_PREEMPT current->preempt_disable_ip = get_lock_parent_ip(); #endif trace_preempt_off(CALLER_ADDR0, get_lock_parent_ip()); } }

Contributors

PersonTokensPropCommitsCommitProp
ingo molnaringo molnar4148.24%111.11%
heiko carstensheiko carstens1214.12%111.11%
steven rostedtsteven rostedt1214.12%111.11%
peter zijlstrapeter zijlstra910.59%444.44%
venkatesh pallipadivenkatesh pallipadi78.24%111.11%
sebastian andrzej siewiorsebastian andrzej siewior44.71%111.11%
Total85100.00%9100.00%

EXPORT_SYMBOL(__local_bh_disable_ip); #endif /* CONFIG_TRACE_IRQFLAGS */
static void __local_bh_enable(unsigned int cnt) { WARN_ON_ONCE(!irqs_disabled()); if (softirq_count() == (cnt & SOFTIRQ_MASK)) trace_softirqs_on(_RET_IP_); preempt_count_sub(cnt); }

Contributors

PersonTokensPropCommitsCommitProp
ingo molnaringo molnar2257.89%120.00%
venkatesh pallipadivenkatesh pallipadi1026.32%120.00%
peter zijlstrapeter zijlstra513.16%240.00%
davidlohr buesodavidlohr bueso12.63%120.00%
Total38100.00%5100.00%

/* * Special-case - softirqs can safely be enabled in * cond_resched_softirq(), or by __do_softirq(), * without processing still-pending softirqs: */
void _local_bh_enable(void) { WARN_ON_ONCE(in_irq()); __local_bh_enable(SOFTIRQ_DISABLE_OFFSET); }

Contributors

PersonTokensPropCommitsCommitProp
venkatesh pallipadivenkatesh pallipadi950.00%133.33%
frederic weisbeckerfrederic weisbecker633.33%133.33%
ingo molnaringo molnar316.67%133.33%
Total18100.00%3100.00%

EXPORT_SYMBOL(_local_bh_enable);
void __local_bh_enable_ip(unsigned long ip, unsigned int cnt) { WARN_ON_ONCE(in_irq() || irqs_disabled()); #ifdef CONFIG_TRACE_IRQFLAGS local_irq_disable(); #endif /* * Are softirqs going to be turned on now: */ if (softirq_count() == SOFTIRQ_DISABLE_OFFSET) trace_softirqs_on(ip); /* * Keep preemption disabled until we are done with * softirq processing: */ preempt_count_sub(cnt - 1); if (unlikely(!in_interrupt() && local_softirq_pending())) { /* * Run softirq if any pending. And do it in its own stack * as we may be calling this deep in a task call stack already. */ do_softirq(); } preempt_count_dec(); #ifdef CONFIG_TRACE_IRQFLAGS local_irq_enable(); #endif preempt_check_resched(); }

Contributors

PersonTokensPropCommitsCommitProp
ingo molnaringo molnar5161.45%112.50%
tim chentim chen1012.05%112.50%
johannes bergjohannes berg910.84%112.50%
peter zijlstrapeter zijlstra89.64%225.00%
frederic weisbeckerfrederic weisbecker33.61%112.50%
joe perchesjoe perches11.20%112.50%
venkatesh pallipadivenkatesh pallipadi11.20%112.50%
Total83100.00%8100.00%

EXPORT_SYMBOL(__local_bh_enable_ip); /* * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times, * but break the loop if need_resched() is set or after 2 ms. * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in * certain cases, such as stop_machine(), jiffies may cease to * increment and so we need the MAX_SOFTIRQ_RESTART limit as * well to make sure we eventually return from this method. * * These limits have been established via experimentation. * The two things to balance is latency against fairness - * we want to handle softirqs as soon as possible, but they * should not be able to lock up the box. */ #define MAX_SOFTIRQ_TIME msecs_to_jiffies(2) #define MAX_SOFTIRQ_RESTART 10 #ifdef CONFIG_TRACE_IRQFLAGS /* * When we run softirqs from irq_exit() and thus on the hardirq stack we need * to keep the lockdep irq context tracking as tight as possible in order to * not miss-qualify lock contexts and miss possible deadlocks. */
static inline bool lockdep_softirq_start(void) { bool in_hardirq = false; if (trace_hardirq_context(current)) { in_hardirq = true; trace_hardirq_exit(); } lockdep_softirq_enter(); return in_hardirq; }

Contributors

PersonTokensPropCommitsCommitProp
peter zijlstrapeter zijlstra1952.78%150.00%
frederic weisbeckerfrederic weisbecker1747.22%150.00%
Total36100.00%2100.00%


static inline void lockdep_softirq_end(bool in_hardirq) { lockdep_softirq_exit(); if (in_hardirq) trace_hardirq_enter(); }

Contributors

PersonTokensPropCommitsCommitProp
peter zijlstrapeter zijlstra1785.00%150.00%
frederic weisbeckerfrederic weisbecker315.00%150.00%
Total20100.00%2100.00%

#else
static inline bool lockdep_softirq_start(void) { return false; }

Contributors

PersonTokensPropCommitsCommitProp
peter zijlstrapeter zijlstra650.00%150.00%
frederic weisbeckerfrederic weisbecker650.00%150.00%
Total12100.00%2100.00%


static inline void lockdep_softirq_end(bool in_hardirq) { }

Contributors

PersonTokensPropCommitsCommitProp
peter zijlstrapeter zijlstra777.78%150.00%
frederic weisbeckerfrederic weisbecker222.22%150.00%
Total9100.00%2100.00%

#endif
asmlinkage __visible void __softirq_entry __do_softirq(void) { unsigned long end = jiffies + MAX_SOFTIRQ_TIME; unsigned long old_flags = current->flags; int max_restart = MAX_SOFTIRQ_RESTART; struct softirq_action *h; bool in_hardirq; __u32 pending; int softirq_bit; /* * Mask out PF_MEMALLOC s current task context is borrowed for the * softirq. A softirq handled such as network RX might set PF_MEMALLOC * again if the socket is related to swap */ current->flags &= ~PF_MEMALLOC; pending = local_softirq_pending(); account_irq_enter_time(current); __local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET); in_hardirq = lockdep_softirq_start(); restart: /* Reset the pending bitmask before enabling irqs */ set_softirq_pending(0); local_irq_enable(); h = softirq_vec; while ((softirq_bit = ffs(pending))) { unsigned int vec_nr; int prev_count; h += softirq_bit - 1; vec_nr = h - softirq_vec; prev_count = preempt_count(); kstat_incr_softirqs_this_cpu(vec_nr); trace_softirq_entry(vec_nr); h->action(h); trace_softirq_exit(vec_nr); if (unlikely(prev_count != preempt_count())) { pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n", vec_nr, softirq_to_name[vec_nr], h->action, prev_count, preempt_count()); preempt_count_set(prev_count); } h++; pending >>= softirq_bit; } rcu_bh_qs(); local_irq_disable(); pending = local_softirq_pending(); if (pending) { if (time_before(jiffies, end) && !need_resched() && --max_restart) goto restart; wakeup_softirqd(); } lockdep_softirq_end(in_hardirq); account_irq_exit_time(current); __local_bh_enable(SOFTIRQ_OFFSET); WARN_ON_ONCE(in_interrupt()); tsk_restore_flags(current, old_flags, PF_MEMALLOC); }

Contributors

PersonTokensPropCommitsCommitProp
thomas gleixnerthomas gleixner4115.59%25.41%
pre-gitpre-git3412.93%410.81%
joe perchesjoe perches2810.65%25.41%
mel gormanmel gorman259.51%12.70%
eric dumazeteric dumazet238.75%25.41%
frederic weisbeckerfrederic weisbecker166.08%38.11%
linus torvaldslinus torvalds166.08%38.11%
peter zijlstrapeter zijlstra155.70%38.11%
jason baronjason baron124.56%25.41%
andrew mortonandrew morton103.80%38.11%
ben greearben greear83.04%12.70%
paul mackerraspaul mackerras83.04%12.70%
venkatesh pallipadivenkatesh pallipadi62.28%12.70%
david s. millerdavid s. miller51.90%12.70%
ingo molnaringo molnar51.90%25.41%
keika kobayashikeika kobayashi41.52%12.70%
andi kleenandi kleen41.52%25.41%
dipankar sarmadipankar sarma10.38%12.70%
davidlohr buesodavidlohr bueso10.38%12.70%
alexander potapenkoalexander potapenko10.38%12.70%
Total263100.00%37100.00%


asmlinkage __visible void do_softirq(void) { __u32 pending; unsigned long flags; if (in_interrupt()) return; local_irq_save(flags); pending = local_softirq_pending(); if (pending) do_softirq_own_stack(); local_irq_restore(flags); }

Contributors

PersonTokensPropCommitsCommitProp
andrew mortonandrew morton3681.82%125.00%
linus torvaldslinus torvalds613.64%125.00%
frederic weisbeckerfrederic weisbecker12.27%125.00%
andi kleenandi kleen12.27%125.00%
Total44100.00%4100.00%

/* * Enter an interrupt context. */
void irq_enter(void) { rcu_irq_enter(); if (is_idle_task(current) && !in_interrupt()) { /* * Prevent raise_softirq from needlessly waking up ksoftirqd * here, as softirq will be serviced on return from interrupt. */ local_bh_disable(); tick_irq_enter(); _local_bh_enable(); } __irq_enter(); }

Contributors

PersonTokensPropCommitsCommitProp
venkatesh pallipadivenkatesh pallipadi1541.67%222.22%
ingo molnaringo molnar822.22%111.11%
thomas gleixnerthomas gleixner616.67%222.22%
frederic weisbeckerfrederic weisbecker411.11%333.33%
paul e. mckenneypaul e. mckenney38.33%111.11%
Total36100.00%9100.00%


static inline void invoke_softirq(void) { if (!force_irqthreads) { #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK /* * We can safely execute softirq on the current stack if * it is the irq stack, because it should be near empty * at this stage. */ __do_softirq(); #else /* * Otherwise, irq_exit() is called on the task stack that can * be potentially deep already. So call softirq in its own stack * to prevent from any overrun. */ do_softirq_own_stack(); #endif } else { wakeup_softirqd(); } }

Contributors

PersonTokensPropCommitsCommitProp
thomas gleixnerthomas gleixner1951.35%120.00%
frederic weisbeckerfrederic weisbecker1745.95%360.00%
christoph hellwigchristoph hellwig12.70%120.00%
Total37100.00%5100.00%


static inline void tick_irq_exit(void) { #ifdef CONFIG_NO_HZ_COMMON int cpu = smp_processor_id(); /* Make sure that timer wheel updates are propagated */ if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) { if (!in_interrupt()) tick_nohz_irq_exit(); } #endif }

Contributors

PersonTokensPropCommitsCommitProp
frederic weisbeckerfrederic weisbecker50100.00%1100.00%
Total50100.00%1100.00%

/* * Exit an interrupt context. Process softirqs if needed and possible: */
void irq_exit(void) { #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED local_irq_disable(); #else WARN_ON_ONCE(!irqs_disabled()); #endif account_irq_exit_time(current); preempt_count_sub(HARDIRQ_OFFSET); if (!in_interrupt() && local_softirq_pending()) invoke_softirq(); tick_irq_exit(); rcu_irq_exit(); trace_hardirq_exit(); /* must be last! */ }

Contributors

PersonTokensPropCommitsCommitProp
christoph hellwigchristoph hellwig2035.71%17.69%
thomas gleixnerthomas gleixner1628.57%215.38%
frederic weisbeckerfrederic weisbecker916.07%646.15%
peter zijlstrapeter zijlstra58.93%215.38%
martin schwidefskymartin schwidefsky47.14%17.69%
ingo molnaringo molnar23.57%17.69%
Total56100.00%13100.00%

/* * This function must run with irqs disabled! */
inline void raise_softirq_irqoff(unsigned int nr) { __raise_softirq_irqoff(nr); /* * If we're in an interrupt or softirq, we're done * (this also catches softirq-disabled code). We will * actually run the softirq once we return from * the irq or softirq. * * Otherwise we wake up ksoftirqd to make sure we * schedule the softirq soon. */ if (!in_interrupt()) wakeup_softirqd(); }

Contributors

PersonTokensPropCommitsCommitProp
linus torvaldslinus torvalds1976.00%120.00%
ingo molnaringo molnar312.00%240.00%
rusty russellrusty russell312.00%240.00%
Total25100.00%5100.00%


void raise_softirq(unsigned int nr) { unsigned long flags; local_irq_save(flags); raise_softirq_irqoff(nr); local_irq_restore(flags); }

Contributors

PersonTokensPropCommitsCommitProp
linus torvaldslinus torvalds2589.29%350.00%
rusty russellrusty russell13.57%116.67%
robert loverobert love13.57%116.67%
pre-gitpre-git13.57%116.67%
Total28100.00%6100.00%


void __raise_softirq_irqoff(unsigned int nr) { trace_softirq_raise(nr); or_softirq_pending(1UL << nr); }

Contributors

PersonTokensPropCommitsCommitProp
steven rostedtsteven rostedt21100.00%1100.00%
Total21100.00%1100.00%


void open_softirq(int nr, void (*action)(struct softirq_action *)) { softirq_vec[nr].action = action; }

Contributors

PersonTokensPropCommitsCommitProp
pre-gitpre-git28100.00%3100.00%
Total28100.00%3100.00%

/* * Tasklets */ struct tasklet_head { struct tasklet_struct *head; struct tasklet_struct **tail; }; static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec); static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
void __tasklet_schedule(struct tasklet_struct *t) { unsigned long flags; local_irq_save(flags); t->next = NULL; *__this_cpu_read(tasklet_vec.tail) = t; __this_cpu_write(tasklet_vec.tail, &(t->next)); raise_softirq_irqoff(TASKLET_SOFTIRQ); local_irq_restore(flags); }

Contributors

PersonTokensPropCommitsCommitProp
linus torvaldslinus torvalds3966.10%233.33%
olof johanssonolof johansson1220.34%116.67%
christoph lameterchristoph lameter58.47%116.67%
rusty russellrusty russell35.08%233.33%
Total59100.00%6100.00%

EXPORT_SYMBOL(__tasklet_schedule);
void __tasklet_hi_schedule(struct tasklet_struct *t) { unsigned long flags; local_irq_save(flags); t->next = NULL; *__this_cpu_read(tasklet_hi_vec.tail) = t; __this_cpu_write(tasklet_hi_vec.tail, &(t->next)); raise_softirq_irqoff(HI_SOFTIRQ); local_irq_restore(flags); }

Contributors

PersonTokensPropCommitsCommitProp
linus torvaldslinus torvalds3966.10%233.33%
olof johanssonolof johansson1220.34%116.67%
christoph lameterchristoph lameter58.47%116.67%
rusty russellrusty russell35.08%233.33%
Total59100.00%6100.00%

EXPORT_SYMBOL(__tasklet_hi_schedule);
void __tasklet_hi_schedule_first(struct tasklet_struct *t) { BUG_ON(!irqs_disabled()); t->next = __this_cpu_read(tasklet_hi_vec.head); __this_cpu_write(tasklet_hi_vec.head, t); __raise_softirq_irqoff(HI_SOFTIRQ); }

Contributors

PersonTokensPropCommitsCommitProp
vegard nossumvegard nossum3788.10%150.00%
christoph lameterchristoph lameter511.90%150.00%
Total42100.00%2100.00%

EXPORT_SYMBOL(__tasklet_hi_schedule_first);
static void tasklet_action(struct softirq_action *a) { struct tasklet_struct *list; local_irq_disable(); list = __this_cpu_read(tasklet_vec.head); __this_cpu_write(tasklet_vec.head, NULL); __this_cpu_write(tasklet_vec.tail, this_cpu_ptr(&tasklet_vec.head)); local_irq_enable(); while (list) { struct tasklet_struct *t = list; list = list->next; if (tasklet_trylock(t)) { if (!atomic_read(&t->count)) { if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) BUG(); t->func(t->data); tasklet_unlock(t); continue; } tasklet_unlock(t); } local_irq_disable(); t->next = NULL; *__this_cpu_read(tasklet_vec.tail) = t; __this_cpu_write(tasklet_vec.tail, &(t->next)); __raise_softirq_irqoff(TASKLET_SOFTIRQ); local_irq_enable(); } }

Contributors

PersonTokensPropCommitsCommitProp
pre-gitpre-git8951.45%535.71%
linus torvaldslinus torvalds4023.12%428.57%
olof johanssonolof johansson2313.29%17.14%
christoph lameterchristoph lameter169.25%214.29%
rusty russellrusty russell52.89%214.29%
Total173100.00%14100.00%


static void tasklet_hi_action(struct softirq_action *a) { struct tasklet_struct *list; local_irq_disable(); list = __this_cpu_read(tasklet_hi_vec.head); __this_cpu_write(tasklet_hi_vec.head, NULL); __this_cpu_write(tasklet_hi_vec.tail, this_cpu_ptr(&tasklet_hi_vec.head)); local_irq_enable(); while (list) { struct tasklet_struct *t = list; list = list->next; if (tasklet_trylock(t)) { if (!atomic_read(&t->count)) { if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) BUG(); t->func(t->data); tasklet_unlock(t); continue; } tasklet_unlock(t); } local_irq_disable(); t->next = NULL; *__this_cpu_read(tasklet_hi_vec.tail) = t; __this_cpu_write(tasklet_hi_vec.tail, &(t->next)); __raise_softirq_irqoff(HI_SOFTIRQ); local_irq_enable(); } }

Contributors

PersonTokensPropCommitsCommitProp
pre-gitpre-git8951.45%325.00%
linus torvaldslinus torvalds4023.12%433.33%
olof johanssonolof johansson2313.29%18.33%
christoph lameterchristoph lameter169.25%216.67%
rusty russellrusty russell52.89%216.67%
Total173100.00%12100.00%


void tasklet_init(struct tasklet_struct *t, void (*func)(unsigned long), unsigned long data) { t->next = NULL; t->state = 0; atomic_set(&t->count, 0); t->func = func; t->data = data; }

Contributors

PersonTokensPropCommitsCommitProp
pre-gitpre-git4475.86%150.00%
linus torvaldslinus torvalds1424.14%150.00%
Total58100.00%2100.00%

EXPORT_SYMBOL(tasklet_init);
void tasklet_kill(struct tasklet_struct *t) { if (in_interrupt()) pr_notice("Attempt to kill tasklet from interrupt\n"); while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) { do { yield(); } while (test_bit(TASKLET_STATE_SCHED, &t->state)); } tasklet_unlock_wait(t); clear_bit(TASKLET_STATE_SCHED, &t->state); }

Contributors

PersonTokensPropCommitsCommitProp
pre-gitpre-git6494.12%562.50%
h hartley sweetenh hartley sweeten22.94%112.50%
joe perchesjoe perches11.47%112.50%
linus torvaldslinus torvalds11.47%112.50%
Total68100.00%8100.00%

EXPORT_SYMBOL(tasklet_kill); /* * tasklet_hrtimer */ /* * The trampoline is called when the hrtimer expires. It schedules a tasklet * to run __tasklet_hrtimer_trampoline() which in turn will call the intended * hrtimer callback, but from softirq context. */
static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer) { struct tasklet_hrtimer *ttimer = container_of(timer, struct tasklet_hrtimer, timer); tasklet_hi_schedule(&ttimer->tasklet); return HRTIMER_NORESTART; }

Contributors

PersonTokensPropCommitsCommitProp
peter zijlstrapeter zijlstra38100.00%1100.00%
Total38100.00%1100.00%

/* * Helper function which calls the hrtimer callback from * tasklet/softirq context */
static void __tasklet_hrtimer_trampoline(unsigned long data) { struct tasklet_hrtimer *ttimer = (void *)data; enum hrtimer_restart restart; restart = ttimer->function(&ttimer->timer); if (restart != HRTIMER_NORESTART) hrtimer_restart(&ttimer->timer); }

Contributors

PersonTokensPropCommitsCommitProp
peter zijlstrapeter zijlstra51100.00%1100.00%
Total51100.00%1100.00%

/** * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks * @ttimer: tasklet_hrtimer which is initialized * @function: hrtimer callback function which gets called from softirq context * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME) * @mode: hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL) */ void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer, enum hrtimer_restart (*function)(struct hrtimer *), clockid_t which_clock, enum hrtimer_mode mode) { hrtimer_init(&ttimer->timer, which_clock, mode); ttimer->timer.function = __hrtimer_tasklet_trampoline; tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline, (unsigned long)ttimer); ttimer->function = function; } EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
void __init softirq_init(void) { int cpu; for_each_possible_cpu(cpu) { per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head; per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head; } open_softirq(TASKLET_SOFTIRQ, tasklet_action); open_softirq(HI_SOFTIRQ, tasklet_hi_action); }

Contributors

PersonTokensPropCommitsCommitProp
olof johanssonolof johansson4768.12%125.00%
pre-gitpre-git1927.54%250.00%
linus torvaldslinus torvalds34.35%125.00%
Total69100.00%4100.00%


static int ksoftirqd_should_run(unsigned int cpu) { return local_softirq_pending(); }

Contributors

PersonTokensPropCommitsCommitProp
thomas gleixnerthomas gleixner642.86%125.00%
linus torvaldslinus torvalds535.71%125.00%
david s. millerdavid s. miller214.29%125.00%
zwane mwaikambozwane mwaikambo17.14%125.00%
Total14100.00%4100.00%


static void run_ksoftirqd(unsigned int cpu) { local_irq_disable(); if (local_softirq_pending()) { /* * We can safely run softirq on inline stack, as we are not deep * in the task stack here. */ __do_softirq(); local_irq_enable(); cond_resched_rcu_qs(); return; } local_irq_enable(); }

Contributors

PersonTokensPropCommitsCommitProp
thomas gleixnerthomas gleixner2470.59%228.57%
rusty russellrusty russell514.71%114.29%
linus torvaldslinus torvalds25.88%114.29%
paul e. mckenneypaul e. mckenney12.94%114.29%
calvin owenscalvin owens12.94%114.29%
frederic weisbeckerfrederic weisbecker12.94%114.29%
Total34100.00%7100.00%

#ifdef CONFIG_HOTPLUG_CPU /* * tasklet_kill_immediate is called to remove a tasklet which can already be * scheduled for execution on @cpu. * * Unlike tasklet_kill, this function removes the tasklet * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state. * * When this function is called, @cpu must be in the CPU_DEAD state. */
void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu) { struct tasklet_struct **i; BUG_ON(cpu_online(cpu)); BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state)); if (!test_bit(TASKLET_STATE_SCHED, &t->state)) return; /* CPU is dead, so no lock needed. */ for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) { if (*i == t) { *i = t->next; /* If this was the tail element, move the tail ptr */ if (*i == NULL) per_cpu(tasklet_vec, cpu).tail = i; return; } } BUG(); }

Contributors

PersonTokensPropCommitsCommitProp
rusty russellrusty russell10483.87%150.00%
olof johanssonolof johansson2016.13%150.00%
Total124100.00%2100.00%


static void takeover_tasklets(unsigned int cpu) { /* CPU is dead, so no lock needed. */ local_irq_disable(); /* Find end, append list for that CPU. */ if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) { *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head; this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail); per_cpu(tasklet_vec, cpu).head = NULL; per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head; } raise_softirq_irqoff(TASKLET_SOFTIRQ); if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) { *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head; __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail); per_cpu(tasklet_hi_vec, cpu).head = NULL; per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head; } raise_softirq_irqoff(HI_SOFTIRQ); local_irq_enable(); }

Contributors

PersonTokensPropCommitsCommitProp
olof johanssonolof johansson7236.00%125.00%
rusty russellrusty russell7236.00%125.00%
christian borntraegerchristian borntraeger4623.00%125.00%
christoph lameterchristoph lameter105.00%125.00%
Total200100.00%4100.00%

#endif /* CONFIG_HOTPLUG_CPU */
static int cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { switch (action) { #ifdef CONFIG_HOTPLUG_CPU case CPU_DEAD: case CPU_DEAD_FROZEN: takeover_tasklets((unsigned long)hcpu); break; #endif /* CONFIG_HOTPLUG_CPU */ } return NOTIFY_OK; }

Contributors

PersonTokensPropCommitsCommitProp
rusty russellrusty russell3060.00%333.33%
linus torvaldslinus torvalds714.00%111.11%
thomas gleixnerthomas gleixner510.00%111.11%
andrew mortonandrew morton510.00%333.33%
rafael j. wysockirafael j. wysocki36.00%111.11%
Total50100.00%9100.00%

static struct notifier_block cpu_nfb = { .notifier_call = cpu_callback }; static struct smp_hotplug_thread softirq_threads = { .store = &ksoftirqd, .thread_should_run = ksoftirqd_should_run, .thread_fn = run_ksoftirqd, .thread_comm = "ksoftirqd/%u", };
static __init int spawn_ksoftirqd(void) { register_cpu_notifier(&cpu_nfb); BUG_ON(smpboot_register_percpu_thread(&softirq_threads)); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
rusty russellrusty russell1037.04%116.67%
andrew mortonandrew morton622.22%116.67%
linus torvaldslinus torvalds414.81%116.67%
thomas gleixnerthomas gleixner414.81%116.67%
akinobu mitaakinobu mita27.41%116.67%
eduard gabriel munteanueduard gabriel munteanu13.70%116.67%
Total27100.00%6100.00%

early_initcall(spawn_ksoftirqd); /* * [ These __weak aliases are kept in a separate compilation unit, so that * GCC does not inline them incorrectly. ] */
int __init __weak early_irq_init(void) { return 0; }

Contributors

PersonTokensPropCommitsCommitProp
yinghai luyinghai lu12100.00%1100.00%
Total12100.00%1100.00%


int __init __weak arch_probe_nr_irqs(void) { return NR_IRQS_LEGACY; }

Contributors

PersonTokensPropCommitsCommitProp
yinghai luyinghai lu1191.67%150.00%
thomas gleixnerthomas gleixner18.33%150.00%
Total12100.00%2100.00%


int __init __weak arch_early_irq_init(void) { return 0; }

Contributors

PersonTokensPropCommitsCommitProp
yinghai luyinghai lu12100.00%1100.00%
Total12100.00%1100.00%


unsigned int __weak arch_dynirq_lower_bound(unsigned int from) { return from; }

Contributors

PersonTokensPropCommitsCommitProp
thomas gleixnerthomas gleixner14100.00%1100.00%
Total14100.00%1100.00%


Overall Contributors

PersonTokensPropCommitsCommitProp
pre-gitpre-git39115.16%107.46%
rusty russellrusty russell30011.63%96.72%
linus torvaldslinus torvalds29311.36%85.97%
peter zijlstrapeter zijlstra27010.47%75.22%
olof johanssonolof johansson2168.38%10.75%
thomas gleixnerthomas gleixner1746.75%128.96%
ingo molnaringo molnar1505.82%64.48%
frederic weisbeckerfrederic weisbecker1365.27%1611.94%
andrew mortonandrew morton833.22%96.72%
christoph lameterchristoph lameter582.25%21.49%
venkatesh pallipadivenkatesh pallipadi501.94%32.24%
christian borntraegerchristian borntraeger461.78%10.75%
steven rostedtsteven rostedt451.74%42.99%
vegard nossumvegard nossum421.63%10.75%
joe perchesjoe perches391.51%32.24%
yinghai luyinghai lu361.40%21.49%
jason baronjason baron291.12%21.49%
mel gormanmel gorman250.97%10.75%
eric dumazeteric dumazet250.97%21.49%
christoph hellwigchristoph hellwig230.89%21.49%
arnaldo carvalho de meloarnaldo carvalho de melo200.78%10.75%
tim chentim chen190.74%10.75%
heiko carstensheiko carstens150.58%21.49%
ben greearben greear130.50%10.75%
johannes bergjohannes berg90.35%10.75%
david s. millerdavid s. miller90.35%21.49%
paul mackerraspaul mackerras80.31%10.75%
rafael j. wysockirafael j. wysocki60.23%21.49%
eduard gabriel munteanueduard gabriel munteanu60.23%10.75%
andi kleenandi kleen50.19%21.49%
sebastian andrzej siewiorsebastian andrzej siewior40.16%10.75%
paul e. mckenneypaul e. mckenney40.16%21.49%
dipankar sarmadipankar sarma40.16%10.75%
keika kobayashikeika kobayashi40.16%10.75%
martin schwidefskymartin schwidefsky40.16%10.75%
li zefanli zefan20.08%10.75%
akinobu mitaakinobu mita20.08%10.75%
davidlohr buesodavidlohr bueso20.08%10.75%
h hartley sweetenh hartley sweeten20.08%10.75%
li shaohuali shaohua20.08%10.75%
alexander potapenkoalexander potapenko10.04%10.75%
alexey dobriyanalexey dobriyan10.04%10.75%
lucas de marchilucas de marchi10.04%10.75%
paul gortmakerpaul gortmaker10.04%10.75%
calvin owenscalvin owens10.04%10.75%
zwane mwaikambozwane mwaikambo10.04%10.75%
dave jonesdave jones10.04%10.75%
robert loverobert love10.04%10.75%
Total2579100.00%134100.00%
Directory: kernel
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
{% endraw %}