cregit-Linux how code gets into the kernel

Release 4.14 arch/xtensa/kernel/smp.c

/*
 * Xtensa SMP support functions.
 *
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) 2008 - 2013 Tensilica Inc.
 *
 * Chris Zankel <chris@zankel.net>
 * Joe Taylor <joe@tensilica.com>
 * Pete Delaney <piet@tensilica.com
 */

#include <linux/cpu.h>
#include <linux/cpumask.h>
#include <linux/delay.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/irqdomain.h>
#include <linux/irq.h>
#include <linux/kdebug.h>
#include <linux/module.h>
#include <linux/sched/mm.h>
#include <linux/sched/hotplug.h>
#include <linux/sched/task_stack.h>
#include <linux/reboot.h>
#include <linux/seq_file.h>
#include <linux/smp.h>
#include <linux/thread_info.h>

#include <asm/cacheflush.h>
#include <asm/kdebug.h>
#include <asm/mmu_context.h>
#include <asm/mxregs.h>
#include <asm/platform.h>
#include <asm/tlbflush.h>
#include <asm/traps.h>

#ifdef CONFIG_SMP
# if XCHAL_HAVE_S32C1I == 0
#  error "The S32C1I option is required for SMP."
# endif
#endif

static void system_invalidate_dcache_range(unsigned long start,
		unsigned long size);
static void system_flush_invalidate_dcache_range(unsigned long start,
		unsigned long size);

/* IPI (Inter Process Interrupt) */


#define IPI_IRQ	0

static irqreturn_t ipi_interrupt(int irq, void *dev_id);

static struct irqaction ipi_irqaction = {
	.handler =	ipi_interrupt,
	.flags =	IRQF_PERCPU,
	.name =		"ipi",
};


void ipi_init(void) { unsigned irq = irq_create_mapping(NULL, IPI_IRQ); setup_irq(irq, &ipi_irqaction); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov25100.00%1100.00%
Total25100.00%1100.00%


static inline unsigned int get_core_count(void) { /* Bits 18..21 of SYSCFGID contain the core count minus 1. */ unsigned int syscfgid = get_er(SYSCFGID); return ((syscfgid >> 18) & 0xf) + 1; }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov33100.00%1100.00%
Total33100.00%1100.00%


static inline int get_core_id(void) { /* Bits 0...18 of SYSCFGID contain the core id */ unsigned int core_id = get_er(SYSCFGID); return core_id & 0x3fff; }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov24100.00%1100.00%
Total24100.00%1100.00%


void __init smp_prepare_cpus(unsigned int max_cpus) { unsigned i; for (i = 0; i < max_cpus; ++i) set_cpu_present(i, true); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov33100.00%1100.00%
Total33100.00%1100.00%


void __init smp_init_cpus(void) { unsigned i; unsigned int ncpus = get_core_count(); unsigned int core_id = get_core_id(); pr_info("%s: Core Count = %d\n", __func__, ncpus); pr_info("%s: Core Id = %d\n", __func__, core_id); for (i = 0; i < ncpus; ++i) set_cpu_possible(i, true); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov63100.00%1100.00%
Total63100.00%1100.00%


void __init smp_prepare_boot_cpu(void) { unsigned int cpu = smp_processor_id(); BUG_ON(cpu != 0); cpu_asid_cache(cpu) = ASID_USER_FIRST; }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov29100.00%1100.00%
Total29100.00%1100.00%


void __init smp_cpus_done(unsigned int max_cpus) { }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov9100.00%1100.00%
Total9100.00%1100.00%

static int boot_secondary_processors = 1; /* Set with xt-gdb via .xt-gdb */ static DECLARE_COMPLETION(cpu_running);
void secondary_start_kernel(void) { struct mm_struct *mm = &init_mm; unsigned int cpu = smp_processor_id(); init_mmu(); #ifdef CONFIG_DEBUG_KERNEL if (boot_secondary_processors == 0) { pr_debug("%s: boot_secondary_processors:%d; Hanging cpu:%d\n", __func__, boot_secondary_processors, cpu); for (;;) __asm__ __volatile__ ("waiti " __stringify(LOCKLEVEL)); } pr_debug("%s: boot_secondary_processors:%d; Booting cpu:%d\n", __func__, boot_secondary_processors, cpu); #endif /* Init EXCSAVE1 */ secondary_trap_init(); /* All kernel threads share the same mm context. */ mmget(mm); mmgrab(mm); current->active_mm = mm; cpumask_set_cpu(cpu, mm_cpumask(mm)); enter_lazy_tlb(mm, current); preempt_disable(); trace_hardirqs_off(); calibrate_delay(); notify_cpu_starting(cpu); secondary_init_irq(); local_timer_setup(cpu); set_cpu_online(cpu, true); local_irq_enable(); complete(&cpu_running); cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov14395.97%120.00%
Kirill V Tkhai32.01%120.00%
Vegard Nossum21.34%240.00%
Thomas Gleixner10.67%120.00%
Total149100.00%5100.00%


static void mx_cpu_start(void *p) { unsigned cpu = (unsigned)p; unsigned long run_stall_mask = get_er(MPSCORE); set_er(run_stall_mask & ~(1u << cpu), MPSCORE); pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n", __func__, cpu, run_stall_mask, get_er(MPSCORE)); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov57100.00%1100.00%
Total57100.00%1100.00%


static void mx_cpu_stop(void *p) { unsigned cpu = (unsigned)p; unsigned long run_stall_mask = get_er(MPSCORE); set_er(run_stall_mask | (1u << cpu), MPSCORE); pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n", __func__, cpu, run_stall_mask, get_er(MPSCORE)); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov56100.00%1100.00%
Total56100.00%1100.00%

#ifdef CONFIG_HOTPLUG_CPU unsigned long cpu_start_id __cacheline_aligned; #endif unsigned long cpu_start_ccount;
static int boot_secondary(unsigned int cpu, struct task_struct *ts) { unsigned long timeout = jiffies + msecs_to_jiffies(1000); unsigned long ccount; int i; #ifdef CONFIG_HOTPLUG_CPU cpu_start_id = cpu; system_flush_invalidate_dcache_range( (unsigned long)&cpu_start_id, sizeof(cpu_start_id)); #endif smp_call_function_single(0, mx_cpu_start, (void *)cpu, 1); for (i = 0; i < 2; ++i) { do ccount = get_ccount(); while (!ccount); cpu_start_ccount = ccount; while (time_before(jiffies, timeout)) { mb(); if (!cpu_start_ccount) break; } if (cpu_start_ccount) { smp_call_function_single(0, mx_cpu_stop, (void *)cpu, 1); cpu_start_ccount = 0; return -EIO; } } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov155100.00%2100.00%
Total155100.00%2100.00%


int __cpu_up(unsigned int cpu, struct task_struct *idle) { int ret = 0; if (cpu_asid_cache(cpu) == 0) cpu_asid_cache(cpu) = ASID_USER_FIRST; start_info.stack = (unsigned long)task_pt_regs(idle); wmb(); pr_debug("%s: Calling wakeup_secondary(cpu:%d, idle:%p, sp: %08lx)\n", __func__, cpu, idle, start_info.stack); ret = boot_secondary(cpu, idle); if (ret == 0) { wait_for_completion_timeout(&cpu_running, msecs_to_jiffies(1000)); if (!cpu_online(cpu)) ret = -EIO; } if (ret) pr_err("CPU %u failed to boot\n", cpu); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov121100.00%1100.00%
Total121100.00%1100.00%

#ifdef CONFIG_HOTPLUG_CPU /* * __cpu_disable runs on the processor to be shutdown. */
int __cpu_disable(void) { unsigned int cpu = smp_processor_id(); /* * Take this CPU offline. Once we clear this, we can't return, * and we must not schedule until we're ready to give up the cpu. */ set_cpu_online(cpu, false); /* * OK - migrate IRQs away from this CPU */ migrate_irqs(); /* * Flush user cache and TLB mappings, and then remove this CPU * from the vm mask set of all processes. */ local_flush_cache_all(); local_flush_tlb_all(); invalidate_page_directory(); clear_tasks_mm_cpumask(cpu); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov44100.00%1100.00%
Total44100.00%1100.00%


static void platform_cpu_kill(unsigned int cpu) { smp_call_function_single(0, mx_cpu_stop, (void *)cpu, true); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov25100.00%1100.00%
Total25100.00%1100.00%

/* * called on the thread which is asking for a CPU to be shutdown - * waits until shutdown has completed, or it is timed out. */
void __cpu_die(unsigned int cpu) { unsigned long timeout = jiffies + msecs_to_jiffies(1000); while (time_before(jiffies, timeout)) { system_invalidate_dcache_range((unsigned long)&cpu_start_id, sizeof(cpu_start_id)); if (cpu_start_id == -cpu) { platform_cpu_kill(cpu); return; } } pr_err("CPU%u: unable to kill\n", cpu); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov68100.00%1100.00%
Total68100.00%1100.00%


void arch_cpu_idle_dead(void) { cpu_die(); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov10100.00%1100.00%
Total10100.00%1100.00%

/* * Called from the idle thread for the CPU which has been shutdown. * * Note that we disable IRQs here, but do not re-enable them * before returning to the caller. This is also the behaviour * of the other hotplug-cpu capable cores, so presumably coming * out of idle fixes this. */
void __ref cpu_die(void) { idle_task_exit(); local_irq_disable(); __asm__ __volatile__( " movi a2, cpu_restart\n" " jx a2\n"); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov17100.00%1100.00%
Total17100.00%1100.00%

#endif /* CONFIG_HOTPLUG_CPU */ enum ipi_msg_type { IPI_RESCHEDULE = 0, IPI_CALL_FUNC, IPI_CPU_STOP, IPI_MAX }; static const struct { const char *short_text; const char *long_text; } ipi_text[] = { { .short_text = "RES", .long_text = "Rescheduling interrupts" }, { .short_text = "CAL", .long_text = "Function call interrupts" }, { .short_text = "DIE", .long_text = "CPU shutdown interrupts" }, }; struct ipi_data { unsigned long ipi_count[IPI_MAX]; }; static DEFINE_PER_CPU(struct ipi_data, ipi_data);
static void send_ipi_message(const struct cpumask *callmask, enum ipi_msg_type msg_id) { int index; unsigned long mask = 0; for_each_cpu(index, callmask) if (index != smp_processor_id()) mask |= 1 << index; set_er(mask, MIPISET(msg_id)); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov54100.00%1100.00%
Total54100.00%1100.00%


void arch_send_call_function_ipi_mask(const struct cpumask *mask) { send_ipi_message(mask, IPI_CALL_FUNC); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov18100.00%1100.00%
Total18100.00%1100.00%


void arch_send_call_function_single_ipi(int cpu) { send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov18100.00%1100.00%
Total18100.00%1100.00%


void smp_send_reschedule(int cpu) { send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov18100.00%1100.00%
Total18100.00%1100.00%


void smp_send_stop(void) { struct cpumask targets; cpumask_copy(&targets, cpu_online_mask); cpumask_clear_cpu(smp_processor_id(), &targets); send_ipi_message(&targets, IPI_CPU_STOP); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov36100.00%1100.00%
Total36100.00%1100.00%


static void ipi_cpu_stop(unsigned int cpu) { set_cpu_online(cpu, false); machine_halt(); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov20100.00%1100.00%
Total20100.00%1100.00%


irqreturn_t ipi_interrupt(int irq, void *dev_id) { unsigned int cpu = smp_processor_id(); struct ipi_data *ipi = &per_cpu(ipi_data, cpu); unsigned int msg; unsigned i; msg = get_er(MIPICAUSE(cpu)); for (i = 0; i < IPI_MAX; i++) if (msg & (1 << i)) { set_er(1 << i, MIPICAUSE(cpu)); ++ipi->ipi_count[i]; } if (msg & (1 << IPI_RESCHEDULE)) scheduler_ipi(); if (msg & (1 << IPI_CALL_FUNC)) generic_smp_call_function_interrupt(); if (msg & (1 << IPI_CPU_STOP)) ipi_cpu_stop(cpu); return IRQ_HANDLED; }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov138100.00%1100.00%
Total138100.00%1100.00%


void show_ipi_list(struct seq_file *p, int prec) { unsigned int cpu; unsigned i; for (i = 0; i < IPI_MAX; ++i) { seq_printf(p, "%*s:", prec, ipi_text[i].short_text); for_each_online_cpu(cpu) seq_printf(p, " %10lu", per_cpu(ipi_data, cpu).ipi_count[i]); seq_printf(p, " %s\n", ipi_text[i].long_text); } }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov88100.00%1100.00%
Total88100.00%1100.00%


int setup_profiling_timer(unsigned int multiplier) { pr_debug("setup_profiling_timer %d\n", multiplier); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov19100.00%1100.00%
Total19100.00%1100.00%

/* TLB flush functions */ struct flush_data { struct vm_area_struct *vma; unsigned long addr1; unsigned long addr2; };
static void ipi_flush_tlb_all(void *arg) { local_flush_tlb_all(); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov13100.00%1100.00%
Total13100.00%1100.00%


void flush_tlb_all(void) { on_each_cpu(ipi_flush_tlb_all, NULL, 1); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov16100.00%1100.00%
Total16100.00%1100.00%


static void ipi_flush_tlb_mm(void *arg) { local_flush_tlb_mm(arg); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov15100.00%1100.00%
Total15100.00%1100.00%


void flush_tlb_mm(struct mm_struct *mm) { on_each_cpu(ipi_flush_tlb_mm, mm, 1); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov19100.00%1100.00%
Total19100.00%1100.00%


static void ipi_flush_tlb_page(void *arg) { struct flush_data *fd = arg; local_flush_tlb_page(fd->vma, fd->addr1); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov28100.00%1100.00%
Total28100.00%1100.00%


void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) { struct flush_data fd = { .vma = vma, .addr1 = addr, }; on_each_cpu(ipi_flush_tlb_page, &fd, 1); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov40100.00%1100.00%
Total40100.00%1100.00%


static void ipi_flush_tlb_range(void *arg) { struct flush_data *fd = arg; local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov32100.00%1100.00%
Total32100.00%1100.00%


void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { struct flush_data fd = { .vma = vma, .addr1 = start, .addr2 = end, }; on_each_cpu(ipi_flush_tlb_range, &fd, 1); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov49100.00%2100.00%
Total49100.00%2100.00%


static void ipi_flush_tlb_kernel_range(void *arg) { struct flush_data *fd = arg; local_flush_tlb_kernel_range(fd->addr1, fd->addr2); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov28100.00%1100.00%
Total28100.00%1100.00%


void flush_tlb_kernel_range(unsigned long start, unsigned long end) { struct flush_data fd = { .addr1 = start, .addr2 = end, }; on_each_cpu(ipi_flush_tlb_kernel_range, &fd, 1); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov39100.00%2100.00%
Total39100.00%2100.00%

/* Cache flush functions */
static void ipi_flush_cache_all(void *arg) { local_flush_cache_all(); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov13100.00%1100.00%
Total13100.00%1100.00%


void flush_cache_all(void) { on_each_cpu(ipi_flush_cache_all, NULL, 1); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov16100.00%1100.00%
Total16100.00%1100.00%


static void ipi_flush_cache_page(void *arg) { struct flush_data *fd = arg; local_flush_cache_page(fd->vma, fd->addr1, fd->addr2); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov32100.00%1100.00%
Total32100.00%1100.00%


void flush_cache_page(struct vm_area_struct *vma, unsigned long address, unsigned long pfn) { struct flush_data fd = { .vma = vma, .addr1 = address, .addr2 = pfn, }; on_each_cpu(ipi_flush_cache_page, &fd, 1); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov49100.00%1100.00%
Total49100.00%1100.00%


static void ipi_flush_cache_range(void *arg) { struct flush_data *fd = arg; local_flush_cache_range(fd->vma, fd->addr1, fd->addr2); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov32100.00%1100.00%
Total32100.00%1100.00%


void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { struct flush_data fd = { .vma = vma, .addr1 = start, .addr2 = end, }; on_each_cpu(ipi_flush_cache_range, &fd, 1); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov49100.00%1100.00%
Total49100.00%1100.00%


static void ipi_flush_icache_range(void *arg) { struct flush_data *fd = arg; local_flush_icache_range(fd->addr1, fd->addr2); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov28100.00%1100.00%
Total28100.00%1100.00%


void flush_icache_range(unsigned long start, unsigned long end) { struct flush_data fd = { .addr1 = start, .addr2 = end, }; on_each_cpu(ipi_flush_icache_range, &fd, 1); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov39100.00%1100.00%
Total39100.00%1100.00%

EXPORT_SYMBOL(flush_icache_range); /* ------------------------------------------------------------------------- */
static void ipi_invalidate_dcache_range(void *arg) { struct flush_data *fd = arg; __invalidate_dcache_range(fd->addr1, fd->addr2); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov28100.00%1100.00%
Total28100.00%1100.00%


static void system_invalidate_dcache_range(unsigned long start, unsigned long size) { struct flush_data fd = { .addr1 = start, .addr2 = size, }; on_each_cpu(ipi_invalidate_dcache_range, &fd, 1); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov40100.00%1100.00%
Total40100.00%1100.00%


static void ipi_flush_invalidate_dcache_range(void *arg) { struct flush_data *fd = arg; __flush_invalidate_dcache_range(fd->addr1, fd->addr2); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov28100.00%1100.00%
Total28100.00%1100.00%


static void system_flush_invalidate_dcache_range(unsigned long start, unsigned long size) { struct flush_data fd = { .addr1 = start, .addr2 = size, }; on_each_cpu(ipi_flush_invalidate_dcache_range, &fd, 1); }

Contributors

PersonTokensPropCommitsCommitProp
Max Filippov40100.00%1100.00%
Total40100.00%1100.00%


Overall Contributors

PersonTokensPropCommitsCommitProp
Max Filippov228399.13%327.27%
Ingo Molnar90.39%327.27%
Pranith Kumar50.22%19.09%
Kirill V Tkhai30.13%19.09%
Vegard Nossum20.09%218.18%
Thomas Gleixner10.04%19.09%
Total2303100.00%11100.00%
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with cregit.