Release 4.13 arch/x86/mm/tlb.c

Directory: arch/x86/mm
#include <linux/init.h>

#include <linux/mm.h>
#include <linux/spinlock.h>
#include <linux/smp.h>
#include <linux/interrupt.h>
#include <linux/export.h>
#include <linux/cpu.h>

#include <asm/tlbflush.h>
#include <asm/mmu_context.h>
#include <asm/cache.h>
#include <asm/apic.h>
#include <asm/uv/uv.h>
#include <linux/debugfs.h>

/*
 *      TLB flushing, formerly SMP-only
 *              c/o Linus Torvalds.
 *
 *      These mean you can really definitely utterly forget about
 *      writing to user space from interrupts. (Its not allowed anyway).
 *
 *      Optimizations Manfred Spraul <manfred@colorfullife.com>
 *
 *      More scalable flush, from Andi Kleen
 *
 *      Implement flush IPI by CALL_FUNCTION_VECTOR, Alex Shi
 */



void leave_mm(int cpu)
{
	struct mm_struct *loaded_mm = this_cpu_read(cpu_tlbstate.loaded_mm);

	/*
         * It's plausible that we're in lazy TLB mode while our mm is init_mm.
         * If so, our callers still expect us to flush the TLB, but there
         * aren't any user TLB entries in init_mm to worry about.
         *
         * This needs to happen before any other sanity checks due to
         * intel_idle's shenanigans.
         */
	if (loaded_mm == &init_mm)
		return;

	if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
		BUG();

	switch_mm(NULL, &init_mm, NULL);
}
Contributors
Person Tokens Prop Commits CommitProp
Glauber de Oliveira Costa 18 33.96% 1 14.29%
Andrew Lutomirski 16 30.19% 1 14.29%
Suresh B. Siddha 11 20.75% 1 14.29%
Brian Gerst 3 5.66% 1 14.29%
Dave Hansen 3 5.66% 1 14.29%
Alex Shi 1 1.89% 1 14.29%
Linus Torvalds 1 1.89% 1 14.29%
Total 53 100.00% 7 100.00%


EXPORT_SYMBOL_GPL(leave_mm);



void switch_mm(struct mm_struct *prev, struct mm_struct *next,
	       struct task_struct *tsk)
{
	unsigned long flags;

	local_irq_save(flags);
	switch_mm_irqs_off(prev, next, tsk);
	local_irq_restore(flags);
}

Contributors
Person Tokens Prop Commits CommitProp
Andrew Lutomirski 43 100.00% 2 100.00%
Total 43 100.00% 2 100.00%



void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
			struct task_struct *tsk)
{
	unsigned cpu = smp_processor_id();
	struct mm_struct *real_prev = this_cpu_read(cpu_tlbstate.loaded_mm);

	/*
         * NB: The scheduler will call us with prev == next when
         * switching from lazy TLB mode to normal mode if active_mm
         * isn't changing.  When this happens, there is no guarantee
         * that CR3 (and hence cpu_tlbstate.loaded_mm) matches next.
         *
         * NB: leave_mm() calls us with prev == NULL and tsk == NULL.
         */

	this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);

	if (real_prev == next) {
		/*
                 * There's nothing to do: we always keep the per-mm control
                 * regs in sync with cpu_tlbstate.loaded_mm.  Just
                 * sanity-check mm_cpumask.
                 */
		if (WARN_ON_ONCE(!cpumask_test_cpu(cpu, mm_cpumask(next))))
			cpumask_set_cpu(cpu, mm_cpumask(next));
		return;
	}

	if (IS_ENABLED(CONFIG_VMAP_STACK)) {
		/*
                 * If our current stack is in vmalloc space and isn't
                 * mapped in the new pgd, we'll double-fault.  Forcibly
                 * map it.
                 */
		unsigned int stack_pgd_index = pgd_index(current_stack_pointer());

		pgd_t *pgd = next->pgd + stack_pgd_index;

		if (unlikely(pgd_none(*pgd)))
			set_pgd(pgd, init_mm.pgd[stack_pgd_index]);
	}

	this_cpu_write(cpu_tlbstate.loaded_mm, next);

	WARN_ON_ONCE(cpumask_test_cpu(cpu, mm_cpumask(next)));
	cpumask_set_cpu(cpu, mm_cpumask(next));

	/*
         * Re-load page tables.
         *
         * This logic has an ordering constraint:
         *
         *  CPU 0: Write to a PTE for 'next'
         *  CPU 0: load bit 1 in mm_cpumask.  if nonzero, send IPI.
         *  CPU 1: set bit 1 in next's mm_cpumask
         *  CPU 1: load from the PTE that CPU 0 writes (implicit)
         *
         * We need to prevent an outcome in which CPU 1 observes
         * the new PTE value and CPU 0 observes bit 1 clear in
         * mm_cpumask.  (If that occurs, then the IPI will never
         * be sent, and CPU 0's TLB will contain a stale entry.)
         *
         * The bad outcome can occur if either CPU's load is
         * reordered before that CPU's store, so both CPUs must
         * execute full barriers to prevent this from happening.
         *
         * Thus, switch_mm needs a full barrier between the
         * store to mm_cpumask and any operation that could load
         * from next->pgd.  TLB fills are special and can happen
         * due to instruction fetches or for no reason at all,
         * and neither LOCK nor MFENCE orders them.
         * Fortunately, load_cr3() is serializing and gives the
         * ordering guarantee we need.
         */
	load_cr3(next->pgd);

	/*
         * This gets called via leave_mm() in the idle path where RCU
         * functions differently.  Tracing normally uses RCU, so we have to
         * call the tracepoint specially here.
         */
	trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);

	/* Stop flush ipis for the previous mm */
	WARN_ON_ONCE(!cpumask_test_cpu(cpu, mm_cpumask(real_prev)) &&
		     real_prev != &init_mm);
	cpumask_clear_cpu(cpu, mm_cpumask(real_prev));

	/* Load per-mm CR4 and LDTR state */
	load_mm_cr4(next);
	switch_ldt(real_prev, next);
}

Contributors
Person Tokens Prop Commits CommitProp
Andrew Lutomirski 228 100.00% 5 100.00%
Total 228 100.00% 5 100.00%



static void flush_tlb_func_common(const struct flush_tlb_info *f,
				  bool local, enum tlb_flush_reason reason)
{
	/* This code cannot presently handle being reentered. */
	VM_WARN_ON(!irqs_disabled());

	if (this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK) {
		leave_mm(smp_processor_id());
		return;
	}

	if (f->end == TLB_FLUSH_ALL) {
		local_flush_tlb();
		if (local)
			count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
		trace_tlb_flush(reason, TLB_FLUSH_ALL);
	} else {
		unsigned long addr;
		unsigned long nr_pages = (f->end - f->start) >> PAGE_SHIFT;
		addr = f->start;
		while (addr < f->end) {
			__flush_tlb_single(addr);
			addr += PAGE_SIZE;
		}
		if (local)
			count_vm_tlb_events(NR_TLB_LOCAL_FLUSH_ONE, nr_pages);
		trace_tlb_flush(reason, nr_pages);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Andrew Lutomirski 57 40.14% 5 41.67%
Alex Shi 32 22.54% 3 25.00%
Dave Hansen 26 18.31% 2 16.67%
Glauber de Oliveira Costa 24 16.90% 1 8.33%
Brian Gerst 3 2.11% 1 8.33%
Total 142 100.00% 12 100.00%



static void flush_tlb_func_local(void *info, enum tlb_flush_reason reason)
{
	const struct flush_tlb_info *f = info;

	flush_tlb_func_common(f, true, reason);
}

Contributors
Person Tokens Prop Commits CommitProp
Andrew Lutomirski 31 100.00% 1 100.00%
Total 31 100.00% 1 100.00%



static void flush_tlb_func_remote(void *info)
{
	const struct flush_tlb_info *f = info;

	inc_irq_stat(irq_tlb_count);

	if (f->mm && f->mm != this_cpu_read(cpu_tlbstate.loaded_mm))
		return;

	count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
	flush_tlb_func_common(f, false, TLB_REMOTE_SHOOTDOWN);
}

Contributors
Person Tokens Prop Commits CommitProp
Andrew Lutomirski 55 100.00% 2 100.00%
Total 55 100.00% 2 100.00%



void native_flush_tlb_others(const struct cpumask *cpumask,
			     const struct flush_tlb_info *info)
{
	count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);
	if (info->end == TLB_FLUSH_ALL)
		trace_tlb_flush(TLB_REMOTE_SEND_IPI, TLB_FLUSH_ALL);
	else
		trace_tlb_flush(TLB_REMOTE_SEND_IPI,
				(info->end - info->start) >> PAGE_SHIFT);

	if (is_uv_system()) {
		unsigned int cpu;

		cpu = smp_processor_id();
		cpumask = uv_flush_tlb_others(cpumask, info);
		if (cpumask)
			smp_call_function_many(cpumask, flush_tlb_func_remote,
					       (void *)info, 1);
		return;
	}
	smp_call_function_many(cpumask, flush_tlb_func_remote,
			       (void *)info, 1);
}

Contributors
Person Tokens Prop Commits CommitProp
Rusty Russell 25 21.74% 1 6.67%
Andrew Lutomirski 20 17.39% 2 13.33%
Nadav Amit 18 15.65% 1 6.67%
Tejun Heo 13 11.30% 1 6.67%
Glauber de Oliveira Costa 10 8.70% 1 6.67%
Mel Gorman 10 8.70% 2 13.33%
Alex Shi 8 6.96% 2 13.33%
Dave Hansen 4 3.48% 1 6.67%
David Shaohua Li 3 2.61% 1 6.67%
Linus Torvalds 2 1.74% 1 6.67%
Xiao Guangrong 1 0.87% 1 6.67%
Mike Travis 1 0.87% 1 6.67%
Total 115 100.00% 15 100.00%

/*
 * See Documentation/x86/tlb.txt for details.  We choose 33
 * because it is large enough to cover the vast majority (at
 * least 95%) of allocations, and is small enough that we are
 * confident it will not cause too much overhead.  Each single
 * flush is about 100 ns, so this caps the maximum overhead at
 * _about_ 3,000 ns.
 *
 * This is in units of pages.
 */

static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33;



void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
				unsigned long end, unsigned long vmflag)
{
	int cpu;

	struct flush_tlb_info info = {
		.mm = mm,
        };

	cpu = get_cpu();

	/* Synchronize with switch_mm. */
	smp_mb();

	/* Should we flush just the requested range? */
	if ((end != TLB_FLUSH_ALL) &&
	    !(vmflag & VM_HUGETLB) &&
	    ((end - start) >> PAGE_SHIFT) <= tlb_single_page_flush_ceiling) {
		info.start = start;
		info.end = end;
	} else {
		info.start = 0UL;
		info.end = TLB_FLUSH_ALL;
	}

	if (mm == this_cpu_read(cpu_tlbstate.loaded_mm)) {
		VM_WARN_ON(irqs_disabled());
		local_irq_disable();
		flush_tlb_func_local(&info, TLB_LOCAL_MM_SHOOTDOWN);
		local_irq_enable();
	}

	if (cpumask_any_but(mm_cpumask(mm), cpu) < nr_cpu_ids)
		flush_tlb_others(mm_cpumask(mm), &info);
	put_cpu();
}


Contributors
Person Tokens Prop Commits CommitProp
Andrew Lutomirski 102 62.58% 5 45.45%
Alex Shi 50 30.67% 3 27.27%
Dave Hansen 6 3.68% 2 18.18%
Glauber de Oliveira Costa 5 3.07% 1 9.09%
Total 163 100.00% 11 100.00%



static void do_flush_tlb_all(void *info)
{
	count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
	__flush_tlb_all();
	if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
		leave_mm(smp_processor_id());
}

Contributors
Person Tokens Prop Commits CommitProp
Glauber de Oliveira Costa 24 68.57% 1 16.67%
Dave Hansen 4 11.43% 1 16.67%
Brian Gerst 3 8.57% 1 16.67%
Borislav Petkov 2 5.71% 1 16.67%
Alex Shi 1 2.86% 1 16.67%
Mel Gorman 1 2.86% 1 16.67%
Total 35 100.00% 6 100.00%



void flush_tlb_all(void)
{
	count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);
	on_each_cpu(do_flush_tlb_all, NULL, 1);
}

Contributors
Person Tokens Prop Commits CommitProp
Glauber de Oliveira Costa 16 76.19% 1 33.33%
Dave Hansen 4 19.05% 1 33.33%
Mel Gorman 1 4.76% 1 33.33%
Total 21 100.00% 3 100.00%



static void do_kernel_range_flush(void *info)
{
	struct flush_tlb_info *f = info;
	unsigned long addr;

	/* flush range by one by one 'invlpg' */
	for (addr = f->start; addr < f->end; addr += PAGE_SIZE)
		__flush_tlb_single(addr);
}

Contributors
Person Tokens Prop Commits CommitProp
Alex Shi 43 95.56% 1 50.00%
Andrew Lutomirski 2 4.44% 1 50.00%
Total 45 100.00% 2 100.00%



void flush_tlb_kernel_range(unsigned long start, unsigned long end)
{

	/* Balance as user space task's flush, a bit conservative */
	if (end == TLB_FLUSH_ALL ||
	    (end - start) > tlb_single_page_flush_ceiling << PAGE_SHIFT) {
		on_each_cpu(do_flush_tlb_all, NULL, 1);
	} else {
		struct flush_tlb_info info;
		info.start = start;
		info.end = end;
		on_each_cpu(do_kernel_range_flush, &info, 1);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Alex Shi 59 84.29% 1 25.00%
Dave Hansen 7 10.00% 1 25.00%
Andrew Lutomirski 4 5.71% 2 50.00%
Total 70 100.00% 4 100.00%



void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
{
	struct flush_tlb_info info = {
		.mm = NULL,
		.start = 0UL,
		.end = TLB_FLUSH_ALL,
        };

	int cpu = get_cpu();

	if (cpumask_test_cpu(cpu, &batch->cpumask)) {
		VM_WARN_ON(irqs_disabled());
		local_irq_disable();
		flush_tlb_func_local(&info, TLB_LOCAL_SHOOTDOWN);
		local_irq_enable();
	}

	if (cpumask_any_but(&batch->cpumask, cpu) < nr_cpu_ids)
		flush_tlb_others(&batch->cpumask, &info);
	cpumask_clear(&batch->cpumask);

	put_cpu();
}

Contributors
Person Tokens Prop Commits CommitProp
Andrew Lutomirski 107 100.00% 4 100.00%
Total 107 100.00% 4 100.00%



static ssize_t tlbflush_read_file(struct file *file, char __user *user_buf,
			     size_t count, loff_t *ppos)
{
	char buf[32];
	unsigned int len;

	len = sprintf(buf, "%ld\n", tlb_single_page_flush_ceiling);
	return simple_read_from_buffer(user_buf, count, ppos, buf, len);
}

Contributors
Person Tokens Prop Commits CommitProp
Dave Hansen 58 100.00% 1 100.00%
Total 58 100.00% 1 100.00%



static ssize_t tlbflush_write_file(struct file *file,
		 const char __user *user_buf, size_t count, loff_t *ppos)
{
	char buf[32];
	ssize_t len;
	int ceiling;

	len = min(count, sizeof(buf) - 1);
	if (copy_from_user(buf, user_buf, len))
		return -EFAULT;

	buf[len] = '\0';
	if (kstrtoint(buf, 0, &ceiling))
		return -EINVAL;

	if (ceiling < 0)
		return -EINVAL;

	tlb_single_page_flush_ceiling = ceiling;
	return count;
}

Contributors
Person Tokens Prop Commits CommitProp
Dave Hansen 105 100.00% 1 100.00%
Total 105 100.00% 1 100.00%


static const struct file_operations fops_tlbflush = {
	.read = tlbflush_read_file,
	.write = tlbflush_write_file,
	.llseek = default_llseek,
};



static int __init create_tlb_single_page_flush_ceiling(void)
{
	debugfs_create_file("tlb_single_page_flush_ceiling", S_IRUSR | S_IWUSR,
			    arch_debugfs_dir, NULL, &fops_tlbflush);
	return 0;
}
Contributors
Person Tokens Prop Commits CommitProp
Dave Hansen 28 100.00% 1 100.00%
Total 28 100.00% 1 100.00%


late_initcall(create_tlb_single_page_flush_ceiling);
Overall Contributors
Person Tokens Prop Commits CommitProp
Andrew Lutomirski 666 48.23% 14 29.17%
Dave Hansen 280 20.28% 8 16.67%
Alex Shi 197 14.27% 7 14.58%
Glauber de Oliveira Costa 129 9.34% 2 4.17%
Rusty Russell 25 1.81% 1 2.08%
Nadav Amit 18 1.30% 1 2.08%
Tejun Heo 15 1.09% 2 4.17%
Mel Gorman 12 0.87% 2 4.17%
Suresh B. Siddha 11 0.80% 1 2.08%
Brian Gerst 9 0.65% 1 2.08%
David Shaohua Li 6 0.43% 1 2.08%
Linus Torvalds 3 0.22% 2 4.17%
Jan Beulich 3 0.22% 1 2.08%
Jeremiah Mahler 2 0.14% 1 2.08%
Borislav Petkov 2 0.14% 1 2.08%
Mike Travis 1 0.07% 1 2.08%
Xiao Guangrong 1 0.07% 1 2.08%
Paul Gortmaker 1 0.07% 1 2.08%
Total 1381 100.00% 48 100.00%
Directory: arch/x86/mm

Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with cregit.
Person	Tokens	Prop	Commits	CommitProp
Glauber de Oliveira Costa	18	33.96%	1	14.29%
Andrew Lutomirski	16	30.19%	1	14.29%
Suresh B. Siddha	11	20.75%	1	14.29%
Brian Gerst	3	5.66%	1	14.29%
Dave Hansen	3	5.66%	1	14.29%
Alex Shi	1	1.89%	1	14.29%
Linus Torvalds	1	1.89%	1	14.29%
Total	53	100.00%	7	100.00%
cregit-Linux how code gets into the kernel

Release 4.13 arch/x86/mm/tlb.c

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Overall Contributors
