Release 4.14 arch/mn10300/mm/tlb-smp.c

/* SMP TLB support routines.
 *
 * Copyright (C) 2006-2008 Panasonic Corporation
 * All Rights Reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * version 2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 */
#include <linux/interrupt.h>
#include <linux/spinlock.h>
#include <linux/init.h>
#include <linux/jiffies.h>
#include <linux/cpumask.h>
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/delay.h>
#include <linux/sched/mm.h>
#include <linux/profile.h>
#include <linux/smp.h>
#include <asm/tlbflush.h>
#include <asm/bitops.h>
#include <asm/processor.h>
#include <asm/bug.h>
#include <asm/exceptions.h>
#include <asm/hardirq.h>
#include <asm/fpu.h>
#include <asm/mmu_context.h>
#include <asm/thread_info.h>
#include <asm/cpu-regs.h>
#include <asm/intctl-regs.h>

/*
 * For flush TLB
 */

#define FLUSH_ALL	0xffffffff


static cpumask_t flush_cpumask;

static struct mm_struct *flush_mm;

static unsigned long flush_va;
static DEFINE_SPINLOCK(tlbstate_lock);

DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate) = {
	&init_mm, 0
};

static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
			     unsigned long va);
static void do_flush_tlb_all(void *info);

/**
 * smp_flush_tlb - Callback to invalidate the TLB.
 * @unused: Callback context (ignored).
 */


void smp_flush_tlb(void *unused)
{
	unsigned long cpu_id;

	cpu_id = get_cpu();

	if (!cpumask_test_cpu(cpu_id, &flush_cpumask))
		/* This was a BUG() but until someone can quote me the line
                 * from the intel manual that guarantees an IPI to multiple
                 * CPUs is retried _only_ on the erroring CPUs its staying as a
                 * return
                 *
                 * BUG();
                 */
		goto out;

	if (flush_va == FLUSH_ALL)
		local_flush_tlb();
	else
		local_flush_tlb_page(flush_mm, flush_va);

	smp_mb__before_atomic();
	cpumask_clear_cpu(cpu_id, &flush_cpumask);
	smp_mb__after_atomic();
out:
	put_cpu();
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Akira Takeuchi
63
91.30%
1
33.33%
Motohiro Kosaki
4
5.80%
1
33.33%
Peter Zijlstra
2
2.90%
1
33.33%
Total
69
100.00%
3
100.00%

/**
 * flush_tlb_others - Tell the specified CPUs to invalidate their TLBs
 * @cpumask: The list of CPUs to target.
 * @mm: The VM context to flush from (if va!=FLUSH_ALL).
 * @va: Virtual address to flush or FLUSH_ALL to flush everything.
 */


static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
			     unsigned long va)
{
	cpumask_t tmp;

	/* A couple of sanity checks (to be removed):
         * - mask must not be empty
         * - current CPU must not be in mask
         * - we do not send IPIs to as-yet unbooted CPUs.
         */
	BUG_ON(!mm);
	BUG_ON(cpumask_empty(&cpumask));
	BUG_ON(cpumask_test_cpu(smp_processor_id(), &cpumask));

	cpumask_and(&tmp, &cpumask, cpu_online_mask);
	BUG_ON(!cpumask_equal(&cpumask, &tmp));

	/* I'm not happy about this global shared spinlock in the MM hot path,
         * but we'll see how contended it is.
         *
         * Temporarily this turns IRQs off, so that lockups are detected by the
         * NMI watchdog.
         */
	spin_lock(&tlbstate_lock);

	flush_mm = mm;
	flush_va = va;
#if NR_CPUS <= BITS_PER_LONG
	atomic_or(cpumask.bits[0], (atomic_t *)&flush_cpumask.bits[0]);
#else
#error Not supported.
#endif

	/* FIXME: if NR_CPUS>=3, change send_IPI_mask */
	smp_call_function(smp_flush_tlb, NULL, 1);

	while (!cpumask_empty(&flush_cpumask))
		/* Lockup detection does not belong here */
		smp_mb();

	flush_mm = NULL;
	flush_va = 0;
	spin_unlock(&tlbstate_lock);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Akira Takeuchi
141
88.68%
1
33.33%
Motohiro Kosaki
13
8.18%
1
33.33%
Peter Zijlstra
5
3.14%
1
33.33%
Total
159
100.00%
3
100.00%

/**
 * flush_tlb_mm - Invalidate TLB of specified VM context
 * @mm: The VM context to invalidate.
 */


void flush_tlb_mm(struct mm_struct *mm)
{
	cpumask_t cpu_mask;

	preempt_disable();
	cpumask_copy(&cpu_mask, mm_cpumask(mm));
	cpumask_clear_cpu(smp_processor_id(), &cpu_mask);

	local_flush_tlb();
	if (!cpumask_empty(&cpu_mask))
		flush_tlb_others(cpu_mask, mm, FLUSH_ALL);

	preempt_enable();
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Akira Takeuchi
48
80.00%
1
50.00%
Motohiro Kosaki
12
20.00%
1
50.00%
Total
60
100.00%
2
100.00%

/**
 * flush_tlb_current_task - Invalidate TLB of current task
 */


void flush_tlb_current_task(void)
{
	struct mm_struct *mm = current->mm;
	cpumask_t cpu_mask;

	preempt_disable();
	cpumask_copy(&cpu_mask, mm_cpumask(mm));
	cpumask_clear_cpu(smp_processor_id(), &cpu_mask);

	local_flush_tlb();
	if (!cpumask_empty(&cpu_mask))
		flush_tlb_others(cpu_mask, mm, FLUSH_ALL);

	preempt_enable();
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Akira Takeuchi
54
81.82%
1
50.00%
Motohiro Kosaki
12
18.18%
1
50.00%
Total
66
100.00%
2
100.00%

/**
 * flush_tlb_page - Invalidate TLB of page
 * @vma: The VM context to invalidate the page for.
 * @va: The virtual address of the page to invalidate.
 */


void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
{
	struct mm_struct *mm = vma->vm_mm;
	cpumask_t cpu_mask;

	preempt_disable();
	cpumask_copy(&cpu_mask, mm_cpumask(mm));
	cpumask_clear_cpu(smp_processor_id(), &cpu_mask);

	local_flush_tlb_page(mm, va);
	if (!cpumask_empty(&cpu_mask))
		flush_tlb_others(cpu_mask, mm, va);

	preempt_enable();
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Akira Takeuchi
65
84.42%
1
50.00%
Motohiro Kosaki
12
15.58%
1
50.00%
Total
77
100.00%
2
100.00%

/**
 * do_flush_tlb_all - Callback to completely invalidate a TLB
 * @unused: Callback context (ignored).
 */


static void do_flush_tlb_all(void *unused)
{
	local_flush_tlb_all();
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Akira Takeuchi
13
100.00%
1
100.00%
Total
13
100.00%
1
100.00%

/**
 * flush_tlb_all - Completely invalidate TLBs on all CPUs
 */


void flush_tlb_all(void)
{
	on_each_cpu(do_flush_tlb_all, 0, 1);
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Akira Takeuchi
16
100.00%
1
100.00%
Total
16
100.00%
1
100.00%

Overall Contributors
Person
Tokens
Prop
Commits
CommitProp
Akira Takeuchi
540
89.85%
1
20.00%
Motohiro Kosaki
53
8.82%
1
20.00%
Peter Zijlstra
7
1.16%
2
40.00%
Ingo Molnar
1
0.17%
1
20.00%
Total
601
100.00%
5
100.00%
Directory: arch/mn10300/mm

Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.