Release 4.10 arch/arm64/include/asm/tlbflush.h

/*
 * Based on arch/arm/include/asm/tlbflush.h
 *
 * Copyright (C) 1999-2003 Russell King
 * Copyright (C) 2012 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
#ifndef __ASM_TLBFLUSH_H

#define __ASM_TLBFLUSH_H

#ifndef __ASSEMBLY__

#include <linux/sched.h>
#include <asm/cputype.h>

/*
 * Raw TLBI operations.
 *
 * Where necessary, use the __tlbi() macro to avoid asm()
 * boilerplate. Drivers and most kernel code should use the TLB
 * management routines in preference to the macro below.
 *
 * The macro can be used as __tlbi(op) or __tlbi(op, arg), depending
 * on whether a particular TLBI operation takes an argument or
 * not. The macros handles invoking the asm with or without the
 * register argument as appropriate.
 */

#define __TLBI_0(op, arg)		asm ("tlbi " #op)

#define __TLBI_1(op, arg)		asm ("tlbi " #op ", %0" : : "r" (arg))

#define __TLBI_N(op, arg, n, ...)	__TLBI_##n(op, arg)


#define __tlbi(op, ...)		__TLBI_N(op, ##__VA_ARGS__, 1, 0)

/*
 *      TLB Management
 *      ==============
 *
 *      The TLB specific code is expected to perform whatever tests it needs
 *      to determine if it should invalidate the TLB for each call.  Start
 *      addresses are inclusive and end addresses are exclusive; it is safe to
 *      round these addresses down.
 *
 *      flush_tlb_all()
 *
 *              Invalidate the entire TLB.
 *
 *      flush_tlb_mm(mm)
 *
 *              Invalidate all TLB entries in a particular address space.
 *              - mm    - mm_struct describing address space
 *
 *      flush_tlb_range(mm,start,end)
 *
 *              Invalidate a range of TLB entries in the specified address
 *              space.
 *              - mm    - mm_struct describing address space
 *              - start - start address (may not be aligned)
 *              - end   - end address (exclusive, may not be aligned)
 *
 *      flush_tlb_page(vaddr,vma)
 *
 *              Invalidate the specified page in the specified address range.
 *              - vaddr - virtual address (may not be aligned)
 *              - vma   - vma_struct describing address range
 *
 *      flush_kern_tlb_page(kaddr)
 *
 *              Invalidate the TLB entry for the specified page.  The address
 *              will be in the kernels virtual memory space.  Current uses
 *              only require the D-TLB to be invalidated.
 *              - kaddr - Kernel virtual memory address
 */


static inline void local_flush_tlb_all(void)
{
	dsb(nshst);
	__tlbi(vmalle1);
	dsb(nsh);
	isb();
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
will deacon
will deacon
22
81.48%
1
50.00%
mark rutland
mark rutland
5
18.52%
1
50.00%
Total
27
100.00%
2
100.00%



static inline void flush_tlb_all(void)
{
	dsb(ishst);
	__tlbi(vmalle1is);
	dsb(ish);
	isb();
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
catalin marinas
catalin marinas
16
59.26%
1
33.33%
will deacon
will deacon
6
22.22%
1
33.33%
mark rutland
mark rutland
5
18.52%
1
33.33%
Total
27
100.00%
3
100.00%



static inline void flush_tlb_mm(struct mm_struct *mm)
{
	unsigned long asid = ASID(mm) << 48;

	dsb(ishst);
	__tlbi(aside1is, asid);
	dsb(ish);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
catalin marinas
catalin marinas
27
67.50%
1
33.33%
mark rutland
mark rutland
7
17.50%
1
33.33%
will deacon
will deacon
6
15.00%
1
33.33%
Total
40
100.00%
3
100.00%



static inline void flush_tlb_page(struct vm_area_struct *vma,
				  unsigned long uaddr)
{
	unsigned long addr = uaddr >> 12 | (ASID(vma->vm_mm) << 48);

	dsb(ishst);
	__tlbi(vale1is, addr);
	dsb(ish);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
catalin marinas
catalin marinas
39
75.00%
1
33.33%
mark rutland
mark rutland
7
13.46%
1
33.33%
will deacon
will deacon
6
11.54%
1
33.33%
Total
52
100.00%
3
100.00%

/*
 * This is meant to avoid soft lock-ups on large TLB flushing ranges and not
 * necessarily a performance improvement.
 */

#define MAX_TLB_RANGE	(1024UL << PAGE_SHIFT)



static inline void __flush_tlb_range(struct vm_area_struct *vma,
				     unsigned long start, unsigned long end,
				     bool last_level)
{
	unsigned long asid = ASID(vma->vm_mm) << 48;
	unsigned long addr;

	if ((end - start) > MAX_TLB_RANGE) {
		flush_tlb_mm(vma->vm_mm);
		return;
	}

	start = asid | (start >> 12);
	end = asid | (end >> 12);

	dsb(ishst);
	for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12)) {
		if (last_level)
			__tlbi(vale1is, addr);
		else
			__tlbi(vae1is, addr);
	}
	dsb(ish);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
steve capper
steve capper
79
60.31%
1
20.00%
catalin marinas
catalin marinas
38
29.01%
3
60.00%
mark rutland
mark rutland
14
10.69%
1
20.00%
Total
131
100.00%
5
100.00%



static inline void flush_tlb_range(struct vm_area_struct *vma,
				   unsigned long start, unsigned long end)
{
	__flush_tlb_range(vma, start, end, false);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
catalin marinas
catalin marinas
31
100.00%
1
100.00%
Total
31
100.00%
1
100.00%



static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
	unsigned long addr;

	if ((end - start) > MAX_TLB_RANGE) {
		flush_tlb_all();
		return;
	}

	start >>= 12;
	end >>= 12;

	dsb(ishst);
	for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12))
		__tlbi(vaae1is, addr);
	dsb(ish);
	isb();
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
steve capper
steve capper
53
63.86%
1
20.00%
catalin marinas
catalin marinas
23
27.71%
3
60.00%
mark rutland
mark rutland
7
8.43%
1
20.00%
Total
83
100.00%
5
100.00%

/*
 * Used to invalidate the TLB (walk caches) corresponding to intermediate page
 * table levels (pgd/pud/pmd).
 */


static inline void __flush_tlb_pgtable(struct mm_struct *mm,
				       unsigned long uaddr)
{
	unsigned long addr = uaddr >> 12 | (ASID(mm) << 48);

	__tlbi(vae1is, addr);
	dsb(ish);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
catalin marinas
catalin marinas
38
84.44%
1
50.00%
mark rutland
mark rutland
7
15.56%
1
50.00%
Total
45
100.00%
2
100.00%

#endif

#endif
Overall Contributors
Person
Tokens
Prop
Commits
CommitProp
catalin marinas
catalin marinas
238
47.22%
5
50.00%
steve capper
steve capper
132
26.19%
1
10.00%
mark rutland
mark rutland
93
18.45%
1
10.00%
will deacon
will deacon
40
7.94%
2
20.00%
vladimir murzin
vladimir murzin
1
0.20%
1
10.00%
Total
504
100.00%
10
100.00%
Directory: arch/arm64/include/asm

Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.