Release 4.9 arch/arm/include/asm/tlb.h

Directory: arch/arm/include/asm
/*
 *  arch/arm/include/asm/tlb.h
 *
 *  Copyright (C) 2002 Russell King
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 *  Experimentation shows that on a StrongARM, it appears to be faster
 *  to use the "invalidate whole tlb" rather than "invalidate single
 *  tlb" for this.
 *
 *  This appears true for both the process fork+exit case, as well as
 *  the munmap-large-area case.
 */
#ifndef __ASMARM_TLB_H

#define __ASMARM_TLB_H

#include <asm/cacheflush.h>

#ifndef CONFIG_MMU

#include <linux/pagemap.h>


#define tlb_flush(tlb)	((void) tlb)

#include <asm-generic/tlb.h>

#else /* !CONFIG_MMU */

#include <linux/swap.h>
#include <asm/pgalloc.h>
#include <asm/tlbflush.h>


#define MMU_GATHER_BUNDLE	8

#ifdef CONFIG_HAVE_RCU_TABLE_FREE


static inline void __tlb_remove_table(void *_table)
{
	free_page_and_swap_cache((struct page *)_table);
}

Contributors
Person Tokens Prop Commits CommitProp
steve capper steve capper 21 100.00% 1 100.00%
Total 21 100.00% 1 100.00%


struct mmu_table_batch {
	
struct rcu_head		rcu;
	
unsigned int		nr;
	
void			*tables[0];
};


#define MAX_TABLE_BATCH		\
	((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))

extern void tlb_table_flush(struct mmu_gather *tlb);
extern void tlb_remove_table(struct mmu_gather *tlb, void *table);


#define tlb_remove_entry(tlb, entry)	tlb_remove_table(tlb, entry)
#else

#define tlb_remove_entry(tlb, entry)	tlb_remove_page(tlb, entry)
#endif /* CONFIG_HAVE_RCU_TABLE_FREE */

/*
 * TLB handling.  This allows us to remove pages from the page
 * tables, and efficiently handle the TLB issues.
 */

struct mmu_gather {
	
struct mm_struct	*mm;
#ifdef CONFIG_HAVE_RCU_TABLE_FREE
	
struct mmu_table_batch	*batch;
	
unsigned int		need_flush;
#endif
	
unsigned int		fullmm;
	
struct vm_area_struct	*vma;
	

unsigned long		start, end;
	
unsigned long		range_start;
	
unsigned long		range_end;
	
unsigned int		nr;
	
unsigned int		max;
	
struct page		**pages;
	
struct page		*local[MMU_GATHER_BUNDLE];
};

DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);

/*
 * This is unnecessarily complex.  There's three ways the TLB shootdown
 * code is used:
 *  1. Unmapping a range of vmas.  See zap_page_range(), unmap_region().
 *     tlb->fullmm = 0, and tlb_start_vma/tlb_end_vma will be called.
 *     tlb->vma will be non-NULL.
 *  2. Unmapping all vmas.  See exit_mmap().
 *     tlb->fullmm = 1, and tlb_start_vma/tlb_end_vma will be called.
 *     tlb->vma will be non-NULL.  Additionally, page tables will be freed.
 *  3. Unmapping argument pages.  See shift_arg_pages().
 *     tlb->fullmm = 0, but tlb_start_vma/tlb_end_vma will not be called.
 *     tlb->vma will be NULL.
 */


static inline void tlb_flush(struct mmu_gather *tlb)
{
	if (tlb->fullmm || !tlb->vma)
		flush_tlb_mm(tlb->mm);
	else if (tlb->range_end > 0) {
		flush_tlb_range(tlb->vma, tlb->range_start, tlb->range_end);
		tlb->range_start = TASK_SIZE;
		tlb->range_end = 0;
	}
}

Contributors
Person Tokens Prop Commits CommitProp
russell king russell king 68 100.00% 1 100.00%
Total 68 100.00% 1 100.00%



static inline void tlb_add_flush(struct mmu_gather *tlb, unsigned long addr)
{
	if (!tlb->fullmm) {
		if (addr < tlb->range_start)
			tlb->range_start = addr;
		if (addr + PAGE_SIZE > tlb->range_end)
			tlb->range_end = addr + PAGE_SIZE;
	}
}

Contributors
Person Tokens Prop Commits CommitProp
russell king russell king 57 100.00% 1 100.00%
Total 57 100.00% 1 100.00%



static inline void __tlb_alloc_page(struct mmu_gather *tlb)
{
	unsigned long addr = __get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);

	if (addr) {
		tlb->pages = (void *)addr;
		tlb->max = PAGE_SIZE / sizeof(struct page *);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
peter zijlstra peter zijlstra 54 100.00% 1 100.00%
Total 54 100.00% 1 100.00%



static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
{
	tlb_flush(tlb);
#ifdef CONFIG_HAVE_RCU_TABLE_FREE
	tlb_table_flush(tlb);
#endif
}

Contributors
Person Tokens Prop Commits CommitProp
russell king russell king 15 55.56% 1 33.33%
steve capper steve capper 10 37.04% 1 33.33%
linus torvalds linus torvalds 2 7.41% 1 33.33%
Total 27 100.00% 3 100.00%



static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
{
	free_pages_and_swap_cache(tlb->pages, tlb->nr);
	tlb->nr = 0;
	if (tlb->pages == tlb->local)
		__tlb_alloc_page(tlb);
}

Contributors
Person Tokens Prop Commits CommitProp
russell king russell king 17 38.64% 1 33.33%
peter zijlstra peter zijlstra 14 31.82% 1 33.33%
linus torvalds linus torvalds 13 29.55% 1 33.33%
Total 44 100.00% 3 100.00%



static inline void tlb_flush_mmu(struct mmu_gather *tlb)
{
	tlb_flush_mmu_tlbonly(tlb);
	tlb_flush_mmu_free(tlb);
}

Contributors
Person Tokens Prop Commits CommitProp
linus torvalds linus torvalds 20 90.91% 1 33.33%
russell king russell king 1 4.55% 1 33.33%
peter zijlstra peter zijlstra 1 4.55% 1 33.33%
Total 22 100.00% 3 100.00%


static inline void

tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end)
{
	tlb->mm = mm;
	tlb->fullmm = !(start | (end+1));
	tlb->start = start;
	tlb->end = end;
	tlb->vma = NULL;
	tlb->max = ARRAY_SIZE(tlb->local);
	tlb->pages = tlb->local;
	tlb->nr = 0;
	__tlb_alloc_page(tlb);

#ifdef CONFIG_HAVE_RCU_TABLE_FREE
	tlb->batch = NULL;
#endif
}

Contributors
Person Tokens Prop Commits CommitProp
russell king russell king 37 35.24% 3 50.00%
peter zijlstra peter zijlstra 29 27.62% 1 16.67%
linus torvalds linus torvalds 28 26.67% 1 16.67%
steve capper steve capper 11 10.48% 1 16.67%
Total 105 100.00% 6 100.00%


static inline void

tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
{
	tlb_flush_mmu(tlb);

	/* keep the page table cache within bounds */
	check_pgt_cache();

	if (tlb->pages != tlb->local)
		free_pages((unsigned long)tlb->pages, 0);
}

Contributors
Person Tokens Prop Commits CommitProp
russell king russell king 26 50.00% 2 33.33%
peter zijlstra peter zijlstra 20 38.46% 1 16.67%
hugh dickins hugh dickins 4 7.69% 2 33.33%
andrew morton andrew morton 2 3.85% 1 16.67%
Total 52 100.00% 6 100.00%

/*
 * Memorize the range for the TLB flush.
 */

static inline void

tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr)
{
	tlb_add_flush(tlb, addr);
}

Contributors
Person Tokens Prop Commits CommitProp
aaro koskinen aaro koskinen 16 59.26% 1 25.00%
russell king russell king 11 40.74% 3 75.00%
Total 27 100.00% 4 100.00%

/*
 * In the case of tlb vma handling, we can optimise these away in the
 * case where we're doing a full MM flush.  When we're doing a munmap,
 * the vmas are adjusted to only cover the region to be torn down.
 */

static inline void

tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
{
	if (!tlb->fullmm) {
		flush_cache_range(vma, vma->vm_start, vma->vm_end);
		tlb->vma = vma;
		tlb->range_start = TASK_SIZE;
		tlb->range_end = 0;
	}
}

Contributors
Person Tokens Prop Commits CommitProp
russell king russell king 43 75.44% 3 75.00%
aaro koskinen aaro koskinen 14 24.56% 1 25.00%
Total 57 100.00% 4 100.00%


static inline void

tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
{
	if (!tlb->fullmm)
		tlb_flush(tlb);
}

Contributors
Person Tokens Prop Commits CommitProp
russell king russell king 28 96.55% 3 75.00%
aaro koskinen aaro koskinen 1 3.45% 1 25.00%
Total 29 100.00% 4 100.00%



static inline bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
{
	if (tlb->nr == tlb->max)
		return true;
	tlb->pages[tlb->nr++] = page;
	return false;
}

Contributors
Person Tokens Prop Commits CommitProp
russell king russell king 24 53.33% 2 40.00%
aneesh kumar aneesh kumar 15 33.33% 1 20.00%
peter zijlstra peter zijlstra 4 8.89% 1 20.00%
aaro koskinen aaro koskinen 2 4.44% 1 20.00%
Total 45 100.00% 5 100.00%



static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
{
	if (__tlb_remove_page(tlb, page)) {
		tlb_flush_mmu(tlb);
		__tlb_remove_page(tlb, page);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
peter zijlstra peter zijlstra 24 60.00% 1 25.00%
aneesh kumar aneesh kumar 10 25.00% 1 25.00%
russell king russell king 6 15.00% 2 50.00%
Total 40 100.00% 4 100.00%



static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
					  struct page *page, int page_size)
{
	return __tlb_remove_page(tlb, page);
}

Contributors
Person Tokens Prop Commits CommitProp
aneesh kumar aneesh kumar 28 100.00% 1 100.00%
Total 28 100.00% 1 100.00%



static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb,
					 struct page *page)
{
	return __tlb_remove_page(tlb, page);
}

Contributors
Person Tokens Prop Commits CommitProp
aneesh kumar aneesh kumar 24 96.00% 1 50.00%
russell king russell king 1 4.00% 1 50.00%
Total 25 100.00% 2 100.00%



static inline void tlb_remove_page_size(struct mmu_gather *tlb,
					struct page *page, int page_size)
{
	return tlb_remove_page(tlb, page);
}

Contributors
Person Tokens Prop Commits CommitProp
aneesh kumar aneesh kumar 28 100.00% 1 100.00%
Total 28 100.00% 1 100.00%



static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
	unsigned long addr)
{
	pgtable_page_dtor(pte);

#ifdef CONFIG_ARM_LPAE
	tlb_add_flush(tlb, addr);
#else
	/*
         * With the classic ARM MMU, a pte page has two corresponding pmd
         * entries, each covering 1MB.
         */
	addr &= PMD_MASK;
	tlb_add_flush(tlb, addr + SZ_1M - PAGE_SIZE);
	tlb_add_flush(tlb, addr + SZ_1M);
#endif

	tlb_remove_entry(tlb, pte);
}

Contributors
Person Tokens Prop Commits CommitProp
russell king russell king 37 52.86% 2 40.00%
catalin marinas catalin marinas 18 25.71% 1 20.00%
will deacon will deacon 14 20.00% 1 20.00%
steve capper steve capper 1 1.43% 1 20.00%
Total 70 100.00% 5 100.00%



static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
				  unsigned long addr)
{
#ifdef CONFIG_ARM_LPAE
	tlb_add_flush(tlb, addr);
	tlb_remove_entry(tlb, virt_to_page(pmdp));
#endif
}

Contributors
Person Tokens Prop Commits CommitProp
catalin marinas catalin marinas 41 97.62% 1 50.00%
steve capper steve capper 1 2.38% 1 50.00%
Total 42 100.00% 2 100.00%


static inline void

tlb_remove_pmd_tlb_entry(struct mmu_gather *tlb, pmd_t *pmdp, unsigned long addr)
{
	tlb_add_flush(tlb, addr);
}

Contributors
Person Tokens Prop Commits CommitProp
catalin marinas catalin marinas 27 100.00% 1 100.00%
Total 27 100.00% 1 100.00%


#define pte_free_tlb(tlb, ptep, addr)	__pte_free_tlb(tlb, ptep, addr)

#define pmd_free_tlb(tlb, pmdp, addr)	__pmd_free_tlb(tlb, pmdp, addr)

#define pud_free_tlb(tlb, pudp, addr)	pud_free((tlb)->mm, pudp)


#define tlb_migrate_finish(mm)		do { } while (0)

#endif /* CONFIG_MMU */
#endif
Overall Contributors
Person Tokens Prop Commits CommitProp
russell king russell king 466 41.94% 10 37.04%
peter zijlstra peter zijlstra 161 14.49% 1 3.70%
steve capper steve capper 133 11.97% 1 3.70%
aneesh kumar aneesh kumar 105 9.45% 2 7.41%
catalin marinas catalin marinas 87 7.83% 3 11.11%
linus torvalds linus torvalds 69 6.21% 2 7.41%
aaro koskinen aaro koskinen 42 3.78% 1 3.70%
hyok s. choi hyok s. choi 15 1.35% 1 3.70%
will deacon will deacon 14 1.26% 1 3.70%
andrew morton andrew morton 11 0.99% 2 7.41%
hugh dickins hugh dickins 4 0.36% 2 7.41%
benjamin herrenschmidt benjamin herrenschmidt 4 0.36% 1 3.70%
Total 1111 100.00% 27 100.00%
  Directory: arch/arm/include/asm
	Person	Tokens	Prop	Commits	CommitProp
steve capper	steve capper	21	100.00%	1	100.00%
	Total	21	100.00%	1	100.00%
cregit-Linux how code gets into the kernel

Release 4.9 arch/arm/include/asm/tlb.h

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Overall Contributors
