Release 4.15 arch/x86/include/asm/pgtable_64.h

Directory: arch/x86/include/asm
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_PGTABLE_64_H

#define _ASM_X86_PGTABLE_64_H

#include <linux/const.h>
#include <asm/pgtable_64_types.h>

#ifndef __ASSEMBLY__

/*
 * This file contains the functions and defines necessary to modify and use
 * the x86-64 page table tree.
 */
#include <asm/processor.h>
#include <linux/bitops.h>
#include <linux/threads.h>

extern p4d_t level4_kernel_pgt[512];
extern p4d_t level4_ident_pgt[512];
extern pud_t level3_kernel_pgt[512];
extern pud_t level3_ident_pgt[512];
extern pmd_t level2_kernel_pgt[512];
extern pmd_t level2_fixmap_pgt[512];
extern pmd_t level2_ident_pgt[512];
extern pte_t level1_fixmap_pgt[512];
extern pgd_t init_top_pgt[];


#define swapper_pg_dir init_top_pgt

extern void paging_init(void);


#define pte_ERROR(e)					\
	pr_err("%s:%d: bad pte %p(%016lx)\n",           \
               __FILE__, __LINE__, &(e), pte_val(e))

#define pmd_ERROR(e)					\
	pr_err("%s:%d: bad pmd %p(%016lx)\n",           \
               __FILE__, __LINE__, &(e), pmd_val(e))

#define pud_ERROR(e)					\
	pr_err("%s:%d: bad pud %p(%016lx)\n",           \
               __FILE__, __LINE__, &(e), pud_val(e))

#if CONFIG_PGTABLE_LEVELS >= 5

#define p4d_ERROR(e)					\
	pr_err("%s:%d: bad p4d %p(%016lx)\n",           \
               __FILE__, __LINE__, &(e), p4d_val(e))
#endif


#define pgd_ERROR(e)					\
	pr_err("%s:%d: bad pgd %p(%016lx)\n",           \
               __FILE__, __LINE__, &(e), pgd_val(e))

struct mm_struct;

void set_pte_vaddr_p4d(p4d_t *p4d_page, unsigned long vaddr, pte_t new_pte);
void set_pte_vaddr_pud(pud_t *pud_page, unsigned long vaddr, pte_t new_pte);



static inline void native_pte_clear(struct mm_struct *mm, unsigned long addr,
				    pte_t *ptep)
{
	*ptep = native_make_pte(0);
}

Contributors
Person Tokens Prop Commits CommitProp
Jeremy Fitzhardinge 15 53.57% 1 50.00%
Andi Kleen 13 46.43% 1 50.00%
Total 28 100.00% 2 100.00%



static inline void native_set_pte(pte_t *ptep, pte_t pte)
{
	*ptep = pte;
}

Contributors
Person Tokens Prop Commits CommitProp
Jeremy Fitzhardinge 15 78.95% 1 50.00%
David S. Miller 4 21.05% 1 50.00%
Total 19 100.00% 2 100.00%



static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte)
{
	native_set_pte(ptep, pte);
}

Contributors
Person Tokens Prop Commits CommitProp
Ingo Molnar 21 100.00% 1 100.00%
Total 21 100.00% 1 100.00%



static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
{
	*pmdp = pmd;
}

Contributors
Person Tokens Prop Commits CommitProp
Andrea Arcangeli 19 100.00% 1 100.00%
Total 19 100.00% 1 100.00%



static inline void native_pmd_clear(pmd_t *pmd)
{
	native_set_pmd(pmd, native_make_pmd(0));
}

Contributors
Person Tokens Prop Commits CommitProp
Andrea Arcangeli 21 100.00% 1 100.00%
Total 21 100.00% 1 100.00%



static inline pte_t native_ptep_get_and_clear(pte_t *xp)
{
#ifdef CONFIG_SMP
	return native_make_pte(xchg(&xp->pte, 0));
#else
	/* native_local_ptep_get_and_clear,
           but duplicated because of cyclic dependency */
	pte_t ret = *xp;
	native_pte_clear(NULL, 0, xp);
	return ret;
#endif
}

Contributors
Person Tokens Prop Commits CommitProp
Jeremy Fitzhardinge 41 80.39% 1 33.33%
Andi Kleen 9 17.65% 1 33.33%
Joe Perches 1 1.96% 1 33.33%
Total 51 100.00% 3 100.00%



static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp)
{
#ifdef CONFIG_SMP
	return native_make_pmd(xchg(&xp->pmd, 0));
#else
	/* native_local_pmdp_get_and_clear,
           but duplicated because of cyclic dependency */
	pmd_t ret = *xp;
	native_pmd_clear(xp);
	return ret;
#endif
}

Contributors
Person Tokens Prop Commits CommitProp
Andrea Arcangeli 31 65.96% 1 33.33%
Andi Kleen 13 27.66% 1 33.33%
Jeremy Fitzhardinge 3 6.38% 1 33.33%
Total 47 100.00% 3 100.00%



static inline void native_set_pud(pud_t *pudp, pud_t pud)
{
	*pudp = pud;
}

Contributors
Person Tokens Prop Commits CommitProp
Andi Kleen 12 63.16% 2 66.67%
Jeremy Fitzhardinge 7 36.84% 1 33.33%
Total 19 100.00% 3 100.00%



static inline void native_pud_clear(pud_t *pud)
{
	native_set_pud(pud, native_make_pud(0));
}

Contributors
Person Tokens Prop Commits CommitProp
Andi Kleen 14 66.67% 1 33.33%
Jeremy Fitzhardinge 6 28.57% 1 33.33%
Adrian Bunk 1 4.76% 1 33.33%
Total 21 100.00% 3 100.00%



static inline pud_t native_pudp_get_and_clear(pud_t *xp)
{
#ifdef CONFIG_SMP
	return native_make_pud(xchg(&xp->pud, 0));
#else
	/* native_local_pudp_get_and_clear,
         * but duplicated because of cyclic dependency
         */
	pud_t ret = *xp;

	native_pud_clear(xp);
	return ret;
#endif
}

Contributors
Person Tokens Prop Commits CommitProp
Matthew Wilcox 47 100.00% 1 100.00%
Total 47 100.00% 1 100.00%

#ifdef CONFIG_PAGE_TABLE_ISOLATION
/*
 * All top-level PAGE_TABLE_ISOLATION page tables are order-1 pages
 * (8k-aligned and 8k in size).  The kernel one is at the beginning 4k and
 * the user one is in the last 4k.  To switch between them, you
 * just need to flip the 12th bit in their addresses.
 */

#define PTI_PGTABLE_SWITCH_BIT	PAGE_SHIFT

/*
 * This generates better code than the inline assembly in
 * __set_bit().
 */


static inline void *ptr_set_bit(void *ptr, int bit)
{
	unsigned long __ptr = (unsigned long)ptr;

	__ptr |= BIT(bit);
	return (void *)__ptr;
}
Contributors
Person Tokens Prop Commits CommitProp
Dave Hansen 39 100.00% 1 100.00%
Total 39 100.00% 1 100.00%



static inline void *ptr_clear_bit(void *ptr, int bit)
{
	unsigned long __ptr = (unsigned long)ptr;

	__ptr &= ~BIT(bit);
	return (void *)__ptr;
}

Contributors
Person Tokens Prop Commits CommitProp
Dave Hansen 40 100.00% 1 100.00%
Total 40 100.00% 1 100.00%



static inline pgd_t *kernel_to_user_pgdp(pgd_t *pgdp)
{
	return ptr_set_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
}

Contributors
Person Tokens Prop Commits CommitProp
Dave Hansen 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%



static inline pgd_t *user_to_kernel_pgdp(pgd_t *pgdp)
{
	return ptr_clear_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
}

Contributors
Person Tokens Prop Commits CommitProp
Dave Hansen 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%



static inline p4d_t *kernel_to_user_p4dp(p4d_t *p4dp)
{
	return ptr_set_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
}

Contributors
Person Tokens Prop Commits CommitProp
Dave Hansen 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%



static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp)
{
	return ptr_clear_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
}
Contributors
Person Tokens Prop Commits CommitProp
Dave Hansen 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%

#endif /* CONFIG_PAGE_TABLE_ISOLATION */

/*
 * Page table pages are page-aligned.  The lower half of the top
 * level is used for userspace and the top half for the kernel.
 *
 * Returns true for parts of the PGD that map userspace and
 * false for the parts that map the kernel.
 */


static inline bool pgdp_maps_userspace(void *__ptr)
{
	unsigned long ptr = (unsigned long)__ptr;

	return (ptr & ~PAGE_MASK) < (PAGE_SIZE / 2);
}

Contributors
Person Tokens Prop Commits CommitProp
Dave Hansen 35 100.00% 1 100.00%
Total 35 100.00% 1 100.00%

#ifdef CONFIG_PAGE_TABLE_ISOLATION
pgd_t __pti_set_user_pgd(pgd_t *pgdp, pgd_t pgd);

/*
 * Take a PGD location (pgdp) and a pgd value that needs to be set there.
 * Populates the user and returns the resulting PGD that must be set in
 * the kernel copy of the page tables.
 */


static inline pgd_t pti_set_user_pgd(pgd_t *pgdp, pgd_t pgd)
{
	if (!static_cpu_has(X86_FEATURE_PTI))
		return pgd;
	return __pti_set_user_pgd(pgdp, pgd);
}
Contributors
Person Tokens Prop Commits CommitProp
Dave Hansen 33 100.00% 1 100.00%
Total 33 100.00% 1 100.00%

#else


static inline pgd_t pti_set_user_pgd(pgd_t *pgdp, pgd_t pgd)
{
	return pgd;
}
Contributors
Person Tokens Prop Commits CommitProp
Dave Hansen 17 100.00% 1 100.00%
Total 17 100.00% 1 100.00%

#endif



static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d)
{
#if defined(CONFIG_PAGE_TABLE_ISOLATION) && !defined(CONFIG_X86_5LEVEL)
	p4dp->pgd = pti_set_user_pgd(&p4dp->pgd, p4d.pgd);
#else
	*p4dp = p4d;
#endif
}

Contributors
Person Tokens Prop Commits CommitProp
Dave Hansen 32 62.75% 1 50.00%
Kirill A. Shutemov 19 37.25% 1 50.00%
Total 51 100.00% 2 100.00%



static inline void native_p4d_clear(p4d_t *p4d)
{
#ifdef CONFIG_X86_5LEVEL
	native_set_p4d(p4d, native_make_p4d(0));
#else
	native_set_p4d(p4d, (p4d_t) { .pgd = native_make_pgd(0)});
#endif
}

Contributors
Person Tokens Prop Commits CommitProp
Kirill A. Shutemov 46 100.00% 2 100.00%
Total 46 100.00% 2 100.00%



static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd)
{
#ifdef CONFIG_PAGE_TABLE_ISOLATION
	*pgdp = pti_set_user_pgd(pgdp, pgd);
#else
	*pgdp = pgd;
#endif
}

Contributors
Person Tokens Prop Commits CommitProp
Dave Hansen 17 47.22% 1 33.33%
Zachary Amsden 11 30.56% 1 33.33%
Jeremy Fitzhardinge 8 22.22% 1 33.33%
Total 36 100.00% 3 100.00%



static inline void native_pgd_clear(pgd_t *pgd)
{
	native_set_pgd(pgd, native_make_pgd(0));
}

Contributors
Person Tokens Prop Commits CommitProp
Jeremy Fitzhardinge 16 76.19% 1 50.00%
Zachary Amsden 5 23.81% 1 50.00%
Total 21 100.00% 2 100.00%

extern void sync_global_pgds(unsigned long start, unsigned long end);

/*
 * Conversion functions: convert a page and protection to a page entry,
 * and a page entry and page directory to the page they refer to.
 */

/*
 * Level 4 access.
 */


static inline int pgd_large(pgd_t pgd) { return 0; }
Contributors
Person Tokens Prop Commits CommitProp
H. Peter Anvin 13 100.00% 1 100.00%
Total 13 100.00% 1 100.00%


#define mk_kernel_pgd(address) __pgd((address) | _KERNPG_TABLE)

/* PUD - Level3 access */

/* PMD  - Level 2 access */

/* PTE - Level 1 access. */

/* x86-64 always has all page tables mapped. */

#define pte_offset_map(dir, address) pte_offset_kernel((dir), (address))

#define pte_unmap(pte) ((void)(pte))
/* NOP */

/*
 * Encode and de-code a swap entry
 *
 * |     ...            | 11| 10|  9|8|7|6|5| 4| 3|2| 1|0| <- bit number
 * |     ...            |SW3|SW2|SW1|G|L|D|A|CD|WT|U| W|P| <- bit names
 * | OFFSET (14->63) | TYPE (9-13)  |0|0|X|X| X| X|X|SD|0| <- swp entry
 *
 * G (8) is aliased and used as a PROT_NONE indicator for
 * !present ptes.  We need to start storing swap entries above
 * there.  We also need to avoid using A and D because of an
 * erratum where they can be incorrectly set by hardware on
 * non-present PTEs.
 *
 * SD (1) in swp entry is used to store soft dirty bit, which helps us
 * remember soft dirty over page migration
 *
 * Bit 7 in swp entry should be 0 because pmd_present checks not only P,
 * but also L and G.
 */

#define SWP_TYPE_FIRST_BIT (_PAGE_BIT_PROTNONE + 1)

#define SWP_TYPE_BITS 5
/* Place the offset above the type: */

#define SWP_OFFSET_FIRST_BIT (SWP_TYPE_FIRST_BIT + SWP_TYPE_BITS)


#define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS)


#define __swp_type(x)			(((x).val >> (SWP_TYPE_FIRST_BIT)) \
                                         & ((1U << SWP_TYPE_BITS) - 1))

#define __swp_offset(x)			((x).val >> SWP_OFFSET_FIRST_BIT)

#define __swp_entry(type, offset)	((swp_entry_t) { \
                                         ((type) << (SWP_TYPE_FIRST_BIT)) \
                                         | ((offset) << SWP_OFFSET_FIRST_BIT) })

#define __pte_to_swp_entry(pte)		((swp_entry_t) { pte_val((pte)) })

#define __pmd_to_swp_entry(pmd)		((swp_entry_t) { pmd_val((pmd)) })

#define __swp_entry_to_pte(x)		((pte_t) { .pte = (x).val })

#define __swp_entry_to_pmd(x)		((pmd_t) { .pmd = (x).val })

extern int kern_addr_valid(unsigned long addr);
extern void cleanup_highmap(void);


#define HAVE_ARCH_UNMAPPED_AREA

#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN


#define pgtable_cache_init()   do { } while (0)

#define check_pgt_cache()      do { } while (0)


#define PAGE_AGP    PAGE_KERNEL_NOCACHE

#define HAVE_PAGE_AGP 1

/* fs/proc/kcore.c */

#define	kc_vaddr_to_offset(v) ((v) & __VIRTUAL_MASK)

#define	kc_offset_to_vaddr(o) ((o) | ~__VIRTUAL_MASK)


#define __HAVE_ARCH_PTE_SAME


#define vmemmap ((struct page *)VMEMMAP_START)

extern void init_extra_mapping_uc(unsigned long phys, unsigned long size);
extern void init_extra_mapping_wb(unsigned long phys, unsigned long size);


#define gup_fast_permitted gup_fast_permitted


static inline bool gup_fast_permitted(unsigned long start, int nr_pages,
		int write)
{
	unsigned long len, end;

	len = (unsigned long)nr_pages << PAGE_SHIFT;
	end = start + len;
	if (end < start)
		return false;
	if (end >> __VIRTUAL_MASK_SHIFT)
		return false;
	return true;
}

Contributors
Person Tokens Prop Commits CommitProp
Kirill A. Shutemov 60 100.00% 1 100.00%
Total 60 100.00% 1 100.00%

#endif /* !__ASSEMBLY__ */
#endif /* _ASM_X86_PGTABLE_64_H */
Overall Contributors
Person Tokens Prop Commits CommitProp
Dave Hansen 335 28.27% 3 6.12%
Andi Kleen 232 19.58% 7 14.29%
Kirill A. Shutemov 176 14.85% 6 12.24%
Jeremy Fitzhardinge 132 11.14% 4 8.16%
Andrea Arcangeli 71 5.99% 1 2.04%
Matthew Wilcox 47 3.97% 1 2.04%
Alexander Duyck 30 2.53% 1 2.04%
Ingo Molnar 21 1.77% 1 2.04%
Eduardo Pereira Habkost 16 1.35% 2 4.08%
H. Peter Anvin 16 1.35% 2 4.08%
Zachary Amsden 16 1.35% 1 2.04%
Zi Yan 14 1.18% 1 2.04%
Haicheng Li 13 1.10% 1 2.04%
Jan Beulich 10 0.84% 1 2.04%
Vivek Goyal 8 0.68% 1 2.04%
Andrew Morton 8 0.68% 2 4.08%
Joe Perches 7 0.59% 2 4.08%
Stefan Bader 7 0.59% 1 2.04%
Thomas Gleixner 7 0.59% 1 2.04%
Linus Torvalds 6 0.51% 2 4.08%
David S. Miller 4 0.34% 1 2.04%
Jiri Kosina 3 0.25% 1 2.04%
Naoya Horiguchi 1 0.08% 1 2.04%
Greg Kroah-Hartman 1 0.08% 1 2.04%
Zwane Mwaikambo 1 0.08% 1 2.04%
Randy Dunlap 1 0.08% 1 2.04%
Jiri Slaby 1 0.08% 1 2.04%
Adrian Bunk 1 0.08% 1 2.04%
Total 1185 100.00% 49 100.00%
Directory: arch/x86/include/asm

Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with cregit.
Person	Tokens	Prop	Commits	CommitProp
Jeremy Fitzhardinge	15	53.57%	1	50.00%
Andi Kleen	13	46.43%	1	50.00%
Total	28	100.00%	2	100.00%
cregit-Linux how code gets into the kernel

Release 4.15 arch/x86/include/asm/pgtable_64.h

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Overall Contributors
