| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| H. Peter Anvin | 690 | 87.01% | 2 | 15.38% |
| Zhu Guihua | 64 | 8.07% | 2 | 15.38% |
| Kirill A. Shutemov | 23 | 2.90% | 2 | 15.38% |
| Mark Rutland | 5 | 0.63% | 1 | 7.69% |
| Tom Lendacky | 4 | 0.50% | 1 | 7.69% |
| Andrew Lutomirski | 3 | 0.38% | 2 | 15.38% |
| Boris Ostrovsky | 2 | 0.25% | 1 | 7.69% |
| Alexander (Sasha) Levin | 1 | 0.13% | 1 | 7.69% |
| Dave Hansen | 1 | 0.13% | 1 | 7.69% |
| Total | 793 | 13 |
/* ----------------------------------------------------------------------- * * * Copyright 2014 Intel Corporation; author: H. Peter Anvin * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, * version 2, as published by the Free Software Foundation. * * This program is distributed in the hope it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * * ----------------------------------------------------------------------- */ /* * The IRET instruction, when returning to a 16-bit segment, only * restores the bottom 16 bits of the user space stack pointer. This * causes some 16-bit software to break, but it also leaks kernel state * to user space. * * This works around this by creating percpu "ministacks", each of which * is mapped 2^16 times 64K apart. When we detect that the return SS is * on the LDT, we copy the IRET frame to the ministack and use the * relevant alias to return to userspace. The ministacks are mapped * readonly, so if the IRET fault we promote #GP to #DF which is an IST * vector and thus has its own stack; we then do the fixup in the #DF * handler. * * This file sets up the ministacks and the related page tables. The * actual ministack invocation is in entry_64.S. */ #include <linux/init.h> #include <linux/init_task.h> #include <linux/kernel.h> #include <linux/percpu.h> #include <linux/gfp.h> #include <linux/random.h> #include <asm/pgtable.h> #include <asm/pgalloc.h> #include <asm/setup.h> #include <asm/espfix.h> /* * Note: we only need 6*8 = 48 bytes for the espfix stack, but round * it up to a cache line to avoid unnecessary sharing. */ #define ESPFIX_STACK_SIZE (8*8UL) #define ESPFIX_STACKS_PER_PAGE (PAGE_SIZE/ESPFIX_STACK_SIZE) /* There is address space for how many espfix pages? */ #define ESPFIX_PAGE_SPACE (1UL << (P4D_SHIFT-PAGE_SHIFT-16)) #define ESPFIX_MAX_CPUS (ESPFIX_STACKS_PER_PAGE * ESPFIX_PAGE_SPACE) #if CONFIG_NR_CPUS > ESPFIX_MAX_CPUS # error "Need more virtual address space for the ESPFIX hack" #endif #define PGALLOC_GFP (GFP_KERNEL | __GFP_ZERO) /* This contains the *bottom* address of the espfix stack */ DEFINE_PER_CPU_READ_MOSTLY(unsigned long, espfix_stack); DEFINE_PER_CPU_READ_MOSTLY(unsigned long, espfix_waddr); /* Initialization mutex - should this be a spinlock? */ static DEFINE_MUTEX(espfix_init_mutex); /* Page allocation bitmap - each page serves ESPFIX_STACKS_PER_PAGE CPUs */ #define ESPFIX_MAX_PAGES DIV_ROUND_UP(CONFIG_NR_CPUS, ESPFIX_STACKS_PER_PAGE) static void *espfix_pages[ESPFIX_MAX_PAGES]; static __page_aligned_bss pud_t espfix_pud_page[PTRS_PER_PUD] __aligned(PAGE_SIZE); static unsigned int page_random, slot_random; /* * This returns the bottom address of the espfix stack for a specific CPU. * The math allows for a non-power-of-two ESPFIX_STACK_SIZE, in which case * we have to account for some amount of padding at the end of each page. */ static inline unsigned long espfix_base_addr(unsigned int cpu) { unsigned long page, slot; unsigned long addr; page = (cpu / ESPFIX_STACKS_PER_PAGE) ^ page_random; slot = (cpu + slot_random) % ESPFIX_STACKS_PER_PAGE; addr = (page << PAGE_SHIFT) + (slot * ESPFIX_STACK_SIZE); addr = (addr & 0xffffUL) | ((addr & ~0xffffUL) << 16); addr += ESPFIX_BASE_ADDR; return addr; } #define PTE_STRIDE (65536/PAGE_SIZE) #define ESPFIX_PTE_CLONES (PTRS_PER_PTE/PTE_STRIDE) #define ESPFIX_PMD_CLONES PTRS_PER_PMD #define ESPFIX_PUD_CLONES (65536/(ESPFIX_PTE_CLONES*ESPFIX_PMD_CLONES)) #define PGTABLE_PROT ((_KERNPG_TABLE & ~_PAGE_RW) | _PAGE_NX) static void init_espfix_random(void) { unsigned long rand; /* * This is run before the entropy pools are initialized, * but this is hopefully better than nothing. */ if (!arch_get_random_long(&rand)) { /* The constant is an arbitrary large prime */ rand = rdtsc(); rand *= 0xc345c6b72fd16123UL; } slot_random = rand % ESPFIX_STACKS_PER_PAGE; page_random = (rand / ESPFIX_STACKS_PER_PAGE) & (ESPFIX_PAGE_SPACE - 1); } void __init init_espfix_bsp(void) { pgd_t *pgd; p4d_t *p4d; /* Install the espfix pud into the kernel page directory */ pgd = &init_top_pgt[pgd_index(ESPFIX_BASE_ADDR)]; p4d = p4d_alloc(&init_mm, pgd, ESPFIX_BASE_ADDR); p4d_populate(&init_mm, p4d, espfix_pud_page); /* Randomize the locations */ init_espfix_random(); /* The rest is the same as for any other processor */ init_espfix_ap(0); } void init_espfix_ap(int cpu) { unsigned int page; unsigned long addr; pud_t pud, *pud_p; pmd_t pmd, *pmd_p; pte_t pte, *pte_p; int n, node; void *stack_page; pteval_t ptemask; /* We only have to do this once... */ if (likely(per_cpu(espfix_stack, cpu))) return; /* Already initialized */ addr = espfix_base_addr(cpu); page = cpu/ESPFIX_STACKS_PER_PAGE; /* Did another CPU already set this up? */ stack_page = READ_ONCE(espfix_pages[page]); if (likely(stack_page)) goto done; mutex_lock(&espfix_init_mutex); /* Did we race on the lock? */ stack_page = READ_ONCE(espfix_pages[page]); if (stack_page) goto unlock_done; node = cpu_to_node(cpu); ptemask = __supported_pte_mask; pud_p = &espfix_pud_page[pud_index(addr)]; pud = *pud_p; if (!pud_present(pud)) { struct page *page = alloc_pages_node(node, PGALLOC_GFP, 0); pmd_p = (pmd_t *)page_address(page); pud = __pud(__pa(pmd_p) | (PGTABLE_PROT & ptemask)); paravirt_alloc_pmd(&init_mm, __pa(pmd_p) >> PAGE_SHIFT); for (n = 0; n < ESPFIX_PUD_CLONES; n++) set_pud(&pud_p[n], pud); } pmd_p = pmd_offset(&pud, addr); pmd = *pmd_p; if (!pmd_present(pmd)) { struct page *page = alloc_pages_node(node, PGALLOC_GFP, 0); pte_p = (pte_t *)page_address(page); pmd = __pmd(__pa(pte_p) | (PGTABLE_PROT & ptemask)); paravirt_alloc_pte(&init_mm, __pa(pte_p) >> PAGE_SHIFT); for (n = 0; n < ESPFIX_PMD_CLONES; n++) set_pmd(&pmd_p[n], pmd); } pte_p = pte_offset_kernel(&pmd, addr); stack_page = page_address(alloc_pages_node(node, GFP_KERNEL, 0)); /* * __PAGE_KERNEL_* includes _PAGE_GLOBAL, which we want since * this is mapped to userspace. */ pte = __pte(__pa(stack_page) | ((__PAGE_KERNEL_RO | _PAGE_ENC) & ptemask)); for (n = 0; n < ESPFIX_PTE_CLONES; n++) set_pte(&pte_p[n*PTE_STRIDE], pte); /* Job is done for this CPU and any CPU which shares this page */ WRITE_ONCE(espfix_pages[page], stack_page); unlock_done: mutex_unlock(&espfix_init_mutex); done: per_cpu(espfix_stack, cpu) = addr; per_cpu(espfix_waddr, cpu) = (unsigned long)stack_page + (addr & ~PAGE_MASK); }
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1