cregit-Linux how code gets into the kernel

Release 4.10 mm/mprotect.c

Directory: mm
/*
 *  mm/mprotect.c
 *
 *  (C) Copyright 1994 Linus Torvalds
 *  (C) Copyright 2002 Christoph Hellwig
 *
 *  Address space accounting code       <alan@lxorguk.ukuu.org.uk>
 *  (C) Copyright 2002 Red Hat Inc, All Rights Reserved
 */

#include <linux/mm.h>
#include <linux/hugetlb.h>
#include <linux/shm.h>
#include <linux/mman.h>
#include <linux/fs.h>
#include <linux/highmem.h>
#include <linux/security.h>
#include <linux/mempolicy.h>
#include <linux/personality.h>
#include <linux/syscalls.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/mmu_notifier.h>
#include <linux/migrate.h>
#include <linux/perf_event.h>
#include <linux/pkeys.h>
#include <linux/ksm.h>
#include <linux/uaccess.h>
#include <asm/pgtable.h>
#include <asm/cacheflush.h>
#include <asm/mmu_context.h>
#include <asm/tlbflush.h>

#include "internal.h"

/*
 * For a prot_numa update we only hold mmap_sem for read so there is a
 * potential race with faulting where a pmd was temporarily none. This
 * function checks for a transhuge pmd under the appropriate lock. It
 * returns a pte if it was successfully locked or NULL if it raced with
 * a transhuge insertion.
 */

static pte_t *lock_pte_protection(struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, int prot_numa, spinlock_t **ptl) { pte_t *pte; spinlock_t *pmdl; /* !prot_numa is protected by mmap_sem held for write */ if (!prot_numa) return pte_offset_map_lock(vma->vm_mm, pmd, addr, ptl); pmdl = pmd_lock(vma->vm_mm, pmd); if (unlikely(pmd_trans_huge(*pmd) || pmd_none(*pmd))) { spin_unlock(pmdl); return NULL; } pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, ptl); spin_unlock(pmdl); return pte; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman117100.00%1100.00%
Total117100.00%1100.00%


static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, unsigned long end, pgprot_t newprot, int dirty_accountable, int prot_numa) { struct mm_struct *mm = vma->vm_mm; pte_t *pte, oldpte; spinlock_t *ptl; unsigned long pages = 0; int target_node = NUMA_NO_NODE; pte = lock_pte_protection(vma, pmd, addr, prot_numa, &ptl); if (!pte) return 0; /* Get target node for single threaded private VMAs */ if (prot_numa && !(vma->vm_flags & VM_SHARED) && atomic_read(&vma->vm_mm->mm_users) == 1) target_node = numa_node_id(); arch_enter_lazy_mmu_mode(); do { oldpte = *pte; if (pte_present(oldpte)) { pte_t ptent; bool preserve_write = prot_numa && pte_write(oldpte); /* * Avoid trapping faults against the zero or KSM * pages. See similar comment in change_huge_pmd. */ if (prot_numa) { struct page *page; page = vm_normal_page(vma, addr, oldpte); if (!page || PageKsm(page)) continue; /* Avoid TLB flush if possible */ if (pte_protnone(oldpte)) continue; /* * Don't mess with PTEs if page is already on the node * a single-threaded process is running on. */ if (target_node == page_to_nid(page)) continue; } ptent = ptep_modify_prot_start(mm, addr, pte); ptent = pte_modify(ptent, newprot); if (preserve_write) ptent = pte_mkwrite(ptent); /* Avoid taking write faults for known dirty pages */ if (dirty_accountable && pte_dirty(ptent) && (pte_soft_dirty(ptent) || !(vma->vm_flags & VM_SOFTDIRTY))) { ptent = pte_mkwrite(ptent); } ptep_modify_prot_commit(mm, addr, pte, ptent); pages++; } else if (IS_ENABLED(CONFIG_MIGRATION)) { swp_entry_t entry = pte_to_swp_entry(oldpte); if (is_write_migration_entry(entry)) { pte_t newpte; /* * A protection check is difficult so * just be safe and disable write */ make_migration_entry_read(&entry); newpte = swp_entry_to_pte(entry); if (pte_swp_soft_dirty(oldpte)) newpte = pte_swp_mksoft_dirty(newpte); set_pte_at(mm, addr, pte, newpte); pages++; } } } while (pte++, addr += PAGE_SIZE, addr != end); arch_leave_lazy_mmu_mode(); pte_unmap_unlock(pte - 1, ptl); return pages; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman9725.13%726.92%
pre-gitpre-git5213.47%415.38%
andi kleenandi kleen4712.18%13.85%
christoph lameterchristoph lameter4712.18%13.85%
aneesh kumaraneesh kumar276.99%13.85%
hugh dickinshugh dickins266.74%27.69%
cyrill gorcunovcyrill gorcunov256.48%13.85%
peter zijlstrapeter zijlstra235.96%27.69%
peter feinerpeter feiner164.15%13.85%
david s. millerdavid s. miller61.55%13.85%
zachary amsdenzachary amsden61.55%13.85%
ingo molnaringo molnar61.55%13.85%
konstantin khlebnikovkonstantin khlebnikov41.04%13.85%
rohit sethrohit seth30.78%13.85%
jeremy fitzhardingejeremy fitzhardinge10.26%13.85%
Total386100.00%26100.00%


static inline unsigned long change_pmd_range(struct vm_area_struct *vma, pud_t *pud, unsigned long addr, unsigned long end, pgprot_t newprot, int dirty_accountable, int prot_numa) { pmd_t *pmd; struct mm_struct *mm = vma->vm_mm; unsigned long next; unsigned long pages = 0; unsigned long nr_huge_updates = 0; unsigned long mni_start = 0; pmd = pmd_offset(pud, addr); do { unsigned long this_pages; next = pmd_addr_end(addr, end); if (!pmd_trans_huge(*pmd) && !pmd_devmap(*pmd) && pmd_none_or_clear_bad(pmd)) continue; /* invoke the mmu notifier if the pmd is populated */ if (!mni_start) { mni_start = addr; mmu_notifier_invalidate_range_start(mm, mni_start, end); } if (pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) { if (next - addr != HPAGE_PMD_SIZE) { __split_huge_pmd(vma, pmd, addr, false, NULL); if (pmd_trans_unstable(pmd)) continue; } else { int nr_ptes = change_huge_pmd(vma, pmd, addr, newprot, prot_numa); if (nr_ptes) { if (nr_ptes == HPAGE_PMD_NR) { pages += HPAGE_PMD_NR; nr_huge_updates++; } /* huge pmd was handled */ continue; } } /* fall through, the trans huge pmd just split */ } this_pages = change_pte_range(vma, pmd, addr, next, newprot, dirty_accountable, prot_numa); pages += this_pages; } while (pmd++, addr = next, addr != end); if (mni_start) mmu_notifier_invalidate_range_end(mm, mni_start, end); if (nr_huge_updates) count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates); return pages; }

Contributors

PersonTokensPropCommitsCommitProp
rik van rielrik van riel6522.34%28.33%
mel gormanmel gorman6020.62%729.17%
johannes weinerjohannes weiner3411.68%28.33%
andi kleenandi kleen279.28%14.17%
hugh dickinshugh dickins237.90%14.17%
pre-gitpre-git237.90%14.17%
peter zijlstrapeter zijlstra206.87%28.33%
dan williamsdan williams134.47%14.17%
kirill a. shutemovkirill a. shutemov124.12%312.50%
david rientjesdavid rientjes51.72%14.17%
andrea arcangeliandrea arcangeli41.37%14.17%
david s. millerdavid s. miller41.37%14.17%
naoya horiguchinaoya horiguchi10.34%14.17%
Total291100.00%24100.00%


static inline unsigned long change_pud_range(struct vm_area_struct *vma, pgd_t *pgd, unsigned long addr, unsigned long end, pgprot_t newprot, int dirty_accountable, int prot_numa) { pud_t *pud; unsigned long next; unsigned long pages = 0; pud = pud_offset(pgd, addr); do { next = pud_addr_end(addr, end); if (pud_none_or_clear_bad(pud)) continue; pages += change_pmd_range(vma, pud, addr, next, newprot, dirty_accountable, prot_numa); } while (pud++, addr = next, addr != end); return pages; }

Contributors

PersonTokensPropCommitsCommitProp
hugh dickinshugh dickins3127.43%218.18%
andi kleenandi kleen2925.66%19.09%
pre-gitpre-git2320.35%327.27%
peter zijlstrapeter zijlstra1815.93%218.18%
mel gormanmel gorman54.42%19.09%
david s. millerdavid s. miller43.54%19.09%
johannes weinerjohannes weiner32.65%19.09%
Total113100.00%11100.00%


static unsigned long change_protection_range(struct vm_area_struct *vma, unsigned long addr, unsigned long end, pgprot_t newprot, int dirty_accountable, int prot_numa) { struct mm_struct *mm = vma->vm_mm; pgd_t *pgd; unsigned long next; unsigned long start = addr; unsigned long pages = 0; BUG_ON(addr >= end); pgd = pgd_offset(mm, addr); flush_cache_range(vma, addr, end); set_tlb_flush_pending(mm); do { next = pgd_addr_end(addr, end); if (pgd_none_or_clear_bad(pgd)) continue; pages += change_pud_range(vma, pgd, addr, next, newprot, dirty_accountable, prot_numa); } while (pgd++, addr = next, addr != end); /* Only flush the TLB if we actually modified any entries: */ if (pages) flush_tlb_range(vma, start, end); clear_tlb_flush_pending(mm); return pages; }

Contributors

PersonTokensPropCommitsCommitProp
hugh dickinshugh dickins4628.22%212.50%
pre-gitpre-git3420.86%425.00%
andi kleenandi kleen2716.56%16.25%
peter zijlstrapeter zijlstra2012.27%212.50%
rik van rielrik van riel106.13%16.25%
kanoj sarcarkanoj sarcar84.91%16.25%
linus torvaldslinus torvalds63.68%16.25%
ingo molnaringo molnar53.07%16.25%
mel gormanmel gorman53.07%16.25%
david s. millerdavid s. miller10.61%16.25%
johannes weinerjohannes weiner10.61%16.25%
Total163100.00%16100.00%


unsigned long change_protection(struct vm_area_struct *vma, unsigned long start, unsigned long end, pgprot_t newprot, int dirty_accountable, int prot_numa) { unsigned long pages; if (is_vm_hugetlb_page(vma)) pages = hugetlb_change_protection(vma, start, end, newprot); else pages = change_protection_range(vma, start, end, newprot, dirty_accountable, prot_numa); return pages; }

Contributors

PersonTokensPropCommitsCommitProp
peter zijlstrapeter zijlstra6791.78%133.33%
mel gormanmel gorman56.85%133.33%
pre-gitpre-git11.37%133.33%
Total73100.00%3100.00%


int mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev, unsigned long start, unsigned long end, unsigned long newflags) { struct mm_struct *mm = vma->vm_mm; unsigned long oldflags = vma->vm_flags; long nrpages = (end - start) >> PAGE_SHIFT; unsigned long charged = 0; pgoff_t pgoff; int error; int dirty_accountable = 0; if (newflags == oldflags) { *pprev = vma; return 0; } /* * If we make a private mapping writable we increase our commit; * but (without finer accounting) cannot reduce our commit if we * make it unwritable again. hugetlb mapping were accounted for * even if read-only so there is no need to account for them here */ if (newflags & VM_WRITE) { /* Check space limits when area turns into data. */ if (!may_expand_vm(mm, newflags, nrpages) && may_expand_vm(mm, oldflags, nrpages)) return -ENOMEM; if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB| VM_SHARED|VM_NORESERVE))) { charged = nrpages; if (security_vm_enough_memory_mm(mm, charged)) return -ENOMEM; newflags |= VM_ACCOUNT; } } /* * First try to merge with previous and/or next vma. */ pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); *pprev = vma_merge(mm, *pprev, start, end, newflags, vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), vma->vm_userfaultfd_ctx); if (*pprev) { vma = *pprev; VM_WARN_ON((vma->vm_flags ^ newflags) & ~VM_SOFTDIRTY); goto success; } *pprev = vma; if (start != vma->vm_start) { error = split_vma(mm, vma, start, 1); if (error) goto fail; } if (end != vma->vm_end) { error = split_vma(mm, vma, end, 0); if (error) goto fail; } success: /* * vm_flags and vm_page_prot are protected by the mmap_sem * held in write mode. */ vma->vm_flags = newflags; dirty_accountable = vma_wants_writenotify(vma, vma->vm_page_prot); vma_set_page_prot(vma); change_protection(vma, start, end, vma->vm_page_prot, dirty_accountable, 0); /* * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major * fault on access. */ if ((oldflags & (VM_WRITE | VM_SHARED | VM_LOCKED)) == VM_LOCKED && (newflags & VM_WRITE)) { populate_vma_page_range(vma, start, end, NULL); } vm_stat_account(mm, oldflags, -nrpages); vm_stat_account(mm, newflags, nrpages); perf_event_mmap(vma); return 0; fail: vm_unacct_memory(charged); return error; }

Contributors

PersonTokensPropCommitsCommitProp
andrew mortonandrew morton17540.70%618.75%
pre-gitpre-git4510.47%515.62%
hugh dickinshugh dickins399.07%26.25%
kirill a. shutemovkirill a. shutemov368.37%13.12%
linus torvaldslinus torvalds337.67%13.12%
konstantin khlebnikovkonstantin khlebnikov266.05%13.12%
andrea arcangeliandrea arcangeli255.81%412.50%
peter zijlstrapeter zijlstra194.42%26.25%
pekka j enbergpekka j enberg51.16%13.12%
chris wrightchris wright51.16%13.12%
david howellsdavid howells40.93%13.12%
mel gormanmel gorman40.93%26.25%
al viroal viro30.70%13.12%
venkatesh pallipadivenkatesh pallipadi30.70%13.12%
peter feinerpeter feiner30.70%13.12%
william lee irwin iiiwilliam lee irwin iii30.70%13.12%
andy whitcroftandy whitcroft20.47%13.12%
Total430100.00%32100.00%

/* * pkey==-1 when doing a legacy mprotect() */
static int do_mprotect_pkey(unsigned long start, size_t len, unsigned long prot, int pkey) { unsigned long nstart, end, tmp, reqprot; struct vm_area_struct *vma, *prev; int error = -EINVAL; const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP); const bool rier = (current->personality & READ_IMPLIES_EXEC) && (prot & PROT_READ); prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP); if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */ return -EINVAL; if (start & ~PAGE_MASK) return -EINVAL; if (!len) return 0; len = PAGE_ALIGN(len); end = start + len; if (end <= start) return -ENOMEM; if (!arch_validate_prot(prot)) return -EINVAL; reqprot = prot; if (down_write_killable(&current->mm->mmap_sem)) return -EINTR; /* * If userspace did not allocate the pkey, do not let * them use it here. */ error = -EINVAL; if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey)) goto out; vma = find_vma(current->mm, start); error = -ENOMEM; if (!vma) goto out; prev = vma->vm_prev; if (unlikely(grows & PROT_GROWSDOWN)) { if (vma->vm_start >= end) goto out; start = vma->vm_start; error = -EINVAL; if (!(vma->vm_flags & VM_GROWSDOWN)) goto out; } else { if (vma->vm_start > start) goto out; if (unlikely(grows & PROT_GROWSUP)) { end = vma->vm_end; error = -EINVAL; if (!(vma->vm_flags & VM_GROWSUP)) goto out; } } if (start > vma->vm_start) prev = vma; for (nstart = start ; ; ) { unsigned long mask_off_old_flags; unsigned long newflags; int new_vma_pkey; /* Here we know that vma->vm_start <= nstart < vma->vm_end. */ /* Does the application expect PROT_READ to imply PROT_EXEC */ if (rier && (vma->vm_flags & VM_MAYEXEC)) prot |= PROT_EXEC; /* * Each mprotect() call explicitly passes r/w/x permissions. * If a permission is not passed to mprotect(), it must be * cleared from the VMA. */ mask_off_old_flags = VM_READ | VM_WRITE | VM_EXEC | ARCH_VM_PKEY_FLAGS; new_vma_pkey = arch_override_mprotect_pkey(vma, prot, pkey); newflags = calc_vm_prot_bits(prot, new_vma_pkey); newflags |= (vma->vm_flags & ~mask_off_old_flags); /* newflags >> 4 shift VM_MAY% in place of VM_% */ if ((newflags & ~(newflags >> 4)) & (VM_READ | VM_WRITE | VM_EXEC)) { error = -EACCES; goto out; } error = security_file_mprotect(vma, reqprot, prot); if (error) goto out; tmp = vma->vm_end; if (tmp > end) tmp = end; error = mprotect_fixup(vma, &prev, nstart, tmp, newflags); if (error) goto out; nstart = tmp; if (nstart < prev->vm_end) nstart = prev->vm_end; if (nstart >= end) goto out; vma = prev->vm_next; if (!vma || vma->vm_start != nstart) { error = -ENOMEM; goto out; } prot = reqprot; } out: up_write(&current->mm->mmap_sem); return error; }

Contributors

PersonTokensPropCommitsCommitProp
pre-gitpre-git20334.88%1031.25%
roland mcgrathroland mcgrath12821.99%13.12%
dave hansendave hansen8214.09%412.50%
andrew mortonandrew morton467.90%39.38%
piotr kwapulinskipiotr kwapulinski396.70%13.12%
linus torvaldslinus torvalds284.81%39.38%
stephen d. smalleystephen d. smalley193.26%26.25%
gordon jingordon jin91.55%13.12%
michal hockomichal hocko81.37%13.12%
paolo giarrussopaolo giarrusso81.37%13.12%
greg kroah-hartmangreg kroah-hartman50.86%26.25%
dave kleikampdave kleikamp40.69%13.12%
hirofumi ogawahirofumi ogawa20.34%13.12%
hugh dickinshugh dickins10.17%13.12%
Total582100.00%32100.00%

SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len, unsigned long, prot) { return do_mprotect_pkey(start, len, prot, -1); } #ifdef CONFIG_ARCH_HAS_PKEYS SYSCALL_DEFINE4(pkey_mprotect, unsigned long, start, size_t, len, unsigned long, prot, int, pkey) { return do_mprotect_pkey(start, len, prot, pkey); } SYSCALL_DEFINE2(pkey_alloc, unsigned long, flags, unsigned long, init_val) { int pkey; int ret; /* No flags supported yet. */ if (flags) return -EINVAL; /* check for unsupported init values */ if (init_val & ~PKEY_ACCESS_MASK) return -EINVAL; down_write(&current->mm->mmap_sem); pkey = mm_pkey_alloc(current->mm); ret = -ENOSPC; if (pkey == -1) goto out; ret = arch_set_user_pkey_access(current, pkey, init_val); if (ret) { mm_pkey_free(current->mm, pkey); goto out; } ret = pkey; out: up_write(&current->mm->mmap_sem); return ret; } SYSCALL_DEFINE1(pkey_free, int, pkey) { int ret; down_write(&current->mm->mmap_sem); ret = mm_pkey_free(current->mm, pkey); up_write(&current->mm->mmap_sem); /* * We could provie warnings or errors if any VMA still * has the pkey set here. */ return ret; } #endif /* CONFIG_ARCH_HAS_PKEYS */

Overall Contributors

PersonTokensPropCommitsCommitProp
pre-gitpre-git39215.88%1915.97%
dave hansendave hansen32513.16%43.36%
mel gormanmel gorman29712.03%1310.92%
andrew mortonandrew morton2279.19%108.40%
peter zijlstrapeter zijlstra1696.84%43.36%
hugh dickinshugh dickins1666.72%54.20%
andi kleenandi kleen1305.27%21.68%
roland mcgrathroland mcgrath1285.18%10.84%
rik van rielrik van riel753.04%32.52%
linus torvaldslinus torvalds712.88%65.04%
christoph lameterchristoph lameter532.15%10.84%
kirill a. shutemovkirill a. shutemov512.07%43.36%
piotr kwapulinskipiotr kwapulinski391.58%10.84%
johannes weinerjohannes weiner381.54%21.68%
andrea arcangeliandrea arcangeli321.30%54.20%
konstantin khlebnikovkonstantin khlebnikov301.22%21.68%
aneesh kumaraneesh kumar271.09%10.84%
cyrill gorcunovcyrill gorcunov251.01%10.84%
stephen d. smalleystephen d. smalley220.89%21.68%
peter feinerpeter feiner190.77%10.84%
ingo molnaringo molnar170.69%43.36%
david s. millerdavid s. miller150.61%10.84%
dan williamsdan williams130.53%10.84%
gordon jingordon jin90.36%10.84%
kanoj sarcarkanoj sarcar80.32%10.84%
paolo giarrussopaolo giarrusso80.32%10.84%
michal hockomichal hocko80.32%10.84%
richard hendersonrichard henderson70.28%10.84%
heiko carstensheiko carstens60.24%10.84%
zachary amsdenzachary amsden60.24%10.84%
pekka j enbergpekka j enberg50.20%10.84%
david rientjesdavid rientjes50.20%10.84%
greg kroah-hartmangreg kroah-hartman50.20%21.68%
chris wrightchris wright50.20%10.84%
dave kleikampdave kleikamp40.16%10.84%
david howellsdavid howells40.16%10.84%
arnd bergmannarnd bergmann30.12%10.84%
rohit sethrohit seth30.12%10.84%
venkatesh pallipadivenkatesh pallipadi30.12%10.84%
kosaki motohirokosaki motohiro30.12%10.84%
dave jonesdave jones30.12%10.84%
william lee irwin iiiwilliam lee irwin iii30.12%10.84%
al viroal viro30.12%10.84%
andy whitcroftandy whitcroft20.08%10.84%
hirofumi ogawahirofumi ogawa20.08%10.84%
jeremy fitzhardingejeremy fitzhardinge10.04%10.84%
alan coxalan cox10.04%10.84%
naoya horiguchinaoya horiguchi10.04%10.84%
Total2469100.00%119100.00%
Directory: mm
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.