cregit-Linux how code gets into the kernel

Release 4.17 fs/exec.c

Directory: fs
/*
 *  linux/fs/exec.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 */

/*
 * #!-checking implemented by tytso.
 */
/*
 * Demand-loading implemented 01.12.91 - no need to read anything but
 * the header into memory. The inode of the executable is put into
 * "current->executable", and page faults do the actual loading. Clean.
 *
 * Once more I can proudly say that linux stood up to being changed: it
 * was less than 2 hours work to get demand-loading completely implemented.
 *
 * Demand loading changed July 1993 by Eric Youngdale.   Use mmap instead,
 * current->executable is only used by the procfs.  This allows a dispatch
 * table to check for several different types  of binary formats.  We keep
 * trying until we recognize the file or we run out of supported binary
 * formats.
 */

#include <linux/slab.h>
#include <linux/file.h>
#include <linux/fdtable.h>
#include <linux/mm.h>
#include <linux/vmacache.h>
#include <linux/stat.h>
#include <linux/fcntl.h>
#include <linux/swap.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/sched/mm.h>
#include <linux/sched/coredump.h>
#include <linux/sched/signal.h>
#include <linux/sched/numa_balancing.h>
#include <linux/sched/task.h>
#include <linux/pagemap.h>
#include <linux/perf_event.h>
#include <linux/highmem.h>
#include <linux/spinlock.h>
#include <linux/key.h>
#include <linux/personality.h>
#include <linux/binfmts.h>
#include <linux/utsname.h>
#include <linux/pid_namespace.h>
#include <linux/module.h>
#include <linux/namei.h>
#include <linux/mount.h>
#include <linux/security.h>
#include <linux/syscalls.h>
#include <linux/tsacct_kern.h>
#include <linux/cn_proc.h>
#include <linux/audit.h>
#include <linux/tracehook.h>
#include <linux/kmod.h>
#include <linux/fsnotify.h>
#include <linux/fs_struct.h>
#include <linux/pipe_fs_i.h>
#include <linux/oom.h>
#include <linux/compat.h>
#include <linux/vmalloc.h>

#include <linux/uaccess.h>
#include <asm/mmu_context.h>
#include <asm/tlb.h>

#include <trace/events/task.h>
#include "internal.h"

#include <trace/events/sched.h>


int suid_dumpable = 0;

static LIST_HEAD(formats);
static DEFINE_RWLOCK(binfmt_lock);


void __register_binfmt(struct linux_binfmt * fmt, int insert) { BUG_ON(!fmt); if (WARN_ON(!fmt->load_binary)) return; write_lock(&binfmt_lock); insert ? list_add(&fmt->lh, &formats) : list_add_tail(&fmt->lh, &formats); write_unlock(&binfmt_lock); }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)2842.42%450.00%
Ivan Kokshaysky1725.76%112.50%
Oleg Nesterov1116.67%112.50%
Al Viro57.58%112.50%
Alexey Dobriyan57.58%112.50%
Total66100.00%8100.00%

EXPORT_SYMBOL(__register_binfmt);
void unregister_binfmt(struct linux_binfmt * fmt) { write_lock(&binfmt_lock); list_del(&fmt->lh); write_unlock(&binfmt_lock); }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)2583.33%360.00%
Alexey Dobriyan516.67%240.00%
Total30100.00%5100.00%

EXPORT_SYMBOL(unregister_binfmt);
static inline void put_binfmt(struct linux_binfmt * fmt) { module_put(fmt->module); }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)1894.74%150.00%
Christoph Hellwig15.26%150.00%
Total19100.00%2100.00%


bool path_noexec(const struct path *path) { return (path->mnt->mnt_flags & MNT_NOEXEC) || (path->mnt->mnt_sb->s_iflags & SB_I_NOEXEC); }

Contributors

PersonTokensPropCommitsCommitProp
Eric W. Biedermann34100.00%1100.00%
Total34100.00%1100.00%

#ifdef CONFIG_USELIB /* * Note that a shared library must be both readable and executable due to * security reasons. * * Also note that we take the address to load from from the file itself. */ SYSCALL_DEFINE1(uselib, const char __user *, library) { struct linux_binfmt *fmt; struct file *file; struct filename *tmp = getname(library); int error = PTR_ERR(tmp); static const struct open_flags uselib_flags = { .open_flag = O_LARGEFILE | O_RDONLY | __FMODE_EXEC, .acc_mode = MAY_READ | MAY_EXEC, .intent = LOOKUP_OPEN, .lookup_flags = LOOKUP_FOLLOW, }; if (IS_ERR(tmp)) goto out; file = do_filp_open(AT_FDCWD, tmp, &uselib_flags); putname(tmp); error = PTR_ERR(file); if (IS_ERR(file)) goto out; error = -EINVAL; if (!S_ISREG(file_inode(file)->i_mode)) goto exit; error = -EACCES; if (path_noexec(&file->f_path)) goto exit; fsnotify_open(file); error = -ENOEXEC; read_lock(&binfmt_lock); list_for_each_entry(fmt, &formats, lh) { if (!fmt->load_shlib) continue; if (!try_module_get(fmt->module)) continue; read_unlock(&binfmt_lock); error = fmt->load_shlib(file); read_lock(&binfmt_lock); put_binfmt(fmt); if (error != -ENOEXEC) break; } read_unlock(&binfmt_lock); exit: fput(file); out: return error; } #endif /* #ifdef CONFIG_USELIB */ #ifdef CONFIG_MMU /* * The nascent bprm->mm is not visible until exec_mmap() but it can * use a lot of memory, account these pages in current->mm temporary * for oom_badness()->get_mm_rss(). Once exec succeeds or fails, we * change the counter back via acct_arg_size(0). */
static void acct_arg_size(struct linux_binprm *bprm, unsigned long pages) { struct mm_struct *mm = current->mm; long diff = (long)(pages - bprm->vma_pages); if (!mm || !diff) return; bprm->vma_pages = pages; add_mm_counter(mm, MM_ANONPAGES, diff); }

Contributors

PersonTokensPropCommitsCommitProp
Oleg Nesterov62100.00%2100.00%
Total62100.00%2100.00%


static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, int write) { struct page *page; int ret; unsigned int gup_flags = FOLL_FORCE; #ifdef CONFIG_STACK_GROWSUP if (write) { ret = expand_downwards(bprm->vma, pos); if (ret < 0) return NULL; } #endif if (write) gup_flags |= FOLL_WRITE; /* * We are doing an exec(). 'current' is the process * doing the exec and bprm->mm is the new process's mm. */ ret = get_user_pages_remote(current, bprm->mm, pos, 1, gup_flags, &page, NULL, NULL); if (ret <= 0) return NULL; if (write) { unsigned long size = bprm->vma->vm_end - bprm->vma->vm_start; unsigned long ptr_size, limit; /* * Since the stack will hold pointers to the strings, we * must account for them as well. * * The size calculation is the entire vma while each arg page is * built, so each time we get here it's calculating how far it * is currently (rather than each call being just the newly * added size from the arg page). As a result, we need to * always add the entire size of the pointers, so that on the * last call to get_arg_page() we'll actually have the entire * correct size. */ ptr_size = (bprm->argc + bprm->envc) * sizeof(void *); if (ptr_size > ULONG_MAX - size) goto fail; size += ptr_size; acct_arg_size(bprm, size / PAGE_SIZE); /* * We've historically supported up to 32 pages (ARG_MAX) * of argument strings even with small stacks */ if (size <= ARG_MAX) return page; /* * Limit to 1/4 of the max stack size or 3/4 of _STK_LIM * (whichever is smaller) for the argv+env strings. * This ensures that: * - the remaining binfmt code will not run out of stack space, * - the program will have a reasonable amount of stack left * to work from. */ limit = _STK_LIM / 4 * 3; limit = min(limit, bprm->rlim_stack.rlim_cur / 4); if (size > limit) goto fail; } return page; fail: put_page(page); return NULL; }

Contributors

PersonTokensPropCommitsCommitProp
Ollie Wild9138.89%15.88%
Kees Cook7331.20%317.65%
Linus Torvalds (pre-git)2510.68%529.41%
Lorenzo Stoakes177.26%211.76%
Linus Torvalds156.41%211.76%
Oleg Nesterov104.27%211.76%
Dave Hansen20.85%15.88%
Michal Hocko10.43%15.88%
Total234100.00%17100.00%


static void put_arg_page(struct page *page) { put_page(page); }

Contributors

PersonTokensPropCommitsCommitProp
Ollie Wild1381.25%133.33%
Linus Torvalds (pre-git)212.50%133.33%
Adrian Bunk16.25%133.33%
Total16100.00%3100.00%


static void free_arg_pages(struct linux_binprm *bprm) { }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)550.00%266.67%
Ollie Wild550.00%133.33%
Total10100.00%3100.00%


static void flush_arg_page(struct linux_binprm *bprm, unsigned long pos, struct page *page) { flush_cache_page(bprm->vma, pos, page_to_pfn(page)); }

Contributors

PersonTokensPropCommitsCommitProp
Ollie Wild2985.29%125.00%
Linus Torvalds (pre-git)411.76%250.00%
Andrew Morton12.94%125.00%
Total34100.00%4100.00%


static int __bprm_mm_init(struct linux_binprm *bprm) { int err; struct vm_area_struct *vma = NULL; struct mm_struct *mm = bprm->mm; bprm->vma = vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL); if (!vma) return -ENOMEM; if (down_write_killable(&mm->mmap_sem)) { err = -EINTR; goto err_free; } vma->vm_mm = mm; /* * Place the stack at the largest stack address the architecture * supports. Later, we'll move this to an appropriate place. We don't * use STACK_TOP because that can depend on attributes which aren't * configured yet. */ BUILD_BUG_ON(VM_STACK_FLAGS & VM_STACK_INCOMPLETE_SETUP); vma->vm_end = STACK_TOP_MAX; vma->vm_start = vma->vm_end - PAGE_SIZE; vma->vm_flags = VM_SOFTDIRTY | VM_STACK_FLAGS | VM_STACK_INCOMPLETE_SETUP; vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); INIT_LIST_HEAD(&vma->anon_vma_chain); err = insert_vm_struct(mm, vma); if (err) goto err; mm->stack_vm = mm->total_vm = 1; arch_bprm_mm_init(mm, vma); up_write(&mm->mmap_sem); bprm->p = vma->vm_end - sizeof(void *); return 0; err: up_write(&mm->mmap_sem); err_free: bprm->vma = NULL; kmem_cache_free(vm_area_cachep, vma); return err; }

Contributors

PersonTokensPropCommitsCommitProp
Ollie Wild16174.19%111.11%
Michal Hocko167.37%222.22%
Luiz Fernando N. Capitulino125.53%111.11%
Rik Van Riel83.69%111.11%
Mel Gorman83.69%111.11%
Dave Hansen73.23%111.11%
Coly Li31.38%111.11%
Cyrill V. Gorcunov20.92%111.11%
Total217100.00%9100.00%


static bool valid_arg_len(struct linux_binprm *bprm, long len) { return len <= MAX_ARG_STRLEN; }

Contributors

PersonTokensPropCommitsCommitProp
Ollie Wild19100.00%1100.00%
Total19100.00%1100.00%

#else
static inline void acct_arg_size(struct linux_binprm *bprm, unsigned long pages) { }

Contributors

PersonTokensPropCommitsCommitProp
Oleg Nesterov15100.00%2100.00%
Total15100.00%2100.00%


static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, int write) { struct page *page; page = bprm->page[pos / PAGE_SIZE]; if (!page && write) { page = alloc_page(GFP_HIGHUSER|__GFP_ZERO); if (!page) return NULL; bprm->page[pos / PAGE_SIZE] = page; } return page; }

Contributors

PersonTokensPropCommitsCommitProp
Ollie Wild7598.68%150.00%
Oleg Nesterov11.32%150.00%
Total76100.00%2100.00%


static void put_arg_page(struct page *page) { }

Contributors

PersonTokensPropCommitsCommitProp
Ollie Wild10100.00%1100.00%
Total10100.00%1100.00%


static void free_arg_page(struct linux_binprm *bprm, int i) { if (bprm->page[i]) { __free_page(bprm->page[i]); bprm->page[i] = NULL; } }

Contributors

PersonTokensPropCommitsCommitProp
Ollie Wild44100.00%1100.00%
Total44100.00%1100.00%


static void free_arg_pages(struct linux_binprm *bprm) { int i; for (i = 0; i < MAX_ARG_PAGES; i++) free_arg_page(bprm, i); }

Contributors

PersonTokensPropCommitsCommitProp
Ollie Wild34100.00%1100.00%
Total34100.00%1100.00%


static void flush_arg_page(struct linux_binprm *bprm, unsigned long pos, struct page *page) { }

Contributors

PersonTokensPropCommitsCommitProp
Ollie Wild19100.00%1100.00%
Total19100.00%1100.00%


static int __bprm_mm_init(struct linux_binprm *bprm) { bprm->p = PAGE_SIZE * MAX_ARG_PAGES - sizeof(void *); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Ollie Wild28100.00%1100.00%
Total28100.00%1100.00%


static bool valid_arg_len(struct linux_binprm *bprm, long len) { return len <= bprm->p; }

Contributors

PersonTokensPropCommitsCommitProp
Ollie Wild21100.00%1100.00%
Total21100.00%1100.00%

#endif /* CONFIG_MMU */ /* * Create a new mm_struct and populate it with a temporary stack * vm_area_struct. We don't have enough context at this point to set the stack * flags, permissions, and offset, so we use temporary values. We'll update * them later in setup_arg_pages(). */
static int bprm_mm_init(struct linux_binprm *bprm) { int err; struct mm_struct *mm = NULL; bprm->mm = mm = mm_alloc(); err = -ENOMEM; if (!mm) goto err; /* Save current stack limit for all calculations made during exec. */ task_lock(current->group_leader); bprm->rlim_stack = current->signal->rlim[RLIMIT_STACK]; task_unlock(current->group_leader); err = __bprm_mm_init(bprm); if (err) goto err; return 0; err: if (mm) { bprm->mm = NULL; mmdrop(mm); } return err; }

Contributors

PersonTokensPropCommitsCommitProp
Ollie Wild8173.64%133.33%
Kees Cook2825.45%133.33%
Yuanhan Liu10.91%133.33%
Total110100.00%3100.00%

struct user_arg_ptr { #ifdef CONFIG_COMPAT bool is_compat; #endif union { const char __user *const __user *native; #ifdef CONFIG_COMPAT const compat_uptr_t __user *compat; #endif } ptr; };
static const char __user *get_user_arg_ptr(struct user_arg_ptr argv, int nr) { const char __user *native; #ifdef CONFIG_COMPAT if (unlikely(argv.is_compat)) { compat_uptr_t compat; if (get_user(compat, argv.ptr.compat + nr)) return ERR_PTR(-EFAULT); return compat_ptr(compat); } #endif if (get_user(native, argv.ptr.native + nr)) return ERR_PTR(-EFAULT); return native; }

Contributors

PersonTokensPropCommitsCommitProp
Oleg Nesterov8691.49%360.00%
Ollie Wild66.38%120.00%
David Howells22.13%120.00%
Total94100.00%5100.00%

/* * count() counts the number of strings in array ARGV. */
static int count(struct user_arg_ptr argv, int max) { int i = 0; if (argv.ptr.native != NULL) { for (;;) { const char __user *p = get_user_arg_ptr(argv, i); if (!p) break; if (IS_ERR(p)) return -EFAULT; if (i >= max) return -E2BIG; ++i; if (fatal_signal_pending(current)) return -ERESTARTNOHAND; cond_resched(); } } return i; }

Contributors

PersonTokensPropCommitsCommitProp
Ollie Wild6162.89%112.50%
Oleg Nesterov2020.62%337.50%
Roland McGrath1111.34%112.50%
Xi Wang33.09%112.50%
David Howells11.03%112.50%
Jason Baron11.03%112.50%
Total97100.00%8100.00%

/* * 'copy_strings()' copies argument/environment strings from the old * processes's memory to the new process's stack. The call to get_user_pages() * ensures the destination page is created and not swapped out. */
static int copy_strings(int argc, struct user_arg_ptr argv, struct linux_binprm *bprm) { struct page *kmapped_page = NULL; char *kaddr = NULL; unsigned long kpos = 0; int ret; while (argc-- > 0) { const char __user *str; int len; unsigned long pos; ret = -EFAULT; str = get_user_arg_ptr(argv, argc); if (IS_ERR(str)) goto out; len = strnlen_user(str, MAX_ARG_STRLEN); if (!len) goto out; ret = -E2BIG; if (!valid_arg_len(bprm, len)) goto out; /* We're going to work our way backwords. */ pos = bprm->p; str += len; bprm->p -= len; while (len > 0) { int offset, bytes_to_copy; if (fatal_signal_pending(current)) { ret = -ERESTARTNOHAND; goto out; } cond_resched(); offset = pos % PAGE_SIZE; if (offset == 0) offset = PAGE_SIZE; bytes_to_copy = offset; if (bytes_to_copy > len) bytes_to_copy = len; offset -= bytes_to_copy; pos -= bytes_to_copy; str -= bytes_to_copy; len -= bytes_to_copy; if (!kmapped_page || kpos != (pos & PAGE_MASK)) { struct page *page; page = get_arg_page(bprm, pos, 1); if (!page) { ret = -E2BIG; goto out; } if (kmapped_page) { flush_kernel_dcache_page(kmapped_page); kunmap(kmapped_page); put_arg_page(kmapped_page); } kmapped_page = page; kaddr = kmap(kmapped_page); kpos = pos & PAGE_MASK; flush_arg_page(bprm, kpos, kmapped_page); } if (copy_from_user(kaddr+offset, str, bytes_to_copy)) { ret = -EFAULT; goto out; } } } ret = 0; out: if (kmapped_page) { flush_kernel_dcache_page(kmapped_page); kunmap(kmapped_page); put_arg_page(kmapped_page); } return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Ollie Wild16144.35%15.88%
Linus Torvalds (pre-git)9726.72%1058.82%
Andrew Morton5414.88%15.88%
Oleg Nesterov308.26%211.76%
Roland McGrath205.51%211.76%
David Howells10.28%15.88%
Total363100.00%17100.00%

/* * Like copy_strings, but get argv and its values from kernel memory. */
int copy_strings_kernel(int argc, const char *const *__argv, struct linux_binprm *bprm) { int r; mm_segment_t oldfs = get_fs(); struct user_arg_ptr argv = { .ptr.native = (const char __user *const __user *)__argv, }; set_fs(KERNEL_DS); r = copy_strings(argc, argv, bprm); set_fs(oldfs); return r; }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)3749.33%337.50%
Oleg Nesterov2330.67%225.00%
Ollie Wild810.67%112.50%
Linus Torvalds56.67%112.50%
David Howells22.67%112.50%
Total75100.00%8100.00%

EXPORT_SYMBOL(copy_strings_kernel); #ifdef CONFIG_MMU /* * During bprm_mm_init(), we create a temporary stack at STACK_TOP_MAX. Once * the binfmt code determines where the new stack should reside, we shift it to * its final location. The process proceeds as follows: * * 1) Use shift to calculate the new vma endpoints. * 2) Extend vma to cover both the old and new ranges. This ensures the * arguments passed to subsequent functions are consistent. * 3) Move vma's page tables to the new range. * 4) Free up any cleared pgd range. * 5) Shrink the vma to cover only the new range. */
static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift) { struct mm_struct *mm = vma->vm_mm; unsigned long old_start = vma->vm_start; unsigned long old_end = vma->vm_end; unsigned long length = old_end - old_start; unsigned long new_start = old_start - shift; unsigned long new_end = old_end - shift; struct mmu_gather tlb; BUG_ON(new_start > new_end); /* * ensure there are no vmas between where we want to go * and where we are */ if (vma != find_vma(mm, new_start)) return -EFAULT; /* * cover the whole range: [new_start, old_end) */ if (vma_adjust(vma, new_start, old_end, vma->vm_pgoff, NULL)) return -ENOMEM; /* * move the page tables downwards, on failure we rely on * process cleanup to remove whatever mess we made. */ if (length != move_page_tables(vma, old_start, vma, new_start, length, false)) return -ENOMEM; lru_add_drain(); tlb_gather_mmu(&tlb, mm, old_start, old_end); if (new_end > old_start) { /* * when the old and new regions overlap clear from new_end. */ free_pgd_range(&tlb, new_end, old_end, new_end, vma->vm_next ? vma->vm_next->vm_start : USER_PGTABLES_CEILING); } else { /* * otherwise, clean from old_start; this is done to not touch * the address space in [new_end, old_start) some architectures * have constraints on va-space that make this illegal (IA64) - * for the others its just a little faster. */ free_pgd_range(&tlb, old_start, old_end, new_end, vma->vm_next ? vma->vm_next->vm_start : USER_PGTABLES_CEILING); } tlb_finish_mmu(&tlb, old_start, old_end); /* * Shrink the vma to just the new range. Always succeeds. */ vma_adjust(vma, new_start, new_end, vma->vm_pgoff, NULL); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Ollie Wild16467.77%16.67%
Matthew Wilcox4217.36%16.67%
Rik Van Riel83.31%16.67%
Linus Torvalds (pre-git)72.89%426.67%
Peter Zijlstra62.48%16.67%
Linus Torvalds41.65%16.67%
Andrew Morton31.24%213.33%
Hugh Dickins20.83%16.67%
Rusty Russell20.83%16.67%
David Howells20.83%16.67%
Michel Lespinasse20.83%16.67%
Total242100.00%15100.00%

/* * Finalizes the stack vm_area_struct. The flags and permissions are updated, * the stack is optionally relocated, and some extra space is added. */
int setup_arg_pages(struct linux_binprm *bprm, unsigned long stack_top, int executable_stack) { unsigned long ret; unsigned long stack_shift; struct mm_struct *mm = current->mm; struct vm_area_struct *vma = bprm->vma; struct vm_area_struct *prev = NULL; unsigned long vm_flags; unsigned long stack_base; unsigned long stack_size; unsigned long stack_expand; unsigned long rlim_stack; #ifdef CONFIG_STACK_GROWSUP /* Limit stack size */ stack_base = bprm->rlim_stack.rlim_max; if (stack_base > STACK_SIZE_MAX) stack_base = STACK_SIZE_MAX; /* Add space for stack randomization. */ stack_base += (STACK_RND_MASK << PAGE_SHIFT); /* Make sure we didn't let the argument array grow too large. */ if (vma->vm_end - vma->vm_start > stack_base) return -ENOMEM; stack_base = PAGE_ALIGN(stack_top - stack_base); stack_shift = vma->vm_start - stack_base; mm->arg_start = bprm->p - stack_shift; bprm->p = vma->vm_end - stack_shift; #else stack_top = arch_align_stack(stack_top); stack_top = PAGE_ALIGN(stack_top); if (unlikely(stack_top < mmap_min_addr) || unlikely(vma->vm_end - vma->vm_start >= stack_top - mmap_min_addr)) return -ENOMEM; stack_shift = vma->vm_end - stack_top; bprm->p -= stack_shift; mm->arg_start = bprm->p; #endif if (bprm->loader) bprm->loader -= stack_shift; bprm->exec -= stack_shift; if (down_write_killable(&mm->mmap_sem)) return -EINTR; vm_flags = VM_STACK_FLAGS; /* * Adjust stack execute permissions; explicitly enable for * EXSTACK_ENABLE_X, disable for EXSTACK_DISABLE_X and leave alone * (arch default) otherwise. */ if (unlikely(executable_stack == EXSTACK_ENABLE_X)) vm_flags |= VM_EXEC; else if (executable_stack == EXSTACK_DISABLE_X) vm_flags &= ~VM_EXEC; vm_flags |= mm->def_flags; vm_flags |= VM_STACK_INCOMPLETE_SETUP; ret = mprotect_fixup(vma, &prev, vma->vm_start, vma->vm_end, vm_flags); if (ret) goto out_unlock; BUG_ON(prev != vma); /* Move stack pages down in memory. */ if (stack_shift) { ret = shift_arg_pages(vma, stack_shift); if (ret) goto out_unlock; } /* mprotect_fixup is overkill to remove the temporary stack flags */ vma->vm_flags &= ~VM_STACK_INCOMPLETE_SETUP; stack_expand = 131072UL; /* randomly 32*4k (or 2*64k) pages */ stack_size = vma->vm_end - vma->vm_start