cregit-Linux how code gets into the kernel

Release 4.15 mm/vmalloc.c

Directory: mm
/*
 *  linux/mm/vmalloc.c
 *
 *  Copyright (C) 1993  Linus Torvalds
 *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
 *  SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
 *  Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
 *  Numa awareness, Christoph Lameter, SGI, June 2005
 */

#include <linux/vmalloc.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/highmem.h>
#include <linux/sched/signal.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/interrupt.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/debugobjects.h>
#include <linux/kallsyms.h>
#include <linux/list.h>
#include <linux/notifier.h>
#include <linux/rbtree.h>
#include <linux/radix-tree.h>
#include <linux/rcupdate.h>
#include <linux/pfn.h>
#include <linux/kmemleak.h>
#include <linux/atomic.h>
#include <linux/compiler.h>
#include <linux/llist.h>
#include <linux/bitops.h>

#include <linux/uaccess.h>
#include <asm/tlbflush.h>
#include <asm/shmparam.h>

#include "internal.h"


struct vfree_deferred {
	
struct llist_head list;
	
struct work_struct wq;
};
static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred);

static void __vunmap(const void *, int);


static void free_work(struct work_struct *w) { struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq); struct llist_node *t, *llnode; llist_for_each_safe(llnode, t, llist_del_all(&p->list)) __vunmap((void *)llnode, 1); }

Contributors

PersonTokensPropCommitsCommitProp
Al Viro4075.47%150.00%
Byungchul Park1324.53%150.00%
Total53100.00%2100.00%

/*** Page table manipulation functions ***/
static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end) { pte_t *pte; pte = pte_offset_kernel(pmd, addr); do { pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte); WARN_ON(!pte_none(ptent) && !pte_present(ptent)); } while (pte++, addr += PAGE_SIZE, addr != end); }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)4356.58%660.00%
Hugh Dickins2431.58%110.00%
David S. Miller45.26%110.00%
Christoph Hellwig45.26%110.00%
Ingo Molnar11.32%110.00%
Total76100.00%10100.00%


static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end) { pmd_t *pmd; unsigned long next; pmd = pmd_offset(pud, addr); do { next = pmd_addr_end(addr, end); if (pmd_clear_huge(pmd)) continue; if (pmd_none_or_clear_bad(pmd)) continue; vunmap_pte_range(pmd, addr, next); } while (pmd++, addr = next, addr != end); }

Contributors

PersonTokensPropCommitsCommitProp
Hugh Dickins3641.86%222.22%
Linus Torvalds (pre-git)3540.70%444.44%
Toshi Kani89.30%111.11%
Christoph Hellwig44.65%111.11%
Andi Kleen33.49%111.11%
Total86100.00%9100.00%


static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end) { pud_t *pud; unsigned long next; pud = pud_offset(p4d, addr); do { next = pud_addr_end(addr, end); if (pud_clear_huge(pud)) continue; if (pud_none_or_clear_bad(pud)) continue; vunmap_pmd_range(pud, addr, next); } while (pud++, addr = next, addr != end); }

Contributors

PersonTokensPropCommitsCommitProp
Andi Kleen4248.84%120.00%
Hugh Dickins3338.37%240.00%
Toshi Kani89.30%120.00%
Kirill A. Shutemov33.49%120.00%
Total86100.00%5100.00%


static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end) { p4d_t *p4d; unsigned long next; p4d = p4d_offset(pgd, addr); do { next = p4d_addr_end(addr, end); if (p4d_clear_huge(p4d)) continue; if (p4d_none_or_clear_bad(p4d)) continue; vunmap_pud_range(p4d, addr, next); } while (p4d++, addr = next, addr != end); }

Contributors

PersonTokensPropCommitsCommitProp
Kirill A. Shutemov86100.00%1100.00%
Total86100.00%1100.00%


static void vunmap_page_range(unsigned long addr, unsigned long end) { pgd_t *pgd; unsigned long next; BUG_ON(addr >= end); pgd = pgd_offset_k(addr); do { next = pgd_addr_end(addr, end); if (pgd_none_or_clear_bad(pgd)) continue; vunmap_p4d_range(pgd, addr, next); } while (pgd++, addr = next, addr != end); }

Contributors

PersonTokensPropCommitsCommitProp
Hugh Dickins4759.49%220.00%
Linus Torvalds (pre-git)2126.58%440.00%
Benjamin Herrenschmidt67.59%110.00%
Nicholas Piggin33.80%110.00%
Andrew Morton11.27%110.00%
Kirill A. Shutemov11.27%110.00%
Total79100.00%10100.00%


static int vmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, int *nr) { pte_t *pte; /* * nr is a running index into the array which helps higher level * callers keep track of where we're up to. */ pte = pte_alloc_kernel(pmd, addr); if (!pte) return -ENOMEM; do { struct page *page = pages[*nr]; if (WARN_ON(!pte_none(*pte))) return -EBUSY; if (WARN_ON(!page)) return -ENOMEM; set_pte_at(&init_mm, addr, pte, mk_pte(page, prot)); (*nr)++; } while (pte++, addr += PAGE_SIZE, addr != end); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)4835.29%440.00%
Hugh Dickins4533.09%110.00%
Nicholas Piggin2014.71%110.00%
Andrew Morton1712.50%110.00%
Christoph Hellwig42.94%110.00%
Oleg Nesterov10.74%110.00%
Ingo Molnar10.74%110.00%
Total136100.00%10100.00%


static int vmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, int *nr) { pmd_t *pmd; unsigned long next; pmd = pmd_alloc(&init_mm, pud, addr); if (!pmd) return -ENOMEM; do { next = pmd_addr_end(addr, end); if (vmap_pte_range(pmd, addr, next, prot, pages, nr)) return -ENOMEM; } while (pmd++, addr = next, addr != end); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Andi Kleen6861.82%125.00%
Hugh Dickins3430.91%125.00%
Nicholas Piggin65.45%125.00%
Andrew Morton21.82%125.00%
Total110100.00%4100.00%


static int vmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, int *nr) { pud_t *pud; unsigned long next; pud = pud_alloc(&init_mm, p4d, addr); if (!pud) return -ENOMEM; do { next = pud_addr_end(addr, end); if (vmap_pmd_range(pud, addr, next, prot, pages, nr)) return -ENOMEM; } while (pud++, addr = next, addr != end); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Hugh Dickins6458.18%112.50%
Andi Kleen1614.55%112.50%
Christoph Hellwig98.18%112.50%
Nicholas Piggin65.45%112.50%
Linus Torvalds (pre-git)65.45%112.50%
Andrew Morton65.45%225.00%
Kirill A. Shutemov32.73%112.50%
Total110100.00%8100.00%


static int vmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, pgprot_t prot, struct page **pages, int *nr) { p4d_t *p4d; unsigned long next; p4d = p4d_alloc(&init_mm, pgd, addr); if (!p4d) return -ENOMEM; do { next = p4d_addr_end(addr, end); if (vmap_pud_range(p4d, addr, next, prot, pages, nr)) return -ENOMEM; } while (p4d++, addr = next, addr != end); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Kirill A. Shutemov110100.00%1100.00%
Total110100.00%1100.00%

/* * Set up page tables in kva (addr, end). The ptes shall have prot "prot", and * will have pfns corresponding to the "pages" array. * * Ie. pte at addr+N*PAGE_SIZE shall point to pfn corresponding to pages[N] */
static int vmap_page_range_noflush(unsigned long start, unsigned long end, pgprot_t prot, struct page **pages) { pgd_t *pgd; unsigned long next; unsigned long addr = start; int err = 0; int nr = 0; BUG_ON(addr >= end); pgd = pgd_offset_k(addr); do { next = pgd_addr_end(addr, end); err = vmap_p4d_range(pgd, addr, next, prot, pages, &nr); if (err) return err; } while (pgd++, addr = next, addr != end); return nr; }

Contributors

PersonTokensPropCommitsCommitProp
Hugh Dickins3933.91%17.14%
Nicholas Piggin1916.52%17.14%
Christoph Hellwig1815.65%17.14%
Linus Torvalds (pre-git)1210.43%428.57%
Adam Lackorzynski76.09%17.14%
Andrew Morton65.22%17.14%
Andi Kleen65.22%17.14%
Marcus Alanen32.61%17.14%
Figo.zhang32.61%17.14%
Tejun Heo10.87%17.14%
Kirill A. Shutemov10.87%17.14%
Total115100.00%14100.00%


static int vmap_page_range(unsigned long start, unsigned long end, pgprot_t prot, struct page **pages) { int ret; ret = vmap_page_range_noflush(start, end, prot, pages); flush_cache_vmap(start, end); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Tejun Heo49100.00%1100.00%
Total49100.00%1100.00%


int is_vmalloc_or_module_addr(const void *x) { /* * ARM, x86-64 and sparc64 put modules in a special place, * and fall back on vmalloc() if that fails. Others * just put it in the vmalloc space. */ #if defined(CONFIG_MODULES) && defined(MODULES_VADDR) unsigned long addr = (unsigned long)x; if (addr >= MODULES_VADDR && addr < MODULES_END) return 1; #endif return is_vmalloc_addr(x); }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds5298.11%150.00%
Russell King11.89%150.00%
Total53100.00%2100.00%

/* * Walk a vmap address to the struct page it maps. */
struct page *vmalloc_to_page(const void *vmalloc_addr) { unsigned long addr = (unsigned long) vmalloc_addr; struct page *page = NULL; pgd_t *pgd = pgd_offset_k(addr); p4d_t *p4d; pud_t *pud; pmd_t *pmd; pte_t *ptep, pte; /* * XXX we might need to change this if we add VIRTUAL_BUG_ON for * architectures that do not vmalloc module space */ VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr)); if (pgd_none(*pgd)) return NULL; p4d = p4d_offset(pgd, addr); if (p4d_none(*p4d)) return NULL; pud = pud_offset(p4d, addr); /* * Don't dereference bad PUD or PMD (below) entries. This will also * identify huge mappings, which we may encounter on architectures * that define CONFIG_HAVE_ARCH_HUGE_VMAP=y. Such regions will be * identified as vmalloc addresses by is_vmalloc_addr(), but are * not [unambiguously] associated with a struct page, so there is * no correct value to return for them. */ WARN_ON_ONCE(pud_bad(*pud)); if (pud_none(*pud) || pud_bad(*pud)) return NULL; pmd = pmd_offset(pud, addr); WARN_ON_ONCE(pmd_bad(*pmd)); if (pmd_none(*pmd) || pmd_bad(*pmd)) return NULL; ptep = pte_offset_map(pmd, addr); pte = *ptep; if (pte_present(pte)) page = pte_page(pte); pte_unmap(ptep); return page; }

Contributors

PersonTokensPropCommitsCommitProp
Christoph Lameter9948.53%225.00%
Kirill A. Shutemov5225.49%112.50%
Ard Biesheuvel3115.20%112.50%
Vassili Karpov125.88%112.50%
Jiri Slaby83.92%112.50%
Ingo Molnar10.49%112.50%
Linus Torvalds10.49%112.50%
Total204100.00%8100.00%

EXPORT_SYMBOL(vmalloc_to_page); /* * Map a vmalloc()-space virtual address to the physical page frame number. */
unsigned long vmalloc_to_pfn(const void *vmalloc_addr) { return page_to_pfn(vmalloc_to_page(vmalloc_addr)); }

Contributors

PersonTokensPropCommitsCommitProp
Christoph Lameter1575.00%266.67%
Vassili Karpov525.00%133.33%
Total20100.00%3100.00%

EXPORT_SYMBOL(vmalloc_to_pfn); /*** Global kva allocator ***/ #define VM_LAZY_FREE 0x02 #define VM_VM_AREA 0x04 static DEFINE_SPINLOCK(vmap_area_lock); /* Export for kexec only */ LIST_HEAD(vmap_area_list); static LLIST_HEAD(vmap_purge_list); static struct rb_root vmap_area_root = RB_ROOT; /* The vmap cache globals are protected by vmap_area_lock */ static struct rb_node *free_vmap_cache; static unsigned long cached_hole_size; static unsigned long cached_vstart; static unsigned long cached_align; static unsigned long vmap_area_pcpu_hole;
static struct vmap_area *__find_vmap_area(unsigned long addr) { struct rb_node *n = vmap_area_root.rb_node; while (n) { struct vmap_area *va; va = rb_entry(n, struct vmap_area, rb_node); if (addr < va->va_start) n = n->rb_left; else if (addr >= va->va_end) n = n->rb_right; else return va; } return NULL; }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin5467.50%120.00%
James Bottomley2025.00%120.00%
Hirofumi Ogawa33.75%120.00%
Daisuke Hatayama22.50%120.00%
Christoph Hellwig11.25%120.00%
Total80100.00%5100.00%


static void __insert_vmap_area(struct vmap_area *va) { struct rb_node **p = &vmap_area_root.rb_node; struct rb_node *parent = NULL; struct rb_node *tmp; while (*p) { struct vmap_area *tmp_va; parent = *p; tmp_va = rb_entry(parent, struct vmap_area, rb_node); if (va->va_start < tmp_va->va_end) p = &(*p)->rb_left; else if (va->va_end > tmp_va->va_start) p = &(*p)->rb_right; else BUG(); } rb_link_node(&va->rb_node, parent, p); rb_insert_color(&va->rb_node, &vmap_area_root); /* address-sort this list */ tmp = rb_prev(&va->rb_node); if (tmp) { struct vmap_area *prev; prev = rb_entry(tmp, struct vmap_area, rb_node); list_add_rcu(&va->list, &prev->list); } else list_add_rcu(&va->list, &vmap_area_list); }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin15481.05%114.29%
Linus Torvalds (pre-git)189.47%114.29%
James Bottomley105.26%114.29%
Namhyung Kim42.11%114.29%
Russell King21.05%114.29%
Christoph Lameter10.53%114.29%
JoonSoo Kim10.53%114.29%
Total190100.00%7100.00%

static void purge_vmap_area_lazy(void); static BLOCKING_NOTIFIER_HEAD(vmap_notify_list); /* * Allocate a region of KVA of the specified size and alignment, within the * vstart and vend. */
static struct vmap_area *alloc_vmap_area(unsigned long size, unsigned long align, unsigned long vstart, unsigned long vend, int node, gfp_t gfp_mask) { struct vmap_area *va; struct rb_node *n; unsigned long addr; int purged = 0; struct vmap_area *first; BUG_ON(!size); BUG_ON(offset_in_page(size)); BUG_ON(!is_power_of_2(align)); might_sleep(); va = kmalloc_node(sizeof(struct vmap_area), gfp_mask & GFP_RECLAIM_MASK, node); if (unlikely(!va)) return ERR_PTR(-ENOMEM); /* * Only scan the relevant parts containing pointers to other objects * to avoid false negatives. */ kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask & GFP_RECLAIM_MASK); retry: spin_lock(&vmap_area_lock); /* * Invalidate cache if we have more permissive parameters. * cached_hole_size notes the largest hole noticed _below_ * the vmap_area cached in free_vmap_cache: if size fits * into that hole, we want to scan from vstart to reuse * the hole instead of allocating above free_vmap_cache. * Note that __free_vmap_area may update free_vmap_cache * without updating cached_hole_size or cached_align. */ if (!free_vmap_cache || size < cached_hole_size || vstart < cached_vstart || align < cached_align) { nocache: cached_hole_size = 0; free_vmap_cache = NULL; } /* record if we encounter less permissive parameters */ cached_vstart = vstart; cached_align = align; /* find starting point for our search */ if (free_vmap_cache) { first = rb_entry(free_vmap_cache, struct vmap_area, rb_node); addr = ALIGN(first->va_end, align); if (addr < vstart) goto nocache; if (addr + size < addr) goto overflow; } else { addr = ALIGN(vstart, align); if (addr + size < addr) goto overflow; n = vmap_area_root.rb_node; first = NULL; while (n) { struct vmap_area *tmp; tmp = rb_entry(n, struct vmap_area, rb_node); if (tmp->va_end >= addr) { first = tmp; if (tmp->va_start <= addr) break; n = n->rb_left; } else n = n->rb_right; } if (!first) goto found; } /* from the starting point, walk areas until a suitable hole is found */ while (addr + size > first->va_start && addr + size <= vend) { if (addr + cached_hole_size < first->va_start) cached_hole_size = first->va_start - addr; addr = ALIGN(first->va_end, align); if (addr + size < addr) goto overflow; if (list_is_last(&first->list, &vmap_area_list)) goto found; first = list_next_entry(first, list); } found: if (addr + size > vend) goto overflow; va->va_start = addr; va->va_end = addr + size; va->flags = 0; __insert_vmap_area(va); free_vmap_cache = &va->rb_node; spin_unlock(&vmap_area_lock); BUG_ON(!IS_ALIGNED(va->va_start, align)); BUG_ON(va->va_start < vstart); BUG_ON(va->va_end > vend); return va; overflow: spin_unlock(&vmap_area_lock); if (!purged) { purge_vmap_area_lazy(); purged = 1; goto retry; } if (gfpflags_allow_blocking(gfp_mask)) { unsigned long freed = 0; blocking_notifier_call_chain(&vmap_notify_list, 0, &freed); if (freed > 0) { purged = 0; goto retry; } } if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit()) pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n", size); kfree(va); return ERR_PTR(-EBUSY); }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin47580.51%313.04%
Chris Wilson427.12%14.35%
Catalin Marinas152.54%14.35%
Hong Zhi Guo122.03%14.35%
Glauber de Oliveira Costa101.69%28.70%
Florian Fainelli71.19%14.35%
Wang Xiaoqiang50.85%14.35%
Ralph Wuerthner50.85%14.35%
Linus Torvalds (pre-git)40.68%313.04%
James Bottomley40.68%28.70%
Alexander Kuleshov30.51%14.35%
Russell King20.34%14.35%
Christoph Hellwig20.34%14.35%
Johannes Weiner10.17%14.35%
Pintu Kumar10.17%14.35%
Geliang Tang10.17%14.35%
Joe Perches10.17%14.35%
Total590100.00%23100.00%


int register_vmap_purge_notifier(struct notifier_block *nb) { return blocking_notifier_chain_register(&vmap_notify_list, nb); }

Contributors

PersonTokensPropCommitsCommitProp
Chris Wilson19100.00%1100.00%
Total19100.00%1100.00%

EXPORT_SYMBOL_GPL(register_vmap_purge_notifier);
int unregister_vmap_purge_notifier(struct notifier_block *nb) { return blocking_notifier_chain_unregister(&vmap_notify_list, nb); }

Contributors

PersonTokensPropCommitsCommitProp
Chris Wilson19100.00%1100.00%
Total19100.00%1100.00%

EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier);
static void __free_vmap_area(struct vmap_area *va) { BUG_ON(RB_EMPTY_NODE(&va->rb_node)); if (free_vmap_cache) { if (va->va_end < cached_vstart) { free_vmap_cache = NULL; } else { struct vmap_area *cache; cache = rb_entry(free_vmap_cache, struct vmap_area, rb_node); if (va->va_start <= cache->va_start) { free_vmap_cache = rb_prev(&va->rb_node); /* * We don't try to update cached_hole_size or * cached_align, but it won't go very wrong. */ } } } rb_erase(&va->rb_node, &vmap_area_root); RB_CLEAR_NODE(&va->rb_node); list_del_rcu(&va->list); /* * Track the highest possible candidate for pcpu area * allocation. Areas outside of vmalloc area can be returned * here too, consider only end addresses which fall inside * vmalloc area proper. */ if (va->va_end > VMALLOC_START && va->va_end <= VMALLOC_END) vmap_area_pcpu_hole = max(vmap_area_pcpu_hole, va->va_end); kfree_rcu(va, rcu_head); }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin11780.69%250.00%
Tejun Heo2617.93%125.00%
Lai Jiangshan21.38%125.00%
Total145100.00%4100.00%

/* * Free a region of KVA allocated by alloc_vmap_area */
static void free_vmap_area(struct vmap_area *va) { spin_lock(&vmap_area_lock); __free_vmap_area(va); spin_unlock(&vmap_area_lock); }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin28100.00%1100.00%
Total28100.00%1100.00%

/* * Clear the pagetable entries of a given vmap_area */
static void unmap_vmap_area(struct vmap_area *va) { vunmap_page_range(va->va_start, va->va_end); }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin22100.00%1100.00%
Total22100.00%1100.00%


static void vmap_debug_free_range(unsigned long start, unsigned long end) { /* * Unmap page tables and force a TLB flush immediately if pagealloc * debugging is enabled. This catches use after free bugs similarly to * those in linear kernel virtual address space after a page has been * freed. * * All the lazy freeing logic is still retained, in order to minimise * intrusiveness of this debugging feature. * * This is going to be *slow* (linear kernel virtual address debugging * doesn't do a broadcast TLB flush so it is a lot faster). */ if (debug_pagealloc_enabled()) { vunmap_page_range(start, end); flush_tlb_kernel_range(start, end); } }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin2877.78%150.00%
JoonSoo Kim822.22%150.00%
Total36100.00%2100.00%

/* * lazy_max_pages is the maximum amount of virtual address space we gather up * before attempting to purge with a TLB flush. * * There is a tradeoff here: a larger number will cover more kernel page tables * and take slightly longer to purge, but it will linearly reduce the number of * global TLB flushes that must be performed. It would seem natural to scale * this number up linearly with the number of CPUs (because vmapping activity * could also scale linearly with the number of CPUs), however it is likely * that in practice, workloads might be constrained in other ways that mean * vmap activity will not scale linearly with CPUs. Also, I want to be * conservative and not introduce a big latency on huge systems, so go with * a less aggressive log scale. It will still be an improvement over the old * code, and it will be simple to change the scale factor if we find that it * becomes a problem on bigger systems. */
static unsigned long lazy_max_pages(void) { unsigned int log; log = fls(num_online_cpus()); return log * (32UL * 1024 * 1024 / PAGE_SIZE); }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin34100.00%1100.00%
Total34100.00%1100.00%

static atomic_t vmap_lazy_nr = ATOMIC_INIT(0); /* * Serialize vmap purging. There is no actual criticial section protected * by this look, but we want to avoid concurrent calls for performance * reasons and to make the pcpu_get_vm_areas more deterministic. */ static DEFINE_MUTEX(vmap_purge_lock); /* for per-CPU blocks */ static void purge_fragmented_blocks_allcpus(void); /* * called before a call to iounmap() if the caller wants vm_area_struct's * immediately freed. */
void set_iounmap_nonlazy(void) { atomic_set(&vmap_lazy_nr, lazy_max_pages()+1); }

Contributors

PersonTokensPropCommitsCommitProp
Cliff Wickman18100.00%1100.00%
Total18100.00%1100.00%

/* * Purges all lazily-freed vmap areas. */
static bool __purge_vmap_area_lazy(unsigned long start, unsigned long end) { struct llist_node *valist; struct vmap_area *va; struct vmap_area *n_va; bool do_free = false; lockdep_assert_held(&vmap_purge_lock); valist = llist_del_all(&vmap_purge_list); llist_for_each_entry(va, valist, purge_list) { if (va->va_start < start) start = va->va_start; if (va->va_end > end) end = va->va_end; do_free = true; } if (!do_free) return false; flush_tlb_kernel_range(start, end); spin_lock(&vmap_area_lock); llist_for_each_entry_safe(va, n_va, valist, purge_list) { int nr = (va->va_end - va->va_start) >> PAGE_SHIFT; __free_vmap_area(va); atomic_sub(nr, &vmap_lazy_nr); cond_resched_lock(&vmap_area_lock); } spin_unlock(&vmap_area_lock); return true; }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin9557.23%120.00%
Joel Fernandes3822.89%120.00%
Chris Wilson169.64%120.00%
Christoph Hellwig106.02%120.00%
Vegard Nossum74.22%120.00%
Total166100.00%5100.00%

/* * Kick off a purge of the outstanding lazy areas. Don't bother if somebody * is already purging. */
static void try_purge_vmap_area_lazy(void) { if (mutex_trylock(&vmap_purge_lock)) { __purge_vmap_area_lazy(ULONG_MAX, 0); mutex_unlock(&vmap_purge_lock); } }

Contributors

PersonTokensPropCommitsCommitProp
Christoph Hellwig1754.84%266.67%
Nicholas Piggin1445.16%133.33%
Total31100.00%3100.00%

/* * Kick off a purge of the outstanding lazy areas. */
static void purge_vmap_area_lazy(void) { mutex_lock(&vmap_purge_lock); purge_fragmented_blocks_allcpus(); __purge_vmap_area_lazy(ULONG_MAX, 0); mutex_unlock(&vmap_purge_lock); }

Contributors

PersonTokensPropCommitsCommitProp
Christoph Hellwig1653.33%266.67%
Nicholas Piggin1446.67%133.33%
Total30100.00%3100.00%

/* * Free a vmap area, caller ensuring that the area has been unmapped * and flush_cache_vunmap had been called for the correct range * previously. */
static void free_vmap_area_noflush(struct vmap_area *va) { int nr_lazy; nr_lazy = atomic_add_return((va->va_end - va->va_start) >> PAGE_SHIFT, &vmap_lazy_nr); /* After this point, we may free va at any time */ llist_add(&va->purge_list, &vmap_purge_list); if (unlikely(nr_lazy > lazy_max_pages())) try_purge_vmap_area_lazy(); }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin3966.10%250.00%
Chris Wilson1932.20%125.00%
Jeremy Fitzhardinge11.69%125.00%
Total59100.00%4100.00%

/* * Free and unmap a vmap area */
static void free_unmap_vmap_area(struct vmap_area *va) { flush_cache_vunmap(va->va_start, va->va_end); unmap_vmap_area(va); free_vmap_area_noflush(va); }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin2681.25%150.00%
Christoph Hellwig618.75%150.00%
Total32100.00%2100.00%


static struct vmap_area *find_vmap_area(unsigned long addr) { struct vmap_area *va; spin_lock(&vmap_area_lock); va = __find_vmap_area(addr); spin_unlock(&vmap_area_lock); return va; }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin39100.00%1100.00%
Total39100.00%1100.00%

/*** Per cpu kva allocator ***/ /* * vmap space is limited especially on 32 bit architectures. Ensure there is * room for at least 16 percpu vmap blocks per CPU. */ /* * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able * to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess * instead (we just need a rough idea) */ #if BITS_PER_LONG == 32 #define VMALLOC_SPACE (128UL*1024*1024) #else #define VMALLOC_SPACE (128UL*1024*1024*1024) #endif #define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE) #define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */ #define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */ #define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2) #define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */ #define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */ #define VMAP_BBMAP_BITS \ VMAP_MIN(VMAP_BBMAP_BITS_MAX, \ VMAP_MAX(VMAP_BBMAP_BITS_MIN, \ VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16)) #define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE) static bool vmap_initialized __read_mostly = false; struct vmap_block_queue { spinlock_t lock; struct list_head free; }; struct vmap_block { spinlock_t lock; struct vmap_area *va; unsigned long free, dirty; unsigned long dirty_min, dirty_max; /*< dirty range */ struct list_head free_list; struct rcu_head rcu_head; struct list_head purge; }; /* Queue of free and dirty vmap blocks, for allocation and flushing purposes */ static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue); /* * Radix tree of vmap blocks, indexed by address, to quickly find a vmap block * in the free path. Could get rid of this if we change the API to return a * "cookie" from alloc, to be passed to free. But no big deal yet. */ static DEFINE_SPINLOCK(vmap_block_tree_lock); static RADIX_TREE(vmap_block_tree, GFP_ATOMIC); /* * We should probably have a fallback mechanism to allocate virtual memory * out of partially filled vmap blocks. However vmap block sizing should be * fairly reasonable according to the vmalloc size, so it shouldn't be a * big problem. */
static unsigned long addr_to_vb_idx(unsigned long addr) { addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1); addr /= VMAP_BLOCK_SIZE; return addr; }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin29100.00%1100.00%
Total29100.00%1100.00%


static void *vmap_block_vaddr(unsigned long va_start, unsigned long pages_off) { unsigned long addr; addr = va_start + (pages_off << PAGE_SHIFT); BUG_ON(addr_to_vb_idx(addr) != addr_to_vb_idx(va_start)); return (void *)addr; }

Contributors

PersonTokensPropCommitsCommitProp
Roman Peniaev49100.00%1100.00%
Total49100.00%1100.00%

/** * new_vmap_block - allocates new vmap_block and occupies 2^order pages in this * block. Of course pages number can't exceed VMAP_BBMAP_BITS * @order: how many 2^order pages should be occupied in newly allocated block * @gfp_mask: flags for the page level allocator * * Returns: virtual address in a newly allocated block or ERR_PTR(-errno) */
static void *new_vmap_block(unsigned int order, gfp_t gfp_mask) { struct vmap_block_queue *vbq; struct vmap_block *vb; struct vmap_area *va; unsigned long vb_idx; int node, err; void *vaddr; node = numa_node_id(); vb = kmalloc_node(sizeof(struct vmap_block), gfp_mask & GFP_RECLAIM_MASK, node); if (unlikely(!vb)) return ERR_PTR(-ENOMEM); va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE, VMALLOC_START, VMALLOC_END, node, gfp_mask); if (IS_ERR(va)) { kfree(vb); return ERR_CAST(va); } err = radix_tree_preload(gfp_mask); if (unlikely(err)) { kfree(vb); free_vmap_area(va); return ERR_PTR(err); } vaddr = vmap_block_vaddr(va->va_start, 0); spin_lock_init(&vb->lock); vb->va = va; /* At least something should be left free */ BUG_ON(VMAP_BBMAP_BITS <= (1UL << order)); vb->free = VMAP_BBMAP_BITS - (1UL << order); vb->dirty = 0; vb->dirty_min = VMAP_BBMAP_BITS; vb->dirty_max = 0; INIT_LIST_HEAD(&vb->free_list); vb_idx = addr_to_vb_idx(va->va_start); spin_lock(&vmap_block_tree_lock); err = radix_tree_insert(&vmap_block_tree, vb_idx, vb); spin_unlock(&vmap_block_tree_lock); BUG_ON(err); radix_tree_preload_end(); vbq = &get_cpu_var(vmap_block_queue); spin_lock(&vbq->lock); list_add_tail_rcu(&vb->free_list, &vbq->free); spin_unlock(&vbq->lock); put_cpu_var(vmap_block_queue); return vaddr; }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin25983.82%116.67%
Roman Peniaev4815.53%350.00%
Julia Lawall10.32%116.67%
Tejun Heo10.32%116.67%
Total309100.00%6100.00%


static void free_vmap_block(struct vmap_block *vb) { struct vmap_block *tmp; unsigned long vb_idx; vb_idx = addr_to_vb_idx(vb->va->va_start); spin_lock(&vmap_block_tree_lock); tmp = radix_tree_delete(&vmap_block_tree, vb_idx); spin_unlock(&vmap_block_tree_lock); BUG_ON(tmp != vb); free_vmap_area_noflush(vb->va); kfree_rcu(vb, rcu_head); }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin7195.95%133.33%
Lai Jiangshan22.70%133.33%
Jeremy Fitzhardinge11.35%133.33%
Total74100.00%3100.00%


static void purge_fragmented_blocks(int cpu) { LIST_HEAD(purge); struct vmap_block *vb; struct vmap_block *n_vb; struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); rcu_read_lock(); list_for_each_entry_rcu(vb, &vbq->free, free_list) { if (!(vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS)) continue; spin_lock(&vb->lock); if (vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS) { vb->free = 0; /* prevent further allocs after releasing lock */ vb->dirty = VMAP_BBMAP_BITS; /* prevent purging it again */ vb->dirty_min = 0; vb->dirty_max = VMAP_BBMAP_BITS; spin_lock(&vbq->lock); list_del_rcu(&vb->free_list); spin_unlock(&vbq->lock); spin_unlock(&vb->lock); list_add_tail(&vb->purge, &purge); } else spin_unlock(&vb->lock); } rcu_read_unlock(); list_for_each_entry_safe(vb, n_vb, &purge, purge) { list_del(&vb->purge); free_vmap_block(vb); } }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin19896.12%150.00%
Roman Peniaev83.88%150.00%
Total206100.00%2100.00%


static void purge_fragmented_blocks_allcpus(void) { int cpu; for_each_possible_cpu(cpu) purge_fragmented_blocks(cpu); }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin20100.00%1100.00%
Total20100.00%1100.00%


static void *vb_alloc(unsigned long size, gfp_t gfp_mask) { struct vmap_block_queue *vbq; struct vmap_block *vb; void *vaddr = NULL; unsigned int order; BUG_ON(offset_in_page(size)); BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); if (WARN_ON(size == 0)) { /* * Allocating 0 bytes isn't what caller wants since * get_order(0) returns funny result. Just warn and terminate * early. */ return NULL; } order = get_order(size); rcu_read_lock(); vbq = &get_cpu_var(vmap_block_queue); list_for_each_entry_rcu(vb, &vbq->free, free_list) { unsigned long pages_off; spin_lock(&vb->lock); if (vb->free < (1UL << order)) { spin_unlock(&vb->lock); continue; } pages_off = VMAP_BBMAP_BITS - vb->free; vaddr = vmap_block_vaddr(vb->va->va_start, pages_off); vb->free -= 1UL << order; if (vb->free == 0) { spin_lock(&vbq->lock); list_del_rcu(&vb->free_list); spin_unlock(&vbq->lock); } spin_unlock(&vb->lock); break; } put_cpu_var(vmap_block_queue); rcu_read_unlock(); /* Allocate new block if nothing was found */ if (!vaddr) vaddr = new_vmap_block(order, gfp_mask); return vaddr; }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin17677.53%337.50%
Roman Peniaev3113.66%112.50%
Jan Kara156.61%112.50%
Alexander Kuleshov31.32%112.50%
Zhang Yanfei10.44%112.50%
Tejun Heo10.44%112.50%
Total227100.00%8100.00%


static void vb_free(const void *addr, unsigned long size) { unsigned long offset; unsigned long vb_idx; unsigned int order; struct vmap_block *vb; BUG_ON(offset_in_page(size)); BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); flush_cache_vunmap((unsigned long)addr, (unsigned long)addr + size); order = get_order(size); offset = (unsigned long)addr & (VMAP_BLOCK_SIZE - 1); offset >>= PAGE_SHIFT; vb_idx = addr_to_vb_idx((unsigned long)addr); rcu_read_lock(); vb = radix_tree_lookup(&vmap_block_tree, vb_idx); rcu_read_unlock(); BUG_ON(!vb); vunmap_page_range((unsigned long)addr, (unsigned long)addr + size); spin_lock(&vb->lock); /* Expand dirty range */ vb->dirty_min = min(vb->dirty_min, offset); vb->dirty_max = max(vb->dirty_max, offset + (1UL << order)); vb->dirty += 1UL << order; if (vb->dirty == VMAP_BBMAP_BITS) { BUG_ON(vb->free); spin_unlock(&vb->lock); free_vmap_block(vb); } else spin_unlock(&vb->lock); }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin18179.04%350.00%
Roman Peniaev2812.23%116.67%
Jeremy Fitzhardinge177.42%116.67%
Alexander Kuleshov31.31%116.67%
Total229100.00%6100.00%

/** * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer * * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily * to amortize TLB flushing overheads. What this means is that any page you * have now, may, in a former life, have been mapped into kernel virtual * address by the vmap layer and so there might be some CPUs with TLB entries * still referencing that page (additional to the regular 1:1 kernel mapping). * * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can * be sure that none of the pages we have control over will have any aliases * from the vmap layer. */
void vm_unmap_aliases(void) { unsigned long start = ULONG_MAX, end = 0; int cpu; int flush = 0; if (unlikely(!vmap_initialized)) return; might_sleep(); for_each_possible_cpu(cpu) { struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu); struct vmap_block *vb; rcu_read_lock(); list_for_each_entry_rcu(vb, &vbq->free, free_list) { spin_lock(&vb->lock); if (vb->dirty) { unsigned long va_start = vb->va->va_start; unsigned long s, e; s = va_start + (vb->dirty_min << PAGE_SHIFT); e = va_start + (vb->dirty_max << PAGE_SHIFT); start = min(s, start); end = max(e, end); flush = 1; } spin_unlock(&vb->lock); } rcu_read_unlock(); } mutex_lock(&vmap_purge_lock); purge_fragmented_blocks_allcpus(); if (!__purge_vmap_area_lazy(start, end) && flush) flush_tlb_kernel_range(start, end); mutex_unlock(&vmap_purge_lock); }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin12563.45%114.29%
Roman Peniaev3316.75%114.29%
Christoph Hellwig2914.72%342.86%
Jeremy Fitzhardinge94.57%114.29%
JoonSoo Kim10.51%114.29%
Total197100.00%7100.00%

EXPORT_SYMBOL_GPL(vm_unmap_aliases); /** * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram * @mem: the pointer returned by vm_map_ram * @count: the count passed to that vm_map_ram call (cannot unmap partial) */
void vm_unmap_ram(const void *mem, unsigned int count) { unsigned long size = (unsigned long)count << PAGE_SHIFT; unsigned long addr = (unsigned long)mem; struct vmap_area *va; might_sleep(); BUG_ON(!addr); BUG_ON(addr < VMALLOC_START); BUG_ON(addr > VMALLOC_END); BUG_ON(!PAGE_ALIGNED(addr)); debug_check_no_locks_freed(mem, size); vmap_debug_free_range(addr, addr+size); if (likely(count <= VMAP_MAX_ALLOC)) { vb_free(mem, size); return; } va = find_vmap_area(addr); BUG_ON(!va); free_unmap_vmap_area(va); }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin9373.81%228.57%
Christoph Hellwig2519.84%228.57%
Guillermo Julián Moreno43.17%114.29%
Wang Xiaoqiang32.38%114.29%
Shawn Lin10.79%114.29%
Total126100.00%7100.00%

EXPORT_SYMBOL(vm_unmap_ram); /** * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space) * @pages: an array of pointers to the pages to be mapped * @count: number of pages * @node: prefer to allocate data structures on this node * @prot: memory protection to use. PAGE_KERNEL for regular RAM * * If you use this function for less than VMAP_MAX_ALLOC pages, it could be * faster than vmap so it's good. But if you mix long-life and short-life * objects with vm_map_ram(), it could consume lots of address space through * fragmentation (especially on a 32bit machine). You could see failures in * the end. Please use this function for short-lived objects. * * Returns: a pointer to the address that has been mapped, or %NULL on failure */
void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot) { unsigned long size = (unsigned long)count << PAGE_SHIFT; unsigned long addr; void *mem; if (likely(count <= VMAP_MAX_ALLOC)) { mem = vb_alloc(size, GFP_KERNEL); if (IS_ERR(mem)) return NULL; addr = (unsigned long)mem; } else { struct vmap_area *va; va = alloc_vmap_area(size, PAGE_SIZE, VMALLOC_START, VMALLOC_END, node, GFP_KERNEL); if (IS_ERR(va)) return NULL; addr = va->va_start; mem = (void *)addr; } if (vmap_page_range(addr, addr + size, prot, pages) < 0) { vm_unmap_ram(mem, count); return NULL; } return mem; }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin15797.52%150.00%
Guillermo Julián Moreno42.48%150.00%
Total161100.00%2100.00%

EXPORT_SYMBOL(vm_map_ram); static struct vm_struct *vmlist __initdata; /** * vm_area_add_early - add vmap area early during boot * @vm: vm_struct to add * * This function is used to add fixed kernel vm area to vmlist before * vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags * should contain proper values and the other fields should be zero. * * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. */
void __init vm_area_add_early(struct vm_struct *vm) { struct vm_struct *tmp, **p; BUG_ON(vmap_initialized); for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) { if (tmp->addr >= vm->addr) { BUG_ON(tmp->addr < vm->addr + vm->size); break; } else BUG_ON(tmp->addr + tmp->size > vm->addr); } vm->next = *p; *p = vm; }

Contributors

PersonTokensPropCommitsCommitProp
Nico Pitre106100.00%1100.00%
Total106100.00%1100.00%

/** * vm_area_register_early - register vmap area early during boot * @vm: vm_struct to register * @align: requested alignment * * This function is used to register kernel vm area before * vmalloc_init() is called. @vm->size and @vm->flags should contain * proper values on entry and other fields should be zero. On return, * vm->addr contains the allocated address. * * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING. */
void __init vm_area_register_early(struct vm_struct *vm, size_t align) { static size_t vm_init_off __initdata; unsigned long addr; addr = ALIGN(VMALLOC_START + vm_init_off, align); vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START; vm->addr = (void *)addr; vm_area_add_early(vm); }

Contributors

PersonTokensPropCommitsCommitProp
Tejun Heo5995.16%266.67%
Nico Pitre34.84%133.33%
Total62100.00%3100.00%


void __init vmalloc_init(void) { struct vmap_area *va; struct vm_struct *tmp; int i; for_each_possible_cpu(i) { struct vmap_block_queue *vbq; struct vfree_deferred *p; vbq = &per_cpu(vmap_block_queue, i); spin_lock_init(&vbq->lock); INIT_LIST_HEAD(&vbq->free); p = &per_cpu(vfree_deferred, i); init_llist_head(&p->list); INIT_WORK(&p->wq, free_work); } /* Import existing vmlist entries. */ for (tmp = vmlist; tmp; tmp = tmp->next) { va = kzalloc(sizeof(struct vmap_area), GFP_NOWAIT); va->flags = VM_VM_AREA; va->va_start = (unsigned long)tmp->addr; va->va_end = va->va_start + tmp->size; va->vm = tmp; __insert_vmap_area(va); } vmap_area_pcpu_hole = VMALLOC_END; vmap_initialized = true; }

Contributors

PersonTokensPropCommitsCommitProp
Ivan Kokshaysky7242.35%114.29%
Nicholas Piggin4828.24%114.29%
Al Viro3319.41%114.29%
Cho KyongHo63.53%114.29%
Jeremy Fitzhardinge42.35%114.29%
Tejun Heo42.35%114.29%
Pekka J Enberg31.76%114.29%
Total170100.00%7100.00%

/** * map_kernel_range_noflush - map kernel VM area with the specified pages * @addr: start of the VM area to map * @size: size of the VM area to map * @prot: page protection flags to use * @pages: pages to map * * Map PFN_UP(@size) pages at @addr. The VM area @addr and @size * specify should have been allocated using get_vm_area() and its * friends. * * NOTE: * This function does NOT do any cache flushing. The caller is * responsible for calling flush_cache_vmap() on to-be-mapped areas * before calling this function. * * RETURNS: * The number of pages mapped on success, -errno on failure. */
int map_kernel_range_noflush(unsigned long addr, unsigned long size, pgprot_t prot, struct page **pages) { return vmap_page_range_noflush(addr, addr + size, prot, pages); }

Contributors

PersonTokensPropCommitsCommitProp
Tejun Heo36100.00%1100.00%
Total36100.00%1100.00%

/** * unmap_kernel_range_noflush - unmap kernel VM area * @addr: start of the VM area to unmap * @size: size of the VM area to unmap * * Unmap PFN_UP(@size) pages at @addr. The VM area @addr and @size * specify should have been allocated using get_vm_area() and its * friends. * * NOTE: * This function does NOT do any cache flushing. The caller is * responsible for calling flush_cache_vunmap() on to-be-mapped areas * before calling this function and flush_tlb_kernel_range() after. */
void unmap_kernel_range_noflush(unsigned long addr, unsigned long size) { vunmap_page_range(addr, addr + size); }

Contributors

PersonTokensPropCommitsCommitProp
Tejun Heo22100.00%1100.00%
Total22100.00%1100.00%

EXPORT_SYMBOL_GPL(unmap_kernel_range_noflush); /** * unmap_kernel_range - unmap kernel VM area and flush cache and TLB * @addr: start of the VM area to unmap * @size: size of the VM area to unmap * * Similar to unmap_kernel_range_noflush() but flushes vcache before * the unmapping and tlb after. */
void unmap_kernel_range(unsigned long addr, unsigned long size) { unsigned long end = addr + size; flush_cache_vunmap(addr, end); vunmap_page_range(addr, end); flush_tlb_kernel_range(addr, end); }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin3480.95%133.33%
Tejun Heo716.67%133.33%
Linus Torvalds (pre-git)12.38%133.33%
Total42100.00%3100.00%

EXPORT_SYMBOL_GPL(unmap_kernel_range);
int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page **pages) { unsigned long addr = (unsigned long)area->addr; unsigned long end = addr + get_vm_area_size(area); int err; err = vmap_page_range(addr, end, prot, pages); return err > 0 ? 0 : err; }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin5582.09%120.00%
Linus Torvalds (pre-git)57.46%120.00%
WANG Chao34.48%120.00%
Wanpeng Li34.48%120.00%
James Bottomley11.49%120.00%
Total67100.00%5100.00%

EXPORT_SYMBOL_GPL(map_vm_area);
static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va, unsigned long flags, const void *caller) { spin_lock(&vmap_area_lock); vm->flags = flags; vm->addr = (void *)va->va_start; vm->size = va->va_end - va->va_start; vm->caller = caller; va->vm = vm; va->flags |= VM_VM_AREA; spin_unlock(&vmap_area_lock); }

Contributors

PersonTokensPropCommitsCommitProp
Tejun Heo6981.18%120.00%
JoonSoo Kim1214.12%120.00%
Mitsuo Hayasaka22.35%120.00%
MinChan Kim11.18%120.00%
Marek Szyprowski11.18%120.00%
Total85100.00%5100.00%


static void clear_vm_uninitialized_flag(struct vm_struct *vm) { /* * Before removing VM_UNINITIALIZED, * we should make sure that vm has proper values. * Pair with smp_rmb() in show_numa_info(). */ smp_wmb(); vm->flags &= ~VM_UNINITIALIZED; }

Contributors

PersonTokensPropCommitsCommitProp
Mitsuo Hayasaka1568.18%125.00%
Zhang Yanfei313.64%125.00%
JoonSoo Kim313.64%125.00%
Tejun Heo14.55%125.00%
Total22100.00%4100.00%


static struct vm_struct *__get_vm_area_node(unsigned long size, unsigned long align, unsigned long flags, unsigned long start, unsigned long end, int node, gfp_t gfp_mask, const void *caller) { struct vmap_area *va; struct vm_struct *area; BUG_ON(in_interrupt()); size = PAGE_ALIGN(size); if (unlikely(!size)) return NULL; if (flags & VM_IOREMAP) align = 1ul << clamp_t(int, get_count_order_long(size), PAGE_SHIFT, IOREMAP_MAX_ORDER); area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node); if (unlikely(!area)) return NULL; if (!(flags & VM_NO_GUARD)) size += PAGE_SIZE; va = alloc_vmap_area(size, align, start, end, node, gfp_mask); if (IS_ERR(va)) { kfree(area); return NULL; } setup_vmalloc_vm(area, va, flags, caller); return area; }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin11461.29%16.25%
zijun_hu1910.22%16.25%
Mitsuo Hayasaka115.91%16.25%
Zhang Yanfei94.84%16.25%
Andrey Ryabinin94.84%16.25%
Linus Torvalds (pre-git)63.23%318.75%
David S. Miller42.15%16.25%
Christoph Hellwig42.15%16.25%
Toshi Kani31.61%16.25%
Andi Kleen21.08%16.25%
Linus Torvalds21.08%16.25%
Marek Szyprowski10.54%16.25%
Tejun Heo10.54%16.25%
James Bottomley10.54%16.25%
Total186100.00%16100.00%


struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags, unsigned long start, unsigned long end) { return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE, GFP_KERNEL, __builtin_return_address(0)); }

Contributors

PersonTokensPropCommitsCommitProp
Christoph Lameter4189.13%240.00%
David S. Miller24.35%120.00%
Giridhar Pemmasani24.35%120.00%
David Rientjes12.17%120.00%
Total46100.00%5100.00%

EXPORT_SYMBOL_GPL(__get_vm_area);
struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags, unsigned long start, unsigned long end, const void *caller) { return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE, GFP_KERNEL, caller); }

Contributors

PersonTokensPropCommitsCommitProp
Benjamin Herrenschmidt4491.67%125.00%
David S. Miller24.17%125.00%
David Rientjes12.08%125.00%
Marek Szyprowski12.08%125.00%
Total48100.00%4100.00%

/** * get_vm_area - reserve a contiguous kernel virtual area * @size: size of the area * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC * * Search an area of @size in the kernel virtual mapping area, * and reserved it for out purposes. Returns the area descriptor * on success or %NULL on failure. */
struct vm_struct *get_vm_area(unsigned long size, unsigned long flags) { return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END, NUMA_NO_NODE, GFP_KERNEL, __builtin_return_address(0)); }

Contributors

PersonTokensPropCommitsCommitProp
Christoph Lameter2052.63%125.00%
Russell King1539.47%125.00%
David S. Miller25.26%125.00%
David Rientjes12.63%125.00%
Total38100.00%4100.00%


struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags, const void *caller) { return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END, NUMA_NO_NODE, GFP_KERNEL, caller); }

Contributors

PersonTokensPropCommitsCommitProp
Christoph Lameter2562.50%120.00%
Russell King1127.50%120.00%
David S. Miller25.00%120.00%
David Rientjes12.50%120.00%
Marek Szyprowski12.50%120.00%
Total40100.00%5100.00%

/** * find_vm_area - find a continuous kernel virtual area * @addr: base address * * Search for the kernel VM area starting at @addr, and return it. * It is up to the caller to do all required locking to keep the returned * pointer valid. */
struct vm_struct *find_vm_area(const void *addr) { struct vmap_area *va; va = find_vmap_area((unsigned long)addr); if (va && va->flags & VM_VM_AREA) return va->vm; return NULL; }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin4597.83%150.00%
MinChan Kim12.17%150.00%
Total46100.00%2100.00%

/** * remove_vm_area - find and remove a continuous kernel virtual area * @addr: base address * * Search for the kernel VM area starting at @addr, and remove it. * This function returns the found VM area, but using it is NOT safe * on SMP machines, except for its size or flags. */
struct vm_struct *remove_vm_area(const void *addr) { struct vmap_area *va; might_sleep(); va = find_vmap_area((unsigned long)addr); if (va && va->flags & VM_VM_AREA) { struct vm_struct *vm = va->vm; spin_lock(&vmap_area_lock); va->vm = NULL; va->flags &= ~VM_VM_AREA; va->flags |= VM_LAZY_FREE; spin_unlock(&vmap_area_lock); vmap_debug_free_range(va->va_start, va->va_end); kasan_free_shadow(vm); free_unmap_vmap_area(va); return vm; } return NULL; }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin4742.73%218.18%
JoonSoo Kim2522.73%19.09%
Kamezawa Hiroyuki1614.55%19.09%
Andi Kleen65.45%218.18%
Yisheng Xie65.45%19.09%
Andrey Ryabinin54.55%19.09%
Christoph Hellwig32.73%19.09%
MinChan Kim10.91%19.09%
Christoph Lameter10.91%19.09%
Total110100.00%11100.00%


static void __vunmap(const void *addr, int deallocate_pages) { struct vm_struct *area; if (!addr) return; if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n", addr)) return; area = remove_vm_area(addr); if (unlikely(!area)) { WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n", addr); return; } debug_check_no_locks_freed(addr, get_vm_area_size(area)); debug_check_no_obj_freed(addr, get_vm_area_size(area)); if (deallocate_pages) { int i; for (i = 0; i < area->nr_pages; i++) { struct page *page = area->pages[i]; BUG_ON(!page); __free_pages(page, 0); } kvfree(area->pages); } kfree(area); return; }

Contributors

PersonTokensPropCommitsCommitProp
Andrew Morton3422.22%211.11%
Andi Kleen3220.92%15.56%
Christoph Hellwig3120.26%15.56%
Christoph Lameter127.84%211.11%
Linus Torvalds (pre-git)85.23%316.67%
Ingo Molnar74.58%15.56%
Thomas Gleixner74.58%15.56%
Jerome Marchand63.92%15.56%
Daisuke Hatayama63.92%15.56%
Arjan van de Ven53.27%15.56%
Vladimir Davydov31.96%211.11%
Benjamin Herrenschmidt10.65%15.56%
David Rientjes10.65%15.56%
Total153100.00%18100.00%


static inline void __vfree_deferred(const void *addr) { /* * Use raw_cpu_ptr() because this can be called from preemptible * context. Preemption is absolutely fine here, because the llist_add() * implementation is lockless, so it works even if we are adding to * nother cpu's list. schedule_work() should be fine with this too. */ struct vfree_deferred *p = raw_cpu_ptr(&vfree_deferred); if (llist_add((struct llist_node *)addr, &p->list)) schedule_work(&p->wq); }

Contributors

PersonTokensPropCommitsCommitProp
Al Viro3265.31%114.29%
Andrey Ryabinin510.20%114.29%
Tom Rini48.16%114.29%
Christoph Hellwig36.12%114.29%
Oleg Nesterov36.12%114.29%
Christoph Lameter24.08%228.57%
Total49100.00%7100.00%

/** * vfree_atomic - release memory allocated by vmalloc() * @addr: memory base address * * This one is just like vfree() but can be called in any atomic context * except NMIs. */
void vfree_atomic(const void *addr) { BUG_ON(in_nmi()); kmemleak_free(addr); if (!addr) return; __vfree_deferred(addr); }

Contributors

PersonTokensPropCommitsCommitProp
Andrey Ryabinin32100.00%1100.00%
Total32100.00%1100.00%

/** * vfree - release memory allocated by vmalloc() * @addr: memory base address * * Free the virtually continuous memory area starting at @addr, as * obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is * NULL, no operation is performed. * * Must not be called in NMI context (strictly speaking, only if we don't * have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling * conventions for vfree() arch-depenedent would be a really bad idea) * * NOTE: assumes that the object at @addr has a size >= sizeof(llist_node) */
void vfree(const void *addr) { BUG_ON(in_nmi()); kmemleak_free(addr); if (!addr) return; if (unlikely(in_interrupt())) __vfree_deferred(addr); else __vunmap(addr, 1); }

Contributors

PersonTokensPropCommitsCommitProp
Andrey Ryabinin3981.25%125.00%
Christoph Hellwig48.33%125.00%
Tom Rini48.33%125.00%
Al Viro12.08%125.00%
Total48100.00%4100.00%

EXPORT_SYMBOL(vfree); /** * vunmap - release virtual mapping obtained by vmap() * @addr: memory base address * * Free the virtually contiguous memory area starting at @addr, * which was created from the page array passed to vmap(). * * Must not be called in interrupt context. */
void vunmap(const void *addr) { BUG_ON(in_interrupt()); might_sleep(); if (addr) __vunmap(addr, 0); }

Contributors

PersonTokensPropCommitsCommitProp
Tom Rini826.67%116.67%
Christoph Hellwig826.67%116.67%
Andrew Morton620.00%116.67%
Al Viro413.33%116.67%
Peter Zijlstra310.00%116.67%
Christoph Lameter13.33%116.67%
Total30100.00%6100.00%

EXPORT_SYMBOL(vunmap); /** * vmap - map an array of pages into virtually contiguous space * @pages: array of page pointers * @count: number of pages to map * @flags: vm_area->flags * @prot: page protection for the mapping * * Maps @count pages from @pages into contiguous kernel virtual * space. */
void *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot) { struct vm_struct *area; unsigned long size; /* In bytes */ might_sleep(); if (count > totalram_pages) return NULL; size = (unsigned long)count << PAGE_SHIFT; area = get_vm_area_caller(size, flags, __builtin_return_address(0)); if (!area) return NULL; if (map_vm_area(area, prot, pages)) { vunmap(area->addr); return NULL; } return area->addr; }

Contributors

PersonTokensPropCommitsCommitProp
Christoph Hellwig5148.57%112.50%
Andrew Morton1918.10%225.00%
Guillermo Julián Moreno1514.29%112.50%
Tom Rini1110.48%112.50%
Christoph Lameter54.76%112.50%
Peter Zijlstra32.86%112.50%
Jan Beulich10.95%112.50%
Total105100.00%8100.00%

EXPORT_SYMBOL(vmap); static void *__vmalloc_node(unsigned long size, unsigned long align, gfp_t gfp_mask, pgprot_t prot, int node, const void *caller);
static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot, int node) { struct page **pages; unsigned int nr_pages, array_size, i; const gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO; const gfp_t alloc_mask = gfp_mask | __GFP_NOWARN; const gfp_t highmem_mask = (gfp_mask & (GFP_DMA | GFP_DMA32)) ? 0 : __GFP_HIGHMEM; nr_pages = get_vm_area_size(area) >> PAGE_SHIFT; array_size = (nr_pages * sizeof(struct page *)); area->nr_pages = nr_pages; /* Please note that the recursion is strictly bounded. */ if (array_size > PAGE_SIZE) { pages = __vmalloc_node(array_size, 1, nested_gfp|highmem_mask, PAGE_KERNEL, node, area->caller); } else { pages = kmalloc_node(array_size, nested_gfp, node); } area->pages = pages; if (!area->pages) { remove_vm_area(area->addr); kfree(area); return NULL; } for (i = 0; i < area->nr_pages; i++) { struct page *page; if (node == NUMA_NO_NODE) page = alloc_page(alloc_mask|highmem_mask); else page = alloc_pages_node(node, alloc_mask|highmem_mask, 0); if (unlikely(!page)) { /* Successfully allocated i pages, free them in __vunmap() */ area->nr_pages = i; goto fail; } area->pages[i] = page; if (gfpflags_allow_blocking(gfp_mask|highmem_mask)) cond_resched(); } if (map_vm_area(area, prot, pages)) goto fail; return area->addr; fail: warn_alloc(gfp_mask, NULL, "vmalloc: allocation failure, allocated %ld of %ld bytes", (area->nr_pages*PAGE_SIZE), area->size); vfree(area->addr); return NULL; }

Contributors

PersonTokensPropCommitsCommitProp
Andrew Morton11435.74%26.67%
Christoph Lameter4213.17%413.33%
Christoph Hellwig309.40%13.33%
Linus Torvalds (pre-git)288.78%413.33%
Laura Abbott257.84%13.33%
Dave Hansen175.33%13.33%
Jan Beulich134.08%13.33%
David Rientjes113.45%13.33%
Oleg Nesterov72.19%26.67%
Eric Dumazet72.19%13.33%
Michal Hocko61.88%26.67%
Wanpeng Li51.57%26.67%
Mel Gorman30.94%13.33%
Jianguo Wu20.63%13.33%
Jan Kiszka20.63%13.33%
David S. Miller20.63%13.33%
Marcus Alanen20.63%13.33%
Vladimir Davydov10.31%13.33%
Adrian Bunk10.31%13.33%
Al Viro10.31%13.33%
Total319100.00%30100.00%

/** * __vmalloc_node_range - allocate virtually contiguous memory * @size: allocation size * @align: desired alignment * @start: vm area range start * @end: vm area range end * @gfp_mask: flags for the page level allocator * @prot: protection mask for the allocated pages * @vm_flags: additional vm area flags (e.g. %VM_NO_GUARD) * @node: node to use for allocation or NUMA_NO_NODE * @caller: caller's return address * * Allocate enough pages to cover @size from the page level * allocator with @gfp_mask flags. Map them into contiguous * kernel virtual space, using a pagetable protection of @prot. */
void *__vmalloc_node_range(unsigned long size, unsigned long align, unsigned long start, unsigned long end, gfp_t gfp_mask, pgprot_t prot, unsigned long vm_flags, int node, const void *caller) { struct vm_struct *area; void *addr; unsigned long real_size = size; size = PAGE_ALIGN(size); if (!size || (size >> PAGE_SHIFT) > totalram_pages) goto fail; area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNINITIALIZED | vm_flags, start, end, node, gfp_mask, caller); if (!area) goto fail; addr = __vmalloc_area_node(area, gfp_mask, prot, node); if (!addr) return NULL; /* * In this function, newly allocated vm_struct has VM_UNINITIALIZED * flag. It means that vm_struct is not fully initialized. * Now, it is fully initialized, so remove this flag here. */ clear_vm_uninitialized_flag(area); kmemleak_vmalloc(area, size, gfp_mask); return addr; fail: warn_alloc(gfp_mask, NULL, "vmalloc: allocation failure: %lu bytes", real_size); return NULL; }

Contributors

PersonTokensPropCommitsCommitProp
Oleg Nesterov5633.73%15.26%
Catalin Marinas2414.46%210.53%
Christoph Lameter2012.05%210.53%
Joe Perches1810.84%15.26%
David Rientjes116.63%15.26%
Andrey Ryabinin63.61%15.26%
David S. Miller63.61%15.26%
Mel Gorman53.01%15.26%
Mitsuo Hayasaka53.01%15.26%
Michal Hocko42.41%210.53%
Zhang Yanfei31.81%15.26%
Wanpeng Li31.81%15.26%
Giridhar Pemmasani21.20%15.26%
Marek Szyprowski10.60%15.26%
Al Viro10.60%15.26%
Jan Beulich10.60%15.26%
Total166100.00%19100.00%

/** * __vmalloc_node - allocate virtually contiguous memory * @size: allocation size * @align: desired alignment * @gfp_mask: flags for the page level allocator * @prot: protection mask for the allocated pages * @node: node to use for allocation or NUMA_NO_NODE * @caller: caller's return address * * Allocate enough pages to cover @size from the page level * allocator with @gfp_mask flags. Map them into contiguous * kernel virtual space, using a pagetable protection of @prot. * * Reclaim modifiers in @gfp_mask - __GFP_NORETRY, __GFP_RETRY_MAYFAIL * and __GFP_NOFAIL are not supported * * Any use of gfp flags outside of GFP_KERNEL should be consulted * with mm people. * */
static void *__vmalloc_node(unsigned long size, unsigned long align, gfp_t gfp_mask, pgprot_t prot, int node, const void *caller) { return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END, gfp_mask, prot, 0, node, caller); }

Contributors

PersonTokensPropCommitsCommitProp
David Rientjes4792.16%125.00%
Andrey Ryabinin23.92%125.00%
Michal Hocko11.96%125.00%
Marek Szyprowski11.96%125.00%
Total51100.00%4100.00%


void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot) { return __vmalloc_node(size, 1, gfp_mask, prot, NUMA_NO_NODE, __builtin_return_address(0)); }

Contributors

PersonTokensPropCommitsCommitProp
Christoph Lameter2982.86%240.00%
Oleg Nesterov38.57%120.00%
David S. Miller25.71%120.00%
David Rientjes12.86%120.00%
Total35100.00%5100.00%

EXPORT_SYMBOL(__vmalloc);
static inline void *__vmalloc_node_flags(unsigned long size, int node, gfp_t flags) { return __vmalloc_node(size, 1, flags, PAGE_KERNEL, node, __builtin_return_address(0)); }

Contributors

PersonTokensPropCommitsCommitProp
Michal Hocko37100.00%1100.00%
Total37100.00%1100.00%


void *__vmalloc_node_flags_caller(unsigned long size, int node, gfp_t flags, void *caller) { return __vmalloc_node(size, 1, flags, PAGE_KERNEL, node, caller); }

Contributors

PersonTokensPropCommitsCommitProp
Michal Hocko36100.00%1100.00%
Total36100.00%1100.00%

/** * vmalloc - allocate virtually contiguous memory * @size: allocation size * Allocate enough pages to cover @size from the page level * allocator and map them into contiguous kernel virtual space. * * For tight control over page level allocator and protection flags * use __vmalloc() instead. */
void *vmalloc(unsigned long size) { return __vmalloc_node_flags(size, NUMA_NO_NODE, GFP_KERNEL); }

Contributors

PersonTokensPropCommitsCommitProp
Christoph Hellwig1365.00%120.00%
Dave Young420.00%120.00%
David Rientjes15.00%120.00%
Linus Torvalds (pre-git)15.00%120.00%
David S. Miller15.00%120.00%
Total20100.00%5100.00%

EXPORT_SYMBOL(vmalloc); /** * vzalloc - allocate virtually contiguous memory with zero fill * @size: allocation size * Allocate enough pages to cover @size from the page level * allocator and map them into contiguous kernel virtual space. * The memory allocated is set to zero. * * For tight control over page level allocator and protection flags * use __vmalloc() instead. */
void *vzalloc(unsigned long size) { return __vmalloc_node_flags(size, NUMA_NO_NODE, GFP_KERNEL | __GFP_ZERO); }

Contributors

PersonTokensPropCommitsCommitProp
Dave Young1672.73%125.00%
Christoph Hellwig313.64%125.00%
Christoph Lameter29.09%125.00%
David Rientjes14.55%125.00%
Total22100.00%4100.00%

EXPORT_SYMBOL(vzalloc); /** * vmalloc_user - allocate zeroed virtually contiguous memory for userspace * @size: allocation size * * The resulting memory area is zeroed so it can be mapped to userspace * without leaking data. */
void *vmalloc_user(unsigned long size) { struct vm_struct *area; void *ret; ret = __vmalloc_node(size, SHMLBA, GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL, NUMA_NO_NODE, __builtin_return_address(0)); if (ret) { area = find_vm_area(ret); area->flags |= VM_USERMAP; } return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin4774.60%233.33%
Glauber de Oliveira Costa711.11%116.67%
Eric Dumazet69.52%116.67%
David S. Miller23.17%116.67%
David Rientjes11.59%116.67%
Total63100.00%6100.00%

EXPORT_SYMBOL(vmalloc_user); /** * vmalloc_node - allocate memory on a specific node * @size: allocation size * @node: numa node * * Allocate enough pages to cover @size from the page level * allocator and map them into contiguous kernel virtual space. * * For tight control over page level allocator and protection flags * use __vmalloc() instead. */
void *vmalloc_node(unsigned long size, int node) { return __vmalloc_node(size, 1, GFP_KERNEL, PAGE_KERNEL, node, __builtin_return_address(0)); }

Contributors

PersonTokensPropCommitsCommitProp
Christoph Lameter3093.75%266.67%
David S. Miller26.25%133.33%
Total32100.00%3100.00%

EXPORT_SYMBOL(vmalloc_node); /** * vzalloc_node - allocate memory on a specific node with zero fill * @size: allocation size * @node: numa node * * Allocate enough pages to cover @size from the page level * allocator and map them into contiguous kernel virtual space. * The memory allocated is set to zero. * * For tight control over page level allocator and protection flags * use __vmalloc_node() instead. */
void *vzalloc_node(unsigned long size, int node) { return __vmalloc_node_flags(size, node, GFP_KERNEL | __GFP_ZERO); }

Contributors

PersonTokensPropCommitsCommitProp
Dave Young25100.00%1100.00%
Total25100.00%1100.00%

EXPORT_SYMBOL(vzalloc_node); #ifndef PAGE_KERNEL_EXEC # define PAGE_KERNEL_EXEC PAGE_KERNEL #endif /** * vmalloc_exec - allocate virtually contiguous, executable memory * @size: allocation size * * Kernel-internal function to allocate enough pages to cover @size * the page level allocator and map them into contiguous and * executable kernel virtual space. * * For tight control over page level allocator and protection flags * use __vmalloc() instead. */
void *vmalloc_exec(unsigned long size) { return __vmalloc_node(size, 1, GFP_KERNEL, PAGE_KERNEL_EXEC, NUMA_NO_NODE, __builtin_return_address(0)); }

Contributors

PersonTokensPropCommitsCommitProp
Ingo Molnar1965.52%125.00%
Glauber de Oliveira Costa724.14%125.00%
David S. Miller26.90%125.00%
David Rientjes13.45%125.00%
Total29100.00%4100.00%

#if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32) #define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL #elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA) #define GFP_VMALLOC32 GFP_DMA | GFP_KERNEL #else #define GFP_VMALLOC32 GFP_KERNEL #endif /** * vmalloc_32 - allocate virtually contiguous memory (32bit addressable) * @size: allocation size * * Allocate enough 32bit PA addressable pages to cover @size from the * page level allocator and map them into contiguous kernel virtual space. */
void *vmalloc_32(unsigned long size) { return __vmalloc_node(size, 1, GFP_VMALLOC32, PAGE_KERNEL, NUMA_NO_NODE, __builtin_return_address(0)); }

Contributors

PersonTokensPropCommitsCommitProp
Christoph Hellwig1655.17%114.29%
Glauber de Oliveira Costa724.14%114.29%
Linus Torvalds (pre-git)26.90%228.57%
David S. Miller26.90%114.29%
Andi Kleen13.45%114.29%
David Rientjes13.45%114.29%
Total29100.00%7100.00%

EXPORT_SYMBOL(vmalloc_32); /** * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory * @size: allocation size * * The resulting memory area is 32bit addressable and zeroed so it can be * mapped to userspace without leaking data. */
void *vmalloc_32_user(unsigned long size) { struct vm_struct *area; void *ret; ret = __vmalloc_node(size, 1, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL, NUMA_NO_NODE, __builtin_return_address(0)); if (ret) { area = find_vm_area(ret); area->flags |= VM_USERMAP; } return ret; }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin4673.02%228.57%
Glauber de Oliveira Costa711.11%114.29%
Eric Dumazet69.52%114.29%
David S. Miller23.17%114.29%
Andi Kleen11.59%114.29%
David Rientjes11.59%114.29%
Total63100.00%7100.00%

EXPORT_SYMBOL(vmalloc_32_user); /* * small helper routine , copy contents to buf from addr. * If the page is not present, fill zero. */
static int aligned_vread(char *buf, char *addr, unsigned long count) { struct page *p; int copied = 0; while (count) { unsigned long offset, length; offset = offset_in_page(addr); length = PAGE_SIZE - offset; if (length > count) length = count; p = vmalloc_to_page(addr); /* * To do safe access to this _mapped_ area, we need * lock. But adding lock here means that we need to add * overhead of vmalloc()/vfree() calles for this _debug_ * interface, rarely used. Instead of that, we'll use * kmap() and get small overhead in this access function. */ if (p) { /* * we can expect USER0 is not used (see vread/vwrite's * function description) */ void *map = kmap_atomic(p); memcpy(buf, map + offset, length); kunmap_atomic(map); } else memset(buf, 0, length); addr += length; buf += length; copied += length; count -= length; } return copied; }

Contributors

PersonTokensPropCommitsCommitProp
Kamezawa Hiroyuki12997.73%150.00%
Alexander Kuleshov32.27%150.00%
Total132100.00%2100.00%


static int aligned_vwrite(char *buf, char *addr, unsigned long count) { struct page *p; int copied = 0; while (count) { unsigned long offset, length; offset = offset_in_page(addr); length = PAGE_SIZE - offset; if (length > count) length = count; p = vmalloc_to_page(addr); /* * To do safe access to this _mapped_ area, we need * lock. But adding lock here means that we need to add * overhead of vmalloc()/vfree() calles for this _debug_ * interface, rarely used. Instead of that, we'll use * kmap() and get small overhead in this access function. */ if (p) { /* * we can expect USER0 is not used (see vread/vwrite's * function description) */ void *map = kmap_atomic(p); memcpy(map + offset, buf, length); kunmap_atomic(map); } addr += length; buf += length; copied += length; count -= length; } return copied; }

Contributors

PersonTokensPropCommitsCommitProp
Kamezawa Hiroyuki11997.54%150.00%
Alexander Kuleshov32.46%150.00%
Total122100.00%2100.00%

/** * vread() - read vmalloc area in a safe way. * @buf: buffer for reading data * @addr: vm address. * @count: number of bytes to be read. * * Returns # of bytes which addr and buf should be increased. * (same number to @count). Returns 0 if [addr...addr+count) doesn't * includes any intersect with alive vmalloc area. * * This function checks that addr is a valid vmalloc'ed area, and * copy data from that area to a given buffer. If the given memory range * of [addr...addr+count) includes some valid address, data is copied to * proper area of @buf. If there are memory holes, they'll be zero-filled. * IOREMAP area is treated as memory hole and no copy is done. * * If [addr...addr+count) doesn't includes any intersects with alive * vm_struct area, returns 0. @buf should be kernel's buffer. * * Note: In usual ops, vread() is never necessary because the caller * should know vmalloc() area is valid and can use memcpy(). * This is for routines which have to access vmalloc area without * any informaion, as /dev/kmem. * */
long vread(char *buf, char *addr, unsigned long count) { struct vmap_area *va; struct vm_struct *vm; char *vaddr, *buf_start = buf; unsigned long buflen = count; unsigned long n; /* Don't allow overflow */ if ((unsigned long) addr + count < count) count = -(unsigned long) addr; spin_lock(&vmap_area_lock); list_for_each_entry(va, &vmap_area_list, list) { if (!count) break; if (!(va->flags & VM_VM_AREA)) continue; vm = va->vm; vaddr = (char *) vm->addr; if (addr >= vaddr + get_vm_area_size(vm)) continue; while (addr < vaddr) { if (count == 0) goto finished; *buf = '\0'; buf++; addr++; count--; } n = vaddr + get_vm_area_size(vm) - addr; if (n > count) n = count; if (!(vm->flags & VM_IOREMAP)) aligned_vread(buf, addr, n); else /* IOREMAP area is treated as memory hole */ memset(buf, 0, n); buf += n; addr += n; count -= n; } finished: spin_unlock(&vmap_area_lock); if (buf == buf_start) return 0; /* zero-fill memory holes */ if (buf != buf_start + buflen) memset(buf, 0, buflen - (buf - buf_start)); return buflen; }

Contributors

PersonTokensPropCommitsCommitProp
Linus Torvalds (pre-git)14252.79%666.67%
Kamezawa Hiroyuki7728.62%111.11%
JoonSoo Kim4416.36%111.11%
Wanpeng Li62.23%111.11%
Total269100.00%9100.00%

/** * vwrite() - write vmalloc area in a safe way. * @buf: buffer for source data * @addr: vm address. * @count: number of bytes to be read. * * Returns # of bytes which addr and buf should be incresed. * (same number to @count). * If [addr...addr+count) doesn't includes any intersect with valid * vmalloc area, returns 0. * * This function checks that addr is a valid vmalloc'ed area, and * copy data from a buffer to the given addr. If specified range of * [addr...addr+count) includes some valid address, data is copied from * proper area of @buf. If there are memory holes, no copy to hole. * IOREMAP area is treated as memory hole and no copy is done. * * If [addr...addr+count) doesn't includes any intersects with alive * vm_struct area, returns 0. @buf should be kernel's buffer. * * Note: In usual ops, vwrite() is never necessary because the caller * should know vmalloc() area is valid and can use memcpy(). * This is for routines which have to access vmalloc area without * any informaion, as /dev/kmem. */
long vwrite(char *buf, char *addr, unsigned long count) { struct vmap_area *va; struct vm_struct *vm; char *vaddr; unsigned long n, buflen; int copied = 0; /* Don't allow overflow */ if ((unsigned long) addr + count < count) count = -(unsigned long) addr; buflen = count; spin_lock(&vmap_area_lock); list_for_each_entry(va, &vmap_area_list, list) { if (!count) break; if (!(va->flags & VM_VM_AREA)) continue; vm = va->vm; vaddr = (char *) vm->addr; if (addr >= vaddr + get_vm_area_size(vm)) continue; while (addr < vaddr) { if (count == 0) goto finished; buf++; addr++; count--; } n = vaddr + get_vm_area_size(vm) - addr; if (n > count) n = count; if (!(vm->flags & VM_IOREMAP)) { aligned_vwrite(buf, addr, n); copied++; } buf += n; addr += n; count -= n; } finished: spin_unlock(&vmap_area_lock); if (!copied) return 0; return buflen; }

Contributors

PersonTokensPropCommitsCommitProp
Dave Jones12955.36%125.00%
Kamezawa Hiroyuki5423.18%125.00%
JoonSoo Kim4418.88%125.00%
Wanpeng Li62.58%125.00%
Total233100.00%4100.00%

/** * remap_vmalloc_range_partial - map vmalloc pages to userspace * @vma: vma to cover * @uaddr: target user address to start at * @kaddr: virtual address of vmalloc kernel memory * @size: size of map area * * Returns: 0 for success, -Exxx on failure * * This function checks that @kaddr is a valid vmalloc'ed area, * and that it is big enough to cover the range starting at * @uaddr in @vma. Will return failure if that criteria isn't * met. * * Similar to remap_pfn_range() (see mm/memory.c) */
int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr, void *kaddr, unsigned long size) { struct vm_struct *area; size = PAGE_ALIGN(size); if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr)) return -EINVAL; area = find_vm_area(kaddr); if (!area) return -EINVAL; if (!(area->flags & VM_USERMAP)) return -EINVAL; if (kaddr + size > area->addr + area->size) return -EINVAL; do { struct page *page = vmalloc_to_page(kaddr); int ret; ret = vm_insert_page(vma, uaddr, page); if (ret) return ret; uaddr += PAGE_SIZE; kaddr += PAGE_SIZE; size -= PAGE_SIZE; } while (size > 0); vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP; return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin12676.36%250.00%
Daisuke Hatayama3621.82%125.00%
Konstantin Khlebnikov31.82%125.00%
Total165100.00%4100.00%

EXPORT_SYMBOL(remap_vmalloc_range_partial); /** * remap_vmalloc_range - map vmalloc pages to userspace * @vma: vma to cover (map full range of vma) * @addr: vmalloc memory * @pgoff: number of pages into addr before first page to map * * Returns: 0 for success, -Exxx on failure * * This function checks that addr is a valid vmalloc'ed area, and * that it is big enough to cover the vma. Will return failure if * that criteria isn't met. * * Similar to remap_pfn_range() (see mm/memory.c) */
int remap_vmalloc_range(struct vm_area_struct *vma, void *addr, unsigned long pgoff) { return remap_vmalloc_range_partial(vma, vma->vm_start, addr + (pgoff << PAGE_SHIFT), vma->vm_end - vma->vm_start); }

Contributors

PersonTokensPropCommitsCommitProp
Daisuke Hatayama44100.00%1100.00%
Total44100.00%1100.00%

EXPORT_SYMBOL(remap_vmalloc_range); /* * Implement a stub for vmalloc_sync_all() if the architecture chose not to * have one. */
void __weak vmalloc_sync_all(void) { }

Contributors

PersonTokensPropCommitsCommitProp
Christoph Hellwig457.14%150.00%
Gideon Israel Dsouza342.86%150.00%
Total7100.00%2100.00%


static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data) { pte_t ***p = data; if (p) { *(*p) = pte; (*p)++; } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
David Vrabel2853.85%133.33%
Jeremy Fitzhardinge2242.31%133.33%
Martin Schwidefsky23.85%133.33%
Total52100.00%3100.00%

/** * alloc_vm_area - allocate a range of kernel address space * @size: size of the area * @ptes: returns the PTEs for the address space * * Returns: NULL on failure, vm_struct on success * * This function reserves a range of kernel address space, and * allocates pagetables to map that range. No actual mappings * are created. * * If @ptes is non-NULL, pointers to the PTEs (in init_mm) * allocated for the VM area are returned. */
struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes) { struct vm_struct *area; area = get_vm_area_caller(size, VM_IOREMAP, __builtin_return_address(0)); if (area == NULL) return NULL; /* * This ensures that page tables are constructed for this region * of kernel virtual address space and mapped into init_mm. */ if (apply_to_page_range(&init_mm, (unsigned long)area->addr, size, f, ptes ? &ptes : NULL)) { free_vm_area(area); return NULL; } return area; }

Contributors

PersonTokensPropCommitsCommitProp
Jeremy Fitzhardinge6880.95%133.33%
David Vrabel1011.90%133.33%
Christoph Lameter67.14%133.33%
Total84100.00%3100.00%

EXPORT_SYMBOL_GPL(alloc_vm_area);
void free_vm_area(struct vm_struct *area) { struct vm_struct *ret; ret = remove_vm_area(area->addr); BUG_ON(ret != area); kfree(area); }

Contributors

PersonTokensPropCommitsCommitProp
Jeremy Fitzhardinge36100.00%1100.00%
Total36100.00%1100.00%

EXPORT_SYMBOL_GPL(free_vm_area); #ifdef CONFIG_SMP
static struct vmap_area *node_to_va(struct rb_node *n) { return rb_entry_safe(n, struct vmap_area, rb_node); }

Contributors

PersonTokensPropCommitsCommitProp
Tejun Heo2395.83%150.00%
Geliang Tang14.17%150.00%
Total24100.00%2100.00%

/** * pvm_find_next_prev - find the next and prev vmap_area surrounding @end * @end: target address * @pnext: out arg for the next vmap_area * @pprev: out arg for the previous vmap_area * * Returns: %true if either or both of next and prev are found, * %false if no vmap_area exists * * Find vmap_areas end addresses of which enclose @end. ie. if not * NULL, *pnext->va_end > @end and *pprev->va_end <= @end. */
static bool pvm_find_next_prev(unsigned long end, struct vmap_area **pnext, struct vmap_area **pprev) { struct rb_node *n = vmap_area_root.rb_node; struct vmap_area *va = NULL; while (n) { va = rb_entry(n, struct vmap_area, rb_node); if (end < va->va_end) n = n->rb_left; else if (end > va->va_end) n = n->rb_right; else break; } if (!va) return false; if (va->va_end > end) { *pnext = va; *pprev = node_to_va(rb_prev(&(*pnext)->rb_node)); } else { *pprev = va; *pnext = node_to_va(rb_next(&(*pprev)->rb_node)); } return true; }

Contributors

PersonTokensPropCommitsCommitProp
Tejun Heo155100.00%1100.00%
Total155100.00%1100.00%

/** * pvm_determine_end - find the highest aligned address between two vmap_areas * @pnext: in/out arg for the next vmap_area * @pprev: in/out arg for the previous vmap_area * @align: alignment * * Returns: determined end address * * Find the highest aligned address between *@pnext and *@pprev below * VMALLOC_END. *@pnext and *@pprev are adjusted so that the aligned * down address is between the end addresses of the two vmap_areas. * * Please note that the address returned by this function may fall * inside *@pnext vmap_area. The caller is responsible for checking * that. */
static unsigned long pvm_determine_end(struct vmap_area **pnext, struct vmap_area **pprev, unsigned long align) { const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); unsigned long addr; if (*pnext) addr = min((*pnext)->va_start & ~(align - 1), vmalloc_end); else addr = vmalloc_end; while (*pprev && (*pprev)->va_end > addr) { *pnext = *pprev; *pprev = node_to_va(rb_prev(&(*pnext)->rb_node)); } return addr; }

Contributors

PersonTokensPropCommitsCommitProp
Tejun Heo114100.00%1100.00%
Total114100.00%1100.00%

/** * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator * @offsets: array containing offset of each area * @sizes: array containing size of each area * @nr_vms: the number of areas to allocate * @align: alignment, all entries in @offsets and @sizes must be aligned to this * * Returns: kmalloc'd vm_struct pointer array pointing to allocated * vm_structs on success, %NULL on failure * * Percpu allocator wants to use congruent vm areas so that it can * maintain the offsets among percpu areas. This function allocates * congruent vmalloc areas for it with GFP_KERNEL. These areas tend to * be scattered pretty far, distance between two areas easily going up * to gigabytes. To avoid interacting with regular vmallocs, these * areas are allocated from top. * * Despite its complicated look, this allocator is rather simple. It * does everything top-down and scans areas from the end looking for * matching slot. While scanning, if any of the areas overlaps with * existing vmap_area, the base address is pulled down to fit the * area. Scanning is repeated till all the areas fit and then all * necessary data structures are inserted and the result is returned. */
struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, const size_t *sizes, int nr_vms, size_t align) { const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align); const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1); struct vmap_area **vas, *prev, *next; struct vm_struct **vms; int area, area2, last_area, term_area; unsigned long base, start, end, last_end; bool purged = false; /* verify parameters and allocate data structures */ BUG_ON(offset_in_page(align) || !is_power_of_2(align)); for (last_area = 0, area = 0; area < nr_vms; area++) { start = offsets[area]; end = start + sizes[area]; /* is everything aligned properly? */ BUG_ON(!IS_ALIGNED(offsets[area], align)); BUG_ON(!IS_ALIGNED(sizes[area], align)); /* detect the area with the highest address */ if (start > offsets[last_area]) last_area = area; for (area2 = area + 1; area2 < nr_vms; area2++) { unsigned long start2 = offsets[area2]; unsigned long end2 = start2 + sizes[area2]; BUG_ON(start2 < end && start < end2); } } last_end = offsets[last_area] + sizes[last_area]; if (vmalloc_end - vmalloc_start < last_end) { WARN_ON(true); return NULL; } vms = kcalloc(nr_vms, sizeof(vms[0]), GFP_KERNEL); vas = kcalloc(nr_vms, sizeof(vas[0]), GFP_KERNEL); if (!vas || !vms) goto err_free2; for (area = 0; area < nr_vms; area++) { vas[area] = kzalloc(sizeof(struct vmap_area), GFP_KERNEL); vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL); if (!vas[area] || !vms[area]) goto err_free; } retry: spin_lock(&vmap_area_lock); /* start scanning - we scan from the top, begin with the last area */ area = term_area = last_area; start = offsets[area]; end = start + sizes[area]; if (!pvm_find_next_prev(vmap_area_pcpu_hole, &next, &prev)) { base = vmalloc_end - last_end; goto found; } base = pvm_determine_end(&next, &prev, align) - end; while (true) { BUG_ON(next && next->va_end <= base + end); BUG_ON(prev && prev->va_end > base + end); /* * base might have underflowed, add last_end before * comparing. */ if (base + last_end < vmalloc_start + last_end) { spin_unlock(&vmap_area_lock); if (!purged) { purge_vmap_area_lazy(); purged = true; goto retry; } goto err_free; } /* * If next overlaps, move base downwards so that it's * right below next and then recheck. */ if (next && next->va_start < base + end) { base = pvm_determine_end(&next, &prev, align) - end; term_area = area; continue; } /* * If prev overlaps, shift down next and prev and move * base so that it's right below new next and then * recheck. */ if (prev && prev->va_end > base + start) { next = prev; prev = node_to_va(rb_prev(&next->rb_node)); base = pvm_determine_end(&next, &prev, align) - end; term_area = area; continue; } /* * This area fits, move on to the previous one. If * the previous one is the terminal one, we're done. */ area = (area + nr_vms - 1) % nr_vms; if (area == term_area) break; start = offsets[area]; end = start + sizes[area]; pvm_find_next_prev(base + end, &next, &prev); } found: /* we've found a fitting base, insert all va's */ for (area = 0; area < nr_vms; area++) { struct vmap_area *va = vas[area]; va->va_start = base + offsets[area]; va->va_end = va->va_start + sizes[area]; __insert_vmap_area(va); } vmap_area_pcpu_hole = base + offsets[last_area]; spin_unlock(&vmap_area_lock); /* insert all vm's */ for (area = 0; area < nr_vms; area++) setup_vmalloc_vm(vms[area], vas[area], VM_ALLOC, pcpu_get_vm_areas); kfree(vas); return vms; err_free: for (area = 0; area < nr_vms; area++) { kfree(vas[area]); kfree(vms[area]); } err_free2: kfree(vas); kfree(vms); return NULL; }

Contributors

PersonTokensPropCommitsCommitProp
Tejun Heo78897.28%114.29%
Thomas Meyer60.74%114.29%
Wei Yang50.62%114.29%
David Rientjes40.49%114.29%
Kautuk Consul30.37%114.29%
Alexander Kuleshov30.37%114.29%
Zhang Yanfei10.12%114.29%
Total810100.00%7100.00%

/** * pcpu_free_vm_areas - free vmalloc areas for percpu allocator * @vms: vm_struct pointer array returned by pcpu_get_vm_areas() * @nr_vms: the number of allocated areas * * Free vm_structs and the array allocated by pcpu_get_vm_areas(). */
void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms) { int i; for (i = 0; i < nr_vms; i++) free_vm_area(vms[i]); kfree(vms); }

Contributors

PersonTokensPropCommitsCommitProp
Tejun Heo43100.00%1100.00%
Total43100.00%1100.00%

#endif /* CONFIG_SMP */ #ifdef CONFIG_PROC_FS
static void *s_start(struct seq_file *m, loff_t *pos) __acquires(&vmap_area_lock

Contributors

PersonTokensPropCommitsCommitProp
Christoph Lameter1482.35%133.33%
Namhyung Kim211.76%133.33%
JoonSoo Kim15.88%133.33%
Total17100.00%3100.00%

) { spin_lock(&vmap_area_lock); return seq_list_start(&vmap_area_list, *pos); }
static void *s_next(struct seq_file *m, void *p, loff_t *pos) { return seq_list_next(p, &vmap_area_list, pos); }

Contributors

PersonTokensPropCommitsCommitProp
Christoph Lameter2064.52%133.33%
zijun_hu619.35%133.33%
JoonSoo Kim516.13%133.33%
Total31100.00%3100.00%


static void s_stop(struct seq_file *m, void *p) __releases(&vmap_area_lock

Contributors

PersonTokensPropCommitsCommitProp
Christoph Lameter1381.25%133.33%
Namhyung Kim212.50%133.33%
JoonSoo Kim16.25%133.33%
Total16100.00%3100.00%

) { spin_unlock(&vmap_area_lock); }
static void show_numa_info(struct seq_file *m, struct vm_struct *v) { if (IS_ENABLED(CONFIG_NUMA)) { unsigned int nr, *counters = m->private; if (!counters) return; if (v->flags & VM_UNINITIALIZED) return; /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */ smp_rmb(); memset(counters, 0, nr_node_ids * sizeof(unsigned int)); for (nr = 0; nr < v->nr_pages; nr++) counters[page_to_nid(v->pages[nr])]++; for_each_node_state(nr, N_HIGH_MEMORY) if (counters[nr]) seq_printf(m, " N%u=%u", nr, counters[nr]); } }

Contributors

PersonTokensPropCommitsCommitProp
Eric Dumazet10986.51%125.00%
Wanpeng Li97.14%125.00%
Dmitriy Vyukov43.17%125.00%
Kirill A. Shutemov43.17%125.00%
Total126100.00%4100.00%


static int s_show(struct seq_file *m, void *p) { struct vmap_area *va; struct vm_struct *v; va = list_entry(p, struct vmap_area, list); /* * s_show can encounter race with remove_vm_area, !VM_VM_AREA on * behalf of vmap area is being tear down or vm_map_ram allocation. */ if (!(va->flags & VM_VM_AREA)) { seq_printf(m, "0x%pK-0x%pK %7ld %s\n", (void *)va->va_start, (void *)va->va_end, va->va_end - va->va_start, va->flags & VM_LAZY_FREE ? "unpurged vm_area" : "vm_map_ram"); return 0; } v = va->vm; seq_printf(m, "0x%pK-0x%pK %7ld", v->addr, v->addr + v->size, v->size); if (v->caller) seq_printf(m, " %pS", v->caller); if (v->nr_pages) seq_printf(m, " pages=%d", v->nr_pages); if (v->phys_addr) seq_printf(m, " phys=%pa", &v->phys_addr); if (v->flags & VM_IOREMAP) seq_puts(m, " ioremap"); if (v->flags & VM_ALLOC) seq_puts(m, " vmalloc"); if (v->flags & VM_MAP) seq_puts(m, " vmap"); if (v->flags & VM_USERMAP) seq_puts(m, " user"); if (is_vmalloc_addr(v->pages)) seq_puts(m, " vpages"); show_numa_info(m, v); seq_putc(m, '\n'); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Christoph Lameter16561.34%216.67%
Yisheng Xie4315.99%18.33%
JoonSoo Kim2710.04%18.33%
zijun_hu124.46%18.33%
Eric Dumazet72.60%18.33%
Fabian Frederick51.86%18.33%
David Rientjes41.49%18.33%
Miles Chen20.74%18.33%
Joe Perches20.74%18.33%
Wanpeng Li10.37%18.33%
Kees Cook10.37%18.33%
Total269100.00%12100.00%

static const struct seq_operations vmalloc_op = { .start = s_start, .next = s_next, .stop = s_stop, .show = s_show, };
static int vmalloc_open(struct inode *inode, struct file *file) { if (IS_ENABLED(CONFIG_NUMA)) return seq_open_private(file, &vmalloc_op, nr_node_ids * sizeof(unsigned int)); else return seq_open(file, &vmalloc_op); }

Contributors

PersonTokensPropCommitsCommitProp
Alexey Dobriyan3672.00%125.00%
Rob Jones918.00%125.00%
Kirill A. Shutemov48.00%125.00%
Vasiliy Kulikov12.00%125.00%
Total50100.00%4100.00%

static const struct file_operations proc_vmalloc_operations = { .open = vmalloc_open, .read = seq_read, .llseek = seq_lseek, .release = seq_release_private, };
static int __init proc_vmalloc_init(void) { proc_create("vmallocinfo", S_IRUSR, NULL, &proc_vmalloc_operations); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
Alexey Dobriyan24100.00%1100.00%
Total24100.00%1100.00%

module_init(proc_vmalloc_init); #endif

Overall Contributors

PersonTokensPropCommitsCommitProp
Nicholas Piggin333031.54%93.83%
Tejun Heo142113.46%83.40%
Christoph Lameter6285.95%83.40%
Kamezawa Hiroyuki3963.75%20.85%
Linus Torvalds (pre-git)3923.71%239.79%
Christoph Hellwig3273.10%72.98%
Hugh Dickins3223.05%20.85%
Kirill A. Shutemov2642.50%20.85%
Andi Kleen2132.02%41.70%
Andrew Morton2081.97%72.98%
Roman Peniaev2051.94%31.28%
JoonSoo Kim1901.80%72.98%
Jeremy Fitzhardinge1761.67%31.28%
Al Viro1481.40%20.85%
Chris Wilson1401.33%20.85%
Eric Dumazet1351.28%31.28%
Dave Jones1311.24%10.43%
Michal Hocko1131.07%41.70%
Nico Pitre1101.04%10.43%
Andrey Ryabinin1000.95%41.70%
Alexey Dobriyan990.94%20.85%
Daisuke Hatayama950.90%20.85%
David Rientjes890.84%52.13%
Ivan Kokshaysky720.68%10.43%
Oleg Nesterov700.66%31.28%
Dave Young580.55%10.43%
Linus Torvalds560.53%31.28%
Yisheng Xie530.50%10.43%
Benjamin Herrenschmidt530.50%41.70%
Catalin Marinas420.40%31.28%
David S. Miller400.38%20.85%
zijun_hu400.38%20.85%
Ingo Molnar390.37%52.13%
David Vrabel390.37%10.43%
Joel Fernandes380.36%10.43%
James Bottomley380.36%20.85%
Glauber de Oliveira Costa380.36%31.28%
Wanpeng Li330.31%52.13%
Mitsuo Hayasaka330.31%10.43%
Arnaldo Carvalho de Melo320.30%10.43%
Ard Biesheuvel310.29%10.43%
Russell King310.29%20.85%
Tom Rini270.26%10.43%
Laura Abbott250.24%10.43%
Guillermo Julián Moreno230.22%10.43%
Toshi Kani220.21%20.85%
Vassili Karpov210.20%10.43%
Joe Perches210.20%31.28%
Cliff Wickman190.18%10.43%
Alexander Kuleshov180.17%10.43%
Dave Hansen170.16%10.43%
Zhang Yanfei170.16%41.70%
Jan Kara150.14%10.43%
Jan Beulich150.14%20.85%
Byungchul Park130.12%10.43%
Hong Zhi Guo120.11%10.43%
Mel Gorman110.10%31.28%
Namhyung Kim100.09%20.85%
Thomas Gleixner90.09%10.43%
Rob Jones90.09%10.43%
Jiri Slaby80.08%10.43%
Wang Xiaoqiang80.08%10.43%
MinChan Kim80.08%20.85%
Vegard Nossum70.07%10.43%
Florian Fainelli70.07%10.43%
Adam Lackorzynski70.07%10.43%
Marek Szyprowski70.07%20.85%
Cho KyongHo60.06%10.43%
Jerome Marchand60.06%10.43%
Wei Yang60.06%10.43%
Gideon Israel Dsouza60.06%10.43%
Thomas Meyer60.06%10.43%
Peter Zijlstra60.06%10.43%
Rolf Eike Beer50.05%10.43%
Ralph Wuerthner50.05%10.43%
Arjan van de Ven50.05%10.43%
Rusty Russell50.05%10.43%
Marcus Alanen50.05%10.43%
Fabian Frederick50.05%10.43%
Huang Ying50.05%10.43%
Dmitriy Vyukov40.04%10.43%
Vladimir Davydov40.04%20.85%
Giridhar Pemmasani40.04%10.43%
Lai Jiangshan40.04%20.85%
Hirofumi Ogawa30.03%10.43%
Michael Opdenacker30.03%10.43%
WANG Chao30.03%10.43%
Pekka J Enberg30.03%10.43%
Konstantin Khlebnikov30.03%10.43%
Kautuk Consul30.03%10.43%
Figo.zhang30.03%10.43%
Jan Kiszka20.02%10.43%
Geliang Tang20.02%20.85%
Américo Wang20.02%10.43%
Miles Chen20.02%10.43%
Jianguo Wu20.02%10.43%
Martin Schwidefsky20.02%10.43%
Clemens Ladisch20.02%10.43%
Arun Sharma10.01%10.43%
Mauro Carvalho Chehab10.01%10.43%
Kees Cook10.01%10.43%
Shawn Lin10.01%10.43%
Adrian Bunk10.01%10.43%
Johannes Weiner10.01%10.43%
Vasiliy Kulikov10.01%10.43%
Gioh Kim10.01%10.43%
Simon Arlott10.01%10.43%
Julia Lawall10.01%10.43%
Pintu Kumar10.01%10.43%
Total10557100.00%235100.00%
Directory: mm
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with cregit.