cregit-Linux how code gets into the kernel

Release 4.8 mm/page_alloc.c

Directory: mm
/*
 *  linux/mm/page_alloc.c
 *
 *  Manages the free list, the system allocates free pages here.
 *  Note that kmalloc() lives in slab.c
 *
 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 *  Swap reorganised 29.12.95, Stephen Tweedie
 *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
 *  Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
 *  Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
 *  Zone balancing, Kanoj Sarcar, SGI, Jan 2000
 *  Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
 *          (lots of bits borrowed from Ingo Molnar & Andrew Morton)
 */

#include <linux/stddef.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/interrupt.h>
#include <linux/pagemap.h>
#include <linux/jiffies.h>
#include <linux/bootmem.h>
#include <linux/memblock.h>
#include <linux/compiler.h>
#include <linux/kernel.h>
#include <linux/kmemcheck.h>
#include <linux/kasan.h>
#include <linux/module.h>
#include <linux/suspend.h>
#include <linux/pagevec.h>
#include <linux/blkdev.h>
#include <linux/slab.h>
#include <linux/ratelimit.h>
#include <linux/oom.h>
#include <linux/notifier.h>
#include <linux/topology.h>
#include <linux/sysctl.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/memory_hotplug.h>
#include <linux/nodemask.h>
#include <linux/vmalloc.h>
#include <linux/vmstat.h>
#include <linux/mempolicy.h>
#include <linux/memremap.h>
#include <linux/stop_machine.h>
#include <linux/sort.h>
#include <linux/pfn.h>
#include <linux/backing-dev.h>
#include <linux/fault-inject.h>
#include <linux/page-isolation.h>
#include <linux/page_ext.h>
#include <linux/debugobjects.h>
#include <linux/kmemleak.h>
#include <linux/compaction.h>
#include <trace/events/kmem.h>
#include <linux/prefetch.h>
#include <linux/mm_inline.h>
#include <linux/migrate.h>
#include <linux/page_ext.h>
#include <linux/hugetlb.h>
#include <linux/sched/rt.h>
#include <linux/page_owner.h>
#include <linux/kthread.h>
#include <linux/memcontrol.h>

#include <asm/sections.h>
#include <asm/tlbflush.h>
#include <asm/div64.h>
#include "internal.h"

/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
static DEFINE_MUTEX(pcp_batch_high_lock);

#define MIN_PERCPU_PAGELIST_FRACTION	(8)

#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
DEFINE_PER_CPU(int, numa_node);

EXPORT_PER_CPU_SYMBOL(numa_node);
#endif

#ifdef CONFIG_HAVE_MEMORYLESS_NODES
/*
 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
 * defined in <linux/topology.h>.
 */
DEFINE_PER_CPU(int, _numa_mem_);		/* Kernel "local memory" node */

EXPORT_PER_CPU_SYMBOL(_numa_mem_);

int _node_numa_mem_[MAX_NUMNODES];
#endif

/*
 * Array of node states.
 */

nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
	[N_POSSIBLE] = NODE_MASK_ALL,
	[N_ONLINE] = { { [0] = 1UL } },
#ifndef CONFIG_NUMA
	[N_NORMAL_MEMORY] = { { [0] = 1UL } },
#ifdef CONFIG_HIGHMEM
	[N_HIGH_MEMORY] = { { [0] = 1UL } },
#endif
#ifdef CONFIG_MOVABLE_NODE
	[N_MEMORY] = { { [0] = 1UL } },
#endif
	[N_CPU] = { { [0] = 1UL } },
#endif	/* NUMA */
};

EXPORT_SYMBOL(node_states);

/* Protect totalram_pages and zone->managed_pages */
static DEFINE_SPINLOCK(managed_page_count_lock);


unsigned long totalram_pages __read_mostly;

unsigned long totalreserve_pages __read_mostly;

unsigned long totalcma_pages __read_mostly;


int percpu_pagelist_fraction;

gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;

/*
 * A cached value of the page's pageblock's migratetype, used when the page is
 * put on a pcplist. Used to avoid the pageblock migratetype lookup when
 * freeing from pcplists in most cases, at the cost of possibly becoming stale.
 * Also the migratetype set in the page does not necessarily match the pcplist
 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any
 * other index - this ensures that it will be put on the correct CMA freelist.
 */

static inline int get_pcppage_migratetype(struct page *page) { return page->index; }

Contributors

PersonTokensPropCommitsCommitProp
vlastimil babkavlastimil babka17100.00%1100.00%
Total17100.00%1100.00%


static inline void set_pcppage_migratetype(struct page *page, int migratetype) { page->index = migratetype; }

Contributors

PersonTokensPropCommitsCommitProp
vlastimil babkavlastimil babka21100.00%1100.00%
Total21100.00%1100.00%

#ifdef CONFIG_PM_SLEEP /* * The following functions are used by the suspend/hibernate code to temporarily * change gfp_allowed_mask in order to avoid using I/O during memory allocations * while devices are suspended. To avoid races with the suspend/hibernate code, * they should always be called with pm_mutex held (gfp_allowed_mask also should * only be modified with pm_mutex held, unless the suspend/hibernate code is * guaranteed not to run in parallel with that modification). */ static gfp_t saved_gfp_mask;
void pm_restore_gfp_mask(void) { WARN_ON(!mutex_is_locked(&pm_mutex)); if (saved_gfp_mask) { gfp_allowed_mask = saved_gfp_mask; saved_gfp_mask = 0; } }

Contributors

PersonTokensPropCommitsCommitProp
rafael j. wysockirafael j. wysocki31100.00%2100.00%
Total31100.00%2100.00%


void pm_restrict_gfp_mask(void) { WARN_ON(!mutex_is_locked(&pm_mutex)); WARN_ON(saved_gfp_mask); saved_gfp_mask = gfp_allowed_mask; gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS); }

Contributors

PersonTokensPropCommitsCommitProp
rafael j. wysockirafael j. wysocki3085.71%266.67%
mel gormanmel gorman514.29%133.33%
Total35100.00%3100.00%


bool pm_suspended_storage(void) { if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS)) return false; return true; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman31100.00%2100.00%
Total31100.00%2100.00%

#endif /* CONFIG_PM_SLEEP */ #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE unsigned int pageblock_order __read_mostly; #endif static void __free_pages_ok(struct page *page, unsigned int order); /* * results with 256, 32 in the lowmem_reserve sysctl: * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) * 1G machine -> (16M dma, 784M normal, 224M high) * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA * * TBD: should special case ZONE_DMA32 machines here - in those we normally * don't need any ZONE_NORMAL reservation */ int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = { #ifdef CONFIG_ZONE_DMA 256, #endif #ifdef CONFIG_ZONE_DMA32 256, #endif #ifdef CONFIG_HIGHMEM 32, #endif 32, }; EXPORT_SYMBOL(totalram_pages); static char * const zone_names[MAX_NR_ZONES] = { #ifdef CONFIG_ZONE_DMA "DMA", #endif #ifdef CONFIG_ZONE_DMA32 "DMA32", #endif "Normal", #ifdef CONFIG_HIGHMEM "HighMem", #endif "Movable", #ifdef CONFIG_ZONE_DEVICE "Device", #endif }; char * const migratetype_names[MIGRATE_TYPES] = { "Unmovable", "Movable", "Reclaimable", "HighAtomic", #ifdef CONFIG_CMA "CMA", #endif #ifdef CONFIG_MEMORY_ISOLATION "Isolate", #endif }; compound_page_dtor * const compound_page_dtors[] = { NULL, free_compound_page, #ifdef CONFIG_HUGETLB_PAGE free_huge_page, #endif #ifdef CONFIG_TRANSPARENT_HUGEPAGE free_transhuge_page, #endif }; int min_free_kbytes = 1024; int user_min_free_kbytes = -1; int watermark_scale_factor = 10; static unsigned long __meminitdata nr_kernel_pages; static unsigned long __meminitdata nr_all_pages; static unsigned long __meminitdata dma_reserve; #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES]; static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES]; static unsigned long __initdata required_kernelcore; static unsigned long __initdata required_movablecore; static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES]; static bool mirrored_kernelcore; /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ int movable_zone; EXPORT_SYMBOL(movable_zone); #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ #if MAX_NUMNODES > 1 int nr_node_ids __read_mostly = MAX_NUMNODES; int nr_online_nodes __read_mostly = 1; EXPORT_SYMBOL(nr_node_ids); EXPORT_SYMBOL(nr_online_nodes); #endif int page_group_by_mobility_disabled __read_mostly; #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
static inline void reset_deferred_meminit(pg_data_t *pgdat) { pgdat->first_deferred_pfn = ULONG_MAX; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman17100.00%1100.00%
Total17100.00%1100.00%

/* Returns true if the struct page for the pfn is uninitialised */
static inline bool __meminit early_page_uninitialised(unsigned long pfn) { int nid = early_pfn_to_nid(pfn); if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn) return true; return false; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman42100.00%3100.00%
Total42100.00%3100.00%

/* * Returns false when the remaining initialisation should be deferred until * later in the boot cycle when it can be parallelised. */
static inline bool update_defer_init(pg_data_t *pgdat, unsigned long pfn, unsigned long zone_end, unsigned long *nr_initialised) { unsigned long max_initialise; /* Always populate low zones for address-contrained allocations */ if (zone_end < pgdat_end_pfn(pgdat)) return true; /* * Initialise at least 2G of a node but also take into account that * two large system hashes that can take up 1GB for 0.25TB/node. */ max_initialise = max(2UL << (30 - PAGE_SHIFT), (pgdat->node_spanned_pages >> 8)); (*nr_initialised)++; if ((*nr_initialised > max_initialise) && (pfn & (PAGES_PER_SECTION - 1)) == 0) { pgdat->first_deferred_pfn = pfn; return false; } return true; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman7673.08%150.00%
li zhangli zhang2826.92%150.00%
Total104100.00%2100.00%

#else
static inline void reset_deferred_meminit(pg_data_t *pgdat) { }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman10100.00%1100.00%
Total10100.00%1100.00%


static inline bool early_page_uninitialised(unsigned long pfn) { return false; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman14100.00%1100.00%
Total14100.00%1100.00%


static inline bool update_defer_init(pg_data_t *pgdat, unsigned long pfn, unsigned long zone_end, unsigned long *nr_initialised) { return true; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman27100.00%1100.00%
Total27100.00%1100.00%

#endif /* Return a pointer to the bitmap storing bits affecting a block of pages */
static inline unsigned long *get_pageblock_bitmap(struct page *page, unsigned long pfn) { #ifdef CONFIG_SPARSEMEM return __pfn_to_section(pfn)->pageblock_flags; #else return page_zone(page)->pageblock_flags; #endif /* CONFIG_SPARSEMEM */ }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman42100.00%3100.00%
Total42100.00%3100.00%


static inline int pfn_to_bitidx(struct page *page, unsigned long pfn) { #ifdef CONFIG_SPARSEMEM pfn &= (PAGES_PER_SECTION-1); return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; #else pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages); return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; #endif /* CONFIG_SPARSEMEM */ }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman4263.64%111.11%
dave hansendave hansen1218.18%333.33%
william lee irwin iiiwilliam lee irwin iii57.58%111.11%
cody p schafercody p schafer46.06%111.11%
rik van rielrik van riel11.52%111.11%
andrew mortonandrew morton11.52%111.11%
pre-gitpre-git11.52%111.11%
Total66100.00%9100.00%

/** * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages * @page: The page within the block of interest * @pfn: The target page frame number * @end_bitidx: The last bit of interest to retrieve * @mask: mask of bits that the caller is interested in * * Return: pageblock_bits flags */
static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page, unsigned long pfn, unsigned long end_bitidx, unsigned long mask) { unsigned long *bitmap; unsigned long bitidx, word_bitidx; unsigned long word; bitmap = get_pageblock_bitmap(page, pfn); bitidx = pfn_to_bitidx(page, pfn); word_bitidx = bitidx / BITS_PER_LONG; bitidx &= (BITS_PER_LONG-1); word = bitmap[word_bitidx]; bitidx += end_bitidx; return (word >> (BITS_PER_LONG - bitidx - 1)) & mask; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman8081.63%125.00%
dave hansendave hansen1212.24%125.00%
william lee irwin iiiwilliam lee irwin iii33.06%125.00%
kamezawa hiroyukikamezawa hiroyuki33.06%125.00%
Total98100.00%4100.00%


unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn, unsigned long end_bitidx, unsigned long mask) { return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask); }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman2262.86%133.33%
dave hansendave hansen1131.43%133.33%
william lee irwin iiiwilliam lee irwin iii25.71%133.33%
Total35100.00%3100.00%


static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn) { return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK); }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman1657.14%150.00%
nick pigginnick piggin1242.86%150.00%
Total28100.00%2100.00%

/** * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages * @page: The page within the block of interest * @flags: The flags to set * @pfn: The target page frame number * @end_bitidx: The last bit of interest * @mask: mask of bits that the caller is interested in */
void set_pfnblock_flags_mask(struct page *page, unsigned long flags, unsigned long pfn, unsigned long end_bitidx, unsigned long mask) { unsigned long *bitmap; unsigned long bitidx, word_bitidx; unsigned long old_word, word; BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); bitmap = get_pageblock_bitmap(page, pfn); bitidx = pfn_to_bitidx(page, pfn); word_bitidx = bitidx / BITS_PER_LONG; bitidx &= (BITS_PER_LONG-1); VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page); bitidx += end_bitidx; mask <<= (BITS_PER_LONG - bitidx - 1); flags <<= (BITS_PER_LONG - bitidx - 1); word = READ_ONCE(bitmap[word_bitidx]); for (;;) { old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags); if (word == old_word) break; word = old_word; } }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman11768.02%17.14%
hugh dickinshugh dickins2514.53%214.29%
andrew mortonandrew morton126.98%214.29%
vlastimil babkavlastimil babka52.91%214.29%
dave hansendave hansen42.33%17.14%
fengguang wufengguang wu31.74%17.14%
randy dunlaprandy dunlap21.16%17.14%
nick pigginnick piggin10.58%17.14%
rusty russellrusty russell10.58%17.14%
william lee irwin iiiwilliam lee irwin iii10.58%17.14%
dave jonesdave jones10.58%17.14%
Total172100.00%14100.00%


void set_pageblock_migratetype(struct page *page, int migratetype) { if (unlikely(page_group_by_mobility_disabled && migratetype < MIGRATE_PCPTYPES)) migratetype = MIGRATE_UNMOVABLE; set_pageblock_flags_group(page, (unsigned long)migratetype, PB_migrate, PB_migrate_end); }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman2967.44%133.33%
hugh dickinshugh dickins1330.23%133.33%
christoph lameterchristoph lameter12.33%133.33%
Total43100.00%3100.00%

#ifdef CONFIG_DEBUG_VM
static int page_outside_zone_boundaries(struct zone *zone, struct page *page) { int ret = 0; unsigned seq; unsigned long pfn = page_to_pfn(page); unsigned long sp, start_pfn; do { seq = zone_span_seqbegin(zone); start_pfn = zone->zone_start_pfn; sp = zone->spanned_pages; if (!zone_spans_pfn(zone, pfn)) ret = 1; } while (zone_span_seqretry(zone, seq)); if (ret) pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", pfn, zone_to_nid(zone), zone->name, start_pfn, start_pfn + sp); return ret; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman8171.05%112.50%
andrew mortonandrew morton108.77%112.50%
andy whitcroftandy whitcroft97.89%112.50%
kirill a. shutemovkirill a. shutemov97.89%337.50%
youquan songyouquan song32.63%112.50%
david rientjesdavid rientjes21.75%112.50%
Total114100.00%8100.00%


static int page_is_consistent(struct zone *zone, struct page *page) { if (!pfn_valid_within(page_to_pfn(page))) return 0; if (zone != page_zone(page)) return 0; return 1; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman2453.33%133.33%
joonsoo kimjoonsoo kim1635.56%133.33%
christian borntraegerchristian borntraeger511.11%133.33%
Total45100.00%3100.00%

/* * Temporary debugging check for pages not lying within a given zone. */
static int bad_range(struct zone *zone, struct page *page) { if (page_outside_zone_boundaries(zone, page)) return 1; if (!page_is_consistent(zone, page)) return 1; return 0; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman3170.45%133.33%
joonsoo kimjoonsoo kim1329.55%266.67%
Total44100.00%3100.00%

#else
static inline int bad_range(struct zone *zone, struct page *page) { return 0; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman1575.00%150.00%
joonsoo kimjoonsoo kim525.00%150.00%
Total20100.00%2100.00%

#endif
static void bad_page(struct page *page, const char *reason, unsigned long bad_flags) { static unsigned long resume; static unsigned long nr_shown; static unsigned long nr_unshown; /* * Allow a burst of 60 reports, then keep quiet for that minute; * or allow a steady drip of one report per second. */ if (nr_shown == 60) { if (time_before(jiffies, resume)) { nr_unshown++; goto out; } if (nr_unshown) { pr_alert( "BUG: Bad page state: %lu messages suppressed\n", nr_unshown); nr_unshown = 0; } nr_shown = 0; } if (nr_shown++ == 0) resume = jiffies + 60 * HZ; pr_alert("BUG: Bad page state in process %s pfn:%05lx\n", current->comm, page_to_pfn(page)); __dump_page(page, reason); bad_flags &= page->flags; if (bad_flags) pr_alert("bad because of flags: %#lx(%pGp)\n", bad_flags, &bad_flags); dump_page_owner(page); print_modules(); dump_stack(); out: /* Leave bad fields for debug, except PageBuddy could make trouble */ page_mapcount_reset(page); /* remove PageBuddy */ add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman12072.73%125.00%
stanislaw gruszkastanislaw gruszka2313.94%125.00%
joonsoo kimjoonsoo kim2213.33%250.00%
Total165100.00%4100.00%

/* * Higher-order pages are called "compound pages". They are structured thusly: * * The first PAGE_SIZE page is called the "head page" and have PG_head set. * * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded * in bit 0 of page->compound_head. The rest of bits is pointer to head page. * * The first tail page's ->compound_dtor holds the offset in array of compound * page destructors. See compound_page_dtors. * * The first tail page's ->compound_order holds the order of allocation. * This usage means that zero-order pages may not be compound. */
void free_compound_page(struct page *page) { __free_pages_ok(page, compound_order(page)); }

Contributors

PersonTokensPropCommitsCommitProp
stanislaw gruszkastanislaw gruszka1050.00%125.00%
joonsoo kimjoonsoo kim735.00%250.00%
mel gormanmel gorman315.00%125.00%
Total20100.00%4100.00%


void prep_compound_page(struct page *page, unsigned int order) { int i; int nr_pages = 1 << order; set_compound_page_dtor(page, COMPOUND_PAGE_DTOR); set_compound_order(page, order); __SetPageHead(page); for (i = 1; i < nr_pages; i++) { struct page *p = page + i; set_page_count(p, 0); p->mapping = TAIL_MAPPING; set_compound_head(p, page); } atomic_set(compound_mapcount_ptr(page), -1); }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman6061.22%120.00%
kamezawa hiroyukikamezawa hiroyuki2020.41%120.00%
stanislaw gruszkastanislaw gruszka77.14%120.00%
hugh dickinshugh dickins66.12%120.00%
joonsoo kimjoonsoo kim55.10%120.00%
Total98100.00%5100.00%

#ifdef CONFIG_DEBUG_PAGEALLOC unsigned int _debug_guardpage_minorder; bool _debug_pagealloc_enabled __read_mostly = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT); EXPORT_SYMBOL(_debug_pagealloc_enabled); bool _debug_guardpage_enabled __read_mostly;
static int __init early_debug_pagealloc(char *buf) { if (!buf) return -EINVAL; return kstrtobool(buf, &_debug_pagealloc_enabled); }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman2275.86%240.00%
nick pigginnick piggin310.34%120.00%
minfei huangminfei huang310.34%120.00%
stanislaw gruszkastanislaw gruszka13.45%120.00%
Total29100.00%5100.00%

early_param("debug_pagealloc", early_debug_pagealloc);
static bool need_debug_guardpage(void) { /* If we don't use debug_pagealloc, we don't need guard page */ if (!debug_pagealloc_enabled()) return false; return true; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman1885.71%150.00%
stanislaw gruszkastanislaw gruszka314.29%150.00%
Total21100.00%2100.00%


static void init_debug_guardpage(void) { if (!debug_pagealloc_enabled()) return; _debug_guardpage_enabled = true; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman1684.21%133.33%
stanislaw gruszkastanislaw gruszka210.53%133.33%
nick pigginnick piggin15.26%133.33%
Total19100.00%3100.00%

struct page_ext_operations debug_guardpage_ops = { .need = need_debug_guardpage, .init = init_debug_guardpage, };
static int __init debug_guardpage_minorder_setup(char *buf) { unsigned long res; if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) { pr_err("Bad debug_guardpage_minorder value\n"); return 0; } _debug_guardpage_minorder = res; pr_info("Setting debug_guardpage_minorder to %lu\n", res); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman5186.44%250.00%
weijie yangweijie yang46.78%125.00%
kamezawa hiroyukikamezawa hiroyuki46.78%125.00%
Total59100.00%4100.00%

__setup("debug_guardpage_minorder=", debug_guardpage_minorder_setup);
static inline void set_page_guard(struct zone *zone, struct page *page, unsigned int order, int migratetype) { struct page_ext *page_ext; if (!debug_guardpage_enabled()) return; page_ext = lookup_page_ext(page); if (unlikely(!page_ext)) return; __set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags); INIT_LIST_HEAD(&page->lru); set_page_private(page, order); /* Guard pages are not available for any usage */ __mod_zone_freepage_state(zone, -(1 << order), migratetype); }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman4953.26%420.00%
yang shiyang shi99.78%15.00%
pre-gitpre-git88.70%315.00%
andrew mortonandrew morton55.43%315.00%
joonsoo kimjoonsoo kim55.43%210.00%
vlastimil babkavlastimil babka44.35%15.00%
cody p schafercody p schafer33.26%15.00%
kirill a. shutemovkirill a. shutemov33.26%15.00%
kamezawa hiroyukikamezawa hiroyuki22.17%15.00%
corrado zoccolocorrado zoccolo22.17%15.00%
hugh dickinshugh dickins11.09%15.00%
sasha levinsasha levin11.09%15.00%
Total92100.00%20100.00%


static inline void clear_page_guard(struct zone *zone, struct page *page, unsigned int order, int migratetype) { struct page_ext *page_ext; if (!debug_guardpage_enabled()) return; page_ext = lookup_page_ext(page); if (unlikely(!page_ext)) return; __clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags); set_page_private(page, 0); if (!is_migrate_isolate(migratetype)) __mod_zone_freepage_state(zone, (1 << order), migratetype); }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman5358.89%16.67%
stanislaw gruszkastanislaw gruszka1213.33%16.67%
yang shiyang shi910.00%16.67%
pre-gitpre-git44.44%426.67%
dave hansendave hansen22.22%16.67%
kamezawa hiroyukikamezawa hiroyuki22.22%16.67%
bartlomiej zolnierkiewiczbartlomiej zolnierkiewicz22.22%16.67%
andy whitcroftandy whitcroft22.22%16.67%
sasha levinsasha levin11.11%16.67%
kyongho chokyongho cho11.11%16.67%
martin blighmartin bligh11.11%16.67%
andrew mortonandrew morton11.11%16.67%
Total90100.00%15100.00%

#else struct page_ext_operations debug_guardpage_ops = { NULL, };
static inline void set_page_guard(struct zone *zone, struct page *page, unsigned int order, int migratetype) {}

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman23100.00%1100.00%
Total23100.00%1100.00%


static inline void clear_page_guard(struct zone *zone, struct page *page, unsigned int order, int migratetype) {}

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman23100.00%1100.00%
Total23100.00%1100.00%

#endif
static inline void set_page_order(struct page *page, unsigned int order) { set_page_private(page, order); __SetPageBuddy(page); }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman2382.14%125.00%
kamezawa hiroyukikamezawa hiroyuki27.14%125.00%
pre-gitpre-git27.14%125.00%
stanislaw gruszkastanislaw gruszka13.57%125.00%
Total28100.00%4100.00%


static inline void rmv_page_order(struct page *page) { __ClearPageBuddy(page); set_page_private(page, 0); }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman1979.17%125.00%
dave hansendave hansen28.33%125.00%
pre-gitpre-git28.33%125.00%
kamezawa hiroyukikamezawa hiroyuki14.17%125.00%
Total24100.00%4100.00%

/* * This function checks whether a page is free && is the buddy * we can do coalesce a page and its buddy if * (a) the buddy is not in a hole && * (b) the buddy is in the buddy system && * (c) a page and its buddy have the same order && * (d) a page and its buddy are in the same zone. * * For recording whether a page is in the buddy system, we set ->_mapcount * PAGE_BUDDY_MAPCOUNT_VALUE. * Setting, clearing, and testing _mapcount PAGE_BUDDY_MAPCOUNT_VALUE is * serialized by zone->lock. * * For recording page's order, we use page_private(page). */
static inline int page_is_buddy(struct page *page, struct page *buddy, unsigned int order) { if (!pfn_valid_within(page_to_pfn(buddy))) return 0; if (page_is_guard(buddy) && page_order(buddy) == order) { if (page_zone_id(page) != page_zone_id(buddy)) return 0; VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); return 1; } if (PageBuddy(buddy) && page_order(buddy) == order) { /* * zone check is done late to avoid uselessly * calculating zone/node ids for pages that could * never merge. */ if (page_zone_id(page) != page_zone_id(buddy)) return 0; VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); return 1; } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman9975.57%125.00%
vlastimil babkavlastimil babka2015.27%125.00%
corrado zoccolocorrado zoccolo86.11%125.00%
kamezawa hiroyukikamezawa hiroyuki43.05%125.00%
Total131100.00%4100.00%

/* * Freeing function for a buddy system allocator. * * The concept of a buddy system is to maintain direct-mapped table * (containing bit values) for memory blocks of various "orders". * The bottom level table contains the map for the smallest allocatable * units of memory (here, pages), and each level above it describes * pairs of units from the levels below, hence, "buddies". * At a high level, all that happens here is marking the table entry * at the bottom level available, and propagating the changes upward * as necessary, plus some accounting needed to play nicely with other * parts of the VM system. * At each level, we keep a list of pages, which are heads of continuous * free pages of length of (1 << order) and marked with _mapcount * PAGE_BUDDY_MAPCOUNT_VALUE. Page's order is recorded in page_private(page) * field. * So when we are allocating or freeing one, we can derive the state of the * other. That is, if we allocate a small block, and both were * free, the remainder of the region must be split into blocks. * If a block is freed, and its buddy is also free, then this * triggers coalescing into a block of larger size. * * -- nyc */
static inline void __free_one_page(struct page *page, unsigned long pfn, struct zone *zone, unsigned int order, int migratetype) { unsigned long page_idx; unsigned long combined_idx; unsigned long uninitialized_var(buddy_idx); struct page *buddy; unsigned int max_order; max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1