Release 4.8 mm/page_alloc.c

Directory: mm
/*
 *  linux/mm/page_alloc.c
 *
 *  Manages the free list, the system allocates free pages here.
 *  Note that kmalloc() lives in slab.c
 *
 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 *  Swap reorganised 29.12.95, Stephen Tweedie
 *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
 *  Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
 *  Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
 *  Zone balancing, Kanoj Sarcar, SGI, Jan 2000
 *  Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
 *          (lots of bits borrowed from Ingo Molnar & Andrew Morton)
 */

#include <linux/stddef.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/interrupt.h>
#include <linux/pagemap.h>
#include <linux/jiffies.h>
#include <linux/bootmem.h>
#include <linux/memblock.h>
#include <linux/compiler.h>
#include <linux/kernel.h>
#include <linux/kmemcheck.h>
#include <linux/kasan.h>
#include <linux/module.h>
#include <linux/suspend.h>
#include <linux/pagevec.h>
#include <linux/blkdev.h>
#include <linux/slab.h>
#include <linux/ratelimit.h>
#include <linux/oom.h>
#include <linux/notifier.h>
#include <linux/topology.h>
#include <linux/sysctl.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/memory_hotplug.h>
#include <linux/nodemask.h>
#include <linux/vmalloc.h>
#include <linux/vmstat.h>
#include <linux/mempolicy.h>
#include <linux/memremap.h>
#include <linux/stop_machine.h>
#include <linux/sort.h>
#include <linux/pfn.h>
#include <linux/backing-dev.h>
#include <linux/fault-inject.h>
#include <linux/page-isolation.h>
#include <linux/page_ext.h>
#include <linux/debugobjects.h>
#include <linux/kmemleak.h>
#include <linux/compaction.h>
#include <trace/events/kmem.h>
#include <linux/prefetch.h>
#include <linux/mm_inline.h>
#include <linux/migrate.h>
#include <linux/page_ext.h>
#include <linux/hugetlb.h>
#include <linux/sched/rt.h>
#include <linux/page_owner.h>
#include <linux/kthread.h>
#include <linux/memcontrol.h>

#include <asm/sections.h>
#include <asm/tlbflush.h>
#include <asm/div64.h>
#include "internal.h"

/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
static DEFINE_MUTEX(pcp_batch_high_lock);

#define MIN_PERCPU_PAGELIST_FRACTION	(8)

#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
DEFINE_PER_CPU(int, numa_node);

EXPORT_PER_CPU_SYMBOL(numa_node);
#endif

#ifdef CONFIG_HAVE_MEMORYLESS_NODES
/*
 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
 * defined in <linux/topology.h>.
 */
DEFINE_PER_CPU(int, _numa_mem_);		/* Kernel "local memory" node */

EXPORT_PER_CPU_SYMBOL(_numa_mem_);

int _node_numa_mem_[MAX_NUMNODES];
#endif

/*
 * Array of node states.
 */

nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
	[N_POSSIBLE] = NODE_MASK_ALL,
	[N_ONLINE] = { { [0] = 1UL } },
#ifndef CONFIG_NUMA
	[N_NORMAL_MEMORY] = { { [0] = 1UL } },
#ifdef CONFIG_HIGHMEM
	[N_HIGH_MEMORY] = { { [0] = 1UL } },
#endif
#ifdef CONFIG_MOVABLE_NODE
	[N_MEMORY] = { { [0] = 1UL } },
#endif
	[N_CPU] = { { [0] = 1UL } },
#endif	/* NUMA */
};

EXPORT_SYMBOL(node_states);

/* Protect totalram_pages and zone->managed_pages */
static DEFINE_SPINLOCK(managed_page_count_lock);


unsigned long totalram_pages __read_mostly;

unsigned long totalreserve_pages __read_mostly;

unsigned long totalcma_pages __read_mostly;


int percpu_pagelist_fraction;

gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;

/*
 * A cached value of the page's pageblock's migratetype, used when the page is
 * put on a pcplist. Used to avoid the pageblock migratetype lookup when
 * freeing from pcplists in most cases, at the cost of possibly becoming stale.
 * Also the migratetype set in the page does not necessarily match the pcplist
 * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any
 * other index - this ensures that it will be put on the correct CMA freelist.
 */


static inline int get_pcppage_migratetype(struct page *page)
{
	return page->index;
}

Contributors
Person Tokens Prop Commits CommitProp
vlastimil babka vlastimil babka 17 100.00% 1 100.00%
Total 17 100.00% 1 100.00%



static inline void set_pcppage_migratetype(struct page *page, int migratetype)
{
	page->index = migratetype;
}

Contributors
Person Tokens Prop Commits CommitProp
vlastimil babka vlastimil babka 21 100.00% 1 100.00%
Total 21 100.00% 1 100.00%

#ifdef CONFIG_PM_SLEEP
/*
 * The following functions are used by the suspend/hibernate code to temporarily
 * change gfp_allowed_mask in order to avoid using I/O during memory allocations
 * while devices are suspended.  To avoid races with the suspend/hibernate code,
 * they should always be called with pm_mutex held (gfp_allowed_mask also should
 * only be modified with pm_mutex held, unless the suspend/hibernate code is
 * guaranteed not to run in parallel with that modification).
 */


static gfp_t saved_gfp_mask;



void pm_restore_gfp_mask(void)
{
	WARN_ON(!mutex_is_locked(&pm_mutex));
	if (saved_gfp_mask) {
		gfp_allowed_mask = saved_gfp_mask;
		saved_gfp_mask = 0;
	}
}

Contributors
Person Tokens Prop Commits CommitProp
rafael j. wysocki rafael j. wysocki 31 100.00% 2 100.00%
Total 31 100.00% 2 100.00%



void pm_restrict_gfp_mask(void)
{
	WARN_ON(!mutex_is_locked(&pm_mutex));
	WARN_ON(saved_gfp_mask);
	saved_gfp_mask = gfp_allowed_mask;
	gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS);
}

Contributors
Person Tokens Prop Commits CommitProp
rafael j. wysocki rafael j. wysocki 30 85.71% 2 66.67%
mel gorman mel gorman 5 14.29% 1 33.33%
Total 35 100.00% 3 100.00%



bool pm_suspended_storage(void)
{
	if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
		return false;
	return true;
}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 31 100.00% 2 100.00%
Total 31 100.00% 2 100.00%

#endif /* CONFIG_PM_SLEEP */

#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE

unsigned int pageblock_order __read_mostly;
#endif

static void __free_pages_ok(struct page *page, unsigned int order);

/*
 * results with 256, 32 in the lowmem_reserve sysctl:
 *      1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
 *      1G machine -> (16M dma, 784M normal, 224M high)
 *      NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
 *      HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
 *      HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA
 *
 * TBD: should special case ZONE_DMA32 machines here - in those we normally
 * don't need any ZONE_NORMAL reservation
 */

int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = {
#ifdef CONFIG_ZONE_DMA
	 256,
#endif
#ifdef CONFIG_ZONE_DMA32
	 256,
#endif
#ifdef CONFIG_HIGHMEM
	 32,
#endif
	 32,
};


EXPORT_SYMBOL(totalram_pages);


static char * const zone_names[MAX_NR_ZONES] = {
#ifdef CONFIG_ZONE_DMA
	 "DMA",
#endif
#ifdef CONFIG_ZONE_DMA32
	 "DMA32",
#endif
	 "Normal",
#ifdef CONFIG_HIGHMEM
	 "HighMem",
#endif
	 "Movable",
#ifdef CONFIG_ZONE_DEVICE
	 "Device",
#endif
};


char * const migratetype_names[MIGRATE_TYPES] = {
	"Unmovable",
	"Movable",
	"Reclaimable",
	"HighAtomic",
#ifdef CONFIG_CMA
	"CMA",
#endif
#ifdef CONFIG_MEMORY_ISOLATION
	"Isolate",
#endif
};


compound_page_dtor * const compound_page_dtors[] = {
	NULL,
	free_compound_page,
#ifdef CONFIG_HUGETLB_PAGE
	free_huge_page,
#endif
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	free_transhuge_page,
#endif
};


int min_free_kbytes = 1024;

int user_min_free_kbytes = -1;

int watermark_scale_factor = 10;


static unsigned long __meminitdata nr_kernel_pages;

static unsigned long __meminitdata nr_all_pages;

static unsigned long __meminitdata dma_reserve;

#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP

static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES];

static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES];

static unsigned long __initdata required_kernelcore;

static unsigned long __initdata required_movablecore;

static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];

static bool mirrored_kernelcore;

/* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */

int movable_zone;

EXPORT_SYMBOL(movable_zone);
#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */

#if MAX_NUMNODES > 1

int nr_node_ids __read_mostly = MAX_NUMNODES;

int nr_online_nodes __read_mostly = 1;

EXPORT_SYMBOL(nr_node_ids);

EXPORT_SYMBOL(nr_online_nodes);
#endif


int page_group_by_mobility_disabled __read_mostly;

#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT


static inline void reset_deferred_meminit(pg_data_t *pgdat)
{
	pgdat->first_deferred_pfn = ULONG_MAX;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 17 100.00% 1 100.00%
Total 17 100.00% 1 100.00%

/* Returns true if the struct page for the pfn is uninitialised */


static inline bool __meminit early_page_uninitialised(unsigned long pfn)
{
	int nid = early_pfn_to_nid(pfn);

	if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)
		return true;

	return false;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 42 100.00% 3 100.00%
Total 42 100.00% 3 100.00%

/*
 * Returns false when the remaining initialisation should be deferred until
 * later in the boot cycle when it can be parallelised.
 */


static inline bool update_defer_init(pg_data_t *pgdat,
				unsigned long pfn, unsigned long zone_end,
				unsigned long *nr_initialised)
{
	unsigned long max_initialise;

	/* Always populate low zones for address-contrained allocations */
	if (zone_end < pgdat_end_pfn(pgdat))
		return true;
	/*
         * Initialise at least 2G of a node but also take into account that
         * two large system hashes that can take up 1GB for 0.25TB/node.
         */
	max_initialise = max(2UL << (30 - PAGE_SHIFT),
		(pgdat->node_spanned_pages >> 8));

	(*nr_initialised)++;
	if ((*nr_initialised > max_initialise) &&
	    (pfn & (PAGES_PER_SECTION - 1)) == 0) {
		pgdat->first_deferred_pfn = pfn;
		return false;
	}

	return true;
}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 76 73.08% 1 50.00%
li zhang li zhang 28 26.92% 1 50.00%
Total 104 100.00% 2 100.00%

#else


static inline void reset_deferred_meminit(pg_data_t *pgdat)
{
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 10 100.00% 1 100.00%
Total 10 100.00% 1 100.00%



static inline bool early_page_uninitialised(unsigned long pfn)
{
	return false;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 14 100.00% 1 100.00%
Total 14 100.00% 1 100.00%



static inline bool update_defer_init(pg_data_t *pgdat,
				unsigned long pfn, unsigned long zone_end,
				unsigned long *nr_initialised)
{
	return true;
}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 27 100.00% 1 100.00%
Total 27 100.00% 1 100.00%

#endif

/* Return a pointer to the bitmap storing bits affecting a block of pages */


static inline unsigned long *get_pageblock_bitmap(struct page *page,
							unsigned long pfn)
{
#ifdef CONFIG_SPARSEMEM
	return __pfn_to_section(pfn)->pageblock_flags;
#else
	return page_zone(page)->pageblock_flags;
#endif /* CONFIG_SPARSEMEM */
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 42 100.00% 3 100.00%
Total 42 100.00% 3 100.00%



static inline int pfn_to_bitidx(struct page *page, unsigned long pfn)
{
#ifdef CONFIG_SPARSEMEM
	pfn &= (PAGES_PER_SECTION-1);
	return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
#else
	pfn = pfn - round_down(page_zone(page)->zone_start_pfn, pageblock_nr_pages);
	return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
#endif /* CONFIG_SPARSEMEM */
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 42 63.64% 1 11.11%
dave hansen dave hansen 12 18.18% 3 33.33%
william lee irwin iii william lee irwin iii 5 7.58% 1 11.11%
cody p schafer cody p schafer 4 6.06% 1 11.11%
rik van riel rik van riel 1 1.52% 1 11.11%
andrew morton andrew morton 1 1.52% 1 11.11%
pre-git pre-git 1 1.52% 1 11.11%
Total 66 100.00% 9 100.00%

/**
 * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages
 * @page: The page within the block of interest
 * @pfn: The target page frame number
 * @end_bitidx: The last bit of interest to retrieve
 * @mask: mask of bits that the caller is interested in
 *
 * Return: pageblock_bits flags
 */


static __always_inline unsigned long __get_pfnblock_flags_mask(struct page *page,
					unsigned long pfn,
					unsigned long end_bitidx,
					unsigned long mask)
{
	unsigned long *bitmap;
	unsigned long bitidx, word_bitidx;
	unsigned long word;

	bitmap = get_pageblock_bitmap(page, pfn);
	bitidx = pfn_to_bitidx(page, pfn);
	word_bitidx = bitidx / BITS_PER_LONG;
	bitidx &= (BITS_PER_LONG-1);

	word = bitmap[word_bitidx];
	bitidx += end_bitidx;
	return (word >> (BITS_PER_LONG - bitidx - 1)) & mask;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 80 81.63% 1 25.00%
dave hansen dave hansen 12 12.24% 1 25.00%
william lee irwin iii william lee irwin iii 3 3.06% 1 25.00%
kamezawa hiroyuki kamezawa hiroyuki 3 3.06% 1 25.00%
Total 98 100.00% 4 100.00%



unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn,
					unsigned long end_bitidx,
					unsigned long mask)
{
	return __get_pfnblock_flags_mask(page, pfn, end_bitidx, mask);
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 22 62.86% 1 33.33%
dave hansen dave hansen 11 31.43% 1 33.33%
william lee irwin iii william lee irwin iii 2 5.71% 1 33.33%
Total 35 100.00% 3 100.00%



static __always_inline int get_pfnblock_migratetype(struct page *page, unsigned long pfn)
{
	return __get_pfnblock_flags_mask(page, pfn, PB_migrate_end, MIGRATETYPE_MASK);
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 16 57.14% 1 50.00%
nick piggin nick piggin 12 42.86% 1 50.00%
Total 28 100.00% 2 100.00%

/**
 * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
 * @page: The page within the block of interest
 * @flags: The flags to set
 * @pfn: The target page frame number
 * @end_bitidx: The last bit of interest
 * @mask: mask of bits that the caller is interested in
 */


void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
					unsigned long pfn,
					unsigned long end_bitidx,
					unsigned long mask)
{
	unsigned long *bitmap;
	unsigned long bitidx, word_bitidx;
	unsigned long old_word, word;

	BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);

	bitmap = get_pageblock_bitmap(page, pfn);
	bitidx = pfn_to_bitidx(page, pfn);
	word_bitidx = bitidx / BITS_PER_LONG;
	bitidx &= (BITS_PER_LONG-1);

	VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);

	bitidx += end_bitidx;
	mask <<= (BITS_PER_LONG - bitidx - 1);
	flags <<= (BITS_PER_LONG - bitidx - 1);

	word = READ_ONCE(bitmap[word_bitidx]);
	for (;;) {
		old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);
		if (word == old_word)
			break;
		word = old_word;
	}
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 117 68.02% 1 7.14%
hugh dickins hugh dickins 25 14.53% 2 14.29%
andrew morton andrew morton 12 6.98% 2 14.29%
vlastimil babka vlastimil babka 5 2.91% 2 14.29%
dave hansen dave hansen 4 2.33% 1 7.14%
fengguang wu fengguang wu 3 1.74% 1 7.14%
randy dunlap randy dunlap 2 1.16% 1 7.14%
nick piggin nick piggin 1 0.58% 1 7.14%
rusty russell rusty russell 1 0.58% 1 7.14%
william lee irwin iii william lee irwin iii 1 0.58% 1 7.14%
dave jones dave jones 1 0.58% 1 7.14%
Total 172 100.00% 14 100.00%



void set_pageblock_migratetype(struct page *page, int migratetype)
{
	if (unlikely(page_group_by_mobility_disabled &&
		     migratetype < MIGRATE_PCPTYPES))
		migratetype = MIGRATE_UNMOVABLE;

	set_pageblock_flags_group(page, (unsigned long)migratetype,
					PB_migrate, PB_migrate_end);
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 29 67.44% 1 33.33%
hugh dickins hugh dickins 13 30.23% 1 33.33%
christoph lameter christoph lameter 1 2.33% 1 33.33%
Total 43 100.00% 3 100.00%

#ifdef CONFIG_DEBUG_VM


static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
{
	int ret = 0;
	unsigned seq;
	unsigned long pfn = page_to_pfn(page);
	unsigned long sp, start_pfn;

	do {
		seq = zone_span_seqbegin(zone);
		start_pfn = zone->zone_start_pfn;
		sp = zone->spanned_pages;
		if (!zone_spans_pfn(zone, pfn))
			ret = 1;
	} while (zone_span_seqretry(zone, seq));

	if (ret)
		pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
			pfn, zone_to_nid(zone), zone->name,
			start_pfn, start_pfn + sp);

	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 81 71.05% 1 12.50%
andrew morton andrew morton 10 8.77% 1 12.50%
andy whitcroft andy whitcroft 9 7.89% 1 12.50%
kirill a. shutemov kirill a. shutemov 9 7.89% 3 37.50%
youquan song youquan song 3 2.63% 1 12.50%
david rientjes david rientjes 2 1.75% 1 12.50%
Total 114 100.00% 8 100.00%



static int page_is_consistent(struct zone *zone, struct page *page)
{
	if (!pfn_valid_within(page_to_pfn(page)))
		return 0;
	if (zone != page_zone(page))
		return 0;

	return 1;
}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 24 53.33% 1 33.33%
joonsoo kim joonsoo kim 16 35.56% 1 33.33%
christian borntraeger christian borntraeger 5 11.11% 1 33.33%
Total 45 100.00% 3 100.00%

/*
 * Temporary debugging check for pages not lying within a given zone.
 */


static int bad_range(struct zone *zone, struct page *page)
{
	if (page_outside_zone_boundaries(zone, page))
		return 1;
	if (!page_is_consistent(zone, page))
		return 1;

	return 0;
}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 31 70.45% 1 33.33%
joonsoo kim joonsoo kim 13 29.55% 2 66.67%
Total 44 100.00% 3 100.00%

#else


static inline int bad_range(struct zone *zone, struct page *page)
{
	return 0;
}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 15 75.00% 1 50.00%
joonsoo kim joonsoo kim 5 25.00% 1 50.00%
Total 20 100.00% 2 100.00%

#endif



static void bad_page(struct page *page, const char *reason,
		unsigned long bad_flags)
{
	static unsigned long resume;
	static unsigned long nr_shown;
	static unsigned long nr_unshown;

	/*
         * Allow a burst of 60 reports, then keep quiet for that minute;
         * or allow a steady drip of one report per second.
         */
	if (nr_shown == 60) {
		if (time_before(jiffies, resume)) {
			nr_unshown++;
			goto out;
		}
		if (nr_unshown) {
			pr_alert(
			      "BUG: Bad page state: %lu messages suppressed\n",
				nr_unshown);
			nr_unshown = 0;
		}
		nr_shown = 0;
	}
	if (nr_shown++ == 0)
		resume = jiffies + 60 * HZ;

	pr_alert("BUG: Bad page state in process %s  pfn:%05lx\n",
		current->comm, page_to_pfn(page));
	__dump_page(page, reason);
	bad_flags &= page->flags;
	if (bad_flags)
		pr_alert("bad because of flags: %#lx(%pGp)\n",
						bad_flags, &bad_flags);
	dump_page_owner(page);

	print_modules();
	dump_stack();
out:
	/* Leave bad fields for debug, except PageBuddy could make trouble */
	page_mapcount_reset(page); /* remove PageBuddy */
	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 120 72.73% 1 25.00%
stanislaw gruszka stanislaw gruszka 23 13.94% 1 25.00%
joonsoo kim joonsoo kim 22 13.33% 2 50.00%
Total 165 100.00% 4 100.00%

/*
 * Higher-order pages are called "compound pages".  They are structured thusly:
 *
 * The first PAGE_SIZE page is called the "head page" and have PG_head set.
 *
 * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded
 * in bit 0 of page->compound_head. The rest of bits is pointer to head page.
 *
 * The first tail page's ->compound_dtor holds the offset in array of compound
 * page destructors. See compound_page_dtors.
 *
 * The first tail page's ->compound_order holds the order of allocation.
 * This usage means that zero-order pages may not be compound.
 */



void free_compound_page(struct page *page)
{
	__free_pages_ok(page, compound_order(page));
}

Contributors
Person Tokens Prop Commits CommitProp
stanislaw gruszka stanislaw gruszka 10 50.00% 1 25.00%
joonsoo kim joonsoo kim 7 35.00% 2 50.00%
mel gorman mel gorman 3 15.00% 1 25.00%
Total 20 100.00% 4 100.00%



void prep_compound_page(struct page *page, unsigned int order)
{
	int i;
	int nr_pages = 1 << order;

	set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
	set_compound_order(page, order);
	__SetPageHead(page);
	for (i = 1; i < nr_pages; i++) {
		struct page *p = page + i;
		set_page_count(p, 0);
		p->mapping = TAIL_MAPPING;
		set_compound_head(p, page);
	}
	atomic_set(compound_mapcount_ptr(page), -1);
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 60 61.22% 1 20.00%
kamezawa hiroyuki kamezawa hiroyuki 20 20.41% 1 20.00%
stanislaw gruszka stanislaw gruszka 7 7.14% 1 20.00%
hugh dickins hugh dickins 6 6.12% 1 20.00%
joonsoo kim joonsoo kim 5 5.10% 1 20.00%
Total 98 100.00% 5 100.00%

#ifdef CONFIG_DEBUG_PAGEALLOC

unsigned int _debug_guardpage_minorder;

bool _debug_pagealloc_enabled __read_mostly
			= IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT);

EXPORT_SYMBOL(_debug_pagealloc_enabled);

bool _debug_guardpage_enabled __read_mostly;



static int __init early_debug_pagealloc(char *buf)
{
	if (!buf)
		return -EINVAL;
	return kstrtobool(buf, &_debug_pagealloc_enabled);
}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 22 75.86% 2 40.00%
nick piggin nick piggin 3 10.34% 1 20.00%
minfei huang minfei huang 3 10.34% 1 20.00%
stanislaw gruszka stanislaw gruszka 1 3.45% 1 20.00%
Total 29 100.00% 5 100.00%

early_param("debug_pagealloc", early_debug_pagealloc);



static bool need_debug_guardpage(void)
{
	/* If we don't use debug_pagealloc, we don't need guard page */
	if (!debug_pagealloc_enabled())
		return false;

	return true;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 18 85.71% 1 50.00%
stanislaw gruszka stanislaw gruszka 3 14.29% 1 50.00%
Total 21 100.00% 2 100.00%



static void init_debug_guardpage(void)
{
	if (!debug_pagealloc_enabled())
		return;

	_debug_guardpage_enabled = true;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 16 84.21% 1 33.33%
stanislaw gruszka stanislaw gruszka 2 10.53% 1 33.33%
nick piggin nick piggin 1 5.26% 1 33.33%
Total 19 100.00% 3 100.00%


struct page_ext_operations debug_guardpage_ops = {
	.need = need_debug_guardpage,
	.init = init_debug_guardpage,
};



static int __init debug_guardpage_minorder_setup(char *buf)
{
	unsigned long res;

	if (kstrtoul(buf, 10, &res) < 0 ||  res > MAX_ORDER / 2) {
		pr_err("Bad debug_guardpage_minorder value\n");
		return 0;
	}
	_debug_guardpage_minorder = res;
	pr_info("Setting debug_guardpage_minorder to %lu\n", res);
	return 0;
}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 51 86.44% 2 50.00%
weijie yang weijie yang 4 6.78% 1 25.00%
kamezawa hiroyuki kamezawa hiroyuki 4 6.78% 1 25.00%
Total 59 100.00% 4 100.00%

__setup("debug_guardpage_minorder=", debug_guardpage_minorder_setup);



static inline void set_page_guard(struct zone *zone, struct page *page,
				unsigned int order, int migratetype)
{
	struct page_ext *page_ext;

	if (!debug_guardpage_enabled())
		return;

	page_ext = lookup_page_ext(page);
	if (unlikely(!page_ext))
		return;

	__set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);

	INIT_LIST_HEAD(&page->lru);
	set_page_private(page, order);
	/* Guard pages are not available for any usage */
	__mod_zone_freepage_state(zone, -(1 << order), migratetype);
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 49 53.26% 4 20.00%
yang shi yang shi 9 9.78% 1 5.00%
pre-git pre-git 8 8.70% 3 15.00%
andrew morton andrew morton 5 5.43% 3 15.00%
joonsoo kim joonsoo kim 5 5.43% 2 10.00%
vlastimil babka vlastimil babka 4 4.35% 1 5.00%
cody p schafer cody p schafer 3 3.26% 1 5.00%
kirill a. shutemov kirill a. shutemov 3 3.26% 1 5.00%
kamezawa hiroyuki kamezawa hiroyuki 2 2.17% 1 5.00%
corrado zoccolo corrado zoccolo 2 2.17% 1 5.00%
hugh dickins hugh dickins 1 1.09% 1 5.00%
sasha levin sasha levin 1 1.09% 1 5.00%
Total 92 100.00% 20 100.00%



static inline void clear_page_guard(struct zone *zone, struct page *page,
				unsigned int order, int migratetype)
{
	struct page_ext *page_ext;

	if (!debug_guardpage_enabled())
		return;

	page_ext = lookup_page_ext(page);
	if (unlikely(!page_ext))
		return;

	__clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);

	set_page_private(page, 0);
	if (!is_migrate_isolate(migratetype))
		__mod_zone_freepage_state(zone, (1 << order), migratetype);
}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 53 58.89% 1 6.67%
stanislaw gruszka stanislaw gruszka 12 13.33% 1 6.67%
yang shi yang shi 9 10.00% 1 6.67%
pre-git pre-git 4 4.44% 4 26.67%
dave hansen dave hansen 2 2.22% 1 6.67%
kamezawa hiroyuki kamezawa hiroyuki 2 2.22% 1 6.67%
bartlomiej zolnierkiewicz bartlomiej zolnierkiewicz 2 2.22% 1 6.67%
andy whitcroft andy whitcroft 2 2.22% 1 6.67%
sasha levin sasha levin 1 1.11% 1 6.67%
kyongho cho kyongho cho 1 1.11% 1 6.67%
martin bligh martin bligh 1 1.11% 1 6.67%
andrew morton andrew morton 1 1.11% 1 6.67%
Total 90 100.00% 15 100.00%

#else

struct page_ext_operations debug_guardpage_ops = { NULL, };


static inline void set_page_guard(struct zone *zone, struct page *page,
				unsigned int order, int migratetype) {}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 23 100.00% 1 100.00%
Total 23 100.00% 1 100.00%



static inline void clear_page_guard(struct zone *zone, struct page *page,
				unsigned int order, int migratetype) {}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 23 100.00% 1 100.00%
Total 23 100.00% 1 100.00%

#endif



static inline void set_page_order(struct page *page, unsigned int order)
{
	set_page_private(page, order);
	__SetPageBuddy(page);
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 23 82.14% 1 25.00%
kamezawa hiroyuki kamezawa hiroyuki 2 7.14% 1 25.00%
pre-git pre-git 2 7.14% 1 25.00%
stanislaw gruszka stanislaw gruszka 1 3.57% 1 25.00%
Total 28 100.00% 4 100.00%



static inline void rmv_page_order(struct page *page)
{
	__ClearPageBuddy(page);
	set_page_private(page, 0);
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 19 79.17% 1 25.00%
dave hansen dave hansen 2 8.33% 1 25.00%
pre-git pre-git 2 8.33% 1 25.00%
kamezawa hiroyuki kamezawa hiroyuki 1 4.17% 1 25.00%
Total 24 100.00% 4 100.00%

/*
 * This function checks whether a page is free && is the buddy
 * we can do coalesce a page and its buddy if
 * (a) the buddy is not in a hole &&
 * (b) the buddy is in the buddy system &&
 * (c) a page and its buddy have the same order &&
 * (d) a page and its buddy are in the same zone.
 *
 * For recording whether a page is in the buddy system, we set ->_mapcount
 * PAGE_BUDDY_MAPCOUNT_VALUE.
 * Setting, clearing, and testing _mapcount PAGE_BUDDY_MAPCOUNT_VALUE is
 * serialized by zone->lock.
 *
 * For recording page's order, we use page_private(page).
 */


static inline int page_is_buddy(struct page *page, struct page *buddy,
							unsigned int order)
{
	if (!pfn_valid_within(page_to_pfn(buddy)))
		return 0;

	if (page_is_guard(buddy) && page_order(buddy) == order) {
		if (page_zone_id(page) != page_zone_id(buddy))
			return 0;

		VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);

		return 1;
	}

	if (PageBuddy(buddy) && page_order(buddy) == order) {
		/*
                 * zone check is done late to avoid uselessly
                 * calculating zone/node ids for pages that could
                 * never merge.
                 */
		if (page_zone_id(page) != page_zone_id(buddy))
			return 0;

		VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy);

		return 1;
	}
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 99 75.57% 1 25.00%
vlastimil babka vlastimil babka 20 15.27% 1 25.00%
corrado zoccolo corrado zoccolo 8 6.11% 1 25.00%
kamezawa hiroyuki kamezawa hiroyuki 4 3.05% 1 25.00%
Total 131 100.00% 4 100.00%

/*
 * Freeing function for a buddy system allocator.
 *
 * The concept of a buddy system is to maintain direct-mapped table
 * (containing bit values) for memory blocks of various "orders".
 * The bottom level table contains the map for the smallest allocatable
 * units of memory (here, pages), and each level above it describes
 * pairs of units from the levels below, hence, "buddies".
 * At a high level, all that happens here is marking the table entry
 * at the bottom level available, and propagating the changes upward
 * as necessary, plus some accounting needed to play nicely with other
 * parts of the VM system.
 * At each level, we keep a list of pages, which are heads of continuous
 * free pages of length of (1 << order) and marked with _mapcount
 * PAGE_BUDDY_MAPCOUNT_VALUE. Page's order is recorded in page_private(page)
 * field.
 * So when we are allocating or freeing one, we can derive the state of the
 * other.  That is, if we allocate a small block, and both were
 * free, the remainder of the region must be split into blocks.
 * If a block is freed, and its buddy is also free, then this
 * triggers coalescing into a block of larger size.
 *
 * -- nyc
 */



static inline void __free_one_page(struct page *page,
		unsigned long pfn,
		struct zone *zone, unsigned int order,
		int migratetype)
{
	unsigned long page_idx;
	unsigned long combined_idx;
	unsigned long uninitialized_var(buddy_idx);
	struct page *buddy;
	unsigned int max_order;

	max_order = min_t(unsigned int, MAX_ORDER, pageblock_order + 1);

	VM_BUG_ON(!zone_is_initialized(zone));
	VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);

	VM_BUG_ON(migratetype == -1);
	if (likely(!is_migrate_isolate(migratetype)))
		__mod_zone_freepage_state(zone, 1 << order, migratetype);

	page_idx = pfn & ((1 << MAX_ORDER) - 1);

	VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page);
	VM_BUG_ON_PAGE(bad_range(zone, page), page);

continue_merging:
	while (order < max_order - 1) {
		buddy_idx = __find_buddy_index(page_idx, order);
		buddy = page + (buddy_idx - page_idx);
		if (!page_is_buddy(page, buddy, order))
			goto done_merging;
		/*
                 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
                 * merge with it and move up one order.
                 */
		if (page_is_guard(buddy)) {
			clear_page_guard(zone, buddy, order, migratetype);
		} else {
			list_del(&buddy->lru);
			zone->free_area[order].nr_free--;
			rmv_page_order(buddy);
		}
		combined_idx = buddy_idx & page_idx;
		page = page + (combined_idx - page_idx);
		page_idx = combined_idx;
		order++;
	}
	if (max_order < MAX_ORDER) {
		/* If we are here, it means order is >= pageblock_order.
                 * We want to prevent merge between freepages on isolate
                 * pageblock and normal pageblock. Without this, pageblock
                 * isolation could cause incorrect freepage or CMA accounting.
                 *
                 * We don't want to hit this code for the more frequent
                 * low-order merging.
                 */
		if (unlikely(has_isolate_pageblock(zone))) {
			int buddy_mt;

			buddy_idx = __find_buddy_index(page_idx, order);
			buddy = page + (buddy_idx - page_idx);
			buddy_mt = get_pageblock_migratetype(buddy);

			if (migratetype != buddy_mt
					&& (is_migrate_isolate(migratetype) ||
						is_migrate_isolate(buddy_mt)))
				goto done_merging;
		}
		max_order++;
		goto continue_merging;
	}

done_merging:
	set_page_order(page, order);

	/*
         * If this is not the largest possible page, check if the buddy
         * of the next-highest order is free. If it is, it's possible
         * that pages are being freed that will coalesce soon. In case,
         * that is happening, add the free page to the tail of the list
         * so it's less likely to be used soon and more likely to be merged
         * as a higher order page
         */
	if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
		struct page *higher_page, *higher_buddy;
		combined_idx = buddy_idx & page_idx;
		higher_page = page + (combined_idx - page_idx);
		buddy_idx = __find_buddy_index(combined_idx, order + 1);
		higher_buddy = higher_page + (buddy_idx - combined_idx);
		if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
			list_add_tail(&page->lru,
				&zone->free_area[order].free_list[migratetype]);
			goto out;
		}
	}

	list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
out:
	zone->free_area[order].nr_free++;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 405 81.49% 2 16.67%
corrado zoccolo corrado zoccolo 50 10.06% 1 8.33%
kamezawa hiroyuki kamezawa hiroyuki 15 3.02% 1 8.33%
kyongho cho kyongho cho 13 2.62% 1 8.33%
pre-git pre-git 7 1.41% 3 25.00%
nick piggin nick piggin 4 0.80% 1 8.33%
andrew morton andrew morton 2 0.40% 2 16.67%
dave hansen dave hansen 1 0.20% 1 8.33%
Total 497 100.00% 12 100.00%

/*
 * A bad page could be due to a number of fields. Instead of multiple branches,
 * try and check multiple fields with one check. The caller must do a detailed
 * check if necessary.
 */


static inline bool page_expected_state(struct page *page,
					unsigned long check_flags)
{
	if (unlikely(atomic_read(&page->_mapcount) != -1))
		return false;

	if (unlikely((unsigned long)page->mapping |
			page_ref_count(page) |
#ifdef CONFIG_MEMCG
			(unsigned long)page->mem_cgroup |
#endif
			(page->flags & check_flags)))
		return false;

	return true;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 80 100.00% 1 100.00%
Total 80 100.00% 1 100.00%



static void free_pages_check_bad(struct page *page)
{
	const char *bad_reason;
	unsigned long bad_flags;

	bad_reason = NULL;
	bad_flags = 0;

	if (unlikely(atomic_read(&page->_mapcount) != -1))
		bad_reason = "nonzero mapcount";
	if (unlikely(page->mapping != NULL))
		bad_reason = "non-NULL mapping";
	if (unlikely(page_ref_count(page) != 0))
		bad_reason = "nonzero _refcount";
	if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) {
		bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
		bad_flags = PAGE_FLAGS_CHECK_AT_FREE;
	}
#ifdef CONFIG_MEMCG
	if (unlikely(page->mem_cgroup))
		bad_reason = "page still charged to cgroup";
#endif
	bad_page(page, bad_reason, bad_flags);
}

Contributors
Person Tokens Prop Commits CommitProp
dave hansen dave hansen 62 48.82% 1 8.33%
andrew morton andrew morton 30 23.62% 1 8.33%
mel gorman mel gorman 13 10.24% 2 16.67%
kirill a. shutemov kirill a. shutemov 8 6.30% 2 16.67%
johannes weiner johannes weiner 8 6.30% 1 8.33%
nick piggin nick piggin 2 1.57% 1 8.33%
joonsoo kim joonsoo kim 2 1.57% 2 16.67%
daisuke nishimura daisuke nishimura 1 0.79% 1 8.33%
russ anderson russ anderson 1 0.79% 1 8.33%
Total 127 100.00% 12 100.00%



static inline int free_pages_check(struct page *page)
{
	if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE)))
		return 0;

	/* Something has gone sideways, find it */
	free_pages_check_bad(page);
	return 1;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 32 88.89% 1 33.33%
hugh dickins hugh dickins 4 11.11% 2 66.67%
Total 36 100.00% 3 100.00%



static int free_tail_pages_check(struct page *head_page, struct page *page)
{
	int ret = 1;

	/*
         * We rely page->lru.next never has bit 0 set, unless the page
         * is PageTail(). Let's make sure that's true even for poisoned ->lru.
         */
	BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1);

	if (!IS_ENABLED(CONFIG_DEBUG_VM)) {
		ret = 0;
		goto out;
	}
	switch (page - head_page) {
	case 1:
		/* the first tail page: ->mapping is compound_mapcount() */
		if (unlikely(compound_mapcount(page))) {
			bad_page(page, "nonzero compound_mapcount", 0);
			goto out;
		}
		break;
	case 2:
		/*
                 * the second tail page: ->mapping is
                 * page_deferred_list().next -- ignore value.
                 */
		break;
	default:
		if (page->mapping != TAIL_MAPPING) {
			bad_page(page, "corrupted mapping in tail page", 0);
			goto out;
		}
		break;
	}
	if (unlikely(!PageTail(page))) {
		bad_page(page, "PageTail not set", 0);
		goto out;
	}
	if (unlikely(compound_head(page) != head_page)) {
		bad_page(page, "compound_head not consistent", 0);
		goto out;
	}
	ret = 0;
out:
	page->mapping = NULL;
	clear_compound_head(page);
	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 187 100.00% 1 100.00%
Total 187 100.00% 1 100.00%



static __always_inline bool free_pages_prepare(struct page *page,
					unsigned int order, bool check_free)
{
	int bad = 0;

	VM_BUG_ON_PAGE(PageTail(page), page);

	trace_mm_page_free(page, order);
	kmemcheck_free_shadow(page, order);

	/*
         * Check tail pages before head page information is cleared to
         * avoid checking PageCompound for order-0 pages.
         */
	if (unlikely(order)) {
		bool compound = PageCompound(page);
		int i;

		VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);

		if (compound)
			ClearPageDoubleMap(page);
		for (i = 1; i < (1 << order); i++) {
			if (compound)
				bad += free_tail_pages_check(page, page + i);
			if (unlikely(free_pages_check(page + i))) {
				bad++;
				continue;
			}
			(page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
		}
	}
	if (PageMappingFlags(page))
		page->mapping = NULL;
	if (memcg_kmem_enabled() && PageKmemcg(page))
		memcg_kmem_uncharge(page, order);
	if (check_free)
		bad += free_pages_check(page);
	if (bad)
		return false;

	page_cpupid_reset_last(page);
	page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
	reset_page_owner(page, order);

	if (!PageHighMem(page)) {
		debug_check_no_locks_freed(page_address(page),
					   PAGE_SIZE << order);
		debug_check_no_obj_freed(page_address(page),
					   PAGE_SIZE << order);
	}
	arch_free_page(page, order);
	kernel_poison_pages(page, 1 << order, 0);
	kernel_map_pages(page, 1 << order, 0);
	kasan_free_pages(page, order);

	return true;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 261 88.47% 2 33.33%
vladimir davydov vladimir davydov 17 5.76% 1 16.67%
kirill a. shutemov kirill a. shutemov 9 3.05% 1 16.67%
seokhoon yoon seokhoon yoon 7 2.37% 1 16.67%
minchan kim minchan kim 1 0.34% 1 16.67%
Total 295 100.00% 6 100.00%

#ifdef CONFIG_DEBUG_VM


static inline bool free_pcp_prepare(struct page *page)
{
	return free_pages_prepare(page, 0, true);
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 22 100.00% 2 100.00%
Total 22 100.00% 2 100.00%



static inline bool bulkfree_pcp_prepare(struct page *page)
{
	return false;
}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 15 100.00% 1 100.00%
Total 15 100.00% 1 100.00%

#else


static bool free_pcp_prepare(struct page *page)
{
	return free_pages_prepare(page, 0, false);
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 21 100.00% 1 100.00%
Total 21 100.00% 1 100.00%



static bool bulkfree_pcp_prepare(struct page *page)
{
	return free_pages_check(page);
}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 17 100.00% 1 100.00%
Total 17 100.00% 1 100.00%

#endif /* CONFIG_DEBUG_VM */

/*
 * Frees a number of pages from the PCP lists
 * Assumes all pages on list are in same zone, and of same order.
 * count is the number of pages to free.
 *
 * If the zone was previously in an "all pages pinned" state then look to
 * see if this freeing clears that state.
 *
 * And clear the zone's pages_scanned counter, to hold off the "all pages are
 * pinned" detection logic.
 */


static void free_pcppages_bulk(struct zone *zone, int count,
					struct per_cpu_pages *pcp)
{
	int migratetype = 0;
	int batch_free = 0;
	unsigned long nr_scanned;
	bool isolated_pageblocks;

	spin_lock(&zone->lock);
	isolated_pageblocks = has_isolate_pageblock(zone);
	nr_scanned = node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED);
	if (nr_scanned)
		__mod_node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED, -nr_scanned);

	while (count) {
		struct page *page;
		struct list_head *list;

		/*
                 * Remove pages from lists in a round-robin fashion. A
                 * batch_free count is maintained that is incremented when an
                 * empty list is encountered.  This is so more pages are freed
                 * off fuller lists instead of spinning excessively around empty
                 * lists
                 */
		do {
			batch_free++;
			if (++migratetype == MIGRATE_PCPTYPES)
				migratetype = 0;
			list = &pcp->lists[migratetype];
		} while (list_empty(list));

		/* This is the only non-empty list. Free them all. */
		if (batch_free == MIGRATE_PCPTYPES)
			batch_free = count;

		do {
			int mt;	/* migratetype of the to-be-freed page */

			page = list_last_entry(list, struct page, lru);
			/* must delete as __free_one_page list manipulates */
			list_del(&page->lru);

			mt = get_pcppage_migratetype(page);
			/* MIGRATE_ISOLATE page should not go to pcplists */
			VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
			/* Pageblock could have been isolated meanwhile */
			if (unlikely(isolated_pageblocks))
				mt = get_pageblock_migratetype(page);

			if (bulkfree_pcp_prepare(page))
				continue;

			__free_one_page(page, page_to_pfn(page), zone, 0, mt);
			trace_mm_page_pcpu_drain(page, 0, mt);
		} while (--count && --batch_free && !list_empty(list));
	}
	spin_unlock(&zone->lock);
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 120 46.51% 8 36.36%
andrew morton andrew morton 55 21.32% 3 13.64%
joonsoo kim joonsoo kim 25 9.69% 1 4.55%
bartlomiej zolnierkiewicz bartlomiej zolnierkiewicz 18 6.98% 2 9.09%
nick piggin nick piggin 14 5.43% 2 9.09%
vlastimil babka vlastimil babka 13 5.04% 2 9.09%
namhyung kim namhyung kim 10 3.88% 1 4.55%
pre-git pre-git 2 0.78% 2 9.09%
geliang tang geliang tang 1 0.39% 1 4.55%
Total 258 100.00% 22 100.00%



static void free_one_page(struct zone *zone,
				struct page *page, unsigned long pfn,
				unsigned int order,
				int migratetype)
{
	unsigned long nr_scanned;
	spin_lock(&zone->lock);
	nr_scanned = node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED);
	if (nr_scanned)
		__mod_node_page_state(zone->zone_pgdat, NR_PAGES_SCANNED, -nr_scanned);

	if (unlikely(has_isolate_pageblock(zone) ||
		is_migrate_isolate(migratetype))) {
		migratetype = get_pfnblock_migratetype(page, pfn);
	}
	__free_one_page(page, pfn, zone, order, migratetype);
	spin_unlock(&zone->lock);
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 39 34.51% 6 60.00%
joonsoo kim joonsoo kim 33 29.20% 1 10.00%
nick piggin nick piggin 28 24.78% 1 10.00%
christoph lameter christoph lameter 12 10.62% 1 10.00%
andrew morton andrew morton 1 0.88% 1 10.00%
Total 113 100.00% 10 100.00%



static void __meminit __init_single_page(struct page *page, unsigned long pfn,
				unsigned long zone, int nid)
{
	set_page_links(page, zone, nid, pfn);
	init_page_count(page);
	page_mapcount_reset(page);
	page_cpupid_reset_last(page);

	INIT_LIST_HEAD(&page->lru);
#ifdef WANT_PAGE_VIRTUAL
	/* The shift won't overflow because ZONE_NORMAL is below 4G. */
	if (!is_highmem_idx(zone))
		set_page_address(page, __va(pfn << PAGE_SHIFT));
#endif
}

Contributors
Person Tokens Prop Commits CommitProp
robin holt robin holt 83 100.00% 1 100.00%
Total 83 100.00% 1 100.00%



static void __meminit __init_single_pfn(unsigned long pfn, unsigned long zone,
					int nid)
{
	return __init_single_page(pfn_to_page(pfn), pfn, zone, nid);
}

Contributors
Person Tokens Prop Commits CommitProp
robin holt robin holt 33 100.00% 1 100.00%
Total 33 100.00% 1 100.00%

#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT


static void init_reserved_page(unsigned long pfn)
{
	pg_data_t *pgdat;
	int nid, zid;

	if (!early_page_uninitialised(pfn))
		return;

	nid = early_pfn_to_nid(pfn);
	pgdat = NODE_DATA(nid);

	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
		struct zone *zone = &pgdat->node_zones[zid];

		if (pfn >= zone->zone_start_pfn && pfn < zone_end_pfn(zone))
			break;
	}
	__init_single_pfn(pfn, zid, nid);
}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 95 100.00% 1 100.00%
Total 95 100.00% 1 100.00%

#else


static inline void init_reserved_page(unsigned long pfn)
{
}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 10 100.00% 1 100.00%
Total 10 100.00% 1 100.00%

#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */

/*
 * Initialised pages do not have PageReserved set. This function is
 * called for each range allocated by the bootmem allocator and
 * marks the pages PageReserved. The remaining valid pages are later
 * sent to the buddy page allocator.
 */


void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end)
{
	unsigned long start_pfn = PFN_DOWN(start);
	unsigned long end_pfn = PFN_UP(end);

	for (; start_pfn < end_pfn; start_pfn++) {
		if (pfn_valid(start_pfn)) {
			struct page *page = pfn_to_page(start_pfn);

			init_reserved_page(start_pfn);

			/* Avoid false-positive PageTail() */
			INIT_LIST_HEAD(&page->lru);

			SetPageReserved(page);
		}
	}
}

Contributors
Person Tokens Prop Commits CommitProp
nathan zimmer nathan zimmer 50 62.50% 1 25.00%
mel gorman mel gorman 19 23.75% 1 25.00%
kirill a. shutemov kirill a. shutemov 9 11.25% 1 25.00%
stefan bader stefan bader 2 2.50% 1 25.00%
Total 80 100.00% 4 100.00%



static void __free_pages_ok(struct page *page, unsigned int order)
{
	unsigned long flags;
	int migratetype;
	unsigned long pfn = page_to_pfn(page);

	if (!free_pages_prepare(page, order, true))
		return;

	migratetype = get_pfnblock_migratetype(page, pfn);
	local_irq_save(flags);
	__count_vm_events(PGFREE, 1 << order);
	free_one_page(page_zone(page), page, pfn, order, migratetype);
	local_irq_restore(flags);
}

Contributors
Person Tokens Prop Commits CommitProp
kosaki motohiro kosaki motohiro 30 34.09% 1 7.14%
mel gorman mel gorman 23 26.14% 4 28.57%
nick piggin nick piggin 19 21.59% 3 21.43%
andrew morton andrew morton 7 7.95% 1 7.14%
pre-git pre-git 4 4.55% 3 21.43%
minchan kim minchan kim 3 3.41% 1 7.14%
christoph lameter christoph lameter 2 2.27% 1 7.14%
Total 88 100.00% 14 100.00%



static void __init __free_pages_boot_core(struct page *page, unsigned int order)
{
	unsigned int nr_pages = 1 << order;
	struct page *p = page;
	unsigned int loop;

	prefetchw(p);
	for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
		prefetchw(p + 1);
		__ClearPageReserved(p);
		set_page_count(p, 0);
	}
	__ClearPageReserved(p);
	set_page_count(p, 0);

	page_zone(page)->managed_pages += nr_pages;
	set_page_refcounted(page);
	__free_pages(page, order);
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 58 50.88% 1 11.11%
yinghai lu yinghai lu 28 24.56% 1 11.11%
johannes weiner johannes weiner 9 7.89% 1 11.11%
jiang liu jiang liu 8 7.02% 1 11.11%
nick piggin nick piggin 7 6.14% 2 22.22%
mel gorman mel gorman 4 3.51% 3 33.33%
Total 114 100.00% 9 100.00%

#if defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) || \
	defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP)


static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;



int __meminit early_pfn_to_nid(unsigned long pfn)
{
	static DEFINE_SPINLOCK(early_pfn_lock);
	int nid;

	spin_lock(&early_pfn_lock);
	nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
	if (nid < 0)
		nid = first_online_node;
	spin_unlock(&early_pfn_lock);

	return nid;
}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 54 100.00% 3 100.00%
Total 54 100.00% 3 100.00%

#endif

#ifdef CONFIG_NODES_SPAN_OTHER_NODES


static inline bool __meminit meminit_pfn_in_nid(unsigned long pfn, int node,
					struct mminit_pfnnid_cache *state)
{
	int nid;

	nid = __early_pfn_to_nid(pfn, state);
	if (nid >= 0 && nid != node)
		return false;
	return true;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 48 100.00% 1 100.00%
Total 48 100.00% 1 100.00%

/* Only safe to use early in boot when initialisation is single-threaded */


static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
{
	return meminit_pfn_in_nid(pfn, node, &early_pfnnid_cache);
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 26 100.00% 1 100.00%
Total 26 100.00% 1 100.00%

#else



static inline bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
{
	return true;
}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 18 100.00% 1 100.00%
Total 18 100.00% 1 100.00%



static inline bool __meminit meminit_pfn_in_nid(unsigned long pfn, int node,
					struct mminit_pfnnid_cache *state)
{
	return true;
}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 23 100.00% 1 100.00%
Total 23 100.00% 1 100.00%

#endif




void __init __free_pages_bootmem(struct page *page, unsigned long pfn,
							unsigned int order)
{
	if (early_page_uninitialised(pfn))
		return;
	return __free_pages_boot_core(page, order);
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 35 100.00% 2 100.00%
Total 35 100.00% 2 100.00%

/*
 * Check that the whole (or subset of) a pageblock given by the interval of
 * [start_pfn, end_pfn) is valid and within the same zone, before scanning it
 * with the migration of free compaction scanner. The scanners then need to
 * use only pfn_valid_within() check for arches that allow holes within
 * pageblocks.
 *
 * Return struct page pointer of start_pfn, or NULL if checks were not passed.
 *
 * It's possible on some configurations to have a setup like node0 node1 node0
 * i.e. it's possible that all pages within a zones range of pages do not
 * belong to a single zone. We assume that a border between node0 and node1
 * can occur within a single pageblock, but not a node0 node1 node0
 * interleaving within a single pageblock. It is therefore sufficient to check
 * the first and last page of a pageblock and avoid checking each individual
 * page in a pageblock.
 */


struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
				     unsigned long end_pfn, struct zone *zone)
{
	struct page *start_page;
	struct page *end_page;

	/* end_pfn is one past the range we are checking */
	end_pfn--;

	if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn))
		return NULL;

	start_page = pfn_to_page(start_pfn);

	if (page_zone(start_page) != zone)
		return NULL;

	end_page = pfn_to_page(end_pfn);

	/* This gives a shorter code than deriving page_zone(end_page) */
	if (page_zone_id(start_page) != page_zone_id(end_page))
		return NULL;

	return start_page;
}

Contributors
Person Tokens Prop Commits CommitProp
joonsoo kim joonsoo kim 96 100.00% 1 100.00%
Total 96 100.00% 1 100.00%



void set_zone_contiguous(struct zone *zone)
{
	unsigned long block_start_pfn = zone->zone_start_pfn;
	unsigned long block_end_pfn;

	block_end_pfn = ALIGN(block_start_pfn + 1, pageblock_nr_pages);
	for (; block_start_pfn < zone_end_pfn(zone);
			block_start_pfn = block_end_pfn,
			 block_end_pfn += pageblock_nr_pages) {

		block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));

		if (!__pageblock_pfn_to_page(block_start_pfn,
					     block_end_pfn, zone))
			return;
	}

	/* We confirm that there is no hole */
	zone->contiguous = true;
}

Contributors
Person Tokens Prop Commits CommitProp
joonsoo kim joonsoo kim 85 100.00% 1 100.00%
Total 85 100.00% 1 100.00%



void clear_zone_contiguous(struct zone *zone)
{
	zone->contiguous = false;
}

Contributors
Person Tokens Prop Commits CommitProp
joonsoo kim joonsoo kim 16 100.00% 1 100.00%
Total 16 100.00% 1 100.00%

#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT


static void __init deferred_free_range(struct page *page,
					unsigned long pfn, int nr_pages)
{
	int i;

	if (!page)
		return;

	/* Free a large naturally-aligned chunk if possible */
	if (nr_pages == MAX_ORDER_NR_PAGES &&
	    (pfn & (MAX_ORDER_NR_PAGES-1)) == 0) {
		set_pageblock_migratetype(page, MIGRATE_MOVABLE);
		__free_pages_boot_core(page, MAX_ORDER-1);
		return;
	}

	for (i = 0; i < nr_pages; i++, page++)
		__free_pages_boot_core(page, 0);
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 89 100.00% 3 100.00%
Total 89 100.00% 3 100.00%

/* Completion tracking for deferred_init_memmap() threads */

static atomic_t pgdat_init_n_undone __initdata;
static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);



static inline void __init pgdat_init_report_one_done(void)
{
	if (atomic_dec_and_test(&pgdat_init_n_undone))
		complete(&pgdat_init_all_done_comp);
}

Contributors
Person Tokens Prop Commits CommitProp
nicolai stange nicolai stange 24 100.00% 1 100.00%
Total 24 100.00% 1 100.00%

/* Initialise remaining memory on a node */


static int __init deferred_init_memmap(void *data)
{
	pg_data_t *pgdat = data;
	int nid = pgdat->node_id;
	struct mminit_pfnnid_cache nid_init_state = { };
	unsigned long start = jiffies;
	unsigned long nr_pages = 0;
	unsigned long walk_start, walk_end;
	int i, zid;
	struct zone *zone;
	unsigned long first_init_pfn = pgdat->first_deferred_pfn;
	const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);

	if (first_init_pfn == ULONG_MAX) {
		pgdat_init_report_one_done();
		return 0;
	}

	/* Bind memory initialisation thread to a local node if possible */
	if (!cpumask_empty(cpumask))
		set_cpus_allowed_ptr(current, cpumask);

	/* Sanity check boundaries */
	BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
	BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));
	pgdat->first_deferred_pfn = ULONG_MAX;

	/* Only the highest zone is deferred so find it */
	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
		zone = pgdat->node_zones + zid;
		if (first_init_pfn < zone_end_pfn(zone))
			break;
	}

	for_each_mem_pfn_range(i, nid, &walk_start, &walk_end, NULL) {
		unsigned long pfn, end_pfn;
		struct page *page = NULL;
		struct page *free_base_page = NULL;
		unsigned long free_base_pfn = 0;
		int nr_to_free = 0;

		end_pfn = min(walk_end, zone_end_pfn(zone));
		pfn = first_init_pfn;
		if (pfn < walk_start)
			pfn = walk_start;
		if (pfn < zone->zone_start_pfn)
			pfn = zone->zone_start_pfn;

		for (; pfn < end_pfn; pfn++) {
			if (!pfn_valid_within(pfn))
				goto free_range;

			/*
                         * Ensure pfn_valid is checked every
                         * MAX_ORDER_NR_PAGES for memory holes
                         */
			if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0) {
				if (!pfn_valid(pfn)) {
					page = NULL;
					goto free_range;
				}
			}

			if (!meminit_pfn_in_nid(pfn, nid, &nid_init_state)) {
				page = NULL;
				goto free_range;
			}

			/* Minimise pfn page lookups and scheduler checks */
			if (page && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0) {
				page++;
			} else {
				nr_pages += nr_to_free;
				deferred_free_range(free_base_page,
						free_base_pfn, nr_to_free);
				free_base_page = NULL;
				free_base_pfn = nr_to_free = 0;

				page = pfn_to_page(pfn);
				cond_resched();
			}

			if (page->flags) {
				VM_BUG_ON(page_zone(page) != zone);
				goto free_range;
			}

			__init_single_page(page, pfn, zid, nid);
			if (!free_base_page) {
				free_base_page = page;
				free_base_pfn = pfn;
				nr_to_free = 0;
			}
			nr_to_free++;

			/* Where possible, batch up pages for a single free */
			continue;
free_range:
			/* Free the current block of pages to allocator */
			nr_pages += nr_to_free;
			deferred_free_range(free_base_page, free_base_pfn,
								nr_to_free);
			free_base_page = NULL;
			free_base_pfn = nr_to_free = 0;
		}

		first_init_pfn = max(end_pfn, first_init_pfn);
	}

	/* Sanity check that the next zone really is unpopulated */
	WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));

	pr_info("node %d initialised, %lu pages in %ums\n", nid, nr_pages,
					jiffies_to_msecs(jiffies - start));

	pgdat_init_report_one_done();
	return 0;
}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 519 99.24% 4 80.00%
nicolai stange nicolai stange 4 0.76% 1 20.00%
Total 523 100.00% 5 100.00%

#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */



void __init page_alloc_init_late(void)
{
	struct zone *zone;

#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
	int nid;

	/* There will be num_node_state(N_MEMORY) threads */
	atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
	for_each_node_state(nid, N_MEMORY) {
		kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
	}

	/* Block until all are initialised */
	wait_for_completion(&pgdat_init_all_done_comp);

	/* Reinit limits that are based on free pages after the kernel is up */
	files_maxfiles_init();
#endif

	for_each_populated_zone(zone)
		set_zone_contiguous(zone);
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 43 57.33% 3 60.00%
joonsoo kim joonsoo kim 18 24.00% 1 20.00%
nicolai stange nicolai stange 14 18.67% 1 20.00%
Total 75 100.00% 5 100.00%

#ifdef CONFIG_CMA
/* Free whole pageblock and set its migration type to MIGRATE_CMA. */


void __init init_cma_reserved_pageblock(struct page *page)
{
	unsigned i = pageblock_nr_pages;
	struct page *p = page;

	do {
		__ClearPageReserved(p);
		set_page_count(p, 0);
	} while (++p, --i);

	set_pageblock_migratetype(page, MIGRATE_CMA);

	if (pageblock_order >= MAX_ORDER) {
		i = pageblock_nr_pages;
		p = page;
		do {
			set_page_refcounted(p);
			__free_pages(p, MAX_ORDER - 1);
			p += MAX_ORDER_NR_PAGES;
		} while (i -= MAX_ORDER_NR_PAGES);
	} else {
		set_page_refcounted(page);
		__free_pages(page, pageblock_order);
	}

	adjust_managed_page_count(page, pageblock_nr_pages);
}
Contributors
Person Tokens Prop Commits CommitProp
michal nazarewicz michal nazarewicz 113 94.17% 2 50.00%
marek szyprowski marek szyprowski 4 3.33% 1 25.00%
jiang liu jiang liu 3 2.50% 1 25.00%
Total 120 100.00% 4 100.00%

#endif

/*
 * The order of subdivision here is critical for the IO subsystem.
 * Please do not alter this order without good reasons and regression
 * testing. Specifically, as large blocks of memory are subdivided,
 * the order in which smaller blocks are delivered depends on the order
 * they're subdivided in this function. This is the primary factor
 * influencing the order in which pages are delivered to the IO
 * subsystem according to empirical testing, and this is also justified
 * by considering the behavior of a buddy system containing a single
 * large block of memory acted on by a series of small allocations.
 * This behavior is a critical factor in sglist merging's success.
 *
 * -- nyc
 */


static inline void expand(struct zone *zone, struct page *page,
	int low, int high, struct free_area *area,
	int migratetype)
{
	unsigned long size = 1 << high;

	while (high > low) {
		area--;
		high--;
		size >>= 1;
		VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);

		if (IS_ENABLED(CONFIG_DEBUG_PAGEALLOC) &&
			debug_guardpage_enabled() &&
			high < debug_guardpage_minorder()) {
			/*
                         * Mark as guard pages (or page), that will allow to
                         * merge back to allocator when buddy will be freed.
                         * Corresponding page table entries will not be touched,
                         * pages will stay not present in virtual address space
                         */
			set_page_guard(zone, &page[size], high, migratetype);
			continue;
		}
		list_add(&page[size].lru, &area->free_list[migratetype]);
		area->nr_free++;
		set_page_order(&page[size], high);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
pre-git pre-git 64 43.84% 5 26.32%
andrew morton andrew morton 23 15.75% 4 21.05%
stanislaw gruszka stanislaw gruszka 21 14.38% 1 5.26%
joonsoo kim joonsoo kim 11 7.53% 2 10.53%
sasha levin sasha levin 7 4.79% 1 5.26%
mel gorman mel gorman 6 4.11% 1 5.26%
nick piggin nick piggin 6 4.11% 2 10.53%
kamezawa hiroyuki kamezawa hiroyuki 5 3.42% 1 5.26%
bartlomiej zolnierkiewicz bartlomiej zolnierkiewicz 2 1.37% 1 5.26%
william lee irwin iii william lee irwin iii 1 0.68% 1 5.26%
Total 146 100.00% 19 100.00%



static void check_new_page_bad(struct page *page)
{
	const char *bad_reason = NULL;
	unsigned long bad_flags = 0;

	if (unlikely(atomic_read(&page->_mapcount) != -1))
		bad_reason = "nonzero mapcount";
	if (unlikely(page->mapping != NULL))
		bad_reason = "non-NULL mapping";
	if (unlikely(page_ref_count(page) != 0))
		bad_reason = "nonzero _count";
	if (unlikely(page->flags & __PG_HWPOISON)) {
		bad_reason = "HWPoisoned (hardware-corrupted)";
		bad_flags = __PG_HWPOISON;
		/* Don't complain about hwpoisoned pages */
		page_mapcount_reset(page); /* remove PageBuddy */
		return;
	}
	if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) {
		bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set";
		bad_flags = PAGE_FLAGS_CHECK_AT_PREP;
	}
#ifdef CONFIG_MEMCG
	if (unlikely(page->mem_cgroup))
		bad_reason = "page still charged to cgroup";
#endif
	bad_page(page, bad_reason, bad_flags);
}

Contributors
Person Tokens Prop Commits CommitProp
dave hansen dave hansen 62 40.79% 1 5.88%
naoya horiguchi naoya horiguchi 29 19.08% 2 11.76%
andrew morton andrew morton 23 15.13% 3 17.65%
hugh dickins hugh dickins 8 5.26% 2 11.76%
kirill a. shutemov kirill a. shutemov 8 5.26% 2 11.76%
johannes weiner johannes weiner 8 5.26% 1 5.88%
vlastimil babka vlastimil babka 6 3.95% 1 5.88%
mel gorman mel gorman 3 1.97% 1 5.88%
nick piggin nick piggin 2 1.32% 1 5.88%
daisuke nishimura daisuke nishimura 1 0.66% 1 5.88%
joonsoo kim joonsoo kim 1 0.66% 1 5.88%
russ anderson russ anderson 1 0.66% 1 5.88%
Total 152 100.00% 17 100.00%

/*
 * This page is about to be returned from the page allocator
 */


static inline int check_new_page(struct page *page)
{
	if (likely(page_expected_state(page,
				PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON)))
		return 0;

	check_new_page_bad(page);
	return 1;
}

Contributors
Person Tokens Prop Commits CommitProp
vlastimil babka vlastimil babka 33 89.19% 1 50.00%
fengguang wu fengguang wu 4 10.81% 1 50.00%
Total 37 100.00% 2 100.00%



static inline bool free_pages_prezeroed(bool poisoned)
{
	return IS_ENABLED(CONFIG_PAGE_POISONING_ZERO) &&
		page_poisoning_enabled() && poisoned;
}

Contributors
Person Tokens Prop Commits CommitProp
laura abbott laura abbott 21 100.00% 1 100.00%
Total 21 100.00% 1 100.00%

#ifdef CONFIG_DEBUG_VM


static bool check_pcp_refill(struct page *page)
{
	return false;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 14 100.00% 1 100.00%
Total 14 100.00% 1 100.00%



static bool check_new_pcp(struct page *page)
{
	return check_new_page(page);
}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 17 100.00% 1 100.00%
Total 17 100.00% 1 100.00%

#else


static bool check_pcp_refill(struct page *page)
{
	return check_new_page(page);
}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 16 94.12% 1 50.00%
fengguang wu fengguang wu 1 5.88% 1 50.00%
Total 17 100.00% 2 100.00%



static bool check_new_pcp(struct page *page)
{
	return false;
}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 14 100.00% 1 100.00%
Total 14 100.00% 1 100.00%

#endif /* CONFIG_DEBUG_VM */



static bool check_new_pages(struct page *page, unsigned int order)
{
	int i;
	for (i = 0; i < (1 << order); i++) {
		struct page *p = page + i;

		if (unlikely(check_new_page(p)))
			return true;
	}

	return false;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 61 98.39% 1 50.00%
fengguang wu fengguang wu 1 1.61% 1 50.00%
Total 62 100.00% 2 100.00%



inline void post_alloc_hook(struct page *page, unsigned int order,
				gfp_t gfp_flags)
{
	set_page_private(page, 0);
	set_page_refcounted(page);

	arch_alloc_page(page, order);
	kernel_map_pages(page, 1 << order, 1);
	kernel_poison_pages(page, 1 << order, 1);
	kasan_alloc_pages(page, order);
	set_page_owner(page, order, gfp_flags);
}

Contributors
Person Tokens Prop Commits CommitProp
joonsoo kim joonsoo kim 75 100.00% 1 100.00%
Total 75 100.00% 1 100.00%



static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
							unsigned int alloc_flags)
{
	int i;
	bool poisoned = true;

	for (i = 0; i < (1 << order); i++) {
		struct page *p = page + i;
		if (poisoned)
			poisoned &= page_is_poisoned(p);
	}

	post_alloc_hook(page, order, gfp_flags);

	if (!free_pages_prezeroed(poisoned) && (gfp_flags & __GFP_ZERO))
		for (i = 0; i < (1 << order); i++)
			clear_highpage(page + i);

	if (order && (gfp_flags & __GFP_COMP))
		prep_compound_page(page, order);

	/*
         * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to
         * allocate the page. The expectation is that the caller is taking
         * steps that will free more memory. The caller should avoid the page
         * being used for !PFMEMALLOC purposes.
         */
	if (alloc_flags & ALLOC_NO_WATERMARKS)
		set_page_pfmemalloc(page);
	else
		clear_page_pfmemalloc(page);
}

Contributors
Person Tokens Prop Commits CommitProp
fengguang wu fengguang wu 47 31.13% 1 7.14%
nick piggin nick piggin 31 20.53% 2 14.29%
laura abbott laura abbott 22 14.57% 1 7.14%
anisse astier anisse astier 20 13.25% 1 7.14%
michal hocko michal hocko 14 9.27% 1 7.14%
vlastimil babka vlastimil babka 7 4.64% 1 7.14%
mel gorman mel gorman 4 2.65% 3 21.43%
joonsoo kim joonsoo kim 2 1.32% 1 7.14%
andrey ryabinin andrey ryabinin 2 1.32% 1 7.14%
andrew morton andrew morton 1 0.66% 1 7.14%
linus torvalds linus torvalds 1 0.66% 1 7.14%
Total 151 100.00% 14 100.00%

/*
 * Go through the free lists for the given migratetype and remove
 * the smallest available page from the freelists
 */

static inline

struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
						int migratetype)
{
	unsigned int current_order;
	struct free_area *area;
	struct page *page;

	/* Find a page of the appropriate size in the preferred list */
	for (current_order = order; current_order < MAX_ORDER; ++current_order) {
		area = &(zone->free_area[current_order]);
		page = list_first_entry_or_null(&area->free_list[migratetype],
							struct page, lru);
		if (!page)
			continue;
		list_del(&page->lru);
		rmv_page_order(page);
		area->nr_free--;
		expand(zone, page, order, current_order, area, migratetype);
		set_pcppage_migratetype(page, migratetype);
		return page;
	}

	return NULL;
}


Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 118 88.72% 2 40.00%
geliang tang geliang tang 8 6.02% 1 20.00%
vlastimil babka vlastimil babka 7 5.26% 2 40.00%
Total 133 100.00% 5 100.00%

/*
 * This array describes the order lists are fallen back to when
 * the free lists for the desirable migrate type are depleted
 */

static int fallbacks[MIGRATE_TYPES][4] = {
	[MIGRATE_UNMOVABLE]   = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE,   MIGRATE_TYPES },
	[MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE,   MIGRATE_MOVABLE,   MIGRATE_TYPES },
	[MIGRATE_MOVABLE]     = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES },
#ifdef CONFIG_CMA
	[MIGRATE_CMA]         = { MIGRATE_TYPES }, /* Never used */
#endif
#ifdef CONFIG_MEMORY_ISOLATION
	[MIGRATE_ISOLATE]     = { MIGRATE_TYPES }, /* Never used */
#endif
};

#ifdef CONFIG_CMA


static struct page *__rmqueue_cma_fallback(struct zone *zone,
					unsigned int order)
{
	return __rmqueue_smallest(zone, order, MIGRATE_CMA);
}
Contributors
Person Tokens Prop Commits CommitProp
joonsoo kim joonsoo kim 27 100.00% 1 100.00%
Total 27 100.00% 1 100.00%

#else


static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
					unsigned int order) { return NULL; }
Contributors
Person Tokens Prop Commits CommitProp
joonsoo kim joonsoo kim 21 100.00% 1 100.00%
Total 21 100.00% 1 100.00%

#endif

/*
 * Move the free pages in a range to the free lists of the requested type.
 * Note that start_page and end_pages are not aligned on a pageblock
 * boundary. If alignment is required, use move_freepages_block()
 */


int move_freepages(struct zone *zone,
			  struct page *start_page, struct page *end_page,
			  int migratetype)
{
	struct page *page;
	unsigned int order;
	int pages_moved = 0;

#ifndef CONFIG_HOLES_IN_ZONE
	/*
         * page_zone is not safe to call in this context when
         * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
         * anyway as we check zone boundaries in move_freepages_block().
         * Remove at a later date when no bug reports exist related to
         * grouping pages by mobility
         */
	VM_BUG_ON(page_zone(start_page) != page_zone(end_page));
#endif

	for (page = start_page; page <= end_page;) {
		/* Make sure we are not inadvertently changing nodes */
		VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);

		if (!pfn_valid_within(page_to_pfn(page))) {
			page++;
			continue;
		}

		if (!PageBuddy(page)) {
			page++;
			continue;
		}

		order = page_order(page);
		list_move(&page->lru,
			  &zone->free_area[order].free_list[migratetype]);
		page += 1 << order;
		pages_moved += 1 << order;
	}

	return pages_moved;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 141 88.68% 4 50.00%
adam litke adam litke 13 8.18% 1 12.50%
sasha levin sasha levin 3 1.89% 1 12.50%
kirill a. shutemov kirill a. shutemov 2 1.26% 2 25.00%
Total 159 100.00% 8 100.00%



int move_freepages_block(struct zone *zone, struct page *page,
				int migratetype)
{
	unsigned long start_pfn, end_pfn;
	struct page *start_page, *end_page;

	start_pfn = page_to_pfn(page);
	start_pfn = start_pfn & ~(pageblock_nr_pages-1);
	start_page = pfn_to_page(start_pfn);
	end_page = start_page + pageblock_nr_pages - 1;
	end_pfn = start_pfn + pageblock_nr_pages - 1;

	/* Do not cross zone boundaries */
	if (!zone_spans_pfn(zone, start_pfn))
		start_page = page;
	if (!zone_spans_pfn(zone, end_pfn))
		return 0;

	return move_freepages(zone, start_page, end_page, migratetype);
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 101 89.38% 2 66.67%
cody p schafer cody p schafer 12 10.62% 1 33.33%
Total 113 100.00% 3 100.00%



static void change_pageblock_range(struct page *pageblock_page,
					int start_order, int migratetype)
{
	int nr_pageblocks = 1 << (start_order - pageblock_order);

	while (nr_pageblocks--) {
		set_pageblock_migratetype(pageblock_page, migratetype);
		pageblock_page += pageblock_nr_pages;
	}
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 46 100.00% 1 100.00%
Total 46 100.00% 1 100.00%

/*
 * When we are falling back to another migratetype during allocation, try to
 * steal extra free pages from the same pageblocks to satisfy further
 * allocations, instead of polluting multiple pageblocks.
 *
 * If we are stealing a relatively large buddy page, it is likely there will
 * be more free pages in the pageblock, so try to steal them all. For
 * reclaimable and unmovable allocations, we steal regardless of page size,
 * as fragmentation caused by those allocations polluting movable pageblocks
 * is worse than movable allocations stealing from unmovable and reclaimable
 * pageblocks.
 */


static bool can_steal_fallback(unsigned int order, int start_mt)
{
	/*
         * Leaving this order check is intended, although there is
         * relaxed order check in next check. The reason is that
         * we can actually steal whole pageblock if this condition met,
         * but, below check doesn't guarantee it and that is just heuristic
         * so could be changed anytime.
         */
	if (order >= pageblock_order)
		return true;

	if (order >= pageblock_order / 2 ||
		start_mt == MIGRATE_RECLAIMABLE ||
		start_mt == MIGRATE_UNMOVABLE ||
		page_group_by_mobility_disabled)
		return true;

	return false;
}

Contributors
Person Tokens Prop Commits CommitProp
joonsoo kim joonsoo kim 47 100.00% 1 100.00%
Total 47 100.00% 1 100.00%

/*
 * This function implements actual steal behaviour. If order is large enough,
 * we can steal whole pageblock. If not, we first move freepages in this
 * pageblock and check whether half of pages are moved or not. If half of
 * pages are moved, we can change migratetype of pageblock and permanently
 * use it's pages as requested migratetype in the future.
 */


static void steal_suitable_fallback(struct zone *zone, struct page *page,
							  int start_type)
{
	unsigned int current_order = page_order(page);
	int pages;

	/* Take ownership for orders >= pageblock_order */
	if (current_order >= pageblock_order) {
		change_pageblock_range(page, current_order, start_type);
		return;
	}

	pages = move_freepages_block(zone, page, start_type);

	/* Claim the whole block if over half of it is free */
	if (pages >= (1 << (pageblock_order-1)) ||
			page_group_by_mobility_disabled)
		set_pageblock_migratetype(page, start_type);
}

Contributors
Person Tokens Prop Commits CommitProp
srivatsa s. bhat srivatsa s. bhat 78 91.76% 1 25.00%
joonsoo kim joonsoo kim 4 4.71% 1 25.00%
vlastimil babka vlastimil babka 2 2.35% 1 25.00%
kirill a. shutemov kirill a. shutemov 1 1.18% 1 25.00%
Total 85 100.00% 4 100.00%

/*
 * Check whether there is a suitable fallback freepage with requested order.
 * If only_stealable is true, this function returns fallback_mt only if
 * we can steal other freepages all together. This would help to reduce
 * fragmentation due to mixed migratetype pages in one pageblock.
 */


int find_suitable_fallback(struct free_area *area, unsigned int order,
			int migratetype, bool only_stealable, bool *can_steal)
{
	int i;
	int fallback_mt;

	if (area->nr_free == 0)
		return -1;

	*can_steal = false;
	for (i = 0;; i++) {
		fallback_mt = fallbacks[migratetype][i];
		if (fallback_mt == MIGRATE_TYPES)
			break;

		if (list_empty(&area->free_list[fallback_mt]))
			continue;

		if (can_steal_fallback(order, migratetype))
			*can_steal = true;

		if (!only_stealable)
			return fallback_mt;

		if (*can_steal)
			return fallback_mt;
	}

	return -1;
}

Contributors
Person Tokens Prop Commits CommitProp
joonsoo kim joonsoo kim 122 98.39% 2 50.00%
srivatsa s. bhat srivatsa s. bhat 1 0.81% 1 25.00%
mel gorman mel gorman 1 0.81% 1 25.00%
Total 124 100.00% 4 100.00%

/*
 * Reserve a pageblock for exclusive use of high-order atomic allocations if
 * there are no empty page blocks that contain a page with a suitable order
 */


static void reserve_highatomic_pageblock(struct page *page, struct zone *zone,
				unsigned int alloc_order)
{
	int mt;
	unsigned long max_managed, flags;

	/*
         * Limit the number reserved to 1 pageblock or roughly 1% of a zone.
         * Check is race-prone but harmless.
         */
	max_managed = (zone->managed_pages / 100) + pageblock_nr_pages;
	if (zone->nr_reserved_highatomic >= max_managed)
		return;

	spin_lock_irqsave(&zone->lock, flags);

	/* Recheck the nr_reserved_highatomic limit under the lock */
	if (zone->nr_reserved_highatomic >= max_managed)
		goto out_unlock;

	/* Yoink! */
	mt = get_pageblock_migratetype(page);
	if (mt != MIGRATE_HIGHATOMIC &&
			!is_migrate_isolate(mt) && !is_migrate_cma(mt)) {
		zone->nr_reserved_highatomic += pageblock_nr_pages;
		set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC);
		move_freepages_block(zone, page, MIGRATE_HIGHATOMIC);
	}

out_unlock:
	spin_unlock_irqrestore(&zone->lock, flags);
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 135 100.00% 1 100.00%
Total 135 100.00% 1 100.00%

/*
 * Used when an allocation is about to fail under memory pressure. This
 * potentially hurts the reliability of high-order allocations when under
 * intense memory pressure but failed atomic allocations should be easier
 * to recover from than an OOM.
 */


static void unreserve_highatomic_pageblock(const struct alloc_context *ac)
{
	struct zonelist *zonelist = ac->zonelist;
	unsigned long flags;
	struct zoneref *z;
	struct zone *zone;
	struct page *page;
	int order;

	for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx,
								ac->nodemask) {
		/* Preserve at least one pageblock */
		if (zone->nr_reserved_highatomic <= pageblock_nr_pages)
			continue;

		spin_lock_irqsave(&zone->lock, flags);
		for (order = 0; order < MAX_ORDER; order++) {
			struct free_area *area = &(zone->free_area[order]);

			page = list_first_entry_or_null(
					&area->free_list[MIGRATE_HIGHATOMIC],
					struct page, lru);
			if (!page)
				continue;

			/*
                         * It should never happen but changes to locking could
                         * inadvertently allow a per-cpu drain to add pages
                         * to MIGRATE_HIGHATOMIC while unreserving so be safe
                         * and watch for underflows.
                         */
			zone->nr_reserved_highatomic -= min(pageblock_nr_pages,
				zone->nr_reserved_highatomic);

			/*
                         * Convert to ac->migratetype and avoid the normal
                         * pageblock stealing heuristics. Minimally, the caller
                         * is doing the work and needs the pages. More
                         * importantly, if the block was always converted to
                         * MIGRATE_UNMOVABLE or another type then the number
                         * of pageblocks that cannot be completely freed
                         * may increase.
                         */
			set_pageblock_migratetype(page, ac->migratetype);
			move_freepages_block(zone, page, ac->migratetype);
			spin_unlock_irqrestore(&zone->lock, flags);
			return;
		}
		spin_unlock_irqrestore(&zone->lock, flags);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 179 95.72% 1 50.00%
geliang tang geliang tang 8 4.28% 1 50.00%
Total 187 100.00% 2 100.00%

/* Remove an element from the buddy allocator from the fallback list */

static inline struct page *

__rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype)
{
	struct free_area *area;
	unsigned int current_order;
	struct page *page;
	int fallback_mt;
	bool can_steal;

	/* Find the largest possible block of pages in the other list */
	for (current_order = MAX_ORDER-1;
				current_order >= order && current_order <= MAX_ORDER-1;
				--current_order) {
		area = &(zone->free_area[current_order]);
		fallback_mt = find_suitable_fallback(area, current_order,
				start_migratetype, false, &can_steal);
		if (fallback_mt == -1)
			continue;

		page = list_first_entry(&area->free_list[fallback_mt],
						struct page, lru);
		if (can_steal)
			steal_suitable_fallback(zone, page, start_migratetype);

		/* Remove the page from the freelists */
		area->nr_free--;
		list_del(&page->lru);
		rmv_page_order(page);

		expand(zone, page, order, current_order, area,
					start_migratetype);
		/*
                 * The pcppage_migratetype may differ from pageblock's
                 * migratetype depending on the decisions in
                 * find_suitable_fallback(). This is OK as long as it does not
                 * differ for MIGRATE_CMA pageblocks. Those can be used as
                 * fallback only via special __rmqueue_cma_fallback() function
                 */
		set_pcppage_migratetype(page, start_migratetype);

		trace_mm_page_alloc_extfrag(page, order, current_order,
			start_migratetype, fallback_mt);

		return page;
	}

	return NULL;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 144 74.61% 7 53.85%
joonsoo kim joonsoo kim 39 20.21% 2 15.38%
vlastimil babka vlastimil babka 7 3.63% 2 15.38%
geliang tang geliang tang 2 1.04% 1 7.69%
srivatsa s. bhat srivatsa s. bhat 1 0.52% 1 7.69%
Total 193 100.00% 13 100.00%

/*
 * Do the hard work of removing an element from the buddy allocator.
 * Call me with the zone->lock already held.
 */


static struct page *__rmqueue(struct zone *zone, unsigned int order,
				int migratetype)
{
	struct page *page;

	page = __rmqueue_smallest(zone, order, migratetype);
	if (unlikely(!page)) {
		if (migratetype == MIGRATE_MOVABLE)
			page = __rmqueue_cma_fallback(zone, order);

		if (!page)
			page = __rmqueue_fallback(zone, order, migratetype);
	}

	trace_mm_page_alloc_zone_locked(page, order, migratetype);
	return page;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 34 38.20% 4 28.57%
pre-git pre-git 24 26.97% 4 28.57%
joonsoo kim joonsoo kim 20 22.47% 1 7.14%
andrew morton andrew morton 7 7.87% 3 21.43%
christoph lameter christoph lameter 3 3.37% 1 7.14%
linus torvalds linus torvalds 1 1.12% 1 7.14%
Total 89 100.00% 14 100.00%

/*
 * Obtain a specified number of elements from the buddy allocator, all under
 * a single hold of the lock, for efficiency.  Add them to the supplied list.
 * Returns the number of new pages which were placed at *list.
 */


static int rmqueue_bulk(struct zone *zone, unsigned int order,
			unsigned long count, struct list_head *list,
			int migratetype, bool cold)
{
	int i;

	spin_lock(&zone->lock);
	for (i = 0; i < count; ++i) {
		struct page *page = __rmqueue(zone, order, migratetype);
		if (unlikely(page == NULL))
			break;

		if (unlikely(check_pcp_refill(page)))
			continue;

		/*
                 * Split buddy pages returned by expand() are received here
                 * in physical page order. The page is added to the callers and
                 * list and the list head then moves forward. From the callers
                 * perspective, the linked list is ordered by page number in
                 * some conditions. This is useful for IO devices that can
                 * merge IO requests if the physical pages are ordered
                 * properly.
                 */
		if (likely(!cold))
			list_add(&page->lru, list);
		else
			list_add_tail(&page->lru, list);
		list = &page->lru;
		if (is_migrate_cma(get_pcppage_migratetype(page)))
			__mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
					      -(1 << order));
	}
	__mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
	spin_unlock(&zone->lock);
	return i;
}

Contributors
Person Tokens Prop Commits CommitProp
andrew morton andrew morton 68 38.42% 3 16.67%
mel gorman mel gorman 61 34.46% 7 38.89%
bartlomiej zolnierkiewicz bartlomiej zolnierkiewicz 20 11.30% 1 5.56%
pre-git pre-git 13 7.34% 3 16.67%
nick piggin nick piggin 11 6.21% 2 11.11%
vlastimil babka vlastimil babka 4 2.26% 2 11.11%
Total 177 100.00% 18 100.00%

#ifdef CONFIG_NUMA
/*
 * Called from the vmstat counter updater to drain pagesets of this
 * currently executing processor on remote nodes after they have
 * expired.
 *
 * Note that this function must be called with the thread pinned to
 * a single processor.
 */


void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
{
	unsigned long flags;
	int to_drain, batch;

	local_irq_save(flags);
	batch = READ_ONCE(pcp->batch);
	to_drain = min(pcp->count, batch);
	if (to_drain > 0) {
		free_pcppages_bulk(zone, to_drain, pcp);
		pcp->count -= to_drain;
	}
	local_irq_restore(flags);
}
Contributors
Person Tokens Prop Commits CommitProp
christoph lameter christoph lameter 47 61.04% 5 45.45%
cody p schafer cody p schafer 10 12.99% 1 9.09%
kosaki motohiro kosaki motohiro 8 10.39% 1 9.09%
michal nazarewicz michal nazarewicz 6 7.79% 1 9.09%
nick piggin nick piggin 4 5.19% 1 9.09%
jason low jason low 1 1.30% 1 9.09%
mel gorman mel gorman 1 1.30% 1 9.09%
Total 77 100.00% 11 100.00%

#endif

/*
 * Drain pcplists of the indicated processor and zone.
 *
 * The processor must either be the current processor and the
 * thread pinned to the current processor or a processor that
 * is not online.
 */


static void drain_pages_zone(unsigned int cpu, struct zone *zone)
{
	unsigned long flags;
	struct per_cpu_pageset *pset;
	struct per_cpu_pages *pcp;

	local_irq_save(flags);
	pset = per_cpu_ptr(zone->pageset, cpu);

	pcp = &pset->pcp;
	if (pcp->count) {
		free_pcppages_bulk(zone, pcp->count, pcp);
		pcp->count = 0;
	}
	local_irq_restore(flags);
}

Contributors
Person Tokens Prop Commits CommitProp
rusty russell rusty russell 36 43.90% 1 11.11%
christoph lameter christoph lameter 16 19.51% 3 33.33%
nick piggin nick piggin 15 18.29% 2 22.22%
david rientjes david rientjes 8 9.76% 1 11.11%
vlastimil babka vlastimil babka 6 7.32% 1 11.11%
mel gorman mel gorman 1 1.22% 1 11.11%
Total 82 100.00% 9 100.00%

/*
 * Drain pcplists of all zones on the indicated processor.
 *
 * The processor must either be the current processor and the
 * thread pinned to the current processor or a processor that
 * is not online.
 */


static void drain_pages(unsigned int cpu)
{
	struct zone *zone;

	for_each_populated_zone(zone) {
		drain_pages_zone(cpu, zone);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
vlastimil babka vlastimil babka 27 96.43% 1 50.00%
rusty russell rusty russell 1 3.57% 1 50.00%
Total 28 100.00% 2 100.00%

/*
 * Spill all of this CPU's per-cpu pages back into the buddy allocator.
 *
 * The CPU has to be pinned. When zone parameter is non-NULL, spill just
 * the single zone's pages.
 */


void drain_local_pages(struct zone *zone)
{
	int cpu = smp_processor_id();

	if (zone)
		drain_pages_zone(cpu, zone);
	else
		drain_pages(cpu);
}

Contributors
Person Tokens Prop Commits CommitProp
vlastimil babka vlastimil babka 22 66.67% 1 50.00%
christoph lameter christoph lameter 11 33.33% 1 50.00%
Total 33 100.00% 2 100.00%

/*
 * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
 *
 * When zone parameter is non-NULL, spill just the single zone's pages.
 *
 * Note that this code is protected against sending an IPI to an offline
 * CPU but does not guarantee sending an IPI to newly hotplugged CPUs:
 * on_each_cpu_mask() blocks hotplug and won't talk to offlined CPUs but
 * nothing keeps CPUs from showing up after we populated the cpumask and
 * before the call to on_each_cpu_mask().
 */


void drain_all_pages(struct zone *zone)
{
	int cpu;

	/*
         * Allocate in the BSS so we wont require allocation in
         * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
         */
	static cpumask_t cpus_with_pcps;

	/*
         * We don't care about racing with CPU hotplug event
         * as offline notification will cause the notified
         * cpu to drain that CPU pcps and on_each_cpu_mask
         * disables preemption as part of its processing
         */
	for_each_online_cpu(cpu) {
		struct per_cpu_pageset *pcp;
		struct zone *z;
		bool has_pcps = false;

		if (zone) {
			pcp = per_cpu_ptr(zone->pageset, cpu);
			if (pcp->pcp.count)
				has_pcps = true;
		} else {
			for_each_populated_zone(z) {
				pcp = per_cpu_ptr(z->pageset, cpu);
				if (pcp->pcp.count) {
					has_pcps = true;
					break;
				}
			}
		}

		if (has_pcps)
			cpumask_set_cpu(cpu, &cpus_with_pcps);
		else
			cpumask_clear_cpu(cpu, &cpus_with_pcps);
	}
	on_each_cpu_mask(&cpus_with_pcps, (smp_call_func_t) drain_local_pages,
								zone, 1);
}

Contributors
Person Tokens Prop Commits CommitProp
gilad ben-yossef gilad ben-yossef 77 55.00% 1 33.33%
vlastimil babka vlastimil babka 52 37.14% 1 33.33%
christoph lameter christoph lameter 11 7.86% 1 33.33%
Total 140 100.00% 3 100.00%

#ifdef CONFIG_HIBERNATION



void mark_free_pages(struct zone *zone)
{
	unsigned long pfn, max_zone_pfn;
	unsigned long flags;
	unsigned int order, t;
	struct page *page;

	if (zone_is_empty(zone))
		return;

	spin_lock_irqsave(&zone->lock, flags);

	max_zone_pfn = zone_end_pfn(zone);
	for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
		if (pfn_valid(pfn)) {
			page = pfn_to_page(pfn);

			if (page_zone(page) != zone)
				continue;

			if (!swsusp_page_is_forbidden(page))
				swsusp_unset_page_free(page);
		}

	for_each_migratetype_order(order, t) {
		list_for_each_entry(page,
				&zone->free_area[order].free_list[t], lru) {
			unsigned long i;

			pfn = page_to_pfn(page);
			for (i = 0; i < (1UL << order); i++)
				swsusp_set_page_free(pfn_to_page(pfn + i));
		}
	}
	spin_unlock_irqrestore(&zone->lock, flags);
}
Contributors
Person Tokens Prop Commits CommitProp
pavel machek pavel machek 78 44.32% 3 21.43%
rafael j. wysocki rafael j. wysocki 41 23.30% 2 14.29%
andrew morton andrew morton 20 11.36% 2 14.29%
mel gorman mel gorman 12 6.82% 2 14.29%
joonsoo kim joonsoo kim 10 5.68% 1 7.14%
geliang tang geliang tang 8 4.55% 1 7.14%
cody p schafer cody p schafer 3 1.70% 1 7.14%
xishi qiu xishi qiu 3 1.70% 1 7.14%
rusty russell rusty russell 1 0.57% 1 7.14%
Total 176 100.00% 14 100.00%

#endif /* CONFIG_PM */

/*
 * Free a 0-order page
 * cold == true ? free a cold page : free a hot page
 */


void free_hot_cold_page(struct page *page, bool cold)
{
	struct zone *zone = page_zone(page);
	struct per_cpu_pages *pcp;
	unsigned long flags;
	unsigned long pfn = page_to_pfn(page);
	int migratetype;

	if (!free_pcp_prepare(page))
		return;

	migratetype = get_pfnblock_migratetype(page, pfn);
	set_pcppage_migratetype(page, migratetype);
	local_irq_save(flags);
	__count_vm_event(PGFREE);

	/*
         * We only track unmovable, reclaimable and movable on pcp lists.
         * Free ISOLATE pages back to the allocator because they are being
         * offlined but treat RESERVE as movable pages so we can get those
         * areas back if necessary. Otherwise, we may have to free
         * excessively into the page allocator
         */
	if (migratetype >= MIGRATE_PCPTYPES) {
		if (unlikely(is_migrate_isolate(migratetype))) {
			free_one_page(zone, page, pfn, 0, migratetype);
			goto out;
		}
		migratetype = MIGRATE_MOVABLE;
	}

	pcp = &this_cpu_ptr(zone->pageset)->pcp;
	if (!cold)
		list_add(&page->lru, &pcp->lists[migratetype]);
	else
		list_add_tail(&page->lru, &pcp->lists[migratetype]);
	pcp->count++;
	if (pcp->count >= pcp->high) {
		unsigned long batch = READ_ONCE(pcp->batch);
		free_pcppages_bulk(zone, batch, pcp);
		pcp->count -= batch;
	}

out:
	local_irq_restore(flags);
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 81 36.82% 6 30.00%
andrew morton andrew morton 59 26.82% 1 5.00%
christoph lameter christoph lameter 46 20.91% 4 20.00%
nick piggin nick piggin 14 6.36% 3 15.00%
cody p schafer cody p schafer 10 4.55% 1 5.00%
hugh dickins hugh dickins 3 1.36% 1 5.00%
minchan kim minchan kim 3 1.36% 1 5.00%
kosaki motohiro kosaki motohiro 2 0.91% 1 5.00%
jason low jason low 1 0.45% 1 5.00%
vlastimil babka vlastimil babka 1 0.45% 1 5.00%
Total 220 100.00% 20 100.00%

/*
 * Free a list of 0-order pages
 */


void free_hot_cold_page_list(struct list_head *list, bool cold)
{
	struct page *page, *next;

	list_for_each_entry_safe(page, next, list, lru) {
		trace_mm_page_free_batched(page, cold);
		free_hot_cold_page(page, cold);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
konstantin khlebnikov konstantin khlebnikov 46 97.87% 2 66.67%
mel gorman mel gorman 1 2.13% 1 33.33%
Total 47 100.00% 3 100.00%

/*
 * split_page takes a non-compound higher-order page, and splits it into
 * n (1<<order) sub-pages: page[0..n]
 * Each sub-page must be freed individually.
 *
 * Note: this is probably too low level an operation for use in drivers.
 * Please consult with lkml before using this in your driver.
 */


void split_page(struct page *page, unsigned int order)
{
	int i;

	VM_BUG_ON_PAGE(PageCompound(page), page);
	VM_BUG_ON_PAGE(!page_count(page), page);

#ifdef CONFIG_KMEMCHECK
	/*
         * Split shadow pages too, because free(page[0]) would
         * otherwise free the whole shadow.
         */
	if (kmemcheck_page_is_tracked(page))
		split_page(virt_to_page(page[0].shadow), order);
#endif

	for (i = 1; i < (1 << order); i++)
		set_page_refcounted(page + i);
	split_page_owner(page, order);
}
Contributors
Person Tokens Prop Commits CommitProp
nick piggin nick piggin 55 56.70% 2 33.33%
vegard nossum vegard nossum 28 28.87% 1 16.67%
joonsoo kim joonsoo kim 8 8.25% 2 33.33%
sasha levin sasha levin 6 6.19% 1 16.67%
Total 97 100.00% 6 100.00%


EXPORT_SYMBOL_GPL(split_page);



int __isolate_free_page(struct page *page, unsigned int order)
{
	unsigned long watermark;
	struct zone *zone;
	int mt;

	BUG_ON(!PageBuddy(page));

	zone = page_zone(page);
	mt = get_pageblock_migratetype(page);

	if (!is_migrate_isolate(mt)) {
		/* Obey watermarks as if the page was being allocated */
		watermark = low_wmark_pages(zone) + (1 << order);
		if (!zone_watermark_ok(zone, 0, watermark, 0, 0))
			return 0;

		__mod_zone_freepage_state(zone, -(1UL << order), mt);
	}

	/* Remove page from free list */
	list_del(&page->lru);
	zone->free_area[order].nr_free--;
	rmv_page_order(page);

	/*
         * Set the pageblock if the isolated page is at least half of a
         * pageblock
         */
	if (order >= pageblock_order - 1) {
		struct page *endpage = page + (1 << order) - 1;
		for (; page < endpage; page += pageblock_nr_pages) {
			int mt = get_pageblock_migratetype(page);
			if (!is_migrate_isolate(mt) && !is_migrate_cma(mt))
				set_pageblock_migratetype(page,
							  MIGRATE_MOVABLE);
		}
	}


	return 1UL << order;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 145 71.43% 3 37.50%
marek szyprowski marek szyprowski 26 12.81% 1 12.50%
michal nazarewicz michal nazarewicz 20 9.85% 1 12.50%
minchan kim minchan kim 8 3.94% 1 12.50%
bartlomiej zolnierkiewicz bartlomiej zolnierkiewicz 3 1.48% 1 12.50%
zhong jiang zhong jiang 1 0.49% 1 12.50%
Total 203 100.00% 8 100.00%

/*
 * Update NUMA hit/miss statistics
 *
 * Must be called with interrupts disabled.
 *
 * When __GFP_OTHER_NODE is set assume the node of the preferred
 * zone is the local node. This is useful for daemons who allocate
 * memory on behalf of other processes.
 */


static inline void zone_statistics(struct zone *preferred_zone, struct zone *z,
								gfp_t flags)
{
#ifdef CONFIG_NUMA
	int local_nid = numa_node_id();
	enum zone_stat_item local_stat = NUMA_LOCAL;

	if (unlikely(flags & __GFP_OTHER_NODE)) {
		local_stat = NUMA_OTHER;
		local_nid = preferred_zone->node;
	}

	if (z->node == local_nid) {
		__inc_zone_state(z, NUMA_HIT);
		__inc_zone_state(z, local_stat);
	} else {
		__inc_zone_state(z, NUMA_MISS);
		__inc_zone_state(preferred_zone, NUMA_FOREIGN);
	}
#endif
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 99 100.00% 1 100.00%
Total 99 100.00% 1 100.00%

/*
 * Allocate a page from the given zone. Use pcplists for order-0 allocations.
 */

static inline

struct page *buffered_rmqueue(struct zone *preferred_zone,
			struct zone *zone, unsigned int order,
			gfp_t gfp_flags, unsigned int alloc_flags,
			int migratetype)
{
	unsigned long flags;
	struct page *page;
	bool cold = ((gfp_flags & __GFP_COLD) != 0);

	if (likely(order == 0)) {
		struct per_cpu_pages *pcp;
		struct list_head *list;

		local_irq_save(flags);
		do {
			pcp = &this_cpu_ptr(zone->pageset)->pcp;
			list = &pcp->lists[migratetype];
			if (list_empty(list)) {
				pcp->count += rmqueue_bulk(zone, 0,
						pcp->batch, list,
						migratetype, cold);
				if (unlikely(list_empty(list)))
					goto failed;
			}

			if (cold)
				page = list_last_entry(list, struct page, lru);
			else
				page = list_first_entry(list, struct page, lru);

			list_del(&page->lru);
			pcp->count--;

		} while (check_new_pcp(page));
	} else {
		/*
                 * We most definitely don't want callers attempting to
                 * allocate greater than order-1 page units with __GFP_NOFAIL.
                 */
		WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
		spin_lock_irqsave(&zone->lock, flags);

		do {
			page = NULL;
			if (alloc_flags & ALLOC_HARDER) {
				page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
				if (page)
					trace_mm_page_alloc_zone_locked(page, order, migratetype);
			}
			if (!page)
				page = __rmqueue(zone, order, migratetype);
		} while (page && check_new_pages(page, order));
		spin_unlock(&zone->lock);
		if (!page)
			goto failed;
		__mod_zone_freepage_state(zone, -(1 << order),
					  get_pcppage_migratetype(page));
	}

	__count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
	zone_statistics(preferred_zone, zone, gfp_flags);
	local_irq_restore(flags);

	VM_BUG_ON_PAGE(bad_range(zone, page), page);
	return page;

failed:
	local_irq_restore(flags);
	return NULL;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 135 35.43% 15 37.50%
andrew morton andrew morton 124 32.55% 6 15.00%
nick piggin nick piggin 46 12.07% 3 7.50%
pavel machek pavel machek 21 5.51% 2 5.00%
christoph lameter christoph lameter 16 4.20% 4 10.00%
johannes weiner johannes weiner 11 2.89% 1 2.50%
vlastimil babka vlastimil babka 9 2.36% 1 2.50%
michal hocko michal hocko 7 1.84% 1 2.50%
sasha levin sasha levin 3 0.79% 1 2.50%
li shaohua li shaohua 2 0.52% 1 2.50%
andi kleen andi kleen 2 0.52% 1 2.50%
geliang tang geliang tang 2 0.52% 1 2.50%
linus torvalds linus torvalds 1 0.26% 1 2.50%
al viro al viro 1 0.26% 1 2.50%
rohit seth rohit seth 1 0.26% 1 2.50%
Total 381 100.00% 40 100.00%

#ifdef CONFIG_FAIL_PAGE_ALLOC

static struct {
	
struct fault_attr attr;

	
bool ignore_gfp_highmem;
	
bool ignore_gfp_reclaim;
	
u32 min_order;
} 
fail_page_alloc = {
	.attr = FAULT_ATTR_INITIALIZER,
	.ignore_gfp_reclaim = true,
	.ignore_gfp_highmem = true,
	.min_order = 1,
};



static int __init setup_fail_page_alloc(char *str)
{
	return setup_fault_attr(&fail_page_alloc.attr, str);
}
Contributors
Person Tokens Prop Commits CommitProp
akinobu mita akinobu mita 22 100.00% 1 100.00%
Total 22 100.00% 1 100.00%

__setup("fail_page_alloc=", setup_fail_page_alloc);



static bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
{
	if (order < fail_page_alloc.min_order)
		return false;
	if (gfp_mask & __GFP_NOFAIL)
		return false;
	if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
		return false;
	if (fail_page_alloc.ignore_gfp_reclaim &&
			(gfp_mask & __GFP_DIRECT_RECLAIM))
		return false;

	return should_fail(&fail_page_alloc.attr, 1 << order);
}

Contributors
Person Tokens Prop Commits CommitProp
akinobu mita akinobu mita 69 90.79% 2 40.00%
gavin shan gavin shan 5 6.58% 1 20.00%
mel gorman mel gorman 2 2.63% 2 40.00%
Total 76 100.00% 5 100.00%

#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS



static int __init fail_page_alloc_debugfs(void)
{
	umode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
	struct dentry *dir;

	dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
					&fail_page_alloc.attr);
	if (IS_ERR(dir))
		return PTR_ERR(dir);

	if (!debugfs_create_bool("ignore-gfp-wait", mode, dir,
				&fail_page_alloc.ignore_gfp_reclaim))
		goto fail;
	if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir,
				&fail_page_alloc.ignore_gfp_highmem))
		goto fail;
	if (!debugfs_create_u32("min-order", mode, dir,
				&fail_page_alloc.min_order))
		goto fail;

	return 0;
fail:
	debugfs_remove_recursive(dir);

	return -ENOMEM;
}

Contributors
Person Tokens Prop Commits CommitProp
akinobu mita akinobu mita 122 98.39% 4 66.67%
al viro al viro 1 0.81% 1 16.67%
mel gorman mel gorman 1 0.81% 1 16.67%
Total 124 100.00% 6 100.00%


late_initcall(fail_page_alloc_debugfs);

#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */

#else /* CONFIG_FAIL_PAGE_ALLOC */



static inline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
{
	return false;
}

Contributors
Person Tokens Prop Commits CommitProp
akinobu mita akinobu mita 15 88.24% 1 50.00%
gavin shan gavin shan 2 11.76% 1 50.00%
Total 17 100.00% 2 100.00%

#endif /* CONFIG_FAIL_PAGE_ALLOC */

/*
 * Return true if free base pages are above 'mark'. For high-order checks it
 * will return true of the order-0 watermark is reached and there is at least
 * one free page of a suitable size. Checking now avoids taking the zone lock
 * to check in the allocation paths if no pages are free.
 */


bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
			 int classzone_idx, unsigned int alloc_flags,
			 long free_pages)
{
	long min = mark;
	int o;
	const bool alloc_harder = (alloc_flags & ALLOC_HARDER);

	/* free_pages may go negative - that's OK */
	free_pages -= (1 << order) - 1;

	if (alloc_flags & ALLOC_HIGH)
		min -= min / 2;

	/*
         * If the caller does not have rights to ALLOC_HARDER then subtract
         * the high-atomic reserves. This will over-estimate the size of the
         * atomic reserve but it avoids a search.
         */
	if (likely(!alloc_harder))
		free_pages -= z->nr_reserved_highatomic;
	else
		min -= min / 4;

#ifdef CONFIG_CMA
	/* If allocation can't use CMA areas don't use free CMA pages */
	if (!(alloc_flags & ALLOC_CMA))
		free_pages -= zone_page_state(z, NR_FREE_CMA_PAGES);
#endif

	/*
         * Check watermarks for an order-0 allocation request. If these
         * are not met, then a high-order request also cannot go ahead
         * even if a suitable page happened to be free.
         */
	if (free_pages <= min + z->lowmem_reserve[classzone_idx])
		return false;

	/* If this is an order-0 request then the watermark is fine */
	if (!order)
		return true;

	/* For a high-order request, check at least one suitable page is free */
	for (o = order; o < MAX_ORDER; o++) {
		struct free_area *area = &z->free_area[o];
		int mt;

		if (!area->nr_free)
			continue;

		if (alloc_harder)
			return true;

		for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
			if (!list_empty(&area->free_list[mt]))
				return true;
		}

#ifdef CONFIG_CMA
		if ((alloc_flags & ALLOC_CMA) &&
		    !list_empty(&area->free_list[MIGRATE_CMA])) {
			return true;
		}
#endif
	}
	return false;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 139 54.94% 6 46.15%
nick piggin nick piggin 82 32.41% 1 7.69%
bartlomiej zolnierkiewicz bartlomiej zolnierkiewicz 22 8.70% 1 7.69%
rohit seth rohit seth 4 1.58% 1 7.69%
tomasz stanislawski tomasz stanislawski 2 0.79% 1 7.69%
andrea arcangeli andrea arcangeli 2 0.79% 1 7.69%
christoph lameter christoph lameter 1 0.40% 1 7.69%
michal hocko michal hocko 1 0.40% 1 7.69%
Total 253 100.00% 13 100.00%



bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
		      int classzone_idx, unsigned int alloc_flags)
{
	return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
					zone_page_state(z, NR_FREE_PAGES));
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 44 95.65% 3 75.00%
nick piggin nick piggin 2 4.35% 1 25.00%
Total 46 100.00% 4 100.00%



static inline bool zone_watermark_fast(struct zone *z, unsigned int order,
		unsigned long mark, int classzone_idx, unsigned int alloc_flags)
{
	long free_pages = zone_page_state(z, NR_FREE_PAGES);
	long cma_pages = 0;

#ifdef CONFIG_CMA
	/* If allocation can't use CMA areas don't use free CMA pages */
	if (!(alloc_flags & ALLOC_CMA))
		cma_pages = zone_page_state(z, NR_FREE_CMA_PAGES);
#endif

	/*
         * Fast check for order-0 only. If this fails then the reserves
         * need to be calculated. There is a corner case where the check
         * passes but only the high-order atomic reserve are free. If
         * the caller is !atomic then it'll uselessly search the free
         * list. That corner case is then slower but it is harmless.
         */
	if (!order && (free_pages - cma_pages) > mark + z->lowmem_reserve[classzone_idx])
		return true;

	return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
					free_pages);
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 106 100.00% 1 100.00%
Total 106 100.00% 1 100.00%



bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
			unsigned long mark, int classzone_idx)
{
	long free_pages = zone_page_state(z, NR_FREE_PAGES);

	if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
		free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);

	return __zone_watermark_ok(z, order, mark, classzone_idx, 0,
								free_pages);
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 65 95.59% 3 75.00%
nick piggin nick piggin 3 4.41% 1 25.00%
Total 68 100.00% 4 100.00%

#ifdef CONFIG_NUMA


static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
{
	return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <
				RECLAIM_DISTANCE;
}
Contributors
Person Tokens Prop Commits CommitProp
david rientjes david rientjes 24 75.00% 1 50.00%
mel gorman mel gorman 8 25.00% 1 50.00%
Total 32 100.00% 2 100.00%

#else	/* CONFIG_NUMA */


static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
{
	return true;
}
Contributors
Person Tokens Prop Commits CommitProp
david rientjes david rientjes 19 100.00% 1 100.00%
Total 19 100.00% 1 100.00%

#endif	/* CONFIG_NUMA */

/*
 * get_page_from_freelist goes through the zonelist trying to allocate
 * a page.
 */

static struct page *

get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
						const struct alloc_context *ac)
{
	struct zoneref *z = ac->preferred_zoneref;
	struct zone *zone;
	struct pglist_data *last_pgdat_dirty_limit = NULL;

	/*
         * Scan zonelist, looking for a zone with enough free.
         * See also __cpuset_node_allowed() comment in kernel/cpuset.c.
         */
	for_next_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
								ac->nodemask) {
		struct page *page;
		unsigned long mark;

		if (cpusets_enabled() &&
			(alloc_flags & ALLOC_CPUSET) &&
			!__cpuset_zone_allowed(zone, gfp_mask))
				continue;
		/*
                 * When allocating a page cache page for writing, we
                 * want to get it from a node that is within its dirty
                 * limit, such that no single node holds more than its
                 * proportional share of globally allowed dirty pages.
                 * The dirty limits take into account the node's
                 * lowmem reserves and high watermark so that kswapd
                 * should be able to balance it without having to
                 * write pages from its LRU list.
                 *
                 * XXX: For now, allow allocations to potentially
                 * exceed the per-node dirty limit in the slowpath
                 * (spread_dirty_pages unset) before going into reclaim,
                 * which is important when on a NUMA setup the allowed
                 * nodes are together not big enough to reach the
                 * global limit.  The proper fix for these situations
                 * will require awareness of nodes in the
                 * dirty-throttling and the flusher threads.
                 */
		if (ac->spread_dirty_pages) {
			if (last_pgdat_dirty_limit == zone->zone_pgdat)
				continue;

			if (!node_dirty_ok(zone->zone_pgdat)) {
				last_pgdat_dirty_limit = zone->zone_pgdat;
				continue;
			}
		}

		mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK];
		if (!zone_watermark_fast(zone, order, mark,
				       ac_classzone_idx(ac), alloc_flags)) {
			int ret;

			/* Checked here to keep the fast path fast */
			BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
			if (alloc_flags & ALLOC_NO_WATERMARKS)
				goto try_this_zone;

			if (node_reclaim_mode == 0 ||
			    !zone_allows_reclaim(ac->preferred_zoneref->zone, zone))
				continue;

			ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);
			switch (ret) {
			case NODE_RECLAIM_NOSCAN:
				/* did not scan */
				continue;
			case NODE_RECLAIM_FULL:
				/* scanned but unreclaimable */
				continue;
			default:
				/* did we reclaim enough */
				if (zone_watermark_ok(zone, order, mark,
						ac_classzone_idx(ac), alloc_flags))
					goto try_this_zone;

				continue;
			}
		}

try_this_zone:
		page = buffered_rmqueue(ac->preferred_zoneref->zone, zone, order,
				gfp_mask, alloc_flags, ac->migratetype);
		if (page) {
			prep_new_page(page, order, gfp_mask, alloc_flags);

			/*
                         * If this is a high-order atomic allocation then check
                         * if the pageblock should be reserved for the future
                         */
			if (unlikely(order && (alloc_flags & ALLOC_HARDER)))
				reserve_highatomic_pageblock(page, zone, order);

			return page;
		}
	}

	return NULL;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 175 54.52% 25 60.98%
rohit seth rohit seth 45 14.02% 1 2.44%
vlastimil babka vlastimil babka 29 9.03% 3 7.32%
christoph lameter christoph lameter 18 5.61% 2 4.88%
johannes weiner johannes weiner 16 4.98% 2 4.88%
nick piggin nick piggin 14 4.36% 2 4.88%
martin hicks martin hicks 12 3.74% 2 4.88%
david rientjes david rientjes 7 2.18% 1 2.44%
paul jackson paul jackson 4 1.25% 2 4.88%
vladimir davydov vladimir davydov 1 0.31% 1 2.44%
Total 321 100.00% 41 100.00%

/*
 * Large machines with many possible nodes should not always dump per-node
 * meminfo in irq context.
 */


static inline bool should_suppress_show_mem(void)
{
	bool ret = false;

#if NODES_SHIFT > 8
	ret = in_interrupt();
#endif
	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
david rientjes david rientjes 29 100.00% 1 100.00%
Total 29 100.00% 1 100.00%

static DEFINE_RATELIMIT_STATE(nopage_rs,
		DEFAULT_RATELIMIT_INTERVAL,
		DEFAULT_RATELIMIT_BURST);



void warn_alloc_failed(gfp_t gfp_mask, unsigned int order, const char *fmt, ...)
{
	unsigned int filter = SHOW_MEM_FILTER_NODES;

	if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) ||
	    debug_guardpage_minorder() > 0)
		return;

	/*
         * This documents exceptions given to allocations in certain
         * contexts that are allowed to allocate outside current's set
         * of allowed nodes.
         */
	if (!(gfp_mask & __GFP_NOMEMALLOC))
		if (test_thread_flag(TIF_MEMDIE) ||
		    (current->flags & (PF_MEMALLOC | PF_EXITING)))
			filter &= ~SHOW_MEM_FILTER_NODES;
	if (in_interrupt() || !(gfp_mask & __GFP_DIRECT_RECLAIM))
		filter &= ~SHOW_MEM_FILTER_NODES;

	if (fmt) {
		struct va_format vaf;
		va_list args;

		va_start(args, fmt);

		vaf.fmt = fmt;
		vaf.va = &args;

		pr_warn("%pV", &vaf);

		va_end(args);
	}

	pr_warn("%s: page allocation failure: order:%u, mode:%#x(%pGg)\n",
		current->comm, order, gfp_mask, &gfp_mask);
	dump_stack();
	if (!should_suppress_show_mem())
		show_mem(filter);
}

Contributors
Person Tokens Prop Commits CommitProp
dave hansen dave hansen 137 79.19% 1 16.67%
joe perches joe perches 25 14.45% 1 16.67%
stanislaw gruszka stanislaw gruszka 5 2.89% 1 16.67%
vlastimil babka vlastimil babka 4 2.31% 1 16.67%
kirill a. shutemov kirill a. shutemov 1 0.58% 1 16.67%
mel gorman mel gorman 1 0.58% 1 16.67%
Total 173 100.00% 6 100.00%


static inline struct page *

__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
	const struct alloc_context *ac, unsigned long *did_some_progress)
{
	struct oom_control oc = {
		.zonelist = ac->zonelist,
		.nodemask = ac->nodemask,
		.memcg = NULL,
		.gfp_mask = gfp_mask,
		.order = order,
        };
	struct page *page;

	*did_some_progress = 0;

	/*
         * Acquire the oom lock.  If that fails, somebody else is
         * making progress for us.
         */
	if (!mutex_trylock(&oom_lock)) {
		*did_some_progress = 1;
		schedule_timeout_uninterruptible(1);
		return NULL;
	}

	/*
         * Go through the zonelist yet one more time, keep very high watermark
         * here, this is only to catch a parallel oom killing, we must fail if
         * we're still under heavy pressure.
         */
	page = get_page_from_freelist(gfp_mask | __GFP_HARDWALL, order,
					ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac);
	if (page)
		goto out;

	if (!(gfp_mask & __GFP_NOFAIL)) {
		/* Coredumps can quickly deplete all memory reserves */
		if (current->flags & PF_DUMPCORE)
			goto out;
		/* The OOM killer will not help higher order allocs */
		if (order > PAGE_ALLOC_COSTLY_ORDER)
			goto out;
		/* The OOM killer does not needlessly kill tasks for lowmem */
		if (ac->high_zoneidx < ZONE_NORMAL)
			goto out;
		if (pm_suspended_storage())
			goto out;
		/*
                 * XXX: GFP_NOFS allocations should rather fail than rely on
                 * other request to make a forward progress.
                 * We are in an unfortunate situation where out_of_memory cannot
                 * do much for this context but let's try it to at least get
                 * access to memory reserved if the current task is killed (see
                 * out_of_memory). Once filesystems are ready to handle allocation
                 * failures more gracefully we should just bail out here.
                 */

		/* The OOM killer may not free memory on a specific node */
		if (gfp_mask & __GFP_THISNODE)
			goto out;
	}
	/* Exhausted what can be done so it's blamo time */
	if (out_of_memory(&oc) || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {
		*did_some_progress = 1;

		if (gfp_mask & __GFP_NOFAIL) {
			page = get_page_from_freelist(gfp_mask, order,
					ALLOC_NO_WATERMARKS|ALLOC_CPUSET, ac);
			/*
                         * fallback to ignore cpuset restriction if our nodes
                         * are depleted
                         */
			if (!page)
				page = get_page_from_freelist(gfp_mask, order,
					ALLOC_NO_WATERMARKS, ac);
		}
	}
out:
	mutex_unlock(&oom_lock);
	return page;
}

Contributors
Person Tokens Prop Commits CommitProp
michal hocko michal hocko 55 20.99% 4 13.33%
johannes weiner johannes weiner 47 17.94% 3 10.00%
david rientjes david rientjes 45 17.18% 4 13.33%
mel gorman mel gorman 29 11.07% 3 10.00%
kamezawa hiroyuki kamezawa hiroyuki 18 6.87% 1 3.33%
pre-git pre-git 11 4.20% 2 6.67%
rohit seth rohit seth 10 3.82% 1 3.33%
christoph lameter christoph lameter 9 3.44% 1 3.33%
martin hicks martin hicks 8 3.05% 1 3.33%
vlastimil babka vlastimil babka 6 2.29% 1 3.33%
linus torvalds linus torvalds 6 2.29% 2 6.67%
akinobu mita akinobu mita 5 1.91% 1 3.33%
vladimir davydov vladimir davydov 5 1.91% 1 3.33%
nick piggin nick piggin 3 1.15% 2 6.67%
martin j. bligh martin j. bligh 3 1.15% 1 3.33%
al viro al viro 1 0.38% 1 3.33%
andrew morton andrew morton 1 0.38% 1 3.33%
Total 262 100.00% 30 100.00%

/*
 * Maximum number of compaction retries wit a progress before OOM
 * killer is consider as the only way to move forward.
 */

#define MAX_COMPACT_RETRIES 16

#ifdef CONFIG_COMPACTION
/* Try memory compaction for high-order allocations before reclaim */

static struct page *

__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
		unsigned int alloc_flags, const struct alloc_context *ac,
		enum compact_priority prio, enum compact_result *compact_result)
{
	struct page *page;

	if (!order)
		return NULL;

	current->flags |= PF_MEMALLOC;
	*compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
									prio);
	current->flags &= ~PF_MEMALLOC;

	if (*compact_result <= COMPACT_INACTIVE)
		return NULL;

	/*
         * At least in one zone compaction wasn't deferred or skipped, so let's
         * count a compaction stall
         */
	count_vm_event(COMPACTSTALL);

	page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);

	if (page) {
		struct zone *zone = page_zone(page);

		zone->compact_blockskip_flush = false;
		compaction_defer_reset(zone, order, true);
		count_vm_event(COMPACTSUCCESS);
		return page;
	}

	/*
         * It's bad if compaction run occurs and fails. The most likely reason
         * is that pages exist, but not enough to satisfy watermarks.
         */
	count_vm_event(COMPACTFAIL);

	cond_resched();

	return NULL;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 87 55.77% 7 38.89%
vlastimil babka vlastimil babka 53 33.97% 6 33.33%
michal hocko michal hocko 10 6.41% 2 11.11%
rik van riel rik van riel 3 1.92% 1 5.56%
andrew morton andrew morton 2 1.28% 1 5.56%
david rientjes david rientjes 1 0.64% 1 5.56%
Total 156 100.00% 18 100.00%

#else

static inline struct page *

__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
		unsigned int alloc_flags, const struct alloc_context *ac,
		enum compact_priority prio, enum compact_result *compact_result)
{
	*compact_result = COMPACT_SKIPPED;
	return NULL;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 29 67.44% 4 44.44%
michal hocko michal hocko 8 18.60% 2 22.22%
vlastimil babka vlastimil babka 5 11.63% 2 22.22%
david rientjes david rientjes 1 2.33% 1 11.11%
Total 43 100.00% 9 100.00%

#endif /* CONFIG_COMPACTION */


static inline bool

should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags,
		     enum compact_result compact_result,
		     enum compact_priority *compact_priority,
		     int compaction_retries)
{
	struct zone *zone;
	struct zoneref *z;

	if (!order || order > PAGE_ALLOC_COSTLY_ORDER)
		return false;

	/*
         * There are setups with compaction disabled which would prefer to loop
         * inside the allocator rather than hit the oom killer prematurely.
         * Let's give them a good hope and keep retrying while the order-0
         * watermarks are OK.
         */
	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
					ac->nodemask) {
		if (zone_watermark_ok(zone, 0, min_wmark_pages(zone),
					ac_classzone_idx(ac), alloc_flags))
			return true;
	}
	return false;
}

Contributors
Person Tokens Prop Commits CommitProp
michal hocko michal hocko 93 97.89% 3 75.00%
vlastimil babka vlastimil babka 2 2.11% 1 25.00%
Total 95 100.00% 4 100.00%

/* Perform direct synchronous page reclaim */

static int

__perform_reclaim(gfp_t gfp_mask, unsigned int order,
					const struct alloc_context *ac)
{
	struct reclaim_state reclaim_state;
	int progress;

	cond_resched();

	/* We now go into synchronous reclaim */
	cpuset_memory_pressure_bump();
	current->flags |= PF_MEMALLOC;
	lockdep_set_current_reclaim_state(gfp_mask);
	reclaim_state.reclaimed_slab = 0;
	current->reclaim_state = &reclaim_state;

	progress = try_to_free_pages(ac->zonelist, order, gfp_mask,
								ac->nodemask);

	current->reclaim_state = NULL;
	lockdep_clear_current_reclaim_state();
	current->flags &= ~PF_MEMALLOC;

	cond_resched();

	return progress;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 73 76.04% 2 33.33%
vlastimil babka vlastimil babka 7 7.29% 1 16.67%
marek szyprowski marek szyprowski 7 7.29% 1 16.67%
rohit seth rohit seth 5 5.21% 1 16.67%
andrew morton andrew morton 4 4.17% 1 16.67%
Total 96 100.00% 6 100.00%

/* The really slow allocator path where we enter direct reclaim */

static inline struct page *

__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
		unsigned int alloc_flags, const struct alloc_context *ac,
		unsigned long *did_some_progress)
{
	struct page *page = NULL;
	bool drained = false;

	*did_some_progress = __perform_reclaim(gfp_mask, order, ac);
	if (unlikely(!(*did_some_progress)))
		return NULL;

retry:
	page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);

	/*
         * If an allocation failed after direct reclaim, it could be because
         * pages are pinned on the per-cpu lists or in high alloc reserves.
         * Shrink them them and try again
         */
	if (!page && !drained) {
		unreserve_highatomic_pageblock(ac);
		drain_all_pages(NULL);
		drained = true;
		goto retry;
	}

	return page;
}

Contributors
Person Tokens Prop Commits CommitProp
marek szyprowski marek szyprowski 49 42.61% 1 7.14%
mel gorman mel gorman 44 38.26% 5 35.71%
vlastimil babka vlastimil babka 8 6.96% 2 14.29%
pre-git pre-git 7 6.09% 2 14.29%
rohit seth rohit seth 3 2.61% 1 7.14%
linus torvalds linus torvalds 2 1.74% 1 7.14%
andrew morton andrew morton 1 0.87% 1 7.14%
paul jackson paul jackson 1 0.87% 1 7.14%
Total 115 100.00% 14 100.00%



static void wake_all_kswapds(unsigned int order, const struct alloc_context *ac)
{
	struct zoneref *z;
	struct zone *zone;
	pg_data_t *last_pgdat = NULL;

	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
					ac->high_zoneidx, ac->nodemask) {
		if (last_pgdat != zone->zone_pgdat)
			wakeup_kswapd(zone, order, ac->high_zoneidx);
		last_pgdat = zone->zone_pgdat;
	}
}

Contributors
Person Tokens Prop Commits CommitProp
johannes weiner johannes weiner 38 53.52% 1 20.00%
mel gorman mel gorman 24 33.80% 2 40.00%
vlastimil babka vlastimil babka 7 9.86% 1 20.00%
weijie yang weijie yang 2 2.82% 1 20.00%
Total 71 100.00% 5 100.00%


static inline unsigned int

gfp_to_alloc_flags(gfp_t gfp_mask)
{
	unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;

	/* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */
	BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);

	/*
         * The caller may dip into page reserves a bit more if the caller
         * cannot run direct reclaim, or if the caller has realtime scheduling
         * policy or is asking for __GFP_HIGH memory.  GFP_ATOMIC requests will
         * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH).
         */
	alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH);

	if (gfp_mask & __GFP_ATOMIC) {
		/*
                 * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
                 * if it can't schedule.
                 */
		if (!(gfp_mask & __GFP_NOMEMALLOC))
			alloc_flags |= ALLOC_HARDER;
		/*
                 * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the
                 * comment for __cpuset_node_allowed().
                 */
		alloc_flags &= ~ALLOC_CPUSET;
	} else if (unlikely(rt_task(current)) && !in_interrupt())
		alloc_flags |= ALLOC_HARDER;

#ifdef CONFIG_CMA
	if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
		alloc_flags |= ALLOC_CMA;
#endif
	return alloc_flags;
}

Contributors
Person Tokens Prop Commits CommitProp
peter zijlstra peter zijlstra 48 42.86% 1 8.33%
mel gorman mel gorman 22 19.64% 5 41.67%
vlastimil babka vlastimil babka 20 17.86% 1 8.33%
namhyung kim namhyung kim 10 8.93% 1 8.33%
andrea arcangeli andrea arcangeli 9 8.04% 1 8.33%
vladimir davydov vladimir davydov 1 0.89% 1 8.33%
david rientjes david rientjes 1 0.89% 1 8.33%
andrew morton andrew morton 1 0.89% 1 8.33%
Total 112 100.00% 12 100.00%



bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
{
	if (unlikely(gfp_mask & __GFP_NOMEMALLOC))
		return false;

	if (gfp_mask & __GFP_MEMALLOC)
		return true;
	if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
		return true;
	if (!in_interrupt() &&
			((current->flags & PF_MEMALLOC) ||
			 unlikely(test_thread_flag(TIF_MEMDIE))))
		return true;

	return false;
}

Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 48 64.00% 3 50.00%
vlastimil babka vlastimil babka 21 28.00% 1 16.67%
peter zijlstra peter zijlstra 5 6.67% 1 16.67%
pre-git pre-git 1 1.33% 1 16.67%
Total 75 100.00% 6 100.00%

/*
 * Maximum number of reclaim retries without any progress before OOM killer
 * is consider as the only way to move forward.
 */

#define MAX_RECLAIM_RETRIES 16

/*
 * Checks whether it makes sense to retry the reclaim to make a forward progress
 * for the given allocation request.
 * The reclaim feedback represented by did_some_progress (any progress during
 * the last reclaim round) and no_progress_loops (number of reclaim rounds without
 * any progress in a row) is considered as well as the reclaimable pages on the
 * applicable zone list (with a backoff mechanism which is a function of
 * no_progress_loops).
 *
 * Returns true if a retry is viable or false to enter the oom path.
 */

static inline bool

should_reclaim_retry(gfp_t gfp_mask, unsigned order,
		     struct alloc_context *ac, int alloc_flags,
		     bool did_some_progress, int no_progress_loops)
{
	struct zone *zone;
	struct zoneref *z;

	/*
         * Make sure we converge to OOM if we cannot make any progress
         * several times in the row.
         */
	if (no_progress_loops > MAX_RECLAIM_RETRIES)
		return false;

	/*
         * Keep reclaiming pages while there is a chance this will lead
         * somewhere.  If none of the target zones can satisfy our allocation
         * request even if all reclaimable pages are considered then we are
         * screwed and have to go OOM.
         */
	for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ac->high_zoneidx,
					ac->nodemask) {
		unsigned long available;
		unsigned long reclaimable;

		available = reclaimable = zone_reclaimable_pages(zone);
		available -= DIV_ROUND_UP(no_progress_loops * available,
					  MAX_RECLAIM_RETRIES);
		available += zone_page_state_snapshot(zone, NR_FREE_PAGES);

		/*
                 * Would the allocation succeed if we reclaimed the whole
                 * available?
                 */
		if (__zone_watermark_ok(zone, order, min_wmark_pages(zone),
				ac_classzone_idx(ac), alloc_flags, available)) {
			/*
                         * If we didn't make any progress and have a lot of
                         * dirty + writeback pages then we should wait for
                         * an IO to complete to slow down the reclaim and
                         * prevent from pre mature OOM
                         */
			if (!did_some_progress) {
				unsigned long write_pending;

				write_pending = zone_page_state_snapshot(zone,
							NR_ZONE_WRITE_PENDING);

				if (2 * write_pending > reclaimable) {
					congestion_wait(BLK_RW_ASYNC, HZ/10);
					return true;
				}
			}

			/*
                         * Memory allocation/reclaim might be called from a WQ
                         * context and the current implementation of the WQ
                         * concurrency control doesn't recognize that
                         * a particular WQ is congested if the worker thread is
                         * looping without ever sleeping. Therefore we have to
                         * do a short sleep here rather than calling
                         * cond_resched().
                         */
			if (current->flags & PF_WQ_WORKER)
				schedule_timeout_uninterruptible(1);
			else
				cond_resched();

			return true;
		}
	}

	return false;
}

Contributors
Person Tokens Prop Commits CommitProp
michal hocko michal hocko 171 89.06% 2 40.00%
mel gorman mel gorman 21 10.94% 3 60.00%
Total 192 100.00% 5 100.00%


static inline struct page *

__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
						struct alloc_context *ac)
{
	bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;
	struct page *page = NULL;
	unsigned int alloc_flags;
	unsigned long did_some_progress;
	enum compact_priority compact_priority = DEF_COMPACT_PRIORITY;
	enum compact_result compact_result;
	int compaction_retries = 0;
	int no_progress_loops = 0;

	/*
         * In the slowpath, we sanity check order to avoid ever trying to
         * reclaim >= MAX_ORDER areas which will never succeed. Callers may
         * be using allocators in order of preference for an area that is
         * too large.
         */
	if (order >= MAX_ORDER) {
		WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
		return NULL;
	}

	/*
         * We also sanity check to catch abuse of atomic reserves being used by
         * callers that are not in atomic context.
         */
	if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) ==
				(__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)))
		gfp_mask &= ~__GFP_ATOMIC;

	/*
         * The fast path uses conservative alloc_flags to succeed only until
         * kswapd needs to be woken up, and to avoid the cost of setting up
         * alloc_flags precisely. So we do that now.
         */
	alloc_flags = gfp_to_alloc_flags(gfp_mask);

	if (gfp_mask & __GFP_KSWAPD_RECLAIM)
		wake_all_kswapds(order, ac);

	/*
         * The adjusted alloc_flags might result in immediate success, so try
         * that first
         */
	page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
	if (page)
		goto got_pg;

	/*
         * For costly allocations, try direct compaction first, as it's likely
         * that we have enough base pages and don't need to reclaim. Don't try
         * that for allocations that are allowed to ignore watermarks, as the
         * ALLOC_NO_WATERMARKS attempt didn't yet happen.
         */
	if (can_direct_reclaim && order > PAGE_ALLOC_COSTLY_ORDER &&
		!gfp_pfmemalloc_allowed(gfp_mask)) {
		page = __alloc_pages_direct_compact(gfp_mask, order,
						alloc_flags, ac,
						INIT_COMPACT_PRIORITY,
						&compact_result);
		if (page)
			goto got_pg;

		/*
                 * Checks for costly allocations with __GFP_NORETRY, which
                 * includes THP page fault allocations
                 */
		if (gfp_mask & __GFP_NORETRY) {
			/*
                         * If compaction is deferred for high-order allocations,
                         * it is because sync compaction recently failed. If
                         * this is the case and the caller requested a THP
                         * allocation, we do not want to heavily disrupt the
                         * system, so we fail the allocation instead of entering
                         * direct reclaim.
                         */
			if (compact_result == COMPACT_DEFERRED)
				goto nopage;

			/*
                         * Looks like reclaim/compaction is worth trying, but
                         * sync compaction could be very expensive, so keep
                         * using async compaction.
                         */
			compact_priority = INIT_COMPACT_PRIORITY;
		}
	}

retry:
	/* Ensure kswapd doesn't accidentally go to sleep as long as we loop */
	if (gfp_mask & __GFP_KSWAPD_RECLAIM)
		wake_all_kswapds(order, ac);

	if (gfp_pfmemalloc_allowed(gfp_mask))
		alloc_flags = ALLOC_NO_WATERMARKS;

	/*
         * Reset the zonelist iterators if memory policies can be ignored.
         * These allocations are high priority and system rather than user
         * orientated.
         */
	if (!(alloc_flags & ALLOC_CPUSET) || (alloc_flags & ALLOC_NO_WATERMARKS)) {
		ac->zonelist = node_zonelist(numa_node_id(), gfp_mask);
		ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
					ac->high_zoneidx, ac->nodemask);
	}

	/* Attempt with potentially adjusted zonelist and alloc_flags */
	page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
	if (page)
		goto got_pg;

	/* Caller is not willing to reclaim, we can't balance anything */
	if (!can_direct_reclaim) {
		/*
                 * All existing users of the __GFP_NOFAIL are blockable, so warn
                 * of any new users that actually allow this type of allocation
                 * to fail.
                 */
		WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL);
		goto nopage;
	}

	/* Avoid recursion of direct reclaim */
	if (current->flags & PF_MEMALLOC) {
		/*
                 * __GFP_NOFAIL request from this context is rather bizarre
                 * because we cannot reclaim anything and only can loop waiting
                 * for somebody to do a work for us.
                 */
		if (WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) {
			cond_resched();
			goto retry;
		}
		goto nopage;
	}

	/* Avoid allocations with no watermarks from looping endlessly */
	if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL))
		goto nopage;


	/* Try direct reclaim and then allocating */
	page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac,
							&did_some_progress);
	if (page)
		goto got_pg;

	/* Try direct compaction and then allocating */
	page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac,
					compact_priority, &compact_result);
	if (page)
		goto got_pg;

	if (order && compaction_made_progress(compact_result))
		compaction_retries++;

	/* Do not loop if specifically requested */
	if (gfp_mask & __GFP_NORETRY)
		goto nopage;

	/*
         * Do not retry costly high order allocations unless they are
         * __GFP_REPEAT
         */
	if (order > PAGE_ALLOC_COSTLY_ORDER && !(gfp_mask & __GFP_REPEAT))
		goto nopage;

	/*
         * Costly allocations might have made a progress but this doesn't mean
         * their order will become available due to high fragmentation so
         * always increment the no progress counter for them
         */
	if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER)
		no_progress_loops = 0;
	else
		no_progress_loops++;

	if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags,
				 did_some_progress > 0, no_progress_loops))
		goto retry;

	/*
         * It doesn't make any sense to retry for the compaction if the order-0
         * reclaim is not able to make any progress because the current
         * implementation of the compaction depends on the sufficient amount
         * of free memory (see __compaction_suitable)
         */
	if (did_some_progress > 0 &&
			should_compact_retry(ac, order, alloc_flags,
				compact_result, &compact_priority,
				compaction_retries))
		goto retry;

	/* Reclaim has failed us, start killing things */
	page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress);
	if (page)
		goto got_pg;

	/* Retry as long as the OOM killer is making progress */
	if (did_some_progress) {
		no_progress_loops = 0;
		goto retry;
	}

nopage:
	warn_alloc_failed(gfp_mask, order, NULL);
got_pg:
	return page;
}

Contributors
Person Tokens Prop Commits CommitProp
vlastimil babka vlastimil babka 160 27.03% 7 10.94%
mel gorman mel gorman 154 26.01% 15 23.44%
michal hocko michal hocko 102 17.23% 6 9.38%
johannes weiner johannes weiner 38 6.42% 4 6.25%
david rientjes david rientjes 31 5.24% 5 7.81%
andrew morton andrew morton 30 5.07% 11 17.19%
pre-git pre-git 26 4.39% 6 9.38%
andrea arcangeli andrea arcangeli 19 3.21% 1 1.56%
peter zijlstra peter zijlstra 8 1.35% 1 1.56%
rohit seth rohit seth 7 1.18% 1 1.56%
linus torvalds linus torvalds 5 0.84% 2 3.12%
nishanth aravamudan nishanth aravamudan 5 0.84% 1 1.56%
dave hansen dave hansen 3 0.51% 1 1.56%
nick piggin nick piggin 2 0.34% 1 1.56%
janet morgan janet morgan 1 0.17% 1 1.56%
qiang huang qiang huang 1 0.17% 1 1.56%
Total 592 100.00% 64 100.00%

/*
 * This is the 'heart' of the zoned buddy allocator.
 */

struct page *

__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
			struct zonelist *zonelist, nodemask_t *nodemask)
{
	struct page *page;
	unsigned int cpuset_mems_cookie;
	unsigned int alloc_flags = ALLOC_WMARK_LOW;
	gfp_t alloc_mask = gfp_mask; /* The gfp_t that was actually used for allocation */
	struct alloc_context ac = {
		.high_zoneidx = gfp_zone(gfp_mask),
		.zonelist = zonelist,
		.nodemask = nodemask,
		.migratetype = gfpflags_to_migratetype(gfp_mask),
        };

	if (cpusets_enabled()) {
		alloc_mask |= __GFP_HARDWALL;
		alloc_flags |= ALLOC_CPUSET;
		if (!ac.nodemask)
			ac.nodemask = &cpuset_current_mems_allowed;
	}

	gfp_mask &= gfp_allowed_mask;

	lockdep_trace_alloc(gfp_mask);

	might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM);

	if (should_fail_alloc_page(gfp_mask, order))
		return NULL;

	/*
         * Check the zones suitable for the gfp_mask contain at least one
         * valid zone. It's possible to have an empty zonelist as a result
         * of __GFP_THISNODE and a memoryless node
         */
	if (unlikely(!zonelist->_zonerefs->zone))
		return NULL;

	if (IS_ENABLED(CONFIG_CMA) && ac.migratetype == MIGRATE_MOVABLE)
		alloc_flags |= ALLOC_CMA;

retry_cpuset:
	cpuset_mems_cookie = read_mems_allowed_begin();

	/* Dirty zone balancing only done in the fast path */
	ac.spread_dirty_pages = (gfp_mask & __GFP_WRITE);

	/*
         * The preferred zone is used for statistics but crucially it is
         * also used as the starting point for the zonelist iterator. It
         * may get reset for allocations that ignore memory policies.
         */
	ac.preferred_zoneref = first_zones_zonelist(ac.zonelist,
					ac.high_zoneidx, ac.nodemask);
	if (!ac.preferred_zoneref) {
		page = NULL;
		goto no_zone;
	}

	/* First allocation attempt */
	page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac);
	if (likely(page))
		goto out;

	/*
         * Runtime PM, block IO and its error handling path can deadlock
         * because I/O on the device might not complete.
         */
	alloc_mask = memalloc_noio_flags(gfp_mask);
	ac.spread_dirty_pages = false;

	/*
         * Restore the original nodemask if it was potentially replaced with
         * &cpuset_current_mems_allowed to optimize the fast-path attempt.
         */
	if (cpusets_enabled())
		ac.nodemask = nodemask;
	page = __alloc_pages_slowpath(alloc_mask, order, &ac);

no_zone:
	/*
         * When updating a task's mems_allowed, it is possible to race with
         * parallel threads in such a way that an allocation can fail while
         * the mask is being updated. If a page allocation is about to fail,
         * check if the cpuset changed during allocation and if so, retry.
         */
	if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie))) {
		alloc_mask = gfp_mask;
		goto retry_cpuset;
	}

out:
	if (memcg_kmem_enabled() && (gfp_mask & __GFP_ACCOUNT) && page &&
	    unlikely(memcg_kmem_charge(page, gfp_mask, order) != 0)) {
		__free_pages(page, order);
		page = NULL;
	}

	if (kmemcheck_enabled && page)
		kmemcheck_pagealloc_alloc(page, order, gfp_mask);

	trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype);

	return page;
}
Contributors
Person Tokens Prop Commits CommitProp
mel gorman mel gorman 199 52.23% 16 47.06%
vlastimil babka vlastimil babka 58 15.22% 2 5.88%
vladimir davydov vladimir davydov 40 10.50% 2 5.88%
andrew morton andrew morton 37 9.71% 4 11.76%
xishi qiu xishi qiu 15 3.94% 1 2.94%
vegard nossum vegard nossum 9 2.36% 1 2.94%
ming lei ming lei 7 1.84% 1 2.94%
bartlomiej zolnierkiewicz bartlomiej zolnierkiewicz 6 1.57% 1 2.94%
benjamin herrenschmidt benjamin herrenschmidt 4 1.05% 1 2.94%
linus torvalds linus torvalds 2 0.52% 2 5.88%
miao xie miao xie 2 0.52% 1 2.94%
david rientjes david rientjes 1 0.26% 1 2.94%
nishanth aravamudan nishanth aravamudan 1 0.26% 1 2.94%
Total 381 100.00% 34 100.00%


EXPORT_SYMBOL(__alloc_pages_nodemask);

/*
 * Common helper functions.
 */


unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
{
	struct page *page;

	/*
         * __get_free_pages() returns a 32-bit address, which cannot represent
         * a highmem page
         */
	VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0);

	page = alloc_pages(gfp_mask, order);
	if (!page)
		return 0;
	return (unsigned long) page_address(page);
}
Contributors
Person Tokens Prop Commits CommitProp
pre-git pre-git 43 75.44% 2 40.00%
akinobu mita akinobu mita 12 21.05% 1 20.00%
al viro al viro 1 1.75% 1 20.00%
linus torvalds linus torvalds 1 1.75% 1 20.00%
Total 57 100.00% 5 100.00%


EXPORT_SYMBOL(__get_free_pages);



unsigned long get_zeroed_page(gfp_t gfp_mask)
{
	return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
}
Contributors
Person Tokens Prop Commits CommitProp
pre-git pre-git 14 73.68% 1 25.00%
akinobu mita akinobu mita 2 10.53% 1 25.00%
christoph lameter christoph lameter 2 10.53% 1 25.00%
al viro al viro 1 5.26% 1 25.00%
Total 19 100.00% 4 100.00%


EXPORT_SYMBOL(get_zeroed_page);



void __free_pages(struct page *page, unsigned int order)
{
	if (put_page_testzero(page)) {
		if (order == 0)
			free_hot_cold_page(page, false);
		else
			__free_pages_ok(page, order);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
pre-git pre-git 27 61.36% 1 20.00%
andrew morton andrew morton 13 29.55% 1 20.00%
li hong li hong 2 4.55% 1 20.00%
linus torvalds linus torvalds 1 2.27% 1 20.00%
mel gorman mel gorman 1 2.27% 1 20.00%
Total 44 100.00% 5 100.00%


EXPORT_SYMBOL(__free_pages);



void free_pages(unsigned long addr, unsigned int order)
{
	if (addr != 0) {
		VM_BUG_ON(!virt_addr_valid((void *)addr));
		__free_pages(virt_to_page((void *)addr), order);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
pre-git pre-git 25 52.08% 2 28.57%
roman zippel roman zippel 10 20.83% 1 14.29%
dave hansen dave hansen 8 16.67% 1 14.29%
linus torvalds linus torvalds 4 8.33% 2 28.57%
nick piggin nick piggin 1 2.08% 1 14.29%
Total 48 100.00% 7 100.00%


EXPORT_SYMBOL(free_pages);

/*
 * Page Fragment:
 *  An arbitrary-length arbitrary-offset area of memory which resides
 *  within a 0 or higher order page.  Multiple fragments within that page
 *  are individually refcounted, in the page's reference counter.
 *
 * The page_frag functions below provide a simple allocation framework for
 * page fragments.  This is used by the network stack and network device
 * drivers to provide a backing region of memory for use as either an
 * sk_buff->head, or to be used in the "frags" portion of skb_shared_info.
 */


static struct page *__page_frag_refill(struct page_frag_cache *nc,
				       gfp_t gfp_mask)
{
	struct page *page = NULL;
	gfp_t gfp = gfp_mask;

#if
	Person	Tokens	Prop	Commits	CommitProp
vlastimil babka	vlastimil babka	17	100.00%	1	100.00%
	Total	17	100.00%	1	100.00%
cregit-Linux how code gets into the kernel

Release 4.8 mm/page_alloc.c

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors
