Release 4.11 mm/memory_hotplug.c

Directory: mm
/*
 *  linux/mm/memory_hotplug.c
 *
 *  Copyright (C)
 */

#include <linux/stddef.h>
#include <linux/mm.h>
#include <linux/sched/signal.h>
#include <linux/swap.h>
#include <linux/interrupt.h>
#include <linux/pagemap.h>
#include <linux/compiler.h>
#include <linux/export.h>
#include <linux/pagevec.h>
#include <linux/writeback.h>
#include <linux/slab.h>
#include <linux/sysctl.h>
#include <linux/cpu.h>
#include <linux/memory.h>
#include <linux/memremap.h>
#include <linux/memory_hotplug.h>
#include <linux/highmem.h>
#include <linux/vmalloc.h>
#include <linux/ioport.h>
#include <linux/delay.h>
#include <linux/migrate.h>
#include <linux/page-isolation.h>
#include <linux/pfn.h>
#include <linux/suspend.h>
#include <linux/mm_inline.h>
#include <linux/firmware-map.h>
#include <linux/stop_machine.h>
#include <linux/hugetlb.h>
#include <linux/memblock.h>
#include <linux/bootmem.h>
#include <linux/compaction.h>

#include <asm/tlbflush.h>

#include "internal.h"

/*
 * online_page_callback contains pointer to current page onlining function.
 * Initially it is generic_online_page(). If it is required it could be
 * changed by calling set_online_page_callback() for callback registration
 * and restore_online_page_callback() for generic callback restore.
 */

static void generic_online_page(struct page *page);


static online_page_callback_t online_page_callback = generic_online_page;
static DEFINE_MUTEX(online_page_callback_lock);

/* The same as the cpu_hotplug lock, but for memory hotplug. */
static struct {
	
struct task_struct *active_writer;
	
struct mutex lock; /* Synchronizes accesses to refcount, */
	/*
         * Also blocks the new readers during
         * an ongoing mem hotplug operation.
         */
	
int refcount;

#ifdef CONFIG_DEBUG_LOCK_ALLOC
	
struct lockdep_map dep_map;
#endif
} 
mem_hotplug = {
	.active_writer = NULL,
	.lock = __MUTEX_INITIALIZER(mem_hotplug.lock),
	.refcount = 0,
#ifdef CONFIG_DEBUG_LOCK_ALLOC
	.dep_map = {.name = "mem_hotplug.lock" },
#endif
};

/* Lockdep annotations for get/put_online_mems() and mem_hotplug_begin/end() */

#define memhp_lock_acquire_read() lock_map_acquire_read(&mem_hotplug.dep_map)

#define memhp_lock_acquire()      lock_map_acquire(&mem_hotplug.dep_map)

#define memhp_lock_release()      lock_map_release(&mem_hotplug.dep_map)

#ifndef CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE

bool memhp_auto_online;
#else

bool memhp_auto_online = true;
#endif

EXPORT_SYMBOL_GPL(memhp_auto_online);



static int __init setup_memhp_default_state(char *str)
{
	if (!strcmp(str, "online"))
		memhp_auto_online = true;
	else if (!strcmp(str, "offline"))
		memhp_auto_online = false;

	return 1;
}
Contributors
Person Tokens Prop Commits CommitProp
Vitaly Kuznetsov 43 100.00% 1 100.00%
Total 43 100.00% 1 100.00%

__setup("memhp_default_state=", setup_memhp_default_state);



void get_online_mems(void)
{
	might_sleep();
	if (mem_hotplug.active_writer == current)
		return;
	memhp_lock_acquire_read();
	mutex_lock(&mem_hotplug.lock);
	mem_hotplug.refcount++;
	mutex_unlock(&mem_hotplug.lock);

}

Contributors
Person Tokens Prop Commits CommitProp
Vladimir Davydov 43 100.00% 1 100.00%
Total 43 100.00% 1 100.00%



void put_online_mems(void)
{
	if (mem_hotplug.active_writer == current)
		return;
	mutex_lock(&mem_hotplug.lock);

	if (WARN_ON(!mem_hotplug.refcount))
		mem_hotplug.refcount++; /* try to fix things up */

	if (!--mem_hotplug.refcount && unlikely(mem_hotplug.active_writer))
		wake_up_process(mem_hotplug.active_writer);
	mutex_unlock(&mem_hotplug.lock);
	memhp_lock_release();

}

Contributors
Person Tokens Prop Commits CommitProp
Vladimir Davydov 73 100.00% 1 100.00%
Total 73 100.00% 1 100.00%

/* Serializes write accesses to mem_hotplug.active_writer. */
static DEFINE_MUTEX(memory_add_remove_lock);



void mem_hotplug_begin(void)
{
	mutex_lock(&memory_add_remove_lock);

	mem_hotplug.active_writer = current;

	memhp_lock_acquire();
	for (;;) {
		mutex_lock(&mem_hotplug.lock);
		if (likely(!mem_hotplug.refcount))
			break;
		__set_current_state(TASK_UNINTERRUPTIBLE);
		mutex_unlock(&mem_hotplug.lock);
		schedule();
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Vladimir Davydov 47 73.44% 1 25.00%
Motohiro Kosaki 11 17.19% 1 25.00%
Heiko Carstens 5 7.81% 1 25.00%
Dan J Williams 1 1.56% 1 25.00%
Total 64 100.00% 4 100.00%



void mem_hotplug_done(void)
{
	mem_hotplug.active_writer = NULL;
	mutex_unlock(&mem_hotplug.lock);
	memhp_lock_release();
	mutex_unlock(&memory_add_remove_lock);
}

Contributors
Person Tokens Prop Commits CommitProp
Vladimir Davydov 13 43.33% 1 33.33%
Motohiro Kosaki 11 36.67% 1 33.33%
Heiko Carstens 6 20.00% 1 33.33%
Total 30 100.00% 3 100.00%

/* add this memory to iomem resource */


static struct resource *register_memory_resource(u64 start, u64 size)
{
	struct resource *res;
	res = kzalloc(sizeof(struct resource), GFP_KERNEL);
	if (!res)
		return ERR_PTR(-ENOMEM);

	res->name = "System RAM";
	res->start = start;
	res->end = start + size - 1;
	res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
	if (request_resource(&iomem_resource, res) < 0) {
		pr_debug("System RAM resource %pR cannot be added\n", res);
		kfree(res);
		return ERR_PTR(-EEXIST);
	}
	return res;
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Mannthey 89 80.91% 1 16.67%
Vitaly Kuznetsov 16 14.55% 1 16.67%
Yasunori Goto 2 1.82% 1 16.67%
Toshi Kani 2 1.82% 2 33.33%
Björn Helgaas 1 0.91% 1 16.67%
Total 110 100.00% 6 100.00%



static void release_memory_resource(struct resource *res)
{
	if (!res)
		return;
	release_resource(res);
	kfree(res);
	return;
}

Contributors
Person Tokens Prop Commits CommitProp
Keith Mannthey 28 100.00% 1 100.00%
Total 28 100.00% 1 100.00%

#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE


void get_page_bootmem(unsigned long info,  struct page *page,
		      unsigned long type)
{
	page->freelist = (void *)type;
	SetPagePrivate(page);
	set_page_private(page, info);
	page_ref_inc(page);
}

Contributors
Person Tokens Prop Commits CommitProp
Yasunori Goto 36 80.00% 2 40.00%
Andrea Arcangeli 6 13.33% 1 20.00%
Yasuaki Ishimatsu 2 4.44% 1 20.00%
JoonSoo Kim 1 2.22% 1 20.00%
Total 45 100.00% 5 100.00%



void put_page_bootmem(struct page *page)
{
	unsigned long type;

	type = (unsigned long) page->freelist;
	BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||
	       type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);

	if (page_ref_dec_return(page) == 1) {
		page->freelist = NULL;
		ClearPagePrivate(page);
		set_page_private(page, 0);
		INIT_LIST_HEAD(&page->lru);
		free_reserved_page(page);
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Yasunori Goto 52 67.53% 2 33.33%
Andrea Arcangeli 16 20.78% 1 16.67%
Yasuaki Ishimatsu 7 9.09% 1 16.67%
JoonSoo Kim 1 1.30% 1 16.67%
Jiang Liu 1 1.30% 1 16.67%
Total 77 100.00% 6 100.00%

#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE
#ifndef CONFIG_SPARSEMEM_VMEMMAP


static void register_page_bootmem_info_section(unsigned long start_pfn)
{
	unsigned long *usemap, mapsize, section_nr, i;
	struct mem_section *ms;
	struct page *page, *memmap;

	section_nr = pfn_to_section_nr(start_pfn);
	ms = __nr_to_section(section_nr);

	/* Get section's memmap address */
	memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);

	/*
         * Get page for the memmap's phys address
         * XXX: need more consideration for sparse_vmemmap...
         */
	page = virt_to_page(memmap);
	mapsize = sizeof(struct page) * PAGES_PER_SECTION;
	mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT;

	/* remember memmap's page */
	for (i = 0; i < mapsize; i++, page++)
		get_page_bootmem(section_nr, page, SECTION_INFO);

	usemap = __nr_to_section(section_nr)->pageblock_flags;
	page = virt_to_page(usemap);

	mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT;

	for (i = 0; i < mapsize; i++, page++)
		get_page_bootmem(section_nr, page, MIX_SECTION_INFO);

}
Contributors
Person Tokens Prop Commits CommitProp
Yasunori Goto 163 99.39% 2 66.67%
Adrian Bunk 1 0.61% 1 33.33%
Total 164 100.00% 3 100.00%

#else /* CONFIG_SPARSEMEM_VMEMMAP */


static void register_page_bootmem_info_section(unsigned long start_pfn)
{
	unsigned long *usemap, mapsize, section_nr, i;
	struct mem_section *ms;
	struct page *page, *memmap;

	if (!pfn_valid(start_pfn))
		return;

	section_nr = pfn_to_section_nr(start_pfn);
	ms = __nr_to_section(section_nr);

	memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);

	register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION);

	usemap = __nr_to_section(section_nr)->pageblock_flags;
	page = virt_to_page(usemap);

	mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT;

	for (i = 0; i < mapsize; i++, page++)
		get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
}
Contributors
Person Tokens Prop Commits CommitProp
Yasuaki Ishimatsu 128 100.00% 1 100.00%
Total 128 100.00% 1 100.00%

#endif /* !CONFIG_SPARSEMEM_VMEMMAP */



void __init register_page_bootmem_info_node(struct pglist_data *pgdat)
{
	unsigned long i, pfn, end_pfn, nr_pages;
	int node = pgdat->node_id;
	struct page *page;

	nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT;
	page = virt_to_page(pgdat);

	for (i = 0; i < nr_pages; i++, page++)
		get_page_bootmem(node, page, NODE_INFO);

	pfn = pgdat->node_start_pfn;
	end_pfn = pgdat_end_pfn(pgdat);

	/* register section info */
	for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
		/*
                 * Some platforms can assign the same pfn to multiple nodes - on
                 * node0 as well as nodeN.  To avoid registering a pfn against
                 * multiple nodes we check that this pfn does not already
                 * reside in some other nodes.
                 */
		if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node))
			register_page_bootmem_info_section(pfn);
	}
}
Contributors
Person Tokens Prop Commits CommitProp
Yasunori Goto 103 81.10% 1 16.67%
qiuxishi 17 13.39% 1 16.67%
Cody P Schafer 3 2.36% 1 16.67%
Tang Chen 2 1.57% 1 16.67%
Yang Shi 1 0.79% 1 16.67%
Linus Torvalds 1 0.79% 1 16.67%
Total 127 100.00% 6 100.00%

#endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */



static void __meminit grow_zone_span(struct zone *zone, unsigned long start_pfn,
				     unsigned long end_pfn)
{
	unsigned long old_zone_end_pfn;

	zone_span_writelock(zone);

	old_zone_end_pfn = zone_end_pfn(zone);
	if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn)
		zone->zone_start_pfn = start_pfn;

	zone->spanned_pages = max(old_zone_end_pfn, end_pfn) -
				zone->zone_start_pfn;

	zone_span_writeunlock(zone);
}

Contributors
Person Tokens Prop Commits CommitProp
Heiko Carstens 66 88.00% 1 20.00%
Xishi Qiu 6 8.00% 2 40.00%
Tang Chen 2 2.67% 1 20.00%
Fabian Frederick 1 1.33% 1 20.00%
Total 75 100.00% 5 100.00%



static void resize_zone(struct zone *zone, unsigned long start_pfn,
		unsigned long end_pfn)
{
	zone_span_writelock(zone);

	if (end_pfn - start_pfn) {
		zone->zone_start_pfn = start_pfn;
		zone->spanned_pages = end_pfn - start_pfn;
	} else {
		/*
                 * make it consist as free_area_init_core(),
                 * if spanned_pages = 0, then keep start_pfn = 0
                 */
		zone->zone_start_pfn = 0;
		zone->spanned_pages = 0;
	}

	zone_span_writeunlock(zone);
}

Contributors
Person Tokens Prop Commits CommitProp
Lai Jiangshan 67 100.00% 2 100.00%
Total 67 100.00% 2 100.00%



static void fix_zone_id(struct zone *zone, unsigned long start_pfn,
		unsigned long end_pfn)
{
	enum zone_type zid = zone_idx(zone);
	int nid = zone->zone_pgdat->node_id;
	unsigned long pfn;

	for (pfn = start_pfn; pfn < end_pfn; pfn++)
		set_page_links(pfn_to_page(pfn), zid, nid, pfn);
}

Contributors
Person Tokens Prop Commits CommitProp
Lai Jiangshan 68 100.00% 1 100.00%
Total 68 100.00% 1 100.00%

/* Can fail with -ENOMEM from allocating a wait table with vmalloc() or
 * alloc_bootmem_node_nopanic()/memblock_virt_alloc_node_nopanic() */


static int __ref ensure_zone_is_initialized(struct zone *zone,
			unsigned long start_pfn, unsigned long num_pages)
{
	if (!zone_is_initialized(zone))
		return init_currently_empty_zone(zone, start_pfn, num_pages);

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Cody P Schafer 41 100.00% 1 100.00%
Total 41 100.00% 1 100.00%



static int __meminit move_pfn_range_left(struct zone *z1, struct zone *z2,
		unsigned long start_pfn, unsigned long end_pfn)
{
	int ret;
	unsigned long flags;
	unsigned long z1_start_pfn;

	ret = ensure_zone_is_initialized(z1, start_pfn, end_pfn - start_pfn);
	if (ret)
		return ret;

	pgdat_resize_lock(z1->zone_pgdat, &flags);

	/* can't move pfns which are higher than @z2 */
	if (end_pfn > zone_end_pfn(z2))
		goto out_fail;
	/* the move out part must be at the left most of @z2 */
	if (start_pfn > z2->zone_start_pfn)
		goto out_fail;
	/* must included/overlap */
	if (end_pfn <= z2->zone_start_pfn)
		goto out_fail;

	/* use start_pfn for z1's start_pfn if z1 is empty */
	if (!zone_is_empty(z1))
		z1_start_pfn = z1->zone_start_pfn;
	else
		z1_start_pfn = start_pfn;

	resize_zone(z1, z1_start_pfn, end_pfn);
	resize_zone(z2, end_pfn, zone_end_pfn(z2));

	pgdat_resize_unlock(z1->zone_pgdat, &flags);

	fix_zone_id(z1, start_pfn, end_pfn);

	return 0;
out_fail:
	pgdat_resize_unlock(z1->zone_pgdat, &flags);
	return -1;
}

Contributors
Person Tokens Prop Commits CommitProp
Lai Jiangshan 170 93.41% 2 33.33%
Cody P Schafer 7 3.85% 2 33.33%
Xishi Qiu 4 2.20% 1 16.67%
Jiang Liu 1 0.55% 1 16.67%
Total 182 100.00% 6 100.00%



static int __meminit move_pfn_range_right(struct zone *z1, struct zone *z2,
		unsigned long start_pfn, unsigned long end_pfn)
{
	int ret;
	unsigned long flags;
	unsigned long z2_end_pfn;

	ret = ensure_zone_is_initialized(z2, start_pfn, end_pfn - start_pfn);
	if (ret)
		return ret;

	pgdat_resize_lock(z1->zone_pgdat, &flags);

	/* can't move pfns which are lower than @z1 */
	if (z1->zone_start_pfn > start_pfn)
		goto out_fail;
	/* the move out part mast at the right most of @z1 */
	if (zone_end_pfn(z1) >  end_pfn)
		goto out_fail;
	/* must included/overlap */
	if (start_pfn >= zone_end_pfn(z1))
		goto out_fail;

	/* use end_pfn for z2's end_pfn if z2 is empty */
	if (!zone_is_empty(z2))
		z2_end_pfn = zone_end_pfn(z2);
	else
		z2_end_pfn = end_pfn;

	resize_zone(z1, z1->zone_start_pfn, start_pfn);
	resize_zone(z2, start_pfn, z2_end_pfn);

	pgdat_resize_unlock(z1->zone_pgdat, &flags);

	fix_zone_id(z2, start_pfn, end_pfn);

	return 0;
out_fail:
	pgdat_resize_unlock(z1->zone_pgdat, &flags);
	return -1;
}

Contributors
Person Tokens Prop Commits CommitProp
Lai Jiangshan 169 92.35% 2 40.00%
Cody P Schafer 10 5.46% 2 40.00%
Xishi Qiu 4 2.19% 1 20.00%
Total 183 100.00% 5 100.00%



static struct zone * __meminit move_pfn_range(int zone_shift,
		unsigned long start_pfn, unsigned long end_pfn)
{
	struct zone *zone = page_zone(pfn_to_page(start_pfn));
	int ret = 0;

	if (zone_shift < 0)
		ret = move_pfn_range_left(zone + zone_shift, zone,
					  start_pfn, end_pfn);
	else if (zone_shift)
		ret = move_pfn_range_right(zone, zone + zone_shift,
					   start_pfn, end_pfn);

	if (ret)
		return NULL;

	return zone + zone_shift;
}

Contributors
Person Tokens Prop Commits CommitProp
Reza Arbab 91 100.00% 1 100.00%
Total 91 100.00% 1 100.00%



static void __meminit grow_pgdat_span(struct pglist_data *pgdat, unsigned long start_pfn,
				      unsigned long end_pfn)
{
	unsigned long old_pgdat_end_pfn = pgdat_end_pfn(pgdat);

	if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn)
		pgdat->node_start_pfn = start_pfn;

	pgdat->node_spanned_pages = max(old_pgdat_end_pfn, end_pfn) -
					pgdat->node_start_pfn;
}

Contributors
Person Tokens Prop Commits CommitProp
Heiko Carstens 54 85.71% 1 25.00%
Tang Chen 5 7.94% 1 25.00%
Xishi Qiu 3 4.76% 1 25.00%
Fabian Frederick 1 1.59% 1 25.00%
Total 63 100.00% 4 100.00%



static int __meminit __add_zone(struct zone *zone, unsigned long phys_start_pfn)
{
	struct pglist_data *pgdat = zone->zone_pgdat;
	int nr_pages = PAGES_PER_SECTION;
	int nid = pgdat->node_id;
	int zone_type;
	unsigned long flags, pfn;
	int ret;

	zone_type = zone - pgdat->node_zones;
	ret = ensure_zone_is_initialized(zone, phys_start_pfn, nr_pages);
	if (ret)
		return ret;

	pgdat_resize_lock(zone->zone_pgdat, &flags);
	grow_zone_span(zone, phys_start_pfn, phys_start_pfn + nr_pages);
	grow_pgdat_span(zone->zone_pgdat, phys_start_pfn,
			phys_start_pfn + nr_pages);
	pgdat_resize_unlock(zone->zone_pgdat, &flags);
	memmap_init_zone(nr_pages, nid, zone_type,
			 phys_start_pfn, MEMMAP_HOTPLUG);

	/* online_page_range is called later and expects pages reserved */
	for (pfn = phys_start_pfn; pfn < phys_start_pfn + nr_pages; pfn++) {
		if (!pfn_valid(pfn))
			continue;

		SetPageReserved(pfn_to_page(pfn));
	}
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Dave Hansen 59 34.71% 2 28.57%
Heiko Carstens 57 33.53% 1 14.29%
Mel Gorman 37 21.76% 1 14.29%
Yasunori Goto 12 7.06% 1 14.29%
Cody P Schafer 4 2.35% 1 14.29%
Al Viro 1 0.59% 1 14.29%
Total 170 100.00% 7 100.00%



static int __meminit __add_section(int nid, struct zone *zone,
					unsigned long phys_start_pfn)
{
	int ret;

	if (pfn_valid(phys_start_pfn))
		return -EEXIST;

	ret = sparse_add_one_section(zone, phys_start_pfn);

	if (ret < 0)
		return ret;

	ret = __add_zone(zone, phys_start_pfn);

	if (ret < 0)
		return ret;

	return register_new_memory(nid, __pfn_to_section(phys_start_pfn));
}

Contributors
Person Tokens Prop Commits CommitProp
Dave Hansen 52 65.00% 1 20.00%
Kamezawa Hiroyuki 11 13.75% 1 20.00%
Yasunori Goto 11 13.75% 1 20.00%
Gary Hade 5 6.25% 1 20.00%
Al Viro 1 1.25% 1 20.00%
Total 80 100.00% 5 100.00%

/*
 * Reasonably generic function for adding memory.  It is
 * expected that archs that support memory hotplug will
 * call this function after deciding the zone to which to
 * add the new pages.
 */


int __ref __add_pages(int nid, struct zone *zone, unsigned long phys_start_pfn,
			unsigned long nr_pages)
{
	unsigned long i;
	int err = 0;
	int start_sec, end_sec;
	struct vmem_altmap *altmap;

	clear_zone_contiguous(zone);

	/* during initialize mem_map, align hot-added range to section */
	start_sec = pfn_to_section_nr(phys_start_pfn);
	end_sec = pfn_to_section_nr(phys_start_pfn + nr_pages - 1);

	altmap = to_vmem_altmap((unsigned long) pfn_to_page(phys_start_pfn));
	if (altmap) {
		/*
                 * Validate altmap is within bounds of the total request
                 */
		if (altmap->base_pfn != phys_start_pfn
				|| vmem_altmap_offset(altmap) > nr_pages) {
			pr_warn_once("memory add fail, invalid altmap\n");
			err = -EINVAL;
			goto out;
		}
		altmap->alloc = 0;
	}

	for (i = start_sec; i <= end_sec; i++) {
		err = __add_section(nid, zone, section_nr_to_pfn(i));

		/*
                 * EEXIST is finally dealt with by ioresource collision
                 * check. see add_memory() => register_memory_resource()
                 * Warning will be printed if there is collision.
                 */
		if (err && (err != -EEXIST))
			break;
		err = 0;
	}
	vmemmap_populate_print_last();
out:
	set_zone_contiguous(zone);
	return err;
}
Contributors
Person Tokens Prop Commits CommitProp
David Rientjes 101 55.80% 1 20.00%
Dan J Williams 56 30.94% 1 20.00%
JoonSoo Kim 18 9.94% 1 20.00%
Sheng Yong 3 1.66% 1 20.00%
Zhu Guihua 3 1.66% 1 20.00%
Total 181 100.00% 5 100.00%


EXPORT_SYMBOL_GPL(__add_pages);

#ifdef CONFIG_MEMORY_HOTREMOVE
/* find the smallest valid pfn in the range [start_pfn, end_pfn) */


static int find_smallest_section_pfn(int nid, struct zone *zone,
				     unsigned long start_pfn,
				     unsigned long end_pfn)
{
	struct mem_section *ms;

	for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SECTION) {
		ms = __pfn_to_section(start_pfn);

		if (unlikely(!valid_section(ms)))
			continue;

		if (unlikely(pfn_to_nid(start_pfn) != nid))
			continue;

		if (zone && zone != page_zone(pfn_to_page(start_pfn)))
			continue;

		return start_pfn;
	}

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Yasuaki Ishimatsu 93 100.00% 1 100.00%
Total 93 100.00% 1 100.00%

/* find the biggest valid pfn in the range [start_pfn, end_pfn). */


static int find_biggest_section_pfn(int nid, struct zone *zone,
				    unsigned long start_pfn,
				    unsigned long end_pfn)
{
	struct mem_section *ms;
	unsigned long pfn;

	/* pfn is the end pfn of a memory section. */
	pfn = end_pfn - 1;
	for (; pfn >= start_pfn; pfn -= PAGES_PER_SECTION) {
		ms = __pfn_to_section(pfn);

		if (unlikely(!valid_section(ms)))
			continue;

		if (unlikely(pfn_to_nid(pfn) != nid))
			continue;

		if (zone && zone != page_zone(pfn_to_page(pfn)))
			continue;

		return pfn;
	}

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Yasuaki Ishimatsu 104 100.00% 1 100.00%
Total 104 100.00% 1 100.00%



static void shrink_zone_span(struct zone *zone, unsigned long start_pfn,
			     unsigned long end_pfn)
{
	unsigned long zone_start_pfn = zone->zone_start_pfn;
	unsigned long z = zone_end_pfn(zone); /* zone_end_pfn namespace clash */
	unsigned long zone_end_pfn = z;
	unsigned long pfn;
	struct mem_section *ms;
	int nid = zone_to_nid(zone);

	zone_span_writelock(zone);
	if (zone_start_pfn == start_pfn) {
		/*
                 * If the section is smallest section in the zone, it need
                 * shrink zone->zone_start_pfn and zone->zone_spanned_pages.
                 * In this case, we find second smallest valid mem_section
                 * for shrinking zone.
                 */
		pfn = find_smallest_section_pfn(nid, zone, end_pfn,
						zone_end_pfn);
		if (pfn) {
			zone->zone_start_pfn = pfn;
			zone->spanned_pages = zone_end_pfn - pfn;
		}
	} else if (zone_end_pfn == end_pfn) {
		/*
                 * If the section is biggest section in the zone, it need
                 * shrink zone->spanned_pages.
                 * In this case, we find second biggest valid mem_section for
                 * shrinking zone.
                 */
		pfn = find_biggest_section_pfn(nid, zone, zone_start_pfn,
					       start_pfn);
		if (pfn)
			zone->spanned_pages = pfn - zone_start_pfn + 1;
	}

	/*
         * The section is not biggest or smallest mem_section in the zone, it
         * only creates a hole in the zone. So in this case, we need not
         * change the zone. But perhaps, the zone has only hole data. Thus
         * it check the zone has only hole or not.
         */
	pfn = zone_start_pfn;
	for (; pfn < zone_end_pfn; pfn += PAGES_PER_SECTION) {
		ms = __pfn_to_section(pfn);

		if (unlikely(!valid_section(ms)))
			continue;

		if (page_zone(pfn_to_page(pfn)) != zone)
			continue;

		 /* If the section is current section, it continues the loop */
		if (start_pfn == pfn)
			continue;

		/* If we find valid section, we have nothing to do */
		zone_span_writeunlock(zone);
		return;
	}

	/* The zone has no valid section */
	zone->zone_start_pfn = 0;
	zone->spanned_pages = 0;
	zone_span_writeunlock(zone);
}

Contributors
Person Tokens Prop Commits CommitProp
Yasuaki Ishimatsu 216 95.15% 1 50.00%
Xishi Qiu 11 4.85% 1 50.00%
Total 227 100.00% 2 100.00%



static void shrink_pgdat_span(struct pglist_data *pgdat,
			      unsigned long start_pfn, unsigned long end_pfn)
{
	unsigned long pgdat_start_pfn = pgdat->node_start_pfn;
	unsigned long p = pgdat_end_pfn(pgdat); /* pgdat_end_pfn namespace clash */
	unsigned long pgdat_end_pfn = p;
	unsigned long pfn;
	struct mem_section *ms;
	int nid = pgdat->node_id;

	if (pgdat_start_pfn == start_pfn) {
		/*
                 * If the section is smallest section in the pgdat, it need
                 * shrink pgdat->node_start_pfn and pgdat->node_spanned_pages.
                 * In this case, we find second smallest valid mem_section
                 * for shrinking zone.
                 */
		pfn = find_smallest_section_pfn(nid, NULL, end_pfn,
						pgdat_end_pfn);
		if (pfn) {
			pgdat->node_start_pfn = pfn;
			pgdat->node_spanned_pages = pgdat_end_pfn - pfn;
		}
	} else if (pgdat_end_pfn == end_pfn) {
		/*
                 * If the section is biggest section in the pgdat, it need
                 * shrink pgdat->node_spanned_pages.
                 * In this case, we find second biggest valid mem_section for
                 * shrinking zone.
                 */
		pfn = find_biggest_section_pfn(nid, NULL, pgdat_start_pfn,
					       start_pfn);
		if (pfn)
			pgdat->node_spanned_pages = pfn - pgdat_start_pfn + 1;
	}

	/*
         * If the section is not biggest or smallest mem_section in the pgdat,
         * it only creates a hole in the pgdat. So in this case, we need not
         * change the pgdat.
         * But perhaps, the pgdat has only hole data. Thus it check the pgdat
         * has only hole or not.
         */
	pfn = pgdat_start_pfn;
	for (; pfn < pgdat_end_pfn; pfn += PAGES_PER_SECTION) {
		ms = __pfn_to_section(pfn);

		if (unlikely(!valid_section(ms)))
			continue;

		if (pfn_to_nid(pfn) != nid)
			continue;

		 /* If the section is current section, it continues the loop */
		if (start_pfn == pfn)
			continue;

		/* If we find valid section, we have nothing to do */
		return;
	}

	/* The pgdat has no valid section */
	pgdat->node_start_pfn = 0;
	pgdat->node_spanned_pages = 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Yasuaki Ishimatsu 197 94.71% 1 50.00%
Xishi Qiu 11 5.29% 1 50.00%
Total 208 100.00% 2 100.00%



static void __remove_zone(struct zone *zone, unsigned long start_pfn)
{
	struct pglist_data *pgdat = zone->zone_pgdat;
	int nr_pages = PAGES_PER_SECTION;
	int zone_type;
	unsigned long flags;

	zone_type = zone - pgdat->node_zones;

	pgdat_resize_lock(zone->zone_pgdat, &flags);
	shrink_zone_span(zone, start_pfn, start_pfn + nr_pages);
	shrink_pgdat_span(pgdat, start_pfn, start_pfn + nr_pages);
	pgdat_resize_unlock(zone->zone_pgdat, &flags);
}

Contributors
Person Tokens Prop Commits CommitProp
Yasuaki Ishimatsu 86 100.00% 1 100.00%
Total 86 100.00% 1 100.00%



static int __remove_section(struct zone *zone, struct mem_section *ms,
		unsigned long map_offset)
{
	unsigned long start_pfn;
	int scn_nr;
	int ret = -EINVAL;

	if (!valid_section(ms))
		return ret;

	ret = unregister_memory_section(ms);
	if (ret)
		return ret;

	scn_nr = __section_nr(ms);
	start_pfn = section_nr_to_pfn(scn_nr);
	__remove_zone(zone, start_pfn);

	sparse_remove_one_section(zone, ms, map_offset);
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Badari Pulavarty 57 62.64% 1 33.33%
Yasuaki Ishimatsu 28 30.77% 1 33.33%
Dan J Williams 6 6.59% 1 33.33%
Total 91 100.00% 3 100.00%

/**
 * __remove_pages() - remove sections of pages from a zone
 * @zone: zone from which pages need to be removed
 * @phys_start_pfn: starting pageframe (must be aligned to start of a section)
 * @nr_pages: number of pages to remove (must be multiple of section size)
 *
 * Generic helper function to remove section mappings and sysfs entries
 * for the section of the memory we are removing. Caller needs to make
 * sure that pages are marked reserved and zones are adjust properly by
 * calling offline_pages().
 */


int __remove_pages(struct zone *zone, unsigned long phys_start_pfn,
		 unsigned long nr_pages)
{
	unsigned long i;
	unsigned long map_offset = 0;
	int sections_to_remove, ret = 0;

	/* In the ZONE_DEVICE case device driver owns the memory region */
	if (is_dev_zone(zone)) {
		struct page *page = pfn_to_page(phys_start_pfn);
		struct vmem_altmap *altmap;

		altmap = to_vmem_altmap((unsigned long) page);
		if (altmap)
			map_offset = vmem_altmap_offset(altmap);
	} else {
		resource_size_t start, size;

		start = phys_start_pfn << PAGE_SHIFT;
		size = nr_pages * PAGE_SIZE;

		ret = release_mem_region_adjustable(&iomem_resource, start,
					size);
		if (ret) {
			resource_size_t endres = start + size - 1;

			pr_warn("Unable to release resource <%pa-%pa> (%d)\n",
					&start, &endres, ret);
		}
	}

	clear_zone_contiguous(zone);

	/*
         * We can only remove entire sections
         */
	BUG_ON(phys_start_pfn & ~PAGE_SECTION_MASK);
	BUG_ON(nr_pages % PAGES_PER_SECTION);

	sections_to_remove = nr_pages / PAGES_PER_SECTION;
	for (i = 0; i < sections_to_remove; i++) {
		unsigned long pfn = phys_start_pfn + i*PAGES_PER_SECTION;

		ret = __remove_section(zone, __pfn_to_section(pfn), map_offset);
		map_offset = 0;
		if (ret)
			break;
	}

	set_zone_contiguous(zone);

	return ret;
}
Contributors
Person Tokens Prop Commits CommitProp
Badari Pulavarty 84 37.33% 1 16.67%
Dan J Williams 76 33.78% 1 16.67%
Toshi Kani 32 14.22% 1 16.67%
Randy Dunlap 15 6.67% 1 16.67%
JoonSoo Kim 10 4.44% 1 16.67%
Yasuaki Ishimatsu 8 3.56% 1 16.67%
Total 225 100.00% 6 100.00%

#endif /* CONFIG_MEMORY_HOTREMOVE */



int set_online_page_callback(online_page_callback_t callback)
{
	int rc = -EINVAL;

	get_online_mems();
	mutex_lock(&online_page_callback_lock);

	if (online_page_callback == generic_online_page) {
		online_page_callback = callback;
		rc = 0;
	}

	mutex_unlock(&online_page_callback_lock);
	put_online_mems();

	return rc;
}
Contributors
Person Tokens Prop Commits CommitProp
Daniel Kiper 37 72.55% 1 50.00%
Vladimir Davydov 14 27.45% 1 50.00%
Total 51 100.00% 2 100.00%


EXPORT_SYMBOL_GPL(set_online_page_callback);



int restore_online_page_callback(online_page_callback_t callback)
{
	int rc = -EINVAL;

	get_online_mems();
	mutex_lock(&online_page_callback_lock);

	if (online_page_callback == callback) {
		online_page_callback = generic_online_page;
		rc = 0;
	}

	mutex_unlock(&online_page_callback_lock);
	put_online_mems();

	return rc;
}
Contributors
Person Tokens Prop Commits CommitProp
Daniel Kiper 37 72.55% 1 50.00%
Vladimir Davydov 14 27.45% 1 50.00%
Total 51 100.00% 2 100.00%


EXPORT_SYMBOL_GPL(restore_online_page_callback);



void __online_page_set_limits(struct page *page)
{
}
Contributors
Person Tokens Prop Commits CommitProp
Jeremy Fitzhardinge 7 77.78% 1 33.33%
Daniel Kiper 1 11.11% 1 33.33%
Jiang Liu 1 11.11% 1 33.33%
Total 9 100.00% 3 100.00%


EXPORT_SYMBOL_GPL(__online_page_set_limits);



void __online_page_increment_counters(struct page *page)
{
	adjust_managed_page_count(page, 1);
}
Contributors
Person Tokens Prop Commits CommitProp
Daniel Kiper 10 58.82% 1 33.33%
Jeremy Fitzhardinge 4 23.53% 1 33.33%
Jiang Liu 3 17.65% 1 33.33%
Total 17 100.00% 3 100.00%


EXPORT_SYMBOL_GPL(__online_page_increment_counters);



void __online_page_free(struct page *page)
{
	__free_reserved_page(page);
}
Contributors
Person Tokens Prop Commits CommitProp
Daniel Kiper 9 60.00% 1 33.33%
Jeremy Fitzhardinge 5 33.33% 1 33.33%
Jiang Liu 1 6.67% 1 33.33%
Total 15 100.00% 3 100.00%


EXPORT_SYMBOL_GPL(__online_page_free);



static void generic_online_page(struct page *page)
{
	__online_page_set_limits(page);
	__online_page_increment_counters(page);
	__online_page_free(page);
}

Contributors
Person Tokens Prop Commits CommitProp
Daniel Kiper 26 100.00% 1 100.00%
Total 26 100.00% 1 100.00%



static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages,
			void *arg)
{
	unsigned long i;
	unsigned long onlined_pages = *(unsigned long *)arg;
	struct page *page;
	if (PageReserved(pfn_to_page(start_pfn)))
		for (i = 0; i < nr_pages; i++) {
			page = pfn_to_page(start_pfn + i);
			(*online_page_callback)(page);
			onlined_pages++;
		}
	*(unsigned long *)arg = onlined_pages;
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Kamezawa Hiroyuki 73 75.26% 2 50.00%
Dave Hansen 20 20.62% 1 25.00%
Daniel Kiper 4 4.12% 1 25.00%
Total 97 100.00% 4 100.00%

#ifdef CONFIG_MOVABLE_NODE
/*
 * When CONFIG_MOVABLE_NODE, we permit onlining of a node which doesn't have
 * normal memory.
 */


static bool can_online_high_movable(struct zone *zone)
{
	return true;
}
Contributors
Person Tokens Prop Commits CommitProp
Lai Jiangshan 14 100.00% 1 100.00%
Total 14 100.00% 1 100.00%

#else /* CONFIG_MOVABLE_NODE */
/* ensure every online node has NORMAL memory */


static bool can_online_high_movable(struct zone *zone)
{
	return node_state(zone_to_nid(zone), N_NORMAL_MEMORY);
}
Contributors
Person Tokens Prop Commits CommitProp
Lai Jiangshan 22 100.00% 1 100.00%
Total 22 100.00% 1 100.00%

#endif /* CONFIG_MOVABLE_NODE */

/* check which state of node_states will be changed when online memory */


static void node_states_check_changes_online(unsigned long nr_pages,
	struct zone *zone, struct memory_notify *arg)
{
	int nid = zone_to_nid(zone);
	enum zone_type zone_last = ZONE_NORMAL;

	/*
         * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY]
         * contains nodes which have zones of 0...ZONE_NORMAL,
         * set zone_last to ZONE_NORMAL.
         *
         * If we don't have HIGHMEM nor movable node,
         * node_states[N_NORMAL_MEMORY] contains nodes which have zones of
         * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE.
         */
	if (N_MEMORY == N_NORMAL_MEMORY)
		zone_last = ZONE_MOVABLE;

	/*
         * if the memory to be online is in a zone of 0...zone_last, and
         * the zones of 0...zone_last don't have memory before online, we will
         * need to set the node to node_states[N_NORMAL_MEMORY] after
         * the memory is online.
         */
	if (zone_idx(zone) <= zone_last && !node_state(nid, N_NORMAL_MEMORY))
		arg->status_change_nid_normal = nid;
	else
		arg->status_change_nid_normal = -1;

#ifdef CONFIG_HIGHMEM
	/*
         * If we have movable node, node_states[N_HIGH_MEMORY]
         * contains nodes which have zones of 0...ZONE_HIGHMEM,
         * set zone_last to ZONE_HIGHMEM.
         *
         * If we don't have movable node, node_states[N_NORMAL_MEMORY]
         * contains nodes which have zones of 0...ZONE_MOVABLE,
         * set zone_last to ZONE_MOVABLE.
         */
	zone_last = ZONE_HIGHMEM;
	if (N_MEMORY == N_HIGH_MEMORY)
		zone_last = ZONE_MOVABLE;

	if (zone_idx(zone) <= zone_last && !node_state(nid, N_HIGH_MEMORY))
		arg->status_change_nid_high = nid;
	else
		arg->status_change_nid_high = -1;
#else
	arg->status_change_nid_high = arg->status_change_nid_normal;
#endif

	/*
         * if the node don't have memory befor online, we will need to
         * set the node to node_states[N_MEMORY] after the memory
         * is online.
         */
	if (!node_state(nid, N_MEMORY))
		arg->status_change_nid = nid;
	else
		arg->status_change_nid = -1;
}

Contributors
Person Tokens Prop Commits CommitProp
Lai Jiangshan 163 100.00% 2 100.00%
Total 163 100.00% 2 100.00%



static void node_states_set_node(int node, struct memory_notify *arg)
{
	if (arg->status_change_nid_normal >= 0)
		node_set_state(node, N_NORMAL_MEMORY);

	if (arg->status_change_nid_high >= 0)
		node_set_state(node, N_HIGH_MEMORY);

	node_set_state(node, N_MEMORY);
}

Contributors
Person Tokens Prop Commits CommitProp
Lai Jiangshan 51 100.00% 2 100.00%
Total 51 100.00% 2 100.00%



bool zone_can_shift(unsigned long pfn, unsigned long nr_pages,
		   enum zone_type target, int *zone_shift)
{
	struct zone *zone = page_zone(pfn_to_page(pfn));
	enum zone_type idx = zone_idx(zone);
	int i;

	*zone_shift = 0;

	if (idx < target) {
		/* pages must be at end of current zone */
		if (pfn + nr_pages != zone_end_pfn(zone))
			return false;

		/* no zones in use between current zone and target */
		for (i = idx + 1; i < target; i++)
			if (zone_is_initialized(zone - idx + i))
				return false;
	}

	if (target < idx) {
		/* pages must be at beginning of current zone */
		if (pfn != zone->zone_start_pfn)
			return false;

		/* no zones in use between current zone and target */
		for (i = target + 1; i < idx; i++)
			if (zone_is_initialized(zone - idx + i))
				return false;
	}

	*zone_shift = target - idx;
	return true;
}

Contributors
Person Tokens Prop Commits CommitProp
Reza Arbab 144 87.80% 1 50.00%
Yasuaki Ishimatsu 20 12.20% 1 50.00%
Total 164 100.00% 2 100.00%

/* Must be protected by mem_hotplug_begin() */


int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_type)
{
	unsigned long flags;
	unsigned long onlined_pages = 0;
	struct zone *zone;
	int need_zonelists_rebuild = 0;
	int nid;
	int ret;
	struct memory_notify arg;
	int zone_shift = 0;

	/*
         * This doesn't need a lock to do pfn_to_page().
         * The section can't be removed here because of the
         * memory_block->state_mutex.
         */
	zone = page_zone(pfn_to_page(pfn));

	if ((zone_idx(zone) > ZONE_NORMAL ||
	    online_type == MMOP_ONLINE_MOVABLE) &&
	    !can_online_high_movable(zone))
		return -EINVAL;

	if (online_type == MMOP_ONLINE_KERNEL) {
		if (!zone_can_shift(pfn, nr_pages, ZONE_NORMAL, &zone_shift))
			return -EINVAL;
	} else if (online_type == MMOP_ONLINE_MOVABLE) {
		if (!zone_can_shift(pfn, nr_pages, ZONE_MOVABLE, &zone_shift))
			return -EINVAL;
	}

	zone = move_pfn_range(zone_shift, pfn, pfn + nr_pages);
	if (!zone)
		return -EINVAL;

	arg.start_pfn = pfn;
	arg.nr_pages = nr_pages;
	node_states_check_changes_online(nr_pages, zone, &arg);

	nid = zone_to_nid(zone);

	ret = memory_notify(MEM_GOING_ONLINE, &arg);
	ret = notifier_to_errno(ret);
	if (ret)
		goto failed_addition;

	/*
         * If this zone is not populated, then it is not in zonelist.
         * This means the page allocator ignores this zone.
         * So, zonelist must be updated after online.
         */
	mutex_lock(&zonelists_mutex);
	if (!populated_zone(zone)) {
		need_zonelists_rebuild = 1;
		build_all_zonelists(NULL, zone);
	}

	ret = walk_system_ram_range(pfn, nr_pages, &onlined_pages,
		online_pages_range);
	if (ret) {
		if (need_zonelists_rebuild)
			zone_pcp_reset(zone);
		mutex_unlock(&zonelists_mutex);
		goto failed_addition;
	}

	zone->present_pages += onlined_pages;

	pgdat_resize_lock(zone->zone_pgdat, &flags);
	zone->zone_pgdat->node_present_pages += onlined_pages;
	pgdat_resize_unlock(zone->zone_pgdat, &flags);

	if (onlined_pages) {
		node_states_set_node(nid, &arg);
		if (need_zonelists_rebuild)
			build_all_zonelists(NULL, NULL);
		else
			zone_pcp_update(zone);
	}

	mutex_unlock(&zonelists_mutex);

	init_per_zone_wmark_min();

	if (onlined_pages) {
		kswapd_run(nid);
		kcompactd_run(nid);
	}

	vm_total_pages = nr_free_pagecache_pages();

	writeback_set_ratelimit();

	if (onlined_pages)
		memory_notify(MEM_ONLINE, &arg);
	return 0;

failed_addition:
	pr_debug("online_pages [mem %#010llx-%#010llx] failed\n",
		 (unsigned long long) pfn << PAGE_SHIFT,
		 (((unsigned long long) pfn + nr_pages) << PAGE_SHIFT) - 1);
	memory_notify(MEM_CANCEL_ONLINE, &arg);
	return ret;
}
Contributors
Person Tokens Prop Commits CommitProp
Yasunori Goto 90 20.41% 3 9.09%
Lai Jiangshan 63 14.29% 3 9.09%
Chen Yucong 50 11.34% 1 3.03%
Kamezawa Hiroyuki 35 7.94% 4 12.12%
Reza Arbab 32 7.26% 2 6.06%
Haicheng Li 27 6.12% 2 6.06%
Yasuaki Ishimatsu 26 5.90% 1 3.03%
Cody P Schafer 21 4.76% 1 3.03%
Wen Congyang 19 4.31% 1 3.03%
Jiang Liu 17 3.85% 3 9.09%
Dave Hansen 14 3.17% 2 6.06%
Vlastimil Babka 11 2.49% 2 6.06%
David Rientjes 9 2.04% 1 3.03%
Geoff Levand 8 1.81% 1 3.03%
Christoph Lameter 6 1.36% 1 3.03%
David Shaohua Li 5 1.13% 1 3.03%
Tang Chen 3 0.68% 1 3.03%
Chandra Seetharaman 3 0.68% 1 3.03%
Motohiro Kosaki 2 0.45% 2 6.06%
Total 441 100.00% 33 100.00%

#endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */



static void reset_node_present_pages(pg_data_t *pgdat)
{
	struct zone *z;

	for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
		z->present_pages = 0;

	pgdat->node_present_pages = 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Tang Chen 46 100.00% 1 100.00%
Total 46 100.00% 1 100.00%

/* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */


static pg_data_t __ref *hotadd_new_pgdat(int nid, u64 start)
{
	struct pglist_data *pgdat;
	unsigned long zones_size[MAX_NR_ZONES] = {0};
	unsigned long zholes_size[MAX_NR_ZONES] = {0};
	unsigned long start_pfn = PFN_DOWN(start);

	pgdat = NODE_DATA(nid);
	if (!pgdat) {
		pgdat = arch_alloc_nodedata(nid);
		if (!pgdat)
			return NULL;

		arch_refresh_nodedata(nid, pgdat);
	} else {
		/* Reset the nr_zones, order and classzone_idx before reuse */
		pgdat->nr_zones = 0;
		pgdat->kswapd_order = 0;
		pgdat->kswapd_classzone_idx = 0;
	}

	/* we can use NODE_DATA(nid) from here */

	/* init node's zones as empty zones, we don't have any present pages.*/
	free_area_init_node(nid, zones_size, start_pfn, zholes_size);
	pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat);

	/*
         * The node we allocated has no zone fallback lists. For avoiding
         * to access not-initialized zonelist, build here.
         */
	mutex_lock(&zonelists_mutex);
	build_all_zonelists(pgdat, NULL);
	mutex_unlock(&zonelists_mutex);

	/*
         * zone->managed_pages is set to an approximate value in
         * free_area_init_core(), which will cause
         * /sys/device/system/node/nodeX/meminfo has wrong data.
         * So reset it to 0 before any memory is onlined.
         */
	reset_node_managed_pages(pgdat);

	/*
         * When memory is hot-added, all the memory is in offline state. So
         * clear all zones' present_pages because they will be updated in
         * online_pages() and offline_pages().
         */
	reset_node_present_pages(pgdat);

	return pgdat;
}

Contributors
Person Tokens Prop Commits CommitProp
Yasunori Goto 84 50.60% 1 8.33%
Tang Chen 26 15.66% 3 25.00%
Gu Zheng 14 8.43% 1 8.33%
David Rientjes 12 7.23% 1 8.33%
Reza Arbab 10 6.02% 1 8.33%
Mel Gorman 8 4.82% 1 8.33%
Kamezawa Hiroyuki 6 3.61% 1 8.33%
Fabian Frederick 3 1.81% 1 8.33%
Jiang Liu 2 1.20% 1 8.33%
Hidetoshi Seto 1 0.60% 1 8.33%
Total 166 100.00% 12 100.00%



static void rollback_node_hotadd(int nid, pg_data_t *pgdat)
{
	arch_refresh_nodedata(nid, NULL);
	free_percpu(pgdat->per_cpu_nodestats);
	arch_free_nodedata(pgdat);
	return;
}


Contributors
Person Tokens Prop Commits CommitProp
Yasunori Goto 26 78.79% 1 50.00%
Reza Arbab 7 21.21% 1 50.00%
Total 33 100.00% 2 100.00%

/**
 * try_online_node - online a node if offlined
 *
 * called by cpu_up() to online a node without onlined memory.
 */


int try_online_node(int nid)
{
	pg_data_t	*pgdat;
	int	ret;

	if (node_online(nid))
		return 0;

	mem_hotplug_begin();
	pgdat = hotadd_new_pgdat(nid, 0);
	if (!pgdat) {
		pr_err("Cannot online node %d due to NULL pgdat\n", nid);
		ret = -ENOMEM;
		goto out;
	}
	node_set_online(nid);
	ret = register_one_node(nid);
	BUG_ON(ret);

	if (pgdat->node_zonelists->_zonerefs->zone == NULL) {
		mutex_lock(&zonelists_mutex);
		build_all_zonelists(NULL, NULL);
		mutex_unlock(&zonelists_mutex);
	}

out:
	mem_hotplug_done();
	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
Minskey Guo 63 53.85% 1 25.00%
Toshi Kani 51 43.59% 1 25.00%
Vladimir Davydov 2 1.71% 1 25.00%
David Rientjes 1 0.85% 1 25.00%
Total 117 100.00% 4 100.00%



static int check_hotplug_memory_range(u64 start, u64 size)
{
	u64 start_pfn = PFN_DOWN(start);
	u64 nr_pages = size >> PAGE_SHIFT;

	/* Memory range must be aligned with section */
	if ((start_pfn & ~PAGE_SECTION_MASK) ||
	    (nr_pages % PAGES_PER_SECTION) || (!nr_pages)) {
		pr_err("Section-unaligned hotplug range: start 0x%llx, size 0x%llx\n",
				(unsigned long long)start,
				(unsigned long long)size);
		return -EINVAL;
	}

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Toshi Kani 73 96.05% 1 50.00%
Fabian Frederick 3 3.95% 1 50.00%
Total 76 100.00% 2 100.00%

/*
 * If movable zone has already been setup, newly added memory should be check.
 * If its address is higher than movable zone, it should be added as movable.
 * Without this check, movable zone may overlap with other zone.
 */


static int should_add_memory_movable(int nid, u64 start, u64 size)
{
	unsigned long start_pfn = start >> PAGE_SHIFT;
	pg_data_t *pgdat = NODE_DATA(nid);
	struct zone *movable_zone = pgdat->node_zones + ZONE_MOVABLE;

	if (zone_is_empty(movable_zone))
		return 0;

	if (movable_zone->zone_start_pfn <= start_pfn)
		return 1;

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Wang Nan 67 100.00% 1 100.00%
Total 67 100.00% 1 100.00%



int zone_for_memory(int nid, u64 start, u64 size, int zone_default,
		bool for_device)
{
#ifdef CONFIG_ZONE_DEVICE
	if (for_device)
		return ZONE_DEVICE;
#endif
	if (should_add_memory_movable(nid, start, size))
		return ZONE_MOVABLE;

	return zone_default;
}

Contributors
Person Tokens Prop Commits CommitProp
Wang Nan 34 69.39% 1 50.00%
Dan J Williams 15 30.61% 1 50.00%
Total 49 100.00% 2 100.00%



static int online_memory_block(struct memory_block *mem, void *arg)
{
	return device_online(&mem->dev);
}

Contributors
Person Tokens Prop Commits CommitProp
Vitaly Kuznetsov 20 83.33% 1 50.00%
Nathan Fontenot 4 16.67% 1 50.00%
Total 24 100.00% 2 100.00%

/* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */


int __ref add_memory_resource(int nid, struct resource *res, bool online)
{
	u64 start, size;
	pg_data_t *pgdat = NULL;
	bool new_pgdat;
	bool new_node;
	int ret;

	start = res->start;
	size = resource_size(res);

	ret = check_hotplug_memory_range(start, size);
	if (ret)
		return ret;

	{	/* Stupid hack to suppress address-never-null warning */
		void *p = NODE_DATA(nid);
		new_pgdat = !p;
	}

	mem_hotplug_begin();

	/*
         * Add new range to memblock so that when hotadd_new_pgdat() is called
         * to allocate new pgdat, get_pfn_range_for_nid() will be able to find
         * this new range and calculate total pages correctly.  The range will
         * be removed at hot-remove time.
         */
	memblock_add_node(start, size, nid);

	new_node = !node_online(nid);
	if (new_node) {
		pgdat = hotadd_new_pgdat(nid, start);
		ret = -ENOMEM;
		if (!pgdat)
			goto error;
	}

	/* call arch's memory hotadd */
	ret = arch_add_memory(nid, start, size, false);

	if (ret < 0)
		goto error;

	/* we online node here. we can't roll back from here. */
	node_set_online(nid);

	if (new_node) {
		ret = register_one_node(nid);
		/*
                 * If sysfs file of new node can't create, cpu on the node
                 * can't be hot-added. There is no rollback way now.
                 * So, check by BUG_ON() to catch it reluctantly..
                 */
		BUG_ON(ret);
	}

	/* create new memmap entry */
	firmware_map_add_hotplug(start, start + size, "System RAM");

	/* online pages if requested */
	if (online)
		walk_memory_range(PFN_DOWN(start), PFN_UP(start + size - 1),
				  NULL, online_memory_block);

	goto out;

error:
	/* rollback pgdat allocation and others */
	if (new_pgdat)
		rollback_node_hotadd(nid, pgdat);
	memblock_remove(start, size);

out:
	mem_hotplug_done();
	return ret;
}
Contributors
Person Tokens Prop Commits CommitProp
Yasunori Goto 104 41.60% 3 18.75%
Tang Chen 45 18.00% 2 12.50%
Vitaly Kuznetsov 29 11.60% 1 6.25%
David Vrabel 19 7.60% 1 6.25%
Andi Kleen 14 5.60% 1 6.25%
Andrew Morton 12 4.80% 1 6.25%
Kamezawa Hiroyuki 10 4.00% 1 6.25%
Toshi Kani 6 2.40% 1 6.25%
Nathan Zimmer 5 2.00% 1 6.25%
Dan J Williams 2 0.80% 1 6.25%
Vladimir Davydov 2 0.80% 1 6.25%
Wen Congyang 1 0.40% 1 6.25%
Al Viro 1 0.40% 1 6.25%
Total 250 100.00% 16 100.00%


EXPORT_SYMBOL_GPL(add_memory_resource);



int __ref add_memory(int nid, u64 start, u64 size)
{
	struct resource *res;
	int ret;

	res = register_memory_resource(start, size);
	if (IS_ERR(res))
		return PTR_ERR(res);

	ret = add_memory_resource(nid, res, memhp_auto_online);
	if (ret < 0)
		release_memory_resource(res);
	return ret;
}
Contributors
Person Tokens Prop Commits CommitProp
David Vrabel 61 87.14% 1 33.33%
Vitaly Kuznetsov 9 12.86% 2 66.67%
Total 70 100.00% 3 100.00%


EXPORT_SYMBOL_GPL(add_memory);

#ifdef CONFIG_MEMORY_HOTREMOVE
/*
 * A free page on the buddy free lists (not the per-cpu lists) has PageBuddy
 * set and the size of the free page is given by page_order(). Using this,
 * the function determines if the pageblock contains only free pages.
 * Due to buddy contraints, a free page at least the size of a pageblock will
 * be located at the start of the pageblock
 */


static inline int pageblock_free(struct page *page)
{
	return PageBuddy(page) && page_order(page) >= pageblock_order;
}

Contributors
Person Tokens Prop Commits CommitProp
Badari Pulavarty 25 100.00% 1 100.00%
Total 25 100.00% 1 100.00%

/* Return the start of the next active pageblock after a given page */


static struct page *next_active_pageblock(struct page *page)
{
	/* Ensure the starting page is pageblock-aligned */
	BUG_ON(page_to_pfn(page) & (pageblock_nr_pages - 1));

	/* If the entire pageblock is free, move to the end of free page */
	if (pageblock_free(page)) {
		int order;
		/* be careful. we don't have locks, page_order can be changed.*/
		order = page_order(page);
		if ((order < MAX_ORDER) && (order >= pageblock_order))
			return page + (1 << order);
	}

	return page + pageblock_nr_pages;
}

Contributors
Person Tokens Prop Commits CommitProp
Badari Pulavarty 47 61.04% 1 50.00%
Kamezawa Hiroyuki 30 38.96% 1 50.00%
Total 77 100.00% 2 100.00%

/* Checks if this range of memory is likely to be hot-removable. */


bool is_mem_section_removable(unsigned long start_pfn, unsigned long nr_pages)
{
	struct page *page = pfn_to_page(start_pfn);
	struct page *end_page = page + nr_pages;

	/* Check the starting page of each pageblock within the range */
	for (; page < end_page; page = next_active_pageblock(page)) {
		if (!is_pageblock_removable_nolock(page))
			return false;
		cond_resched();
	}

	/* All pageblocks in the memory block are likely to be hot-removable */
	return true;
}

Contributors
Person Tokens Prop Commits CommitProp
Badari Pulavarty 60 89.55% 1 33.33%
Kamezawa Hiroyuki 4 5.97% 1 33.33%
Yaowei Bai 3 4.48% 1 33.33%
Total 67 100.00% 3 100.00%

/*
 * Confirm all pages in a range [start, end) belong to the same zone.
 * When true, return its valid [start, end).
 */


int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn,
			 unsigned long *valid_start, unsigned long *valid_end)
{
	unsigned long pfn, sec_end_pfn;
	unsigned long start, end;
	struct zone *zone = NULL;
	struct page *page;
	int i;
	for (pfn = start_pfn, sec_end_pfn = SECTION_ALIGN_UP(start_pfn + 1);
	     pfn < end_pfn;
	     pfn = sec_end_pfn, sec_end_pfn += PAGES_PER_SECTION) {
		/* Make sure the memory section is present first */
		if (!present_section_nr(pfn_to_section_nr(pfn)))
			continue;
		for (; pfn < sec_end_pfn && pfn < end_pfn;
		     pfn += MAX_ORDER_NR_PAGES) {
			i = 0;
			/* This is just a CONFIG_HOLES_IN_ZONE check.*/
			while ((i < MAX_ORDER_NR_PAGES) &&
				!pfn_valid_within(pfn + i))
				i++;
			if (i == MAX_ORDER_NR_PAGES || pfn + i >= end_pfn)
				continue;
			page = pfn_to_page(pfn + i);
			if (zone && page_zone(page) != zone)
				return 0;
			if (!zone)
				start = pfn + i;
			zone = page_zone(page);
			end = pfn + MAX_ORDER_NR_PAGES;
		}
	}

	if (zone) {
		*valid_start = start;
		*valid_end = min(end, end_pfn);
		return 1;
	} else {
		return 0;
	}
}

Contributors
Person Tokens Prop Commits CommitProp
Kamezawa Hiroyuki 112 50.22% 1 20.00%
Toshi Kani 57 25.56% 2 40.00%
Andrew Banman 43 19.28% 1 20.00%
zhong jiang 11 4.93% 1 20.00%
Total 223 100.00% 5 100.00%

/*
 * Scan pfn range [start,end) to find movable/migratable pages (LRU pages,
 * non-lru movable pages and hugepages). We scan pfn because it's much
 * easier than scanning over linked list. This function returns the pfn
 * of the first found movable page if it's found, otherwise 0.
 */


static unsigned long scan_movable_pages(unsigned long start, unsigned long end)
{
	unsigned long pfn;
	struct page *page;
	for (pfn = start; pfn < end; pfn++) {
		if (pfn_valid(pfn)) {
			page = pfn_to_page(pfn);
			if (PageLRU(page))
				return pfn;
			if (__PageMovable(page))
				return pfn;
			if (PageHuge(page)) {
				if (page_huge_active(page))
					return pfn;
				else
					pfn = round_up(pfn + 1,
						1 << compound_order(page)) - 1;
			}
		}
	}
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Kamezawa Hiroyuki 66 56.90% 2 33.33%
Naoya Horiguchi 39 33.62% 2 33.33%
Yisheng Xie 10 8.62% 1 16.67%
Andrew Morton 1 0.86% 1 16.67%
Total 116 100.00% 6 100.00%



static struct page *new_node_page(struct page *page, unsigned long private,
		int **result)
{
	gfp_t gfp_mask = GFP_USER | __GFP_MOVABLE;
	int nid = page_to_nid(page);
	nodemask_t nmask = node_states[N_MEMORY];
	struct page *new_page = NULL;

	/*
         * TODO: allocate a destination hugepage from a nearest neighbor node,
         * accordance with memory policy of the user process if possible. For
         * now as a simple work-around, we use the next node for destination.
         */
	if (PageHuge(page))
		return alloc_huge_page_node(page_hstate(compound_head(page)),
					next_node_in(nid, nmask));

	node_clear(nid, nmask);

	if (PageHighMem(page)
	    || (zone_idx(page_zone(page)) == ZONE_MOVABLE))
		gfp_mask |= __GFP_HIGHMEM;

	if (!nodes_empty(nmask))
		new_page = __alloc_pages_nodemask(gfp_mask, 0,
					node_zonelist(nid, gfp_mask), &nmask);
	if (!new_page)
		new_page = __alloc_pages(gfp_mask, 0,
					node_zonelist(nid, gfp_mask));

	return new_page;
}

Contributors
Person Tokens Prop Commits CommitProp
Xishi Qiu 146 91.25% 1 50.00%
Li Zhong 14 8.75% 1 50.00%
Total 160 100.00% 2 100.00%


#define NR_OFFLINE_AT_ONCE_PAGES	(256)

static int

do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
{
	unsigned long pfn;
	struct page *page;
	int move_pages = NR_OFFLINE_AT_ONCE_PAGES;
	int not_managed = 0;
	int ret = 0;
	LIST_HEAD(source);

	for (pfn = start_pfn; pfn < end_pfn && move_pages > 0; pfn++) {
		if (!pfn_valid(pfn))
			continue;
		page = pfn_to_page(pfn);

		if (PageHuge(page)) {
			struct page *head = compound_head(page);
			pfn = page_to_pfn(head) + (1<<compound_order(head)) - 1;
			if (compound_order(head) > PFN_SECTION_SHIFT) {
				ret = -EBUSY;
				break;
			}
			if (isolate_huge_page(page, &source))
				move_pages -= 1 << compound_order(head);
			continue;
		}

		if (!get_page_unless_zero(page))
			continue;
		/*
                 * We can skip free pages. And we can deal with pages on
                 * LRU and non-lru movable pages.
                 */
		if (PageLRU(page))
			ret = isolate_lru_page(page);
		else
			ret = isolate_movable_page(page, ISOLATE_UNEVICTABLE);
		if (!ret) { /* Success */
			put_page(page);
			list_add_tail(&page->lru, &source);
			move_pages--;
			if (!__PageMovable(page))
				inc_node_page_state(page, NR_ISOLATED_ANON +
						    page_is_file_cache(page));

		} else {
#ifdef CONFIG_DEBUG_VM
			pr_alert("failed to isolate pfn %lx\n", pfn);
			dump_page(page, "isolation failed");
#endif
			put_page(page);
			/* Because we don't have big zone->lock. we should
                           check this again here. */
			if (page_count(page)) {
				not_managed++;
				ret = -EBUSY;
				break;
			}
		}
	}
	if (!list_empty(&source)) {
		if (not_managed) {
			putback_movable_pages(&source);
			goto out;
		}

		/* Allocate a new page from the nearest neighbor node */
		ret = migrate_pages(&source, new_node_page, NULL, 0,
					MIGRATE_SYNC, MR_MEMORY_HOTPLUG);
		if (ret)
			putback_movable_pages(&source);
	}
out:
	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
Kamezawa Hiroyuki 156 46.02% 1 5.26%
Naoya Horiguchi 76 22.42% 1 5.26%
Yisheng Xie 28 8.26% 1 5.26%
Bob Liu 22 6.49% 2 10.53%
Nicholas Piggin 11 3.24% 1 5.26%
Motohiro Kosaki 11 3.24% 1 5.26%
Konstantin Khlebnikov 11 3.24% 1 5.26%
MinChan Kim 9 2.65% 1 5.26%
Mel Gorman 4 1.18% 3 15.79%
Fengguang Wu 3 0.88% 1 5.26%
Xishi Qiu 2 0.59% 1 5.26%
David Rientjes 2 0.59% 1 5.26%
Hugh Dickins 1 0.29% 1 5.26%
Lucas De Marchi 1 0.29% 1 5.26%
Chen Yucong 1 0.29% 1 5.26%
Dave Hansen 1 0.29% 1 5.26%
Total 339 100.00% 19 100.00%

/*
 * remove from free_area[] and mark all as Reserved.
 */

static int

offline_isolated_pages_cb(unsigned long start, unsigned long nr_pages,
			void *data)
{
	__offline_isolated_pages(start, start + nr_pages);
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Kamezawa Hiroyuki 30 100.00% 1 100.00%
Total 30 100.00% 1 100.00%


static void

offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
{
	walk_system_ram_range(start_pfn, end_pfn - start_pfn, NULL,
				offline_isolated_pages_cb);
}

Contributors
Person Tokens Prop Commits CommitProp
Kamezawa Hiroyuki 27 100.00% 2 100.00%
Total 27 100.00% 2 100.00%

/*
 * Check all pages in range, recoreded as memory resource, are isolated.
 */

static int

check_pages_isolated_cb(unsigned long start_pfn, unsigned long nr_pages,
			void *data)
{
	int ret;
	long offlined = *(long *)data;
	ret = test_pages_isolated(start_pfn, start_pfn + nr_pages, true);
	offlined = nr_pages;
	if (!ret)
		*(long *)data += offlined;
	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
Kamezawa Hiroyuki 63 96.92% 1 50.00%
Wen Congyang 2 3.08% 1 50.00%
Total 65 100.00% 2 100.00%


static long

check_pages_isolated(unsigned long start_pfn, unsigned long end_pfn)
{
	long offlined = 0;
	int ret;

	ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn, &offlined,
			check_pages_isolated_cb);
	if (ret < 0)
		offlined = (long)ret;
	return offlined;
}

Contributors
Person Tokens Prop Commits CommitProp
Kamezawa Hiroyuki 54 100.00% 2 100.00%
Total 54 100.00% 2 100.00%

#ifdef CONFIG_MOVABLE_NODE
/*
 * When CONFIG_MOVABLE_NODE, we permit offlining of a node which doesn't have
 * normal memory.
 */


static bool can_offline_normal(struct zone *zone, unsigned long nr_pages)
{
	return true;
}
Contributors
Person Tokens Prop Commits CommitProp
Lai Jiangshan 18 100.00% 1 100.00%
Total 18 100.00% 1 100.00%

#else /* CONFIG_MOVABLE_NODE */
/* ensure the node has NORMAL memory if it is still online */


static bool can_offline_normal(struct zone *zone, unsigned long nr_pages)
{
	struct pglist_data *pgdat = zone->zone_pgdat;
	unsigned long present_pages = 0;
	enum zone_type zt;

	for (zt = 0; zt <= ZONE_NORMAL; zt++)
		present_pages += pgdat->node_zones[zt].present_pages;

	if (present_pages > nr_pages)
		return true;

	present_pages = 0;
	for (; zt <= ZONE_MOVABLE; zt++)
		present_pages += pgdat->node_zones[zt].present_pages;

	/*
         * we can't offline the last normal memory until all
         * higher memory is offlined.
         */
	return present_pages == 0;
}
Contributors
Person Tokens Prop Commits CommitProp
Lai Jiangshan 98 100.00% 1 100.00%
Total 98 100.00% 1 100.00%

#endif /* CONFIG_MOVABLE_NODE */



static int __init cmdline_parse_movable_node(char *p)
{
#ifdef CONFIG_MOVABLE_NODE
	movable_node_enabled = true;
#else
	pr_warn("movable_node option not supported\n");
#endif
	return 0;
}
Contributors
Person Tokens Prop Commits CommitProp
Tang Chen 30 100.00% 2 100.00%
Total 30 100.00% 2 100.00%

early_param("movable_node", cmdline_parse_movable_node);

/* check which state of node_states will be changed when offline memory */


static void node_states_check_changes_offline(unsigned long nr_pages,
		struct zone *zone, struct memory_notify *arg)
{
	struct pglist_data *pgdat = zone->zone_pgdat;
	unsigned long present_pages = 0;
	enum zone_type zt, zone_last = ZONE_NORMAL;

	/*
         * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY]
         * contains nodes which have zones of 0...ZONE_NORMAL,
         * set zone_last to ZONE_NORMAL.
         *
         * If we don't have HIGHMEM nor movable node,
         * node_states[N_NORMAL_MEMORY] contains nodes which have zones of
         * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE.
         */
	if (N_MEMORY == N_NORMAL_MEMORY)
		zone_last = ZONE_MOVABLE;

	/*
         * check whether node_states[N_NORMAL_MEMORY] will be changed.
         * If the memory to be offline is in a zone of 0...zone_last,
         * and it is the last present memory, 0...zone_last will
         * become empty after offline , thus we can determind we will
         * need to clear the node from node_states[N_NORMAL_MEMORY].
         */
	for (zt = 0; zt <= zone_last; zt++)
		present_pages += pgdat->node_zones[zt].present_pages;
	if (zone_idx(zone) <= zone_last && nr_pages >= present_pages)
		arg->status_change_nid_normal = zone_to_nid(zone);
	else
		arg->status_change_nid_normal = -1;

#ifdef CONFIG_HIGHMEM
	/*
         * If we have movable node, node_states[N_HIGH_MEMORY]
         * contains nodes which have zones of 0...ZONE_HIGHMEM,
         * set zone_last to ZONE_HIGHMEM.
         *
         * If we don't have movable node, node_states[N_NORMAL_MEMORY]
         * contains nodes which have zones of 0...ZONE_MOVABLE,
         * set zone_last to ZONE_MOVABLE.
         */
	zone_last = ZONE_HIGHMEM;
	if (N_MEMORY == N_HIGH_MEMORY)
		zone_last = ZONE_MOVABLE;

	for (; zt <= zone_last; zt++)
		present_pages += pgdat->node_zones[zt].present_pages;
	if (zone_idx(zone) <= zone_last && nr_pages >= present_pages)
		arg->status_change_nid_high = zone_to_nid(zone);
	else
		arg->status_change_nid_high = -1;
#else
	arg->status_change_nid_high = arg->status_change_nid_normal;
#endif

	/*
         * node_states[N_HIGH_MEMORY] contains nodes which have 0...ZONE_MOVABLE
         */
	zone_last = ZONE_MOVABLE;

	/*
         * check whether node_states[N_HIGH_MEMORY] will be changed
         * If we try to offline the last present @nr_pages from the node,
         * we can determind we will need to clear the node from
         * node_states[N_HIGH_MEMORY].
         */
	for (; zt <= zone_last; zt++)
		present_pages += pgdat->node_zones[zt].present_pages;
	if (nr_pages >= present_pages)
		arg->status_change_nid = zone_to_nid(zone);
	else
		arg->status_change_nid = -1;
}

Contributors
Person Tokens Prop Commits CommitProp
Lai Jiangshan 240 100.00% 2 100.00%
Total 240 100.00% 2 100.00%



static void node_states_clear_node(int node, struct memory_notify *arg)
{
	if (arg->status_change_nid_normal >= 0)
		node_clear_state(node, N_NORMAL_MEMORY);

	if ((N_MEMORY != N_NORMAL_MEMORY) &&
	    (arg->status_change_nid_high >= 0))
		node_clear_state(node, N_HIGH_MEMORY);

	if ((N_MEMORY != N_HIGH_MEMORY) &&
	    (arg->status_change_nid >= 0))
		node_clear_state(node, N_MEMORY);
}

Contributors
Person Tokens Prop Commits CommitProp
Lai Jiangshan 75 100.00% 2 100.00%
Total 75 100.00% 2 100.00%



static int __ref __offline_pages(unsigned long start_pfn,
		  unsigned long end_pfn, unsigned long timeout)
{
	unsigned long pfn, nr_pages, expire;
	long offlined_pages;
	int ret, drain, retry_max, node;
	unsigned long flags;
	unsigned long valid_start, valid_end;
	struct zone *zone;
	struct memory_notify arg;

	/* at least, alignment against pageblock is necessary */
	if (!IS_ALIGNED(start_pfn, pageblock_nr_pages))
		return -EINVAL;
	if (!IS_ALIGNED(end_pfn, pageblock_nr_pages))
		return -EINVAL;
	/* This makes hotplug much easier...and readable.
           we assume this for now. .*/
	if (!test_pages_in_a_zone(start_pfn, end_pfn, &valid_start, &valid_end))
		return -EINVAL;

	zone = page_zone(pfn_to_page(valid_start));
	node = zone_to_nid(zone);
	nr_pages = end_pfn - start_pfn;

	if (zone_idx(zone) <= ZONE_NORMAL && !can_offline_normal(zone, nr_pages))
		return -EINVAL;

	/* set above range as isolated */
	ret = start_isolate_page_range(start_pfn, end_pfn,
				       MIGRATE_MOVABLE, true);
	if (ret)
		return ret;

	arg.start_pfn = start_pfn;
	arg.nr_pages = nr_pages;
	node_states_check_changes_offline(nr_pages, zone, &arg);

	ret = memory_notify(MEM_GOING_OFFLINE, &arg);
	ret = notifier_to_errno(ret);
	if (ret)
		goto failed_removal;

	pfn = start_pfn;
	expire = jiffies + timeout;
	drain = 0;
	retry_max = 5;
repeat:
	/* start memory hot removal */
	ret = -EAGAIN;
	if (time_after(jiffies, expire))
		goto failed_removal;
	ret = -EINTR;
	if (signal_pending(current))
		goto failed_removal;
	ret = 0;
	if (drain) {
		lru_add_drain_all();
		cond_resched();
		drain_all_pages(zone);
	}

	pfn = scan_movable_pages(start_pfn, end_pfn);
	if (pfn) { /* We have movable pages */
		ret = do_migrate_range(pfn, end_pfn);
		if (!ret) {
			drain = 1;
			goto repeat;
		} else {
			if (ret < 0)
				if (--retry_max == 0)
					goto failed_removal;
			yield();
			drain = 1;
			goto repeat;
		}
	}
	/* drain all zone's lru pagevec, this is asynchronous... */
	lru_add_drain_all();
	yield();
	/* drain pcp pages, this is synchronous. */
	drain_all_pages(zone);
	/*
         * dissolve free hugepages in the memory block before doing offlining
         * actually in order to make hugetlbfs's object counting consistent.
         */
	ret = dissolve_free_huge_pages(start_pfn, end_pfn);
	if (ret)
		goto failed_removal;
	/* check again */
	offlined_pages = check_pages_isolated(start_pfn, end_pfn);
	if (offlined_pages < 0) {
		ret = -EBUSY;
		goto failed_removal;
	}
	pr_info("Offlined Pages %ld\n", offlined_pages);
	/* Ok, all of our target is isolated.
           We cannot do rollback at this point. */
	offline_isolated_pages(start_pfn, end_pfn);
	/* reset pagetype flags and makes migrate type to be MOVABLE */
	undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
	/* removal success */
	adjust_managed_page_count(pfn_to_page(start_pfn), -offlined_pages);
	zone->present_pages -= offlined_pages;

	pgdat_resize_lock(zone->zone_pgdat, &flags);
	zone->zone_pgdat->node_present_pages -= offlined_pages;
	pgdat_resize_unlock(zone->zone_pgdat, &flags);

	init_per_zone_wmark_min();

	if (!populated_zone(zone)) {
		zone_pcp_reset(zone);
		mutex_lock(&zonelists_mutex);
		build_all_zonelists(NULL, NULL);
		mutex_unlock(&zonelists_mutex);
	} else
		zone_pcp_update(zone);

	node_states_clear_node(node, &arg);
	if (arg.status_change_nid >= 0) {
		kswapd_stop(node);
		kcompactd_stop(node);
	}

	vm_total_pages = nr_free_pagecache_pages();
	writeback_set_ratelimit();

	memory_notify(MEM_OFFLINE, &arg);
	return 0;

failed_removal:
	pr_debug("memory offlining [mem %#010llx-%#010llx] failed\n",
		 (unsigned long long) start_pfn << PAGE_SHIFT,
		 ((unsigned long long) end_pfn << PAGE_SHIFT) - 1);
	memory_notify(MEM_CANCEL_OFFLINE, &arg);
	/* pushback to free area */
	undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
Kamezawa Hiroyuki 334 53.18% 2 6.90%
Yasunori Goto 81 12.90% 1 3.45%
Lai Jiangshan 35 5.57% 2 6.90%
Xishi Qiu 27 4.30% 1 3.45%
Jiang Liu 23 3.66% 3 10.34%
Cody P Schafer 23 3.66% 1 3.45%
David Rientjes 20 3.18% 2 6.90%
Björn Helgaas 19 3.03% 1 3.45%
Toshi Kani 13 2.07% 1 3.45%
Vlastimil Babka 13 2.07% 3 10.34%
Naoya Horiguchi 10 1.59% 1 3.45%
Gerald Schaefer 9 1.43% 1 3.45%
Michal Nazarewicz 6 0.96% 1 3.45%
Wen Congyang 3 0.48% 2 6.90%
Adam Buchbinder 3 0.48% 1 3.45%
MinChan Kim 2 0.32% 1 3.45%
Motohiro Kosaki 2 0.32% 2 6.90%
Christoph Lameter 2 0.32% 1 3.45%
Chen Yucong 2 0.32% 1 3.45%
Andrew Morton 1 0.16% 1 3.45%
Total 628 100.00% 29 100.00%

/* Must be protected by mem_hotplug_begin() */


int offline_pages(unsigned long start_pfn, unsigned long nr_pages)
{
	return __offline_pages(start_pfn, start_pfn + nr_pages, 120 * HZ);
}
Contributors
Person Tokens Prop Commits CommitProp
Wen Congyang 27 100.00% 1 100.00%
Total 27 100.00% 1 100.00%

#endif /* CONFIG_MEMORY_HOTREMOVE */

/**
 * walk_memory_range - walks through all mem sections in [start_pfn, end_pfn)
 * @start_pfn: start pfn of the memory range
 * @end_pfn: end pfn of the memory range
 * @arg: argument passed to func
 * @func: callback for each memory section walked
 *
 * This function walks through all present mem sections in range
 * [start_pfn, end_pfn) and call func on each mem section.
 *
 * Returns the return value of func.
 */


int walk_memory_range(unsigned long start_pfn, unsigned long end_pfn,
		void *arg, int (*func)(struct memory_block *, void *))
{
	struct memory_block *mem = NULL;
	struct mem_section *section;
	unsigned long pfn, section_nr;
	int ret;

	for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
		section_nr = pfn_to_section_nr(pfn);
		if (!present_section_nr(section_nr))
			continue;

		section = __nr_to_section(section_nr);
		/* same memblock? */
		if (mem)
			if ((section_nr >= mem->start_section_nr) &&
			    (section_nr <= mem->end_section_nr))
				continue;

		mem = find_memory_block_hinted(section, mem);
		if (!mem)
			continue;

		ret = func(mem, arg);
		if (ret) {
			kobject_put(&mem->dev.kobj);
			return ret;
		}
	}

	if (mem)
		kobject_put(&mem->dev.kobj);

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Wen Congyang 164 93.71% 3 75.00%
Badari Pulavarty 11 6.29% 1 25.00%
Total 175 100.00% 4 100.00%

#ifdef CONFIG_MEMORY_HOTREMOVE


static int check_memblock_offlined_cb(struct memory_block *mem, void *arg)
{
	int ret = !is_memblock_offlined(mem);

	if (unlikely(ret)) {
		phys_addr_t beginpa, endpa;

		beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr));
		endpa = PFN_PHYS(section_nr_to_pfn(mem->end_section_nr + 1))-1;
		pr_warn("removing memory fails, because memory [%pa-%pa] is onlined\n",
			&beginpa, &endpa);
	}

	return ret;
}

Contributors
Person Tokens Prop Commits CommitProp
Yasuaki Ishimatsu 34 42.50% 1 20.00%
Wen Congyang 23 28.75% 1 20.00%
Randy Dunlap 21 26.25% 1 20.00%
Xishi Qiu 1 1.25% 1 20.00%
Joe Perches 1 1.25% 1 20.00%
Total 80 100.00% 5 100.00%



static int check_cpu_on_node(pg_data_t *pgdat)
{
	int cpu;

	for_each_present_cpu(cpu) {
		if (cpu_to_node(cpu) == pgdat->node_id)
			/*
                         * the cpu on this node isn't removed, and we can't
                         * offline this node.
                         */
			return -EBUSY;
	}

	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Tang Chen 36 94.74% 1 50.00%
Toshi Kani 2 5.26% 1 50.00%
Total 38 100.00% 2 100.00%



static void unmap_cpu_on_node(pg_data_t *pgdat)
{
#ifdef CONFIG_ACPI_NUMA
	int cpu;

	for_each_possible_cpu(cpu)
		if (cpu_to_node(cpu) == pgdat->node_id)
			numa_clear_node(cpu);
#endif
}

Contributors
Person Tokens Prop Commits CommitProp
Wen Congyang 36 94.74% 1 50.00%
Toshi Kani 2 5.26% 1 50.00%
Total 38 100.00% 2 100.00%



static int check_and_unmap_cpu_on_node(pg_data_t *pgdat)
{
	int ret;

	ret = check_cpu_on_node(pgdat);
	if (ret)
		return ret;

	/*
         * the node will be offlined when we come here, so we can clear
         * the cpu_to_node() now.
         */

	unmap_cpu_on_node(pgdat);
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
Wen Congyang 28 77.78% 1 50.00%
Toshi Kani 8 22.22% 1 50.00%
Total 36 100.00% 2 100.00%

/**
 * try_offline_node
 *
 * Offline a node if all memory sections and cpus of the node are removed.
 *
 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
 * and online/offline operations before this call.
 */


void try_offline_node(int nid)
{
	pg_data_t *pgdat = NODE_DATA(nid);
	unsigned long start_pfn = pgdat->node_start_pfn;
	unsigned long end_pfn = start_pfn + pgdat->node_spanned_pages;
	unsigned long pfn;

	for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
		unsigned long section_nr = pfn_to_section_nr(pfn);

		if (!present_section_nr(section_nr))
			continue;

		if (pfn_to_nid(pfn) != nid)
			continue;

		/*
                 * some memory sections of this node are not removed, and we
                 * can't offline node now.
                 */
		return;
	}

	if (check_and_unmap_cpu_on_node(pgdat))
		return;

	/*
         * all memory/cpu of this node are removed, we can offline this
         * node now.
         */
	node_set_offline(nid);
	unregister_one_node(nid);
}
Contributors
Person Tokens Prop Commits CommitProp
Tang Chen 91 87.50% 1 25.00%
Wen Congyang 12 11.54% 2 50.00%
Toshi Kani 1 0.96% 1 25.00%
Total 104 100.00% 4 100.00%


EXPORT_SYMBOL(try_offline_node);

/**
 * remove_memory
 *
 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
 * and online/offline operations before this call, as required by
 * try_offline_node().
 */


void __ref remove_memory(int nid, u64 start, u64 size)
{
	int ret;

	BUG_ON(check_hotplug_memory_range(start, size));

	mem_hotplug_begin();

	/*
         * All memory blocks must be offlined before removing memory.  Check
         * whether all memory blocks in question are offline and trigger a BUG()
         * if this is not the case.
         */
	ret = walk_memory_range(PFN_DOWN(start), PFN_UP(start + size - 1), NULL,
				check_memblock_offlined_cb);
	if (ret)
		BUG();

	/* remove memmap entry */
	firmware_map_remove(start, start + size, "System RAM");
	memblock_free(start, size);
	memblock_remove(start, size);

	arch_remove_memory(start, size);

	try_offline_node(nid);

	mem_hotplug_done();
}
Contributors
Person Tokens Prop Commits CommitProp
Wen Congyang 31 30.10% 2 16.67%
Yasuaki Ishimatsu 20 19.42% 2 16.67%
Xishi Qiu 15 14.56% 2 16.67%
Toshi Kani 15 14.56% 2 16.67%
Rafael J. Wysocki 11 10.68% 1 8.33%
Tang Chen 8 7.77% 1 8.33%
Vladimir Davydov 2 1.94% 1 8.33%
Badari Pulavarty 1 0.97% 1 8.33%
Total 103 100.00% 12 100.00%


EXPORT_SYMBOL_GPL(remove_memory);
#endif /* CONFIG_MEMORY_HOTREMOVE */
Overall Contributors
Person Tokens Prop Commits CommitProp
Lai Jiangshan 1271 15.05% 6 3.09%
Kamezawa Hiroyuki 1037 12.28% 13 6.70%
Yasuaki Ishimatsu 984 11.65% 7 3.61%
Yasunori Goto 771 9.13% 10 5.15%
Wen Congyang 351 4.16% 11 5.67%
Tang Chen 316 3.74% 11 5.67%
Vladimir Davydov 296 3.50% 1 0.52%
Badari Pulavarty 292 3.46% 3 1.55%
Reza Arbab 284 3.36% 3 1.55%
Toshi Kani 267 3.16% 10 5.15%
Xishi Qiu 230 2.72% 7 3.61%
Heiko Carstens 195 2.31% 2 1.03%
Dave Hansen 194 2.30% 4 2.06%
Daniel Kiper 166 1.97% 1 0.52%
Dan J Williams 159 1.88% 3 1.55%
David Rientjes 159 1.88% 6 3.09%
Vitaly Kuznetsov 144 1.70% 4 2.06%
Naoya Horiguchi 128 1.52% 2 1.03%
Keith Mannthey 124 1.47% 2 1.03%
Cody P Schafer 109 1.29% 6 3.09%
Wang Nan 102 1.21% 1 0.52%
David Vrabel 85 1.01% 1 0.52%
Minskey Guo 63 0.75% 1 0.52%
Chen Yucong 53 0.63% 1 0.52%
Mel Gorman 49 0.58% 5 2.58%
Jiang Liu 49 0.58% 8 4.12%
Motohiro Kosaki 44 0.52% 4 2.06%
Andrew Banman 43 0.51% 1 0.52%
Yisheng Xie 39 0.46% 1 0.52%
Randy Dunlap 36 0.43% 2 1.03%
JoonSoo Kim 30 0.36% 2 1.03%
Vlastimil Babka 27 0.32% 4 2.06%
Haicheng Li 27 0.32% 2 1.03%
Bob Liu 22 0.26% 2 1.03%
Andrea Arcangeli 22 0.26% 1 0.52%
Björn Helgaas 20 0.24% 1 0.52%
Rafael J. Wysocki 20 0.24% 3 1.55%
Andi Kleen 17 0.20% 1 0.52%
qiuxishi 17 0.20% 1 0.52%
Andrew Morton 17 0.20% 3 1.55%
Jeremy Fitzhardinge 16 0.19% 1 0.52%
Li Zhong 14 0.17% 1 0.52%
Gu Zheng 14 0.17% 1 0.52%
Nicholas Piggin 11 0.13% 1 0.52%
MinChan Kim 11 0.13% 2 1.03%
zhong jiang 11 0.13% 1 0.52%
Konstantin Khlebnikov 11 0.13% 1 0.52%
Gerald Schaefer 9 0.11% 1 0.52%
Fabian Frederick 8 0.09% 2 1.03%
Christoph Lameter 8 0.09% 2 1.03%
Geoff Levand 8 0.09% 1 0.52%
Michal Nazarewicz 6 0.07% 1 0.52%
Chandra Seetharaman 6 0.07% 1 0.52%
Nathan Zimmer 5 0.06% 1 0.52%
David Shaohua Li 5 0.06% 1 0.52%
Gary Hade 5 0.06% 1 0.52%
Adrian Bunk 4 0.05% 2 1.03%
Nathan Fontenot 4 0.05% 1 0.52%
Al Viro 4 0.05% 1 0.52%
Sheng Yong 3 0.04% 1 0.52%
Ingo Molnar 3 0.04% 1 0.52%
Adam Buchbinder 3 0.04% 1 0.52%
Zhu Guihua 3 0.04% 1 0.52%
Fengguang Wu 3 0.04% 1 0.52%
Yaowei Bai 3 0.04% 1 0.52%
Hidetoshi Seto 2 0.02% 1 0.52%
Paul Gortmaker 1 0.01% 1 0.52%
Santosh Shilimkar 1 0.01% 1 0.52%
Joe Perches 1 0.01% 1 0.52%
Hugh Dickins 1 0.01% 1 0.52%
Lucas De Marchi 1 0.01% 1 0.52%
Linus Torvalds 1 0.01% 1 0.52%
Yang Shi 1 0.01% 1 0.52%
Total 8446 100.00% 194 100.00%
Directory: mm

Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with cregit.
Person	Tokens	Prop	Commits	CommitProp
Vitaly Kuznetsov	43	100.00%	1	100.00%
Total	43	100.00%	1	100.00%
cregit-Linux how code gets into the kernel

Release 4.11 mm/memory_hotplug.c

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors
