Release 4.11 mm/page_isolation.c

/*
 * linux/mm/page_isolation.c
 */

#include <linux/mm.h>
#include <linux/page-isolation.h>
#include <linux/pageblock-flags.h>
#include <linux/memory.h>
#include <linux/hugetlb.h>
#include <linux/page_owner.h>
#include "internal.h"


#define CREATE_TRACE_POINTS
#include <trace/events/page_isolation.h>



static int set_migratetype_isolate(struct page *page,
				bool skip_hwpoisoned_pages)
{
	struct zone *zone;
	unsigned long flags, pfn;
	struct memory_isolate_notify arg;
	int notifier_ret;
	int ret = -EBUSY;

	zone = page_zone(page);

	spin_lock_irqsave(&zone->lock, flags);

	pfn = page_to_pfn(page);
	arg.start_pfn = pfn;
	arg.nr_pages = pageblock_nr_pages;
	arg.pages_found = 0;

	/*
         * It may be possible to isolate a pageblock even if the
         * migratetype is not MIGRATE_MOVABLE. The memory isolation
         * notifier chain is used by balloon drivers to return the
         * number of pages in a range that are held by the balloon
         * driver to shrink memory. If all the pages are accounted for
         * by balloons, are free, or on the LRU, isolation can continue.
         * Later, for example, when memory hotplug notifier runs, these
         * pages reported as "can be isolated" should be isolated(freed)
         * by the balloon driver through the memory notifier chain.
         */
	notifier_ret = memory_isolate_notify(MEM_ISOLATE_COUNT, &arg);
	notifier_ret = notifier_to_errno(notifier_ret);
	if (notifier_ret)
		goto out;
	/*
         * FIXME: Now, memory hotplug doesn't call shrink_slab() by itself.
         * We just check MOVABLE pages.
         */
	if (!has_unmovable_pages(zone, page, arg.pages_found,
				 skip_hwpoisoned_pages))
		ret = 0;

	/*
         * immobile means "not-on-lru" pages. If immobile is larger than
         * removable-by-driver pages reported by notifier, we'll fail.
         */

out:
	if (!ret) {
		unsigned long nr_pages;
		int migratetype = get_pageblock_migratetype(page);

		set_pageblock_migratetype(page, MIGRATE_ISOLATE);
		zone->nr_isolate_pageblock++;
		nr_pages = move_freepages_block(zone, page, MIGRATE_ISOLATE);

		__mod_zone_freepage_state(zone, -nr_pages, migratetype);
	}

	spin_unlock_irqrestore(&zone->lock, flags);
	if (!ret)
		drain_all_pages(zone);
	return ret;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
MinChan Kim
162
79.41%
1
10.00%
Bartlomiej Zolnierkiewicz
27
13.24%
3
30.00%
Wen Congyang
5
2.45%
1
10.00%
JoonSoo Kim
5
2.45%
1
10.00%
Vlastimil Babka
3
1.47%
2
20.00%
Yisheng Xie
1
0.49%
1
10.00%
Naoya Horiguchi
1
0.49%
1
10.00%
Total
204
100.00%
10
100.00%



static void unset_migratetype_isolate(struct page *page, unsigned migratetype)
{
	struct zone *zone;
	unsigned long flags, nr_pages;
	bool isolated_page = false;
	unsigned int order;
	unsigned long pfn, buddy_pfn;
	struct page *buddy;

	zone = page_zone(page);
	spin_lock_irqsave(&zone->lock, flags);
	if (get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
		goto out;

	/*
         * Because freepage with more than pageblock_order on isolated
         * pageblock is restricted to merge due to freepage counting problem,
         * it is possible that there is free buddy page.
         * move_freepages_block() doesn't care of merge so we need other
         * approach in order to merge them. Isolation and free will make
         * these pages to be merged.
         */
	if (PageBuddy(page)) {
		order = page_order(page);
		if (order >= pageblock_order) {
			pfn = page_to_pfn(page);
			buddy_pfn = __find_buddy_pfn(pfn, order);
			buddy = page + (buddy_pfn - pfn);

			if (pfn_valid_within(buddy_pfn) &&
			    !is_migrate_isolate_page(buddy)) {
				__isolate_free_page(page, order);
				isolated_page = true;
			}
		}
	}

	/*
         * If we isolate freepage with more than pageblock_order, there
         * should be no freepage in the range, so we could avoid costly
         * pageblock scanning for freepage moving.
         */
	if (!isolated_page) {
		nr_pages = move_freepages_block(zone, page, migratetype);
		__mod_zone_freepage_state(zone, nr_pages, migratetype);
	}
	set_pageblock_migratetype(page, migratetype);
	zone->nr_isolate_pageblock--;
out:
	spin_unlock_irqrestore(&zone->lock, flags);
	if (isolated_page) {
		post_alloc_hook(page, order, __GFP_MOVABLE);
		__free_pages(page, order);
	}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
JoonSoo Kim
119
52.89%
5
35.71%
MinChan Kim
78
34.67%
2
14.29%
Bartlomiej Zolnierkiewicz
14
6.22%
3
21.43%
Vlastimil Babka
9
4.00%
2
14.29%
Hui Zhu
4
1.78%
1
7.14%
Naoya Horiguchi
1
0.44%
1
7.14%
Total
225
100.00%
14
100.00%


static inline struct page *

__first_valid_page(unsigned long pfn, unsigned long nr_pages)
{
	int i;
	for (i = 0; i < nr_pages; i++)
		if (pfn_valid_within(pfn + i))
			break;
	if (unlikely(i == nr_pages))
		return NULL;
	return pfn_to_page(pfn + i);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Kamezawa Hiroyuki
63
100.00%
1
100.00%
Total
63
100.00%
1
100.00%

/*
 * start_isolate_page_range() -- make page-allocation-type of range of pages
 * to be MIGRATE_ISOLATE.
 * @start_pfn: The lower PFN of the range to be isolated.
 * @end_pfn: The upper PFN of the range to be isolated.
 * @migratetype: migrate type to set in error recovery.
 *
 * Making page-allocation-type to be MIGRATE_ISOLATE means free pages in
 * the range will never be allocated. Any free pages and pages freed in the
 * future will not be allocated again.
 *
 * start_pfn/end_pfn must be aligned to pageblock_order.
 * Returns 0 on success and -EBUSY if any part of range cannot be isolated.
 */


int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
			     unsigned migratetype, bool skip_hwpoisoned_pages)
{
	unsigned long pfn;
	unsigned long undo_pfn;
	struct page *page;

	BUG_ON(!IS_ALIGNED(start_pfn, pageblock_nr_pages));
	BUG_ON(!IS_ALIGNED(end_pfn, pageblock_nr_pages));

	for (pfn = start_pfn;
	     pfn < end_pfn;
	     pfn += pageblock_nr_pages) {
		page = __first_valid_page(pfn, pageblock_nr_pages);
		if (page &&
		    set_migratetype_isolate(page, skip_hwpoisoned_pages)) {
			undo_pfn = pfn;
			goto undo;
		}
	}
	return 0;
undo:
	for (pfn = start_pfn;
	     pfn < undo_pfn;
	     pfn += pageblock_nr_pages)
		unset_migratetype_isolate(pfn_to_page(pfn), migratetype);

	return -EBUSY;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Kamezawa Hiroyuki
111
84.09%
2
40.00%
Naoya Horiguchi
10
7.58%
1
20.00%
Michal Nazarewicz
6
4.55%
1
20.00%
Wen Congyang
5
3.79%
1
20.00%
Total
132
100.00%
5
100.00%

/*
 * Make isolated pages available again.
 */


int undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
			    unsigned migratetype)
{
	unsigned long pfn;
	struct page *page;

	BUG_ON(!IS_ALIGNED(start_pfn, pageblock_nr_pages));
	BUG_ON(!IS_ALIGNED(end_pfn, pageblock_nr_pages));

	for (pfn = start_pfn;
	     pfn < end_pfn;
	     pfn += pageblock_nr_pages) {
		page = __first_valid_page(pfn, pageblock_nr_pages);
		if (!page || get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
			continue;
		unset_migratetype_isolate(page, migratetype);
	}
	return 0;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Kamezawa Hiroyuki
79
83.16%
2
50.00%
Wang Xiaoqiang
10
10.53%
1
25.00%
Michal Nazarewicz
6
6.32%
1
25.00%
Total
95
100.00%
4
100.00%

/*
 * Test all pages in the range is free(means isolated) or not.
 * all pages in [start_pfn...end_pfn) must be in the same zone.
 * zone->lock must be held before call this.
 *
 * Returns the last tested pfn.
 */

static unsigned long

__test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn,
				  bool skip_hwpoisoned_pages)
{
	struct page *page;

	while (pfn < end_pfn) {
		if (!pfn_valid_within(pfn)) {
			pfn++;
			continue;
		}
		page = pfn_to_page(pfn);
		if (PageBuddy(page))
			/*
                         * If the page is on a free list, it has to be on
                         * the correct MIGRATE_ISOLATE freelist. There is no
                         * simple way to verify that as VM_BUG_ON(), though.
                         */
			pfn += 1 << page_order(page);
		else if (skip_hwpoisoned_pages && PageHWPoison(page))
			/* A HWPoisoned page cannot be also PageBuddy */
			pfn++;
		else
			break;
	}

	return pfn;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Kamezawa Hiroyuki
67
76.14%
1
25.00%
Wen Congyang
16
18.18%
1
25.00%
JoonSoo Kim
3
3.41%
1
25.00%
Vlastimil Babka
2
2.27%
1
25.00%
Total
88
100.00%
4
100.00%

/* Caller should ensure that requested range is in a single zone */


int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn,
			bool skip_hwpoisoned_pages)
{
	unsigned long pfn, flags;
	struct page *page;
	struct zone *zone;

	/*
         * Note: pageblock_nr_pages != MAX_ORDER. Then, chunks of free pages
         * are not aligned to pageblock_nr_pages.
         * Then we just check migratetype first.
         */
	for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
		page = __first_valid_page(pfn, pageblock_nr_pages);
		if (page && get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
			break;
	}
	page = __first_valid_page(start_pfn, end_pfn - start_pfn);
	if ((pfn < end_pfn) || !page)
		return -EBUSY;
	/* Check all pages are free or marked as ISOLATED */
	zone = page_zone(page);
	spin_lock_irqsave(&zone->lock, flags);
	pfn = __test_page_isolated_in_pageblock(start_pfn, end_pfn,
						skip_hwpoisoned_pages);
	spin_unlock_irqrestore(&zone->lock, flags);

	trace_test_pages_isolated(start_pfn, end_pfn, pfn);

	return pfn < end_pfn ? -EBUSY : 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Kamezawa Hiroyuki
79
51.30%
2
25.00%
Gerald Schaefer
53
34.42%
2
25.00%
JoonSoo Kim
15
9.74%
2
25.00%
Wen Congyang
5
3.25%
1
12.50%
Tang Chen
2
1.30%
1
12.50%
Total
154
100.00%
8
100.00%



struct page *alloc_migrate_target(struct page *page, unsigned long private,
				  int **resultp)
{
	gfp_t gfp_mask = GFP_USER | __GFP_MOVABLE;

	/*
         * TODO: allocate a destination hugepage from a nearest neighbor node,
         * accordance with memory policy of the user process if possible. For
         * now as a simple work-around, we use the next node for destination.
         */
	if (PageHuge(page))
		return alloc_huge_page_node(page_hstate(compound_head(page)),
					    next_node_in(page_to_nid(page),
							 node_online_map));

	if (PageHighMem(page))
		gfp_mask |= __GFP_HIGHMEM;

	return alloc_page(gfp_mask);
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
MinChan Kim
45
60.00%
1
33.33%
Naoya Horiguchi
21
28.00%
1
33.33%
Andrew Morton
9
12.00%
1
33.33%
Total
75
100.00%
3
100.00%

Overall Contributors
Person
Tokens
Prop
Commits
CommitProp
Kamezawa Hiroyuki
413
38.67%
2
5.88%
MinChan Kim
288
26.97%
3
8.82%
JoonSoo Kim
152
14.23%
8
23.53%
Gerald Schaefer
53
4.96%
2
5.88%
Bartlomiej Zolnierkiewicz
41
3.84%
3
8.82%
Naoya Horiguchi
36
3.37%
3
8.82%
Wen Congyang
31
2.90%
1
2.94%
Vlastimil Babka
14
1.31%
5
14.71%
Michal Nazarewicz
13
1.22%
1
2.94%
Wang Xiaoqiang
10
0.94%
1
2.94%
Andrew Morton
9
0.84%
1
2.94%
Hui Zhu
4
0.37%
1
2.94%
Tang Chen
2
0.19%
1
2.94%
Neil Zhang
1
0.09%
1
2.94%
Yisheng Xie
1
0.09%
1
2.94%
Total
1068
100.00%
34
100.00%
Directory: mm

Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.