cregit-Linux how code gets into the kernel

Release 4.8 mm/compaction.c

Directory: mm
/*
 * linux/mm/compaction.c
 *
 * Memory compaction for the reduction of external fragmentation. Note that
 * this heavily depends upon page migration to do all the real heavy
 * lifting
 *
 * Copyright IBM Corp. 2007-2010 Mel Gorman <mel@csn.ul.ie>
 */
#include <linux/cpu.h>
#include <linux/swap.h>
#include <linux/migrate.h>
#include <linux/compaction.h>
#include <linux/mm_inline.h>
#include <linux/backing-dev.h>
#include <linux/sysctl.h>
#include <linux/sysfs.h>
#include <linux/page-isolation.h>
#include <linux/kasan.h>
#include <linux/kthread.h>
#include <linux/freezer.h>
#include <linux/page_owner.h>
#include "internal.h"

#ifdef CONFIG_COMPACTION

static inline void count_compact_event(enum vm_event_item item) { count_vm_event(item); }

Contributors

PersonTokensPropCommitsCommitProp
minchan kimminchan kim16100.00%1100.00%
Total16100.00%1100.00%


static inline void count_compact_events(enum vm_event_item item, long delta) { count_vm_events(item, delta); }

Contributors

PersonTokensPropCommitsCommitProp
minchan kimminchan kim21100.00%1100.00%
Total21100.00%1100.00%

#else #define count_compact_event(item) do { } while (0) #define count_compact_events(item, delta) do { } while (0) #endif #if defined CONFIG_COMPACTION || defined CONFIG_CMA #define CREATE_TRACE_POINTS #include <trace/events/compaction.h> #define block_start_pfn(pfn, order) round_down(pfn, 1UL << (order)) #define block_end_pfn(pfn, order) ALIGN((pfn) + 1, 1UL << (order)) #define pageblock_start_pfn(pfn) block_start_pfn(pfn, pageblock_order) #define pageblock_end_pfn(pfn) block_end_pfn(pfn, pageblock_order)
static unsigned long release_freepages(struct list_head *freelist) { struct page *page, *next; unsigned long high_pfn = 0; list_for_each_entry_safe(page, next, freelist, lru) { unsigned long pfn = page_to_pfn(page); list_del(&page->lru); __free_page(page); if (pfn > high_pfn) high_pfn = pfn; } return high_pfn; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman5372.60%150.00%
vlastimil babkavlastimil babka2027.40%150.00%
Total73100.00%2100.00%


static void map_pages(struct list_head *list) { unsigned int i, order, nr_pages; struct page *page, *next; LIST_HEAD(tmp_list); list_for_each_entry_safe(page, next, list, lru) { list_del(&page->lru); order = page_private(page); nr_pages = 1 << order; post_alloc_hook(page, order, __GFP_MOVABLE); if (order) split_page(page, order); for (i = 0; i < nr_pages; i++) { list_add(&page->lru, &tmp_list); page++; } } list_splice(&tmp_list, list); }

Contributors

PersonTokensPropCommitsCommitProp
joonsoo kimjoonsoo kim9477.05%360.00%
michal nazarewiczmichal nazarewicz2520.49%120.00%
andrey ryabininandrey ryabinin32.46%120.00%
Total122100.00%5100.00%


static inline bool migrate_async_suitable(int migratetype) { return is_migrate_cma(migratetype) || migratetype == MIGRATE_MOVABLE; }

Contributors

PersonTokensPropCommitsCommitProp
michal nazarewiczmichal nazarewicz20100.00%1100.00%
Total20100.00%1100.00%

#ifdef CONFIG_COMPACTION
int PageMovable(struct page *page) { struct address_space *mapping; VM_BUG_ON_PAGE(!PageLocked(page), page); if (!__PageMovable(page)) return 0; mapping = page_mapping(page); if (mapping && mapping->a_ops && mapping->a_ops->isolate_page) return 1; return 0; }

Contributors

PersonTokensPropCommitsCommitProp
minchan kimminchan kim64100.00%1100.00%
Total64100.00%1100.00%

EXPORT_SYMBOL(PageMovable);
void __SetPageMovable(struct page *page, struct address_space *mapping) { VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE((unsigned long)mapping & PAGE_MAPPING_MOVABLE, page); page->mapping = (void *)((unsigned long)mapping | PAGE_MAPPING_MOVABLE); }

Contributors

PersonTokensPropCommitsCommitProp
minchan kimminchan kim57100.00%1100.00%
Total57100.00%1100.00%

EXPORT_SYMBOL(__SetPageMovable);
void __ClearPageMovable(struct page *page) { VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(!PageMovable(page), page); /* * Clear registered address_space val with keeping PAGE_MAPPING_MOVABLE * flag so that VM can catch up released page by driver after isolation. * With it, VM migration doesn't try to put it back. */ page->mapping = (void *)((unsigned long)page->mapping & PAGE_MAPPING_MOVABLE); }

Contributors

PersonTokensPropCommitsCommitProp
minchan kimminchan kim53100.00%1100.00%
Total53100.00%1100.00%

EXPORT_SYMBOL(__ClearPageMovable); /* Do not skip compaction more than 64 times */ #define COMPACT_MAX_DEFER_SHIFT 6 /* * Compaction is deferred when compaction fails to result in a page * allocation success. 1 << compact_defer_limit compactions are skipped up * to a limit of 1 << COMPACT_MAX_DEFER_SHIFT */
void defer_compaction(struct zone *zone, int order) { zone->compact_considered = 0; zone->compact_defer_shift++; if (order < zone->compact_order_failed) zone->compact_order_failed = order; if (zone->compact_defer_shift > COMPACT_MAX_DEFER_SHIFT) zone->compact_defer_shift = COMPACT_MAX_DEFER_SHIFT; trace_mm_compaction_defer_compaction(zone, order); }

Contributors

PersonTokensPropCommitsCommitProp
joonsoo kimjoonsoo kim59100.00%1100.00%
Total59100.00%1100.00%

/* Returns true if compaction should be skipped this time */
bool compaction_deferred(struct zone *zone, int order) { unsigned long defer_limit = 1UL << zone->compact_defer_shift; if (order < zone->compact_order_failed) return false; /* Avoid possible overflow */ if (++zone->compact_considered > defer_limit) zone->compact_considered = defer_limit; if (zone->compact_considered >= defer_limit) return false; trace_mm_compaction_deferred(zone, order); return true; }

Contributors

PersonTokensPropCommitsCommitProp
joonsoo kimjoonsoo kim71100.00%1100.00%
Total71100.00%1100.00%

/* * Update defer tracking counters after successful compaction of given order, * which means an allocation either succeeded (alloc_success == true) or is * expected to succeed. */
void compaction_defer_reset(struct zone *zone, int order, bool alloc_success) { if (alloc_success) { zone->compact_considered = 0; zone->compact_defer_shift = 0; } if (order >= zone->compact_order_failed) zone->compact_order_failed = order + 1; trace_mm_compaction_defer_reset(zone, order); }

Contributors

PersonTokensPropCommitsCommitProp
joonsoo kimjoonsoo kim57100.00%1100.00%
Total57100.00%1100.00%

/* Returns true if restarting compaction after many failures */
bool compaction_restarting(struct zone *zone, int order) { if (order < zone->compact_order_failed) return false; return zone->compact_defer_shift == COMPACT_MAX_DEFER_SHIFT && zone->compact_considered >= 1UL << zone->compact_defer_shift; }

Contributors

PersonTokensPropCommitsCommitProp
joonsoo kimjoonsoo kim41100.00%1100.00%
Total41100.00%1100.00%

/* Returns true if the pageblock should be scanned for pages to isolate. */
static inline bool isolation_suitable(struct compact_control *cc, struct page *page) { if (cc->ignore_skip_hint) return true; return !get_pageblock_skip(page); }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman33100.00%1100.00%
Total33100.00%1100.00%


static void reset_cached_positions(struct zone *zone) { zone->compact_cached_migrate_pfn[0] = zone->zone_start_pfn; zone->compact_cached_migrate_pfn[1] = zone->zone_start_pfn; zone->compact_cached_free_pfn = pageblock_start_pfn(zone_end_pfn(zone) - 1); }

Contributors

PersonTokensPropCommitsCommitProp
vlastimil babkavlastimil babka4391.49%266.67%
joonsoo kimjoonsoo kim48.51%133.33%
Total47100.00%3100.00%

/* * This function is called to clear all cached information on pageblocks that * should be skipped for page isolation when the migrate and free page scanner * meet. */
static void __reset_isolation_suitable(struct zone *zone) { unsigned long start_pfn = zone->zone_start_pfn; unsigned long end_pfn = zone_end_pfn(zone); unsigned long pfn; zone->compact_blockskip_flush = false; /* Walk the zone and mark every pageblock as suitable for isolation */ for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { struct page *page; cond_resched(); if (!pfn_valid(pfn)) continue; page = pfn_to_page(pfn); if (zone != page_zone(page)) continue; clear_pageblock_skip(page); } reset_cached_positions(zone); }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman9191.92%250.00%
vlastimil babkavlastimil babka55.05%125.00%
cody p schafercody p schafer33.03%125.00%
Total99100.00%4100.00%


void reset_isolation_suitable(pg_data_t *pgdat) { int zoneid; for (zoneid = 0; zoneid < MAX_NR_ZONES; zoneid++) { struct zone *zone = &pgdat->node_zones[zoneid]; if (!populated_zone(zone)) continue; /* Only flush if a full compaction finished recently */ if (zone->compact_blockskip_flush) __reset_isolation_suitable(zone); } }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman61100.00%1100.00%
Total61100.00%1100.00%

/* * If no pages were isolated then mark this pageblock to be skipped in the * future. The information is later cleared by __reset_isolation_suitable(). */
static void update_pageblock_skip(struct compact_control *cc, struct page *page, unsigned long nr_isolated, bool migrate_scanner) { struct zone *zone = cc->zone; unsigned long pfn; if (cc->ignore_skip_hint) return; if (!page) return; if (nr_isolated) return; set_pageblock_skip(page); pfn = page_to_pfn(page); /* Update where async and sync compaction should restart */ if (migrate_scanner) { if (pfn > zone->compact_cached_migrate_pfn[0]) zone->compact_cached_migrate_pfn[0] = pfn; if (cc->mode != MIGRATE_ASYNC && pfn > zone->compact_cached_migrate_pfn[1]) zone->compact_cached_migrate_pfn[1] = pfn; } else { if (pfn < zone->compact_cached_free_pfn) zone->compact_cached_free_pfn = pfn; } }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman8763.97%240.00%
david rientjesdavid rientjes4230.88%240.00%
joonsoo kimjoonsoo kim75.15%120.00%
Total136100.00%5100.00%

#else
static inline bool isolation_suitable(struct compact_control *cc, struct page *page) { return true; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman20100.00%1100.00%
Total20100.00%1100.00%


static void update_pageblock_skip(struct compact_control *cc, struct page *page, unsigned long nr_isolated, bool migrate_scanner) { }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman22100.00%2100.00%
Total22100.00%2100.00%

#endif /* CONFIG_COMPACTION */ /* * Compaction requires the taking of some coarse locks that are potentially * very heavily contended. For async compaction, back out if the lock cannot * be taken immediately. For sync compaction, spin on the lock if needed. * * Returns true if the lock is held * Returns false if the lock is not held and compaction should abort */
static bool compact_trylock_irqsave(spinlock_t *lock, unsigned long *flags, struct compact_control *cc) { if (cc->mode == MIGRATE_ASYNC) { if (!spin_trylock_irqsave(lock, *flags)) { cc->contended = true; return false; } } else { spin_lock_irqsave(lock, *flags); } return true; }

Contributors

PersonTokensPropCommitsCommitProp
vlastimil babkavlastimil babka5481.82%375.00%
mel gormanmel gorman1218.18%125.00%
Total66100.00%4100.00%

/* * Compaction requires the taking of some coarse locks that are potentially * very heavily contended. The lock should be periodically unlocked to avoid * having disabled IRQs for a long time, even when there is nobody waiting on * the lock. It might also be that allowing the IRQs will result in * need_resched() becoming true. If scheduling is needed, async compaction * aborts. Sync compaction schedules. * Either compaction type will also abort if a fatal signal is pending. * In either case if the lock was locked, it is dropped and not regained. * * Returns true if compaction should abort due to fatal signal pending, or * async compaction due to need_resched() * Returns false when compaction can continue (sync compaction might have * scheduled) */
static bool compact_unlock_should_abort(spinlock_t *lock, unsigned long flags, bool *locked, struct compact_control *cc) { if (*locked) { spin_unlock_irqrestore(lock, flags); *locked = false; } if (fatal_signal_pending(current)) { cc->contended = true; return true; } if (need_resched()) { if (cc->mode == MIGRATE_ASYNC) { cc->contended = true; return true; } cond_resched(); } return false; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman5863.04%125.00%
vlastimil babkavlastimil babka3133.70%250.00%
david rientjesdavid rientjes33.26%125.00%
Total92100.00%4100.00%

/* * Aside from avoiding lock contention, compaction also periodically checks * need_resched() and either schedules in sync compaction or aborts async * compaction. This is similar to what compact_unlock_should_abort() does, but * is used where no lock is concerned. * * Returns false when no scheduling was needed, or sync compaction scheduled. * Returns true when async compaction should abort. */
static inline bool compact_should_abort(struct compact_control *cc) { /* async compaction aborts if contended */ if (need_resched()) { if (cc->mode == MIGRATE_ASYNC) { cc->contended = true; return true; } cond_resched(); } return false; }

Contributors

PersonTokensPropCommitsCommitProp
vlastimil babkavlastimil babka45100.00%2100.00%
Total45100.00%2100.00%

/* * Isolate free pages onto a private freelist. If @strict is true, will abort * returning 0 on any invalid PFNs or non-free pages inside of the pageblock * (even though it may still end up isolating some pages). */
static unsigned long isolate_freepages_block(struct compact_control *cc, unsigned long *start_pfn, unsigned long end_pfn, struct list_head *freelist, bool strict) { int nr_scanned = 0, total_isolated = 0; struct page *cursor, *valid_page = NULL; unsigned long flags = 0; bool locked = false; unsigned long blockpfn = *start_pfn; unsigned int order; cursor = pfn_to_page(blockpfn); /* Isolate free pages. */ for (; blockpfn < end_pfn; blockpfn++, cursor++) { int isolated; struct page *page = cursor; /* * Periodically drop the lock (if held) regardless of its * contention, to give chance to IRQs. Abort if fatal signal * pending or async compaction detects need_resched() */ if (!(blockpfn % SWAP_CLUSTER_MAX) && compact_unlock_should_abort(&cc->zone->lock, flags, &locked, cc)) break; nr_scanned++; if (!pfn_valid_within(blockpfn)) goto isolate_fail; if (!valid_page) valid_page = page; /* * For compound pages such as THP and hugetlbfs, we can save * potentially a lot of iterations if we skip them at once. * The check is racy, but we can consider only valid values * and the only danger is skipping too much. */ if (PageCompound(page)) { unsigned int comp_order = compound_order(page); if (likely(comp_order < MAX_ORDER)) { blockpfn += (1UL << comp_order) - 1; cursor += (1UL << comp_order) - 1; } goto isolate_fail; } if (!PageBuddy(page)) goto isolate_fail; /* * If we already hold the lock, we can skip some rechecking. * Note that if we hold the lock now, checked_pageblock was * already set in some previous iteration (or strict is true), * so it is correct to skip the suitable migration target * recheck as well. */ if (!locked) { /* * The zone lock must be held to isolate freepages. * Unfortunately this is a very coarse lock and can be * heavily contended if there are parallel allocations * or parallel compactions. For async compaction do not * spin on the lock and we acquire the lock as late as * possible. */ locked = compact_trylock_irqsave(&cc->zone->lock, &flags, cc); if (!locked) break; /* Recheck this is a buddy page under lock */ if (!PageBuddy(page)) goto isolate_fail; } /* Found a free page, will break it into order-0 pages */ order = page_order(page); isolated = __isolate_free_page(page, order); if (!isolated) break; set_page_private(page, order); total_isolated += isolated; cc->nr_freepages += isolated; list_add_tail(&page->lru, freelist); if (!strict && cc->nr_migratepages <= cc->nr_freepages) { blockpfn += isolated; break; } /* Advance to the end of split page */ blockpfn += isolated - 1; cursor += isolated - 1; continue; isolate_fail: if (strict) break; else continue; } if (locked) spin_unlock_irqrestore(&cc->zone->lock, flags); /* * There is a tiny chance that we have read bogus compound_order(), * so be careful to not go outside of the pageblock. */ if (unlikely(blockpfn > end_pfn)) blockpfn = end_pfn; trace_mm_compaction_isolate_freepages(*start_pfn, blockpfn, nr_scanned, total_isolated); /* Record how far we have got within the block */ *start_pfn = blockpfn; /* * If strict isolation is requested by CMA then check that all the * pages requested were isolated. If there were any failures, 0 is * returned and CMA will fail. */ if (strict && blockpfn < end_pfn) total_isolated = 0; /* Update the pageblock-skip if the whole pageblock was scanned */ if (blockpfn == end_pfn) update_pageblock_skip(cc, valid_page, total_isolated, false); count_compact_events(COMPACTFREE_SCANNED, nr_scanned); if (total_isolated) count_compact_events(COMPACTISOLATED, total_isolated); return total_isolated; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman21747.69%633.33%
vlastimil babkavlastimil babka11725.71%422.22%
joonsoo kimjoonsoo kim5311.65%316.67%
david rientjesdavid rientjes296.37%15.56%
laura abbottlaura abbott224.84%15.56%
michal nazarewiczmichal nazarewicz132.86%15.56%
minchan kimminchan kim20.44%15.56%
xiubo lixiubo li20.44%15.56%
Total455100.00%18100.00%

/** * isolate_freepages_range() - isolate free pages. * @start_pfn: The first PFN to start isolating. * @end_pfn: The one-past-last PFN. * * Non-free pages, invalid PFNs, or zone boundaries within the * [start_pfn, end_pfn) range are considered errors, cause function to * undo its actions and return zero. * * Otherwise, function returns one-past-the-last PFN of isolated page * (which may be greater then end_pfn if end fell in a middle of * a free page). */
unsigned long isolate_freepages_range(struct compact_control *cc, unsigned long start_pfn, unsigned long end_pfn) { unsigned long isolated, pfn, block_start_pfn, block_end_pfn; LIST_HEAD(freelist); pfn = start_pfn; block_start_pfn = pageblock_start_pfn(pfn); if (block_start_pfn < cc->zone->zone_start_pfn) block_start_pfn = cc->zone->zone_start_pfn; block_end_pfn = pageblock_end_pfn(pfn); for (; pfn < end_pfn; pfn += isolated, block_start_pfn = block_end_pfn, block_end_pfn += pageblock_nr_pages) { /* Protect pfn from changing by isolate_freepages_block */ unsigned long isolate_start_pfn = pfn; block_end_pfn = min(block_end_pfn, end_pfn); /* * pfn could pass the block_end_pfn if isolated freepage * is more than pageblock order. In this case, we adjust * scanning range to right one. */ if (pfn >= block_end_pfn) { block_start_pfn = pageblock_start_pfn(pfn); block_end_pfn = pageblock_end_pfn(pfn); block_end_pfn = min(block_end_pfn, end_pfn); } if (!pageblock_pfn_to_page(block_start_pfn, block_end_pfn, cc->zone)) break; isolated = isolate_freepages_block(cc, &isolate_start_pfn, block_end_pfn, &freelist, true); /* * In strict mode, isolate_freepages_block() returns 0 if * there are any holes in the block (ie. invalid PFNs or * non-free pages). */ if (!isolated) break; /* * If we managed to isolate pages, it is always (1 << n) * * pageblock_nr_pages for some non-negative n. (Max order * page may span two pageblocks). */ } /* __isolate_free_page() does not map the pages */ map_pages(&freelist); if (pfn < end_pfn) { /* Loop terminated early, cleanup. */ release_freepages(&freelist); return 0; } /* We don't use freelists for anything. */ return pfn; }

Contributors

PersonTokensPropCommitsCommitProp
michal nazarewiczmichal nazarewicz9646.15%218.18%
joonsoo kimjoonsoo kim5827.88%327.27%
vlastimil babkavlastimil babka4320.67%327.27%
mel gormanmel gorman115.29%327.27%
Total208100.00%11100.00%

/* Update the number of anon and file isolated pages in the zone */
static void acct_isolated(struct zone *zone, struct compact_control *cc) { struct page *page; unsigned int count[2] = { 0, }; if (list_empty(&cc->migratepages)) return; list_for_each_entry(page, &cc->migratepages, lru) count[!!page_is_file_cache(page)]++; mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON, count[0]); mod_node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE, count[1]); }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman7381.11%360.00%
vlastimil babkavlastimil babka1112.22%120.00%
minchan kimminchan kim66.67%120.00%
Total90100.00%5100.00%

/* Similar to reclaim, but different enough that they don't share logic */
static bool too_many_isolated(struct zone *zone) { unsigned long active, inactive, isolated; inactive = node_page_state(zone->zone_pgdat, NR_INACTIVE_FILE) + node_page_state(zone->zone_pgdat, NR_INACTIVE_ANON); active = node_page_state(zone->zone_pgdat, NR_ACTIVE_FILE) + node_page_state(zone->zone_pgdat, NR_ACTIVE_ANON); isolated = node_page_state(zone->zone_pgdat, NR_ISOLATED_FILE) + node_page_state(zone->zone_pgdat, NR_ISOLATED_ANON); return isolated > (inactive + active) / 2; }

Contributors

PersonTokensPropCommitsCommitProp
mel gormanmel gorman6875.56%266.67%
minchan kimminchan kim2224.44%133.33%
Total90100.00%3100.00%

/** * isolate_migratepages_block() - isolate all migrate-able pages within * a single pageblock * @cc: Compaction control structure. * @low_pfn: The first PFN to isolate * @end_pfn: The one-past-the-last PFN to isolate, within same pageblock * @isolate_mode: Isolation mode to be used. * * Isolate all pages that can be migrated from the range specified by * [low_pfn, end_pfn). The range is expected to be within same pageblock. * Returns zero if there is a fatal signal pending, otherwise PFN of the * first page that was not scanned (which may be both less, equal to or more * than end_pfn). * * The pages are isolated on cc->migratepages list (not required to be empty), * and cc->nr_migratepages is updated accordingly. The cc->migrate_pfn field * is neither read nor updated. */
static unsigned long isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn, unsigned long end_pfn, isolate_mode_t isolate_mode) { struct zone *zone = cc->zone; unsigned long nr_scanned = 0, nr_isolated = 0; struct lruvec *lruvec; unsigned long flags = 0; bool locked = false; struct page *page = NULL, *valid_page = NULL; unsigned long start_pfn = low_pfn; bool skip_on_failure = false; unsigned long next_skip_pfn = 0; /* * Ensure that there are not too many pages isolated from the LRU * list by either parallel reclaimers or compaction. If there are, * delay for some time until fewer pages are isolated */ while (unlikely(too_many_isolated(zone))) { /* async migration should just abort */ if (cc->mode == MIGRATE_ASYNC) return 0; congestion_wait(BLK_RW_ASYNC, HZ/10); if (fatal_signal_pending(current)) return 0; } if (compact_should_abort(cc)) return 0; if (cc->direct_compaction && (cc->mode == MIGRATE_ASYNC)) { skip_on_failure = true; next_skip_pfn = block_end_pfn(low_pfn, cc->order); } /* Time to isolate some pages for migration */ for (; low_pfn < end_pfn; low_pfn++) { if (skip_on_failure && low_pfn >= next_skip_pfn) { /* * We have isolated all migration candidates in the * previous order-aligned block, and did not skip it due * to failure. We should migrate the pages now and * hopefully succeed compaction. */ if (nr_isolated) break; /* * We failed to isolate in the previous order-aligned * block. Set the new boundary to the end of the * current block. Note we can't simply increase * next_skip_pfn by 1 << order, as low_pfn might have * been incremented by a higher number due to skipping * a compound or a high-order buddy page in the * previous loop iteration. */ next_skip_pfn = block_end_pfn(low_pfn, cc->order); } /* * Periodically drop the lock (if held) regardless of its * contention, to give chance to IRQs. Abort async compaction * if contended. */ if (!(low_pfn % SWAP_CLUSTER_MAX) && compact_unlock_should_abort(zone_lru_lock(zone), flags, &locked, cc)) break; if (!pfn_valid_within(low_pfn)) goto isolate_fail; nr_scanned++; page = pfn_to_page(low_pfn); if (!valid_page) valid_page = page; /* * Skip if free. We read page order here without zone lock * which is generally unsafe, but the race window is small and * the worst thing that can happen is that we skip some * potential isolation targets. */ if (PageBuddy(page)) { unsigned long freepage_order = page_order_unsafe(page); /* * Without lock, we cannot be sure that what we got is * a valid page order. Consider only values in the * valid order range to prevent low_pfn overflow. */ if (freepage_order > 0 && freepage_order < MAX_ORDER) low_pfn += (1UL << freepage_order) - 1; continue; } /* * Regardless of being on LRU, compound pages such as THP and * hugetlbfs are not to be compacted. We can potentially save * a lot of iterations if we skip them at once. The check is * racy, but we can consider only valid values and the only * danger is skipping too much. */ if (PageCompound(page)) { unsigned int comp_order = compound_order(page); if (likely(comp_order < MAX_ORDER)) low_pfn += (1UL << comp_order) - 1; goto isolate_fail; } /* * Check may be lockless but that's ok as we recheck later. * It's possible to migrate LRU and non-lru movable pages. * Skip any other type of page */ if (!PageLRU(page)) { /* * __PageMovable can return false positive so we need * to verify it under page_lock. */ if (unlikely(__PageMovable(page)) && !PageIsolated(page)) { if (locked) { spin_unlock_irqrestore(zone_lru_lock(zone), flags); locked = false; } if (isolate_movable_page(page, isolate_mode)) goto isolate_success; } goto isolate_fail; } /* * Migration will fail if an anonymous page is pinned in memory, * so avoid taking lru_lock and isolating it unnecessarily in an * admittedly racy check. */ if (!page_mapping(page) && page_count(page) > page_mapcount(page)) goto isolate_fail; /* If we already hold the lock, we can skip some rechecking */ if (!locked) { locked = compact_trylock_irqsave(zone_lru_lock(zone), &flags, cc); if (!locked) break; /* Recheck PageLRU and PageCompound under lock */ if (!PageLRU(page)) goto isolate_fail; /* * Page become compound since the non-locked check, * and it's on LRU. It can only be a THP so the order * is safe to read and it's 0 for tail pages. */ if (unlikely(PageCompound(page))) { low_pfn += (1UL << compound_order(page)) - 1; goto isolate_fail