cregit-Linux how code gets into the kernel

Release 4.8 mm/migrate.c

Directory: mm
 * Memory Migration functionality - linux/mm/migrate.c
 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
 * Page migration was first developed in the context of the memory hotplug
 * project. The main authors of the migration code are:
 * IWAMOTO Toshihiro <>
 * Hirokazu Takahashi <>
 * Dave Hansen <>
 * Christoph Lameter

#include <linux/migrate.h>
#include <linux/export.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/pagemap.h>
#include <linux/buffer_head.h>
#include <linux/mm_inline.h>
#include <linux/nsproxy.h>
#include <linux/pagevec.h>
#include <linux/ksm.h>
#include <linux/rmap.h>
#include <linux/topology.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
#include <linux/writeback.h>
#include <linux/mempolicy.h>
#include <linux/vmalloc.h>
#include <linux/security.h>
#include <linux/backing-dev.h>
#include <linux/compaction.h>
#include <linux/syscalls.h>
#include <linux/hugetlb.h>
#include <linux/hugetlb_cgroup.h>
#include <linux/gfp.h>
#include <linux/balloon_compaction.h>
#include <linux/mmu_notifier.h>
#include <linux/page_idle.h>
#include <linux/page_owner.h>

#include <asm/tlbflush.h>

#include <trace/events/migrate.h>

#include "internal.h"

 * migrate_prep() needs to be called before we start compiling a list of pages
 * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is
 * undesirable, use migrate_prep_local()

int migrate_prep(void) { /* * Clear the LRU lists so pages can be isolated. * Note that pages may be moved off the LRU after we have * drained them. Those pages will fail to migrate like other * pages that may be busy. */ lru_add_drain_all(); return 0; }


christoph lameterchristoph lameter1178.57%150.00%
mel gormanmel gorman321.43%150.00%

/* Do the necessary work of migrate_prep but not if it involves other CPUs */
int migrate_prep_local(void) { lru_add_drain(); return 0; }


mel gormanmel gorman1076.92%150.00%
christoph lameterchristoph lameter323.08%150.00%

bool isolate_movable_page(struct page *page, isolate_mode_t mode) { struct address_space *mapping; /* * Avoid burning cycles with pages that are yet under __free_pages(), * or just got freed under us. * * In case we 'win' a race for a movable page being freed under us and * raise its refcount preventing __free_pages() from doing its job * the put_page() at the end of this block will take care of * release this page, thus avoiding a nasty leakage. */ if (unlikely(!get_page_unless_zero(page))) goto out; /* * Check PageMovable before holding a PG_lock because page's owner * assumes anybody doesn't touch PG_lock of newly allocated page * so unconditionally grapping the lock ruins page's owner side. */ if (unlikely(!__PageMovable(page))) goto out_putpage; /* * As movable pages are not isolated from LRU lists, concurrent * compaction threads can race against page migration functions * as well as race against the releasing a page. * * In order to avoid having an already isolated movable page * being (wrongly) re-isolated while it is under migration, * or to avoid attempting to isolate pages being released, * lets be sure we have the page lock * before proceeding with the movable page isolation steps. */ if (unlikely(!trylock_page(page))) goto out_putpage; if (!PageMovable(page) || PageIsolated(page)) goto out_no_isolated; mapping = page_mapping(page); VM_BUG_ON_PAGE(!mapping, page); if (!mapping->a_ops->isolate_page(page, mode)) goto out_no_isolated; /* Driver shouldn't use PG_isolated bit of page->flags */ WARN_ON_ONCE(PageIsolated(page)); __SetPageIsolated(page); unlock_page(page); return true; out_no_isolated: unlock_page(page); out_putpage: put_page(page); out: return false; }


minchan kimminchan kim152100.00%1100.00%

/* It should be called on page which is PG_movable */
void putback_movable_page(struct page *page) { struct address_space *mapping; VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(!PageMovable(page), page); VM_BUG_ON_PAGE(!PageIsolated(page), page); mapping = page_mapping(page); mapping->a_ops->putback_page(page); __ClearPageIsolated(page); }


minchan kimminchan kim69100.00%1100.00%

/* * Put previously isolated pages back onto the appropriate lists * from where they were once taken off for compaction/migration. * * This function shall be used whenever the isolated pageset has been * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range() * and isolate_huge_page(). */
void putback_movable_pages(struct list_head *l) { struct page *page; struct page *page2; list_for_each_entry_safe(page, page2, l, lru) { if (unlikely(PageHuge(page))) { putback_active_hugepage(page); continue; } list_del(&page->lru); dec_node_page_state(page, NR_ISOLATED_ANON + page_is_file_cache(page)); /* * We isolated non-lru movable page so here we can use * __PageMovable because LRU page's mapping cannot have * PAGE_MAPPING_MOVABLE. */ if (unlikely(__PageMovable(page))) { VM_BUG_ON_PAGE(!PageIsolated(page), page); lock_page(page); if (PageMovable(page)) putback_movable_page(page); else __ClearPageIsolated(page); unlock_page(page); put_page(page); } else { putback_lru_page(page); } } }


minchan kimminchan kim5943.70%111.11%
christoph lameterchristoph lameter2317.04%222.22%
rafael aquinirafael aquini2216.30%222.22%
naoya horiguchinaoya horiguchi1813.33%111.11%
kosaki motohirokosaki motohiro118.15%111.11%
lee schermerhornlee schermerhorn10.74%111.11%
mel gormanmel gorman10.74%111.11%

/* * Restore a potential migration pte to a working pte entry */
static int remove_migration_pte(struct page *new, struct vm_area_struct *vma, unsigned long addr, void *old) { struct mm_struct *mm = vma->vm_mm; swp_entry_t entry; pmd_t *pmd; pte_t *ptep, pte; spinlock_t *ptl; if (unlikely(PageHuge(new))) { ptep = huge_pte_offset(mm, addr); if (!ptep) goto out; ptl = huge_pte_lockptr(hstate_vma(vma), mm, ptep); } else { pmd = mm_find_pmd(mm, addr); if (!pmd) goto out; ptep = pte_offset_map(pmd, addr); /* * Peek to check is_swap_pte() before taking ptlock? No, we * can race mremap's move_ptes(), which skips anon_vma lock. */ ptl = pte_lockptr(mm, pmd); } spin_lock(ptl); pte = *ptep; if (!is_swap_pte(pte)) goto unlock; entry = pte_to_swp_entry(pte); if (!is_migration_entry(entry) || migration_entry_to_page(entry) != old) goto unlock; get_page(new); pte = pte_mkold(mk_pte(new, vma->vm_page_prot)); if (pte_swp_soft_dirty(*ptep)) pte = pte_mksoft_dirty(pte); /* Recheck VMA as permissions can change since migration started */ if (is_write_migration_entry(entry)) pte = maybe_mkwrite(pte, vma); #ifdef CONFIG_HUGETLB_PAGE if (PageHuge(new)) { pte = pte_mkhuge(pte); pte = arch_make_huge_pte(pte, vma, new, 0); } #endif flush_dcache_page(new); set_pte_at(mm, addr, ptep, pte); if (PageHuge(new)) { if (PageAnon(new)) hugepage_add_anon_rmap(new, vma, addr); else page_dup_rmap(new, true); } else if (PageAnon(new)) page_add_anon_rmap(new, vma, addr, false); else page_add_file_rmap(new, false); if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new)) mlock_vma_page(new); /* No need to invalidate - it was non-present before */ update_mmu_cache(vma, addr, ptep); unlock: pte_unmap_unlock(ptep, ptl); out: return SWAP_AGAIN; }


christoph lameterchristoph lameter19450.79%210.00%
naoya horiguchinaoya horiguchi8221.47%15.00%
hugh dickinshugh dickins338.64%315.00%
kirill a. shutemovkirill a. shutemov225.76%525.00%
tony lutony lu153.93%15.00%
cyrill gorcunovcyrill gorcunov153.93%15.00%
andi kleenandi kleen51.31%15.00%
andrea arcangeliandrea arcangeli41.05%15.00%
mel gormanmel gorman41.05%15.00%
kamezawa hiroyukikamezawa hiroyuki30.79%15.00%
leonid yegoshinleonid yegoshin20.52%15.00%
bob liubob liu20.52%15.00%
russell kingrussell king10.26%15.00%

/* * Get rid of all migration entries and replace them by * references to the indicated page. */
void remove_migration_ptes(struct page *old, struct page *new, bool locked) { struct rmap_walk_control rwc = { .rmap_one = remove_migration_pte, .arg = old, }; if (locked) rmap_walk_locked(new, &rwc); else rmap_walk(new, &rwc); }


christoph lameterchristoph lameter2036.36%125.00%
joonsoo kimjoonsoo kim1832.73%125.00%
kirill a. shutemovkirill a. shutemov1629.09%125.00%
hugh dickinshugh dickins11.82%125.00%

/* * Something used the pte of a page under migration. We need to * get to the page and wait until migration is finished. * When we return from this function the fault will be retried. */
void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep, spinlock_t *ptl) { pte_t pte; swp_entry_t entry; struct page *page; spin_lock(ptl); pte = *ptep; if (!is_swap_pte(pte)) goto out; entry = pte_to_swp_entry(pte); if (!is_migration_entry(entry)) goto out; page = migration_entry_to_page(entry); /* * Once radix-tree replacement of page migration started, page_count * *must* be zero. And, we don't want to call wait_on_page_locked() * against a page without get_page(). * So, we use get_page_unless_zero(), here. Even failed, page fault * will occur again. */ if (!get_page_unless_zero(page)) goto out; pte_unmap_unlock(ptep, ptl); wait_on_page_locked(page); put_page(page); return; out: pte_unmap_unlock(ptep, ptl); }


christoph lameterchristoph lameter9885.96%133.33%
nick pigginnick piggin97.89%133.33%
naoya horiguchinaoya horiguchi76.14%133.33%

void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd, unsigned long address) { spinlock_t *ptl = pte_lockptr(mm, pmd); pte_t *ptep = pte_offset_map(pmd, address); __migration_entry_wait(mm, ptep, ptl); }


naoya horiguchinaoya horiguchi49100.00%1100.00%

void migration_entry_wait_huge(struct vm_area_struct *vma, struct mm_struct *mm, pte_t *pte) { spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), mm, pte); __migration_entry_wait(mm, pte, ptl); }


naoya horiguchinaoya horiguchi2965.91%150.00%
kirill a. shutemovkirill a. shutemov1534.09%150.00%

#ifdef CONFIG_BLOCK /* Returns true if all buffers are successfully locked */
static bool buffer_migrate_lock_buffers(struct buffer_head *head, enum migrate_mode mode) { struct buffer_head *bh = head; /* Simple case, sync compaction */ if (mode != MIGRATE_ASYNC) { do { get_bh(bh); lock_buffer(bh); bh = bh->b_this_page; } while (bh != head); return true; } /* async case, we cannot block on lock_buffer so use trylock_buffer */ do { get_bh(bh); if (!trylock_buffer(bh)) { /* * We failed to lock the buffer and cannot stall in * async migration. Release the taken locks */ struct buffer_head *failed_bh = bh; put_bh(failed_bh); bh = head; while (bh != failed_bh) { unlock_buffer(bh); put_bh(bh); bh = bh->b_this_page; } return false; } bh = bh->b_this_page; } while (bh != head); return true; }


mel gormanmel gorman139100.00%2100.00%

static inline bool buffer_migrate_lock_buffers(struct buffer_head *head, enum migrate_mode mode) { return true; }


mel gormanmel gorman19100.00%2100.00%

#endif /* CONFIG_BLOCK */ /* * Replace the page in the mapping. * * The number of remaining references must be: * 1 for anonymous pages without a mapping * 2 for pages with a mapping * 3 for pages with a mapping and PagePrivate/PagePrivate2 set. */
int migrate_page_move_mapping(struct address_space *mapping, struct page *newpage, struct page *page, struct buffer_head *head, enum migrate_mode mode, int extra_count) { struct zone *oldzone, *newzone; int dirty; int expected_count = 1 + extra_count; void **pslot; if (!mapping) { /* Anonymous page without mapping */ if (page_count(page) != expected_count) return -EAGAIN; /* No turning back from here */ newpage->index = page->index; newpage->mapping = page->mapping; if (PageSwapBacked(page)) __SetPageSwapBacked(newpage); return MIGRATEPAGE_SUCCESS; } oldzone = page_zone(page); newzone = page_zone(newpage); spin_lock_irq(&mapping->tree_lock); pslot = radix_tree_lookup_slot(&mapping->page_tree, page_index(page)); expected_count += 1 + page_has_private(page); if (page_count(page) != expected_count || radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) { spin_unlock_irq(&mapping->tree_lock); return -EAGAIN; } if (!page_ref_freeze(page, expected_count)) { spin_unlock_irq(&mapping->tree_lock); return -EAGAIN; } /* * In the async migration case of moving a page with buffers, lock the * buffers using trylock before the mapping is moved. If the mapping * was moved, we later failed to lock the buffers and could not move * the mapping back due to an elevated page count, we would have to * block waiting on other references to be dropped. */ if (mode == MIGRATE_ASYNC && head && !buffer_migrate_lock_buffers(head, mode)) { page_ref_unfreeze(page, expected_count); spin_unlock_irq(&mapping->tree_lock); return -EAGAIN; } /* * Now we know that no one else is looking at the page: * no turning back from here. */ newpage->index = page->index; newpage->mapping = page->mapping; if (PageSwapBacked(page)) __SetPageSwapBacked(newpage); get_page(newpage); /* add cache reference */ if (PageSwapCache(page)) { SetPageSwapCache(newpage); set_page_private(newpage, page_private(page)); } /* Move dirty while page refs frozen and newpage not yet exposed */ dirty = PageDirty(page); if (dirty) { ClearPageDirty(page); SetPageDirty(newpage); } radix_tree_replace_slot(pslot, newpage); /* * Drop cache reference from old page by unfreezing * to one less reference. * We know this isn't the last reference. */ page_ref_unfreeze(page, expected_count - 1); spin_unlock(&mapping->tree_lock); /* Leave irq disabled to prevent preemption while updating stats */ /* * If moved to a different zone then also account * the page for that zone. Other VM counters will be * taken care of when we establish references to the * new page and drop references to the old page. * * Note that anonymous pages are accounted for * via NR_FILE_PAGES and NR_ANON_MAPPED if they * are mapped to swap space. */ if (newzone != oldzone) { __dec_node_state(oldzone->zone_pgdat, NR_FILE_PAGES); __inc_node_state(newzone->zone_pgdat, NR_FILE_PAGES); if (PageSwapBacked(page) && !PageSwapCache(page)) { __dec_node_state(oldzone->zone_pgdat, NR_SHMEM); __inc_node_state(newzone->zone_pgdat, NR_SHMEM); } if (dirty && mapping_cap_account_dirty(mapping)) { __dec_node_state(oldzone->zone_pgdat, NR_FILE_DIRTY); __dec_zone_state(oldzone, NR_ZONE_WRITE_PENDING); __inc_node_state(newzone->zone_pgdat, NR_FILE_DIRTY); __inc_zone_state(newzone, NR_ZONE_WRITE_PENDING); } } local_irq_enable(); return MIGRATEPAGE_SUCCESS; }


hugh dickinshugh dickins14430.70%311.11%
christoph lameterchristoph lameter13629.00%725.93%
mel gormanmel gorman9420.04%622.22%
nick pigginnick piggin5411.51%311.11%
kosaki motohirokosaki motohiro183.84%13.70%
benjamin lahaisebenjamin lahaise91.92%13.70%
andrea arcangeliandrea arcangeli40.85%13.70%
joonsoo kimjoonsoo kim30.64%13.70%
jacobo giraltjacobo giralt30.64%13.70%
rafael aquinirafael aquini20.43%13.70%
peter zijlstrapeter zijlstra10.21%13.70%
david howellsdavid howells10.21%13.70%

EXPORT_SYMBOL(migrate_page_move_mapping); /* * The expected number of remaining references is the same as that * of migrate_page_move_mapping(). */
int migrate_huge_page_move_mapping(struct address_space *mapping, struct page *newpage, struct page *page) { int expected_count; void **pslot; spin_lock_irq(&mapping->tree_lock); pslot = radix_tree_lookup_slot(&mapping->page_tree, page_index(page)); expected_count = 2 + page_has_private(page); if (page_count(page) != expected_count || radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) { spin_unlock_irq(&mapping->tree_lock); return -EAGAIN; } if (!page_ref_freeze(page, expected_count)) { spin_unlock_irq(&mapping->tree_lock); return -EAGAIN; } newpage->index = page->index; newpage->mapping = page->mapping; get_page(newpage); radix_tree_replace_slot(pslot, newpage); page_ref_unfreeze(page, expected_count - 1); spin_unlock_irq(&mapping->tree_lock); return MIGRATEPAGE_SUCCESS; }


naoya horiguchinaoya horiguchi8450.30%110.00%
christoph lameterchristoph lameter4124.55%110.00%
hugh dickinshugh dickins1710.18%220.00%
lee schermerhornlee schermerhorn105.99%110.00%
mel gormanmel gorman63.59%110.00%
nick pigginnick piggin42.40%110.00%
joonsoo kimjoonsoo kim21.20%110.00%
jacobo giraltjacobo giralt21.20%110.00%
rafael aquinirafael aquini10.60%110.00%

/* * Gigantic pages are so large that we do not guarantee that page++ pointer * arithmetic will work across the entire page. We need something more * specialized. */
static void __copy_gigantic_page(struct page *dst, struct page *src, int nr_pages) { int i; struct page *dst_base = dst; struct page *src_base = src; for (i = 0; i < nr_pages; ) { cond_resched(); copy_highpage(dst, src); i++; dst = mem_map_next(dst, dst_base, i); src = mem_map_next(src, src_base, i); } }


dave hansendave hansen84100.00%1100.00%

static void copy_huge_page(struct page *dst, struct page *src) { int i; int nr_pages; if (PageHuge(src)) { /* hugetlbfs page */ struct hstate *h = page_hstate(src); nr_pages = pages_per_huge_page(h); if (unlikely(nr_pages > MAX_ORDER_NR_PAGES)) { __copy_gigantic_page(dst, src, nr_pages); return; } } else { /* thp page */ BUG_ON(!PageTransHuge(src)); nr_pages = hpage_nr_pages(src); } for (i = 0; i < nr_pages; i++) { cond_resched(); copy_highpage(dst + i, src + i); } }


dave hansendave hansen119100.00%1100.00%

/* * Copy the page to its new location */
void migrate_page_copy(struct page *newpage, struct page *page) { int cpupid; if (PageHuge(page) || PageTransHuge(page)) copy_huge_page(newpage, page); else copy_highpage(newpage, page); if (PageError(page)) SetPageError(newpage); if (PageReferenced(page)) SetPageReferenced(newpage); if (PageUptodate(page)) SetPageUptodate(newpage); if (TestClearPageActive(page)) { VM_BUG_ON_PAGE(PageUnevictable(page), page); SetPageActive(newpage); } else if (TestClearPageUnevictable(page)) SetPageUnevictable(newpage); if (PageChecked(page)) SetPageChecked(newpage); if (PageMappedToDisk(page)) SetPageMappedToDisk(newpage); /* Move dirty on pages not done by migrate_page_move_mapping() */ if (PageDirty(page)) SetPageDirty(newpage); if (page_is_young(page)) set_page_young(newpage); if (page_is_idle(page)) set_page_idle(newpage); /* * Copy NUMA information to the new page, to prevent over-eager * future migrations of this same page. */ cpupid = page_cpupid_xchg_last(page, -1); page_cpupid_xchg_last(newpage, cpupid); ksm_migrate_page(newpage, page); /* * Please do not reorder this without considering how mm/ksm.c's * get_ksm_page() depends upon ksm_migrate_page() and PageSwapCache(). */ if (PageSwapCache(page)) ClearPageSwapCache(page); ClearPagePrivate(page); set_page_private(page, 0); /* * If any waiters have accumulated on the new page then * wake them up. */ if (PageWriteback(newpage)) end_page_writeback(newpage); copy_page_owner(page, newpage); mem_cgroup_migrate(page, newpage); }


naoya horiguchinaoya horiguchi14857.81%215.38%
christoph lameterchristoph lameter3112.11%17.69%
vladimir davydovvladimir davydov249.38%17.69%
rik van rielrik van riel218.20%17.69%
hugh dickinshugh dickins103.91%430.77%
johannes weinerjohannes weiner72.73%17.69%
vlastimil babkavlastimil babka72.73%17.69%
mel gormanmel gorman51.95%17.69%
sasha levinsasha levin31.17%17.69%

EXPORT_SYMBOL(migrate_page_copy); /************************************************************ * Migration functions ***********************************************************/ /* * Common logic to directly migrate a single LRU page suitable for * pages that do not use PagePrivate/PagePrivate2. * * Pages are locked upon entry and exit. */
int migrate_page(struct address_space *mapping, struct page *newpage, struct page *page, enum migrate_mode mode) { int rc; BUG_ON(PageWriteback(page)); /* Writeback must be complete */ rc = migrate_page_move_mapping(mapping, newpage, page, NULL, mode, 0); if (rc != MIGRATEPAGE_SUCCESS) return rc; migrate_page_copy(newpage, page); return MIGRATEPAGE_SUCCESS; }


christoph lameterchristoph lameter5981.94%342.86%
mel gormanmel gorman811.11%228.57%
rafael aquinirafael aquini34.17%114.29%
benjamin lahaisebenjamin lahaise22.78%114.29%

EXPORT_SYMBOL(migrate_page); #ifdef CONFIG_BLOCK /* * Migration function for pages with buffers. This function can only be used * if the underlying filesystem guarantees that no other references to "page" * exist. */
int buffer_migrate_page(struct address_space *mapping, struct page *newpage, struct page *page, enum migrate_mode mode) { struct buffer_head *bh, *head; int rc; if (!page_has_buffers(page)) return migrate_page(mapping, newpage, page, mode); head = page_buffers(page); rc = migrate_page_move_mapping(mapping, newpage, page, head, mode, 0); if (rc != MIGRATEPAGE_SUCCESS) return rc; /* * In the async case, migrate_page_move_mapping locked the buffers * with an IRQ-safe spinlock held. In the sync case, the buffers * need to be locked now */ if (mode != MIGRATE_ASYNC) BUG_ON(!buffer_migrate_lock_buffers(head, mode)); ClearPagePrivate(page); set_page_private(newpage, page_private(page)); set_page_private(page, 0); put_page(page); get_page(newpage); bh = head; do { set_bh_page(bh, newpage, bh_offset(bh)); bh = bh->b_this_page; } while (bh != head); SetPagePrivate(newpage); migrate_page_copy(newpage, page); bh = head; do { unlock_buffer(bh); put_bh(bh); bh = bh->b_this_page; } while (bh != head); return MIGRATEPAGE_SUCCESS; }


christoph lameterchristoph lameter18686.51%450.00%
mel gormanmel gorman2411.16%225.00%
rafael aquinirafael aquini31.40%112.50%
benjamin lahaisebenjamin lahaise20.93%112.50%

EXPORT_SYMBOL(buffer_migrate_page); #endif /* * Writeback a page to clean the dirty state */
static int writeout(struct address_space *mapping, struct page *page) { struct writeback_control wbc = { .sync_mode = WB_SYNC_NONE, .nr_to_write = 1, .range_start = 0, .range_end = LLONG_MAX, .for_reclaim = 1 }; int rc; if (!mapping->a_ops->writepage) /* No write method for the address space */ return -EINVAL; if (!clear_page_dirty_for_io(page)) /* Someone else already triggered a write */ return -EAGAIN; /* * A dirty page may imply that the underlying filesystem has * the page on some queue. So the page must be clean for * migration. Writeout may mean we loose the lock and the * page state is no longer what we checked for earlier. * At this point we know that the migration attempt cannot * be successful. */ remove_migration_ptes(page, page, false); rc = mapping->a_ops->writepage(page, &wbc); if (rc != AOP_WRITEPAGE_ACTIVATE) /* unlocked. Relock */ lock_page(page); return (rc < 0) ? -EIO :