cregit-Linux how code gets into the kernel

Release 4.9 mm/swap_state.c

Directory: mm
/*
 *  linux/mm/swap_state.c
 *
 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 *  Swap reorganised 29.12.95, Stephen Tweedie
 *
 *  Rewritten to use page cache, (C) 1998 Stephen Tweedie
 */
#include <linux/mm.h>
#include <linux/gfp.h>
#include <linux/kernel_stat.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/init.h>
#include <linux/pagemap.h>
#include <linux/backing-dev.h>
#include <linux/blkdev.h>
#include <linux/pagevec.h>
#include <linux/migrate.h>

#include <asm/pgtable.h>

/*
 * swapper_space is a fiction, retained to simplify the path through
 * vmscan's shrink_page_list.
 */

static const struct address_space_operations swap_aops = {
	.writepage	= swap_writepage,
	.set_page_dirty	= swap_set_page_dirty,
#ifdef CONFIG_MIGRATION
	.migratepage	= migrate_page,
#endif
};


struct address_space swapper_spaces[MAX_SWAPFILES] = {
	[0 ... MAX_SWAPFILES - 1] = {
		.page_tree	= RADIX_TREE_INIT(GFP_ATOMIC|__GFP_NOWARN),
		.i_mmap_writable = ATOMIC_INIT(0),
		.a_ops		= &swap_aops,
		/* swap cache doesn't use writeback related tags */
		.flags		= 1 << AS_NO_WRITEBACK_TAGS,
        }
};


#define INC_CACHE_INFO(x)	do { swap_cache_info.x++; } while (0)

static struct {
	
unsigned long add_total;
	
unsigned long del_total;
	
unsigned long find_success;
	
unsigned long find_total;
} 
swap_cache_info;


unsigned long total_swapcache_pages(void) { int i; unsigned long ret = 0; for (i = 0; i < MAX_SWAPFILES; i++) ret += swapper_spaces[i].nrpages; return ret; }

Contributors

PersonTokensPropCommitsCommitProp
shaohua lishaohua li42100.00%1100.00%
Total42100.00%1100.00%

static atomic_t swapin_readahead_hits = ATOMIC_INIT(4);
void show_swap_cache_info(void) { printk("%lu pages in swap cache\n", total_swapcache_pages()); printk("Swap cache stats: add %lu, delete %lu, find %lu/%lu\n", swap_cache_info.add_total, swap_cache_info.del_total, swap_cache_info.find_success, swap_cache_info.find_total); printk("Free swap = %ldkB\n", get_nr_swap_pages() << (PAGE_SHIFT - 10)); printk("Total swap = %lukB\n", total_swap_pages << (PAGE_SHIFT - 10)); }

Contributors

PersonTokensPropCommitsCommitProp
andrea arcangeliandrea arcangeli2438.10%114.29%
pre-gitpre-git1422.22%114.29%
linus torvaldslinus torvalds1320.63%114.29%
johannes weinerjohannes weiner812.70%114.29%
shaohua lishaohua li34.76%228.57%
hugh dickinshugh dickins11.59%114.29%
Total63100.00%7100.00%

/* * __add_to_swap_cache resembles add_to_page_cache_locked on swapper_space, * but sets SwapCache flag and private instead of mapping and index. */
int __add_to_swap_cache(struct page *page, swp_entry_t entry) { int error; struct address_space *address_space; VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(PageSwapCache(page), page); VM_BUG_ON_PAGE(!PageSwapBacked(page), page); get_page(page); SetPageSwapCache(page); set_page_private(page, entry.val); address_space = swap_address_space(entry); spin_lock_irq(&address_space->tree_lock); error = radix_tree_insert(&address_space->page_tree, swp_offset(entry), page); if (likely(!error)) { address_space->nrpages++; __inc_node_page_state(page, NR_FILE_PAGES); INC_CACHE_INFO(add_total); } spin_unlock_irq(&address_space->tree_lock); if (unlikely(error)) { /* * Only the context which have set SWAP_HAS_CACHE flag * would call add_to_swap_cache(). * So add_to_swap_cache() doesn't returns -EEXIST. */ VM_BUG_ON(error == -EEXIST); set_page_private(page, 0UL); ClearPageSwapCache(page); put_page(page); } return error; }

Contributors

PersonTokensPropCommitsCommitProp
nick pigginnick piggin5631.64%315.00%
andrew mortonandrew morton4324.29%210.00%
shaohua lishaohua li2111.86%15.00%
daisuke nishimuradaisuke nishimura147.91%210.00%
pre-gitpre-git116.21%420.00%
sasha levinsasha levin95.08%15.00%
rik van rielrik van riel84.52%15.00%
hugh dickinshugh dickins52.82%15.00%
christoph lameterchristoph lameter31.69%15.00%
huang yinghuang ying31.69%15.00%
kirill a. shutemovkirill a. shutemov21.13%15.00%
linus torvaldslinus torvalds10.56%15.00%
mel gormanmel gorman10.56%15.00%
Total177100.00%20100.00%


int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp_mask) { int error; error = radix_tree_maybe_preload(gfp_mask); if (!error) { error = __add_to_swap_cache(page, entry); radix_tree_preload_end(); } return error; }

Contributors

PersonTokensPropCommitsCommitProp
daisuke nishimuradaisuke nishimura4287.50%133.33%
andrew mortonandrew morton510.42%133.33%
jan karajan kara12.08%133.33%
Total48100.00%3100.00%

/* * This must be called only on pages that have * been verified to be in the swap cache. */
void __delete_from_swap_cache(struct page *page) { swp_entry_t entry; struct address_space *address_space; VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(!PageSwapCache(page), page); VM_BUG_ON_PAGE(PageWriteback(page), page); entry.val = page_private(page); address_space = swap_address_space(entry); radix_tree_delete(&address_space->page_tree, swp_offset(entry)); set_page_private(page, 0); ClearPageSwapCache(page); address_space->nrpages--; __dec_node_page_state(page, NR_FILE_PAGES); INC_CACHE_INFO(del_total); }

Contributors

PersonTokensPropCommitsCommitProp
shaohua lishaohua li2926.85%17.14%
andrew mortonandrew morton2825.93%214.29%
pre-gitpre-git1816.67%321.43%
linus torvaldslinus torvalds1211.11%321.43%
sasha levinsasha levin98.33%17.14%
hugh dickinshugh dickins65.56%17.14%
christoph lameterchristoph lameter32.78%17.14%
huang yinghuang ying21.85%17.14%
mel gormanmel gorman10.93%17.14%
Total108100.00%14100.00%

/** * add_to_swap - allocate swap space for a page * @page: page we want to move to swap * * Allocate swap space for the page and add the page to the * swap cache. Caller needs to hold the page lock. */
int add_to_swap(struct page *page, struct list_head *list) { swp_entry_t entry; int err; VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(!PageUptodate(page), page); entry = get_swap_page(); if (!entry.val) return 0; if (mem_cgroup_try_charge_swap(page, entry)) { swapcache_free(entry); return 0; } if (unlikely(PageTransHuge(page))) if (unlikely(split_huge_page_to_list(page, list))) { swapcache_free(entry); return 0; } /* * Radix-tree node allocations from PF_MEMALLOC contexts could * completely exhaust the page allocator. __GFP_NOMEMALLOC * stops emergency reserves from being allocated. * * TODO: this could cause a theoretical memory reclaim * deadlock in the swap out path. */ /* * Add it to the swap cache. */ err = add_to_swap_cache(page, entry, __GFP_HIGH|__GFP_NOMEMALLOC|__GFP_NOWARN); if (!err) { return 1; } else { /* -ENOMEM radix-tree allocation failure */ /* * add_to_swap_cache() doesn't return -EEXIST, so we can safely * clear SWAP_HAS_CACHE flag. */ swapcache_free(entry); return 0; } }

Contributors

PersonTokensPropCommitsCommitProp
andrew mortonandrew morton5939.60%423.53%
andrea arcangeliandrea arcangeli2919.46%15.88%
vladimir davydovvladimir davydov1912.75%15.88%
nick pigginnick piggin106.71%211.76%
shaohua lishaohua li85.37%15.88%
hugh dickinshugh dickins74.70%317.65%
daisuke nishimuradaisuke nishimura74.70%15.88%
sasha levinsasha levin64.03%15.88%
eric sesterhenneric sesterhenn21.34%15.88%
minchan kimminchan kim10.67%15.88%
kamezawa hiroyukikamezawa hiroyuki10.67%15.88%
Total149100.00%17100.00%

/* * This must be called only on pages that have * been verified to be in the swap cache and locked. * It will never put the page into the free list, * the caller has a reference on the page. */
void delete_from_swap_cache(struct page *page) { swp_entry_t entry; struct address_space *address_space; entry.val = page_private(page); address_space = swap_address_space(entry); spin_lock_irq(&address_space->tree_lock); __delete_from_swap_cache(page); spin_unlock_irq(&address_space->tree_lock); swapcache_free(entry); put_page(page); }

Contributors

PersonTokensPropCommitsCommitProp
pre-gitpre-git2944.62%646.15%
shaohua lishaohua li1624.62%17.69%
linus torvaldslinus torvalds1116.92%17.69%
hugh dickinshugh dickins34.62%17.69%
nick pigginnick piggin23.08%17.69%
andrew mortonandrew morton23.08%17.69%
kirill a. shutemovkirill a. shutemov11.54%17.69%
kamezawa hiroyukikamezawa hiroyuki11.54%17.69%
Total65100.00%13100.00%

/* * If we are the only user, then try to free up the swap cache. * * Its ok to check for PageSwapCache without the page lock * here because we are going to recheck again inside * try_to_free_swap() _with_ the lock. * - Marcelo */
static inline void free_swap_cache(struct page *page) { if (PageSwapCache(page) && !page_mapped(page) && trylock_page(page)) { try_to_free_swap(page); unlock_page(page); } }

Contributors

PersonTokensPropCommitsCommitProp
linus torvaldslinus torvalds2969.05%120.00%
hugh dickinshugh dickins716.67%120.00%
andrew mortonandrew morton511.90%240.00%
nick pigginnick piggin12.38%120.00%
Total42100.00%5100.00%

/* * Perform a free_page(), also freeing any swap cache associated with * this page if it is the last user of the page. */
void free_page_and_swap_cache(struct page *page) { free_swap_cache(page); if (!is_huge_zero_page(page)) put_page(page); }

Contributors

PersonTokensPropCommitsCommitProp
andrew mortonandrew morton1450.00%120.00%
gerald schaefergerald schaefer725.00%120.00%
linus torvaldslinus torvalds517.86%120.00%
aaron luaaron lu13.57%120.00%
kirill a. shutemovkirill a. shutemov13.57%120.00%
Total28100.00%5100.00%

/* * Passed an array of pages, drop them all from swapcache and then release * them. They are removed from the LRU and freed if this is their last use. */
void free_pages_and_swap_cache(struct page **pages, int nr) { struct page **pagep = pages; int i; lru_add_drain(); for (i = 0; i < nr; i++) free_swap_cache(pagep[i]); release_pages(pagep, nr, false); }

Contributors

PersonTokensPropCommitsCommitProp
andrew mortonandrew morton5289.66%250.00%
michal hockomichal hocko58.62%125.00%
mel gormanmel gorman11.72%125.00%
Total58100.00%4100.00%

/* * Lookup a swap entry in the swap cache. A found page will be returned * unlocked and with its refcount incremented - we rely on the kernel * lock getting page table operations atomic even if we drop the page * lock before returning. */
struct page * lookup_swap_cache(swp_entry_t entry) { struct page *page; page = find_get_page(swap_address_space(entry), swp_offset(entry)); if (page) { INC_CACHE_INFO(find_success); if (TestClearPageReadahead(page)) atomic_inc(&swapin_readahead_hits); } INC_CACHE_INFO(find_total); return page; }

Contributors

PersonTokensPropCommitsCommitProp
pre-gitpre-git2641.94%545.45%
shaohua lishaohua li1930.65%218.18%
andrew mortonandrew morton914.52%19.09%
linus torvaldslinus torvalds46.45%19.09%
huang yinghuang ying34.84%19.09%
marcelo tosattimarcelo tosatti11.61%19.09%
Total62100.00%11100.00%


struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr, bool *new_page_allocated) { struct page *found_page, *new_page = NULL; struct address_space *swapper_space = swap_address_space(entry); int err; *new_page_allocated = false; do { /* * First check the swap cache. Since this is normally * called after lookup_swap_cache() failed, re-calling * that would confuse statistics. */ found_page = find_get_page(swapper_space, swp_offset(entry)); if (found_page) break; /* * Get a new page to read into from swap. */ if (!new_page) { new_page = alloc_page_vma(gfp_mask, vma, addr); if (!new_page) break; /* Out of memory */ } /* * call radix_tree_preload() while we can wait. */ err = radix_tree_maybe_preload(gfp_mask & GFP_KERNEL); if (err) break; /* * Swap entry may have been freed since our caller observed it. */ err = swapcache_prepare(entry); if (err == -EEXIST) { radix_tree_preload_end(); /* * We might race against get_swap_page() and stumble * across a SWAP_HAS_CACHE swap_map entry whose page * has not been brought into the swapcache yet, while * the other end is scheduled away waiting on discard * I/O completion at scan_swap_map(). * * In order to avoid turning this transitory state * into a permanent loop around this -EEXIST case * if !CONFIG_PREEMPT and the I/O completion happens * to be waiting on the CPU waitqueue where we are now * busy looping, we just conditionally invoke the * scheduler here, if there are some more important * tasks to run. */ cond_resched(); continue; } if (err) { /* swp entry is obsolete ? */ radix_tree_preload_end(); break; } /* May fail (-ENOMEM) if radix-tree node allocation failed. */ __SetPageLocked(new_page); __SetPageSwapBacked(new_page); err = __add_to_swap_cache(new_page, entry); if (likely(!err)) { radix_tree_preload_end(); /* * Initiate read into locked page and return. */ lru_cache_add_anon(new_page); *new_page_allocated = true; return new_page; } radix_tree_preload_end(); __ClearPageLocked(new_page); /* * add_to_swap_cache() doesn't return -EEXIST, so we can safely * clear SWAP_HAS_CACHE flag. */ swapcache_free(entry); } while (err != -ENOMEM); if (new_page) put_page(new_page); return found_page; }

Contributors

PersonTokensPropCommitsCommitProp
pre-gitpre-git5423.18%310.71%
linus torvaldslinus torvalds4117.60%414.29%
daisuke nishimuradaisuke nishimura3314.16%27.14%
dmitry safonovdmitry safonov2510.73%13.57%
hugh dickinshugh dickins239.87%310.71%
andrew mortonandrew morton208.58%414.29%
kamezawa hiroyukikamezawa hiroyuki177.30%27.14%
rik van rielrik van riel52.15%27.14%
rafael aquinirafael aquini41.72%13.57%
nick pigginnick piggin31.29%13.57%
kirill a. shutemovkirill a. shutemov31.29%27.14%
huang yinghuang ying31.29%13.57%
marcelo tosattimarcelo tosatti10.43%13.57%
jan karajan kara10.43%13.57%
Total233100.00%28100.00%

/* * Locate a page of swap in physical memory, reserving swap cache space * and reading the disk if it is not already cached. * A failure return means that either the page allocation failed or that * the swap entry is no longer in use. */
struct page *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr) { bool page_was_allocated; struct page *retpage = __read_swap_cache_async(entry, gfp_mask, vma, addr, &page_was_allocated); if (page_was_allocated) swap_readpage(retpage); return retpage; }

Contributors

PersonTokensPropCommitsCommitProp
dmitry safonovdmitry safonov56100.00%1100.00%
Total56100.00%1100.00%


static unsigned long swapin_nr_pages(unsigned long offset) { static unsigned long prev_offset; unsigned int pages, max_pages, last_ra; static atomic_t last_readahead_pages; max_pages = 1 << READ_ONCE(page_cluster); if (max_pages <= 1) return 1; /* * This heuristic has been found to work well on both sequential and * random loads, swapping to hard disk or to SSD: please don't ask * what the "+ 2" means, it just happens to work well, that's all. */ pages = atomic_xchg(&swapin_readahead_hits, 0) + 2; if (pages == 2) { /* * We can have no readahead hits to judge by: but must not get * stuck here forever, so check for an adjacent offset instead * (and don't even bother to check whether swap type is same). */ if (offset != prev_offset + 1 && offset != prev_offset - 1) pages = 1; prev_offset = offset; } else { unsigned int roundup = 4; while (roundup < pages) roundup <<= 1; pages = roundup; } if (pages > max_pages) pages = max_pages; /* Don't shrink readahead too fast */ last_ra = atomic_read(&last_readahead_pages) / 2; if (pages < last_ra) pages = last_ra; atomic_set(&last_readahead_pages, pages); return pages; }

Contributors

PersonTokensPropCommitsCommitProp
shaohua lishaohua li15499.35%150.00%
jason lowjason low10.65%150.00%
Total155100.00%2100.00%

/** * swapin_readahead - swap in pages in hope we need them soon * @entry: swap entry of this memory * @gfp_mask: memory allocation flags * @vma: user vma this address belongs to * @addr: target address for mempolicy * * Returns the struct page for entry and addr, after queueing swapin. * * Primitive swap readahead code. We simply read an aligned block of * (1 << page_cluster) entries in the swap area. This method is chosen * because it doesn't cost us any seek time. We also make sure to queue * the 'original' request together with the readahead ones... * * This has been extended to use the NUMA policies from the mm triggering * the readahead. * * Caller must hold down_read on the vma->vm_mm if vma is not NULL. */
struct page *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr) { struct page *page; unsigned long entry_offset = swp_offset(entry); unsigned long offset = entry_offset; unsigned long start_offset, end_offset; unsigned long mask; struct blk_plug plug; mask = swapin_nr_pages(offset) - 1; if (!mask) goto skip; /* Read a page_cluster sized and aligned cluster around offset. */ start_offset = offset & ~mask; end_offset = offset | mask; if (!start_offset) /* First page is swap header. */ start_offset++; blk_start_plug(&plug); for (offset = start_offset; offset <= end_offset ; offset++) { /* Ok, do the async read-ahead now */ page = read_swap_cache_async(swp_entry(swp_type(entry), offset), gfp_mask, vma, addr); if (!page) continue; if (offset != entry_offset) SetPageReadahead(page); put_page(page); } blk_finish_plug(&plug); lru_add_drain(); /* Push any new pages onto the LRU now */ skip: return read_swap_cache_async(entry, gfp_mask, vma, addr); }

Contributors

PersonTokensPropCommitsCommitProp
hugh dickinshugh dickins9752.43%233.33%
shaohua lishaohua li3720.00%116.67%
rik van rielrik van riel3418.38%116.67%
christian ehrhardtchristian ehrhardt168.65%116.67%
kirill a. shutemovkirill a. shutemov10.54%116.67%
Total185100.00%6100.00%


Overall Contributors

PersonTokensPropCommitsCommitProp
shaohua lishaohua li35321.56%43.88%
andrew mortonandrew morton27316.68%1817.48%
pre-gitpre-git19111.67%1817.48%
hugh dickinshugh dickins1619.84%1312.62%
linus torvaldslinus torvalds1348.19%54.85%
daisuke nishimuradaisuke nishimura975.93%21.94%
dmitry safonovdmitry safonov825.01%10.97%
nick pigginnick piggin724.40%65.83%
andrea arcangeliandrea arcangeli533.24%21.94%
rik van rielrik van riel472.87%32.91%
sasha levinsasha levin241.47%10.97%
kamezawa hiroyukikamezawa hiroyuki191.16%21.94%
vladimir davydovvladimir davydov191.16%10.97%
christian ehrhardtchristian ehrhardt191.16%10.97%
huang yinghuang ying191.16%21.94%
christoph lameterchristoph lameter140.86%32.91%
david herrmanndavid herrmann80.49%10.97%
johannes weinerjohannes weiner80.49%10.97%
kirill a. shutemovkirill a. shutemov80.49%21.94%
gerald schaefergerald schaefer70.43%10.97%
michal hockomichal hocko50.31%10.97%
rafael aquinirafael aquini40.24%10.97%
mel gormanmel gorman40.24%32.91%
tejun heotejun heo30.18%10.97%
marcelo tosattimarcelo tosatti20.12%10.97%
jan karajan kara20.12%10.97%
eric sesterhenneric sesterhenn20.12%10.97%
minchan kimminchan kim20.12%21.94%
aaron luaaron lu10.06%10.97%
christoph hellwigchristoph hellwig10.06%10.97%
jens axboejens axboe10.06%10.97%
randy dunlaprandy dunlap10.06%10.97%
jason lowjason low10.06%10.97%
Total1637100.00%103100.00%
Directory: mm