cregit-Linux how code gets into the kernel

Release 4.8 mm/khugepaged.c

Directory: mm

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/mm.h>
#include <linux/sched.h>
#include <linux/mmu_notifier.h>
#include <linux/rmap.h>
#include <linux/swap.h>
#include <linux/mm_inline.h>
#include <linux/kthread.h>
#include <linux/khugepaged.h>
#include <linux/freezer.h>
#include <linux/mman.h>
#include <linux/hashtable.h>
#include <linux/userfaultfd_k.h>
#include <linux/page_idle.h>
#include <linux/swapops.h>
#include <linux/shmem_fs.h>

#include <asm/tlb.h>
#include <asm/pgalloc.h>
#include "internal.h"


enum scan_result {
	
SCAN_FAIL,
	
SCAN_SUCCEED,
	
SCAN_PMD_NULL,
	
SCAN_EXCEED_NONE_PTE,
	
SCAN_PTE_NON_PRESENT,
	
SCAN_PAGE_RO,
	
SCAN_LACK_REFERENCED_PAGE,
	
SCAN_PAGE_NULL,
	
SCAN_SCAN_ABORT,
	
SCAN_PAGE_COUNT,
	
SCAN_PAGE_LRU,
	
SCAN_PAGE_LOCK,
	
SCAN_PAGE_ANON,
	
SCAN_PAGE_COMPOUND,
	
SCAN_ANY_PROCESS,
	
SCAN_VMA_NULL,
	
SCAN_VMA_CHECK,
	
SCAN_ADDRESS_RANGE,
	
SCAN_SWAP_CACHE_PAGE,
	
SCAN_DEL_PAGE_LRU,
	
SCAN_ALLOC_HUGE_PAGE_FAIL,
	
SCAN_CGROUP_CHARGE_FAIL,
	
SCAN_EXCEED_SWAP_PTE,
	
SCAN_TRUNCATED,
};


#define CREATE_TRACE_POINTS
#include <trace/events/huge_memory.h>

/* default scan 8*512 pte (or vmas) every 30 second */

static unsigned int khugepaged_pages_to_scan __read_mostly;

static unsigned int khugepaged_pages_collapsed;

static unsigned int khugepaged_full_scans;

static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000;
/* during fragmentation poll the hugepage allocator once every minute */

static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000;

static unsigned long khugepaged_sleep_expire;
static DEFINE_SPINLOCK(khugepaged_mm_lock);
static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait);
/*
 * default collapse hugepages if there is at least one pte mapped like
 * it would have happened if the vma was large enough during page
 * fault.
 */

static unsigned int khugepaged_max_ptes_none __read_mostly;

static unsigned int khugepaged_max_ptes_swap __read_mostly;


#define MM_SLOTS_HASH_BITS 10
static __read_mostly DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);


static struct kmem_cache *mm_slot_cache __read_mostly;

/**
 * struct mm_slot - hash lookup from mm to mm_slot
 * @hash: hash collision list
 * @mm_node: khugepaged scan list headed in khugepaged_scan.mm_head
 * @mm: the mm that this information is valid for
 */

struct mm_slot {
	
struct hlist_node hash;
	
struct list_head mm_node;
	
struct mm_struct *mm;
};

/**
 * struct khugepaged_scan - cursor for scanning
 * @mm_head: the head of the mm list to scan
 * @mm_slot: the current mm_slot we are scanning
 * @address: the next address inside that to be scanned
 *
 * There is only the one khugepaged_scan instance of this cursor structure.
 */

struct khugepaged_scan {
	
struct list_head mm_head;
	
struct mm_slot *mm_slot;
	
unsigned long address;
};


static struct khugepaged_scan khugepaged_scan = {
	.mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head),
};


static ssize_t scan_sleep_millisecs_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { return sprintf(buf, "%u\n", khugepaged_scan_sleep_millisecs); }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov30100.00%1100.00%
Total30100.00%1100.00%


static ssize_t scan_sleep_millisecs_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { unsigned long msecs; int err; err = kstrtoul(buf, 10, &msecs); if (err || msecs > UINT_MAX) return -EINVAL; khugepaged_scan_sleep_millisecs = msecs; khugepaged_sleep_expire = 0; wake_up_interruptible(&khugepaged_wait); return count; }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov72100.00%1100.00%
Total72100.00%1100.00%

static struct kobj_attribute scan_sleep_millisecs_attr = __ATTR(scan_sleep_millisecs, 0644, scan_sleep_millisecs_show, scan_sleep_millisecs_store);
static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { return sprintf(buf, "%u\n", khugepaged_alloc_sleep_millisecs); }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov30100.00%1100.00%
Total30100.00%1100.00%


static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { unsigned long msecs; int err; err = kstrtoul(buf, 10, &msecs); if (err || msecs > UINT_MAX) return -EINVAL; khugepaged_alloc_sleep_millisecs = msecs; khugepaged_sleep_expire = 0; wake_up_interruptible(&khugepaged_wait); return count; }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov72100.00%1100.00%
Total72100.00%1100.00%

static struct kobj_attribute alloc_sleep_millisecs_attr = __ATTR(alloc_sleep_millisecs, 0644, alloc_sleep_millisecs_show, alloc_sleep_millisecs_store);
static ssize_t pages_to_scan_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { return sprintf(buf, "%u\n", khugepaged_pages_to_scan); }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov30100.00%1100.00%
Total30100.00%1100.00%


static ssize_t pages_to_scan_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { int err; unsigned long pages; err = kstrtoul(buf, 10, &pages); if (err || !pages || pages > UINT_MAX) return -EINVAL; khugepaged_pages_to_scan = pages; return count; }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov65100.00%1100.00%
Total65100.00%1100.00%

static struct kobj_attribute pages_to_scan_attr = __ATTR(pages_to_scan, 0644, pages_to_scan_show, pages_to_scan_store);
static ssize_t pages_collapsed_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { return sprintf(buf, "%u\n", khugepaged_pages_collapsed); }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov30100.00%1100.00%
Total30100.00%1100.00%

static struct kobj_attribute pages_collapsed_attr = __ATTR_RO(pages_collapsed);
static ssize_t full_scans_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { return sprintf(buf, "%u\n", khugepaged_full_scans); }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov30100.00%1100.00%
Total30100.00%1100.00%

static struct kobj_attribute full_scans_attr = __ATTR_RO(full_scans);
static ssize_t khugepaged_defrag_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { return single_hugepage_flag_show(kobj, attr, buf, TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG); }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov32100.00%1100.00%
Total32100.00%1100.00%


static ssize_t khugepaged_defrag_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { return single_hugepage_flag_store(kobj, attr, buf, count, TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG); }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov38100.00%1100.00%
Total38100.00%1100.00%

static struct kobj_attribute khugepaged_defrag_attr = __ATTR(defrag, 0644, khugepaged_defrag_show, khugepaged_defrag_store); /* * max_ptes_none controls if khugepaged should collapse hugepages over * any unmapped ptes in turn potentially increasing the memory * footprint of the vmas. When max_ptes_none is 0 khugepaged will not * reduce the available free memory in the system as it * runs. Increasing max_ptes_none will instead potentially reduce the * free memory in the system during the khugepaged scan. */
static ssize_t khugepaged_max_ptes_none_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { return sprintf(buf, "%u\n", khugepaged_max_ptes_none); }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov30100.00%1100.00%
Total30100.00%1100.00%


static ssize_t khugepaged_max_ptes_none_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { int err; unsigned long max_ptes_none; err = kstrtoul(buf, 10, &max_ptes_none); if (err || max_ptes_none > HPAGE_PMD_NR-1) return -EINVAL; khugepaged_max_ptes_none = max_ptes_none; return count; }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov64100.00%1100.00%
Total64100.00%1100.00%

static struct kobj_attribute khugepaged_max_ptes_none_attr = __ATTR(max_ptes_none, 0644, khugepaged_max_ptes_none_show, khugepaged_max_ptes_none_store);
static ssize_t khugepaged_max_ptes_swap_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { return sprintf(buf, "%u\n", khugepaged_max_ptes_swap); }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov30100.00%1100.00%
Total30100.00%1100.00%


static ssize_t khugepaged_max_ptes_swap_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { int err; unsigned long max_ptes_swap; err = kstrtoul(buf, 10, &max_ptes_swap); if (err || max_ptes_swap > HPAGE_PMD_NR-1) return -EINVAL; khugepaged_max_ptes_swap = max_ptes_swap; return count; }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov64100.00%1100.00%
Total64100.00%1100.00%

static struct kobj_attribute khugepaged_max_ptes_swap_attr = __ATTR(max_ptes_swap, 0644, khugepaged_max_ptes_swap_show, khugepaged_max_ptes_swap_store); static struct attribute *khugepaged_attr[] = { &khugepaged_defrag_attr.attr, &khugepaged_max_ptes_none_attr.attr, &pages_to_scan_attr.attr, &pages_collapsed_attr.attr, &full_scans_attr.attr, &scan_sleep_millisecs_attr.attr, &alloc_sleep_millisecs_attr.attr, &khugepaged_max_ptes_swap_attr.attr, NULL, }; struct attribute_group khugepaged_attr_group = { .attrs = khugepaged_attr, .name = "khugepaged", }; #define VM_NO_KHUGEPAGED (VM_SPECIAL | VM_HUGETLB)
int hugepage_madvise(struct vm_area_struct *vma, unsigned long *vm_flags, int advice) { switch (advice) { case MADV_HUGEPAGE: #ifdef CONFIG_S390 /* * qemu blindly sets MADV_HUGEPAGE on all allocations, but s390 * can't handle this properly after s390_enable_sie, so we simply * ignore the madvise to prevent qemu from causing a SIGSEGV. */ if (mm_has_pgste(vma->vm_mm)) return 0; #endif *vm_flags &= ~VM_NOHUGEPAGE; *vm_flags |= VM_HUGEPAGE; /* * If the vma become good for khugepaged to scan, * register it here without waiting a page fault that * may not happen any time soon. */ if (!(*vm_flags & VM_NO_KHUGEPAGED) && khugepaged_enter_vma_merge(vma, *vm_flags)) return -ENOMEM; break; case MADV_NOHUGEPAGE: *vm_flags &= ~VM_HUGEPAGE; *vm_flags |= VM_NOHUGEPAGE; /* * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning * this vma even if we leave the mm registered in khugepaged if * it got registered before VM_NOHUGEPAGE was set. */ break; } return 0; }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov99100.00%1100.00%
Total99100.00%1100.00%


int __init khugepaged_init(void) { mm_slot_cache = kmem_cache_create("khugepaged_mm_slot", sizeof(struct mm_slot), __alignof__(struct mm_slot), 0, NULL); if (!mm_slot_cache) return -ENOMEM; khugepaged_pages_to_scan = HPAGE_PMD_NR * 8; khugepaged_max_ptes_none = HPAGE_PMD_NR - 1; khugepaged_max_ptes_swap = HPAGE_PMD_NR / 8; return 0; }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov61100.00%1100.00%
Total61100.00%1100.00%


void __init khugepaged_destroy(void) { kmem_cache_destroy(mm_slot_cache); }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov13100.00%1100.00%
Total13100.00%1100.00%


static inline struct mm_slot *alloc_mm_slot(void) { if (!mm_slot_cache) /* initialization failed */ return NULL; return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL); }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov28100.00%1100.00%
Total28100.00%1100.00%


static inline void free_mm_slot(struct mm_slot *mm_slot) { kmem_cache_free(mm_slot_cache, mm_slot); }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov19100.00%1100.00%
Total19100.00%1100.00%


static struct mm_slot *get_mm_slot(struct mm_struct *mm) { struct mm_slot *mm_slot; hash_for_each_possible(mm_slots_hash, mm_slot, hash, (unsigned long)mm) if (mm == mm_slot->mm) return mm_slot; return NULL; }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov42100.00%1100.00%
Total42100.00%1100.00%


static void insert_to_mm_slots_hash(struct mm_struct *mm, struct mm_slot *mm_slot) { mm_slot->mm = mm; hash_add(mm_slots_hash, &mm_slot->hash, (long)mm); }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov37100.00%1100.00%
Total37100.00%1100.00%


static inline int khugepaged_test_exit(struct mm_struct *mm) { return atomic_read(&mm->mm_users) == 0; }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov23100.00%1100.00%
Total23100.00%1100.00%


int __khugepaged_enter(struct mm_struct *mm) { struct mm_slot *mm_slot; int wakeup; mm_slot = alloc_mm_slot(); if (!mm_slot) return -ENOMEM; /* __khugepaged_exit() must not run from under us */ VM_BUG_ON_MM(khugepaged_test_exit(mm), mm); if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) { free_mm_slot(mm_slot); return 0; } spin_lock(&khugepaged_mm_lock); insert_to_mm_slots_hash(mm, mm_slot); /* * Insert just behind the scanning cursor, to let the area settle * down a little. */ wakeup = list_empty(&khugepaged_scan.mm_head); list_add_tail(&mm_slot->mm_node, &khugepaged_scan.mm_head); spin_unlock(&khugepaged_mm_lock); atomic_inc(&mm->mm_count); if (wakeup) wake_up_interruptible(&khugepaged_wait); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov132100.00%1100.00%
Total132100.00%1100.00%


int khugepaged_enter_vma_merge(struct vm_area_struct *vma, unsigned long vm_flags) { unsigned long hstart, hend; if (!vma->anon_vma) /* * Not yet faulted in so we will register later in the * page fault if needed. */ return 0; if (vma->vm_ops || (vm_flags & VM_NO_KHUGEPAGED)) /* khugepaged not yet working on file or special mappings */ return 0; hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK; hend = vma->vm_end & HPAGE_PMD_MASK; if (hstart < hend) return khugepaged_enter(vma, vm_flags); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov85100.00%1100.00%
Total85100.00%1100.00%


void __khugepaged_exit(struct mm_struct *mm) { struct mm_slot *mm_slot; int free = 0; spin_lock(&khugepaged_mm_lock); mm_slot = get_mm_slot(mm); if (mm_slot && khugepaged_scan.mm_slot != mm_slot) { hash_del(&mm_slot->hash); list_del(&mm_slot->mm_node); free = 1; } spin_unlock(&khugepaged_mm_lock); if (free) { clear_bit(MMF_VM_HUGEPAGE, &mm->flags); free_mm_slot(mm_slot); mmdrop(mm); } else if (mm_slot) { /* * This is required to serialize against * khugepaged_test_exit() (which is guaranteed to run * under mmap sem read mode). Stop here (after we * return all pagetables will be destroyed) until * khugepaged has finished working on the pagetables * under the mmap_sem. */ down_write(&mm->mmap_sem); up_write(&mm->mmap_sem); } }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov121100.00%1100.00%
Total121100.00%1100.00%


static void release_pte_page(struct page *page) { /* 0 stands for page_is_file_cache(page) == false */ dec_node_page_state(page, NR_ISOLATED_ANON + 0); unlock_page(page); putback_lru_page(page); }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov3096.77%150.00%
mel gormanmel gorman13.23%150.00%
Total31100.00%2100.00%


static void release_pte_pages(pte_t *pte, pte_t *_pte) { while (--_pte >= pte) { pte_t pteval = *_pte; if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval))) release_pte_page(pte_page(pteval)); } }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov54100.00%1100.00%
Total54100.00%1100.00%


static int __collapse_huge_page_isolate(struct vm_area_struct *vma, unsigned long address, pte_t *pte) { struct page *page = NULL; pte_t *_pte; int none_or_zero = 0, result = 0, referenced = 0; bool writable = false; for (_pte = pte; _pte < pte+HPAGE_PMD_NR; _pte++, address += PAGE_SIZE) { pte_t pteval = *_pte; if (pte_none(pteval) || (pte_present(pteval) && is_zero_pfn(pte_pfn(pteval)))) { if (!userfaultfd_armed(vma) && ++none_or_zero <= khugepaged_max_ptes_none) { continue; } else { result = SCAN_EXCEED_NONE_PTE; goto out; } } if (!pte_present(pteval)) { result = SCAN_PTE_NON_PRESENT; goto out; } page = vm_normal_page(vma, address, pteval); if (unlikely(!page)) { result = SCAN_PAGE_NULL; goto out; } VM_BUG_ON_PAGE(PageCompound(page), page); VM_BUG_ON_PAGE(!PageAnon(page), page); VM_BUG_ON_PAGE(!PageSwapBacked(page), page); /* * We can do it before isolate_lru_page because the * page can't be freed from under us. NOTE: PG_lock * is needed to serialize against split_huge_page * when invoked from the VM. */ if (!trylock_page(page)) { result = SCAN_PAGE_LOCK; goto out; } /* * cannot use mapcount: can't collapse if there's a gup pin. * The page must only be referenced by the scanned process * and page swap cache. */ if (page_count(page) != 1 + !!PageSwapCache(page)) { unlock_page(page); result = SCAN_PAGE_COUNT; goto out; } if (pte_write(pteval)) { writable = true; } else { if (PageSwapCache(page) && !reuse_swap_page(page, NULL)) { unlock_page(page); result = SCAN_SWAP_CACHE_PAGE; goto out; } /* * Page is not in the swap cache. It can be collapsed * into a THP. */ } /* * Isolate the page to avoid collapsing an hugepage * currently in use by the VM. */ if (isolate_lru_page(page)) { unlock_page(page); result = SCAN_DEL_PAGE_LRU; goto out; } /* 0 stands for page_is_file_cache(page) == false */ inc_node_page_state(page, NR_ISOLATED_ANON + 0); VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(PageLRU(page), page); /* There should be enough young pte to collapse the page */ if (pte_young(pteval) || page_is_young(page) || PageReferenced(page) || mmu_notifier_test_young(vma->vm_mm, address)) referenced++; } if (likely(writable)) { if (likely(referenced)) { result = SCAN_SUCCEED; trace_mm_collapse_huge_page_isolate(page, none_or_zero, referenced, writable, result); return 1; } } else { result = SCAN_PAGE_RO; } out: release_pte_pages(pte, _pte); trace_mm_collapse_huge_page_isolate(page, none_or_zero, referenced, writable, result); return 0; }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov44398.44%133.33%
ebru akagunduzebru akagunduz61.33%133.33%
mel gormanmel gorman10.22%133.33%
Total450100.00%3100.00%


static void __collapse_huge_page_copy(pte_t *pte, struct page *page, struct vm_area_struct *vma, unsigned long address, spinlock_t *ptl) { pte_t *_pte; for (_pte = pte; _pte < pte+HPAGE_PMD_NR; _pte++) { pte_t pteval = *_pte; struct page *src_page; if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) { clear_user_highpage(page, address); add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1); if (is_zero_pfn(pte_pfn(pteval))) { /* * ptl mostly unnecessary. */ spin_lock(ptl); /* * paravirt calls inside pte_clear here are * superfluous. */ pte_clear(vma->vm_mm, address, _pte); spin_unlock(ptl); } } else { src_page = pte_page(pteval); copy_user_highpage(page, src_page, address, vma); VM_BUG_ON_PAGE(page_mapcount(src_page) != 1, src_page); release_pte_page(src_page); /* * ptl mostly unnecessary, but preempt has to * be disabled to update the per-cpu stats * inside page_remove_rmap(). */ spin_lock(ptl); /* * paravirt calls inside pte_clear here are * superfluous. */ pte_clear(vma->vm_mm, address, _pte); page_remove_rmap(src_page, false); spin_unlock(ptl); free_page_and_swap_cache(src_page); } address += PAGE_SIZE; page++; } }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov210100.00%1100.00%
Total210100.00%1100.00%


static void khugepaged_alloc_sleep(void) { DEFINE_WAIT(wait); add_wait_queue(&khugepaged_wait, &wait); freezable_schedule_timeout_interruptible( msecs_to_jiffies(khugepaged_alloc_sleep_millisecs)); remove_wait_queue(&khugepaged_wait, &wait); }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov39100.00%1100.00%
Total39100.00%1100.00%

static int khugepaged_node_load[MAX_NUMNODES];
static bool khugepaged_scan_abort(int nid) { int i; /* * If node_reclaim_mode is disabled, then no extra effort is made to * allocate memory locally. */ if (!node_reclaim_mode) return false; /* If there is a count for this node already, it must be acceptable */ if (khugepaged_node_load[nid]) return false; for (i = 0; i < MAX_NUMNODES; i++) { if (!khugepaged_node_load[i]) continue; if (node_distance(nid, i) > RECLAIM_DISTANCE) return true; } return false; }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov7197.26%150.00%
mel gormanmel gorman22.74%150.00%
Total73100.00%2100.00%

/* Defrag for khugepaged will enter direct reclaim/compaction if necessary */
static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void) { return khugepaged_defrag() ? GFP_TRANSHUGE : GFP_TRANSHUGE_LIGHT; }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov1588.24%150.00%
vlastimil babkavlastimil babka211.76%150.00%
Total17100.00%2100.00%

#ifdef CONFIG_NUMA
static int khugepaged_find_target_node(void) { static int last_khugepaged_target_node = NUMA_NO_NODE; int nid, target_node = 0, max_value = 0; /* find first node with max normal pages hit */ for (nid = 0; nid < MAX_NUMNODES; nid++) if (khugepaged_node_load[nid] > max_value) { max_value = khugepaged_node_load[nid]; target_node = nid; } /* do some balance if several nodes have the same hit record */ if (target_node <= last_khugepaged_target_node) for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES; nid++) if (max_value == khugepaged_node_load[nid]) { target_node = nid; break; } last_khugepaged_target_node = target_node; return target_node; }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov106100.00%1100.00%
Total106100.00%1100.00%


static bool khugepaged_prealloc_page(struct page **hpage, bool *wait) { if (IS_ERR(*hpage)) { if (!*wait) return false; *wait = false; *hpage = NULL; khugepaged_alloc_sleep(); } else if (*hpage) { put_page(*hpage); *hpage = NULL; } return true; }

Contributors

PersonTokensPropCommitsCommitProp
kirill a. shutemovkirill a. shutemov70100.00%1100.00%
Total70100.00%1100.00%


static struct page * khugepaged_alloc_page(struct page **hpage, gfp_t gfp, int node) { VM_BUG_ON_PAGE(*hpage, *hpage); *hpage = __alloc_pages_node(node, gfp