Release 4.11 arch/x86/mm/hugetlbpage.c

/*
 * IA-32 Huge TLB Page Support for Kernel.
 *
 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
 */

#include <linux/init.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/sched/mm.h>
#include <linux/hugetlb.h>
#include <linux/pagemap.h>
#include <linux/err.h>
#include <linux/sysctl.h>
#include <asm/mman.h>
#include <asm/tlb.h>
#include <asm/tlbflush.h>
#include <asm/pgalloc.h>

#if 0	/* This is just for testing */
struct page *
follow_huge_addr(struct mm_struct *mm, unsigned long address, int write)
{
        unsigned long start = address;
        int length = 1;
        int nr;
        struct page *page;
        struct vm_area_struct *vma;

        vma = find_vma(mm, addr);
        if (!vma || !is_vm_hugetlb_page(vma))
                return ERR_PTR(-EINVAL);

        pte = huge_pte_offset(mm, address);

	/* hugetlb should be locked, and hence, prefaulted */
	WARN_ON(!pte || pte_none(*pte));

        page = &pte_page(*pte)[vpfn % (HPAGE_SIZE/PAGE_SIZE)];

        WARN_ON(!PageHead(page));

        return page;
}

int pmd_huge(pmd_t pmd)
{
        return 0;
}

int pud_huge(pud_t pud)
{
        return 0;
}

#else

/*
 * pmd_huge() returns 1 if @pmd is hugetlb related entry, that is normal
 * hugetlb entry or non-present (migration or hwpoisoned) hugetlb entry.
 * Otherwise, returns 0.
 */


int pmd_huge(pmd_t pmd)
{
	return !pmd_none(pmd) &&
		(pmd_val(pmd) & (_PAGE_PRESENT|_PAGE_PSE)) != _PAGE_PRESENT;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
David Gibson
19
63.33%
1
50.00%
Naoya Horiguchi
11
36.67%
1
50.00%
Total
30
100.00%
2
100.00%



int pud_huge(pud_t pud)
{
	return !!(pud_val(pud) & _PAGE_PSE);
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Andi Kleen
20
100.00%
2
100.00%
Total
20
100.00%
2
100.00%

#endif

#ifdef CONFIG_HUGETLB_PAGE


static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
		unsigned long addr, unsigned long len,
		unsigned long pgoff, unsigned long flags)
{
	struct hstate *h = hstate_file(file);
	struct vm_unmapped_area_info info;

	info.flags = 0;
	info.length = len;
	info.low_limit = current->mm->mmap_legacy_base;
	info.high_limit = TASK_SIZE;
	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
	info.align_offset = 0;
	return vm_unmapped_area(&info);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Yanmin Zhang
40
42.11%
1
20.00%
Michel Lespinasse
32
33.68%
1
20.00%
Andi Kleen
11
11.58%
1
20.00%
Wolfgang Wander
7
7.37%
1
20.00%
Kirill A. Shutemov
5
5.26%
1
20.00%
Total
95
100.00%
5
100.00%



static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file,
		unsigned long addr0, unsigned long len,
		unsigned long pgoff, unsigned long flags)
{
	struct hstate *h = hstate_file(file);
	struct vm_unmapped_area_info info;
	unsigned long addr;

	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
	info.length = len;
	info.low_limit = PAGE_SIZE;
	info.high_limit = current->mm->mmap_base;
	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
	info.align_offset = 0;
	addr = vm_unmapped_area(&info);

	/*
         * A failed mmap() very likely causes application failure,
         * so fall back to the bottom-up function here. This scenario
         * can happen with large stack limits and large mmap()
         * allocations.
         */
	if (addr & ~PAGE_MASK) {
		VM_BUG_ON(addr != -ENOMEM);
		info.flags = 0;
		info.low_limit = TASK_UNMAPPED_BASE;
		info.high_limit = TASK_SIZE;
		addr = vm_unmapped_area(&info);
	}

	return addr;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Michel Lespinasse
57
38.78%
1
20.00%
Yanmin Zhang
53
36.05%
1
20.00%
Wolfgang Wander
17
11.56%
1
20.00%
Andi Kleen
15
10.20%
1
20.00%
Xiao Guangrong
5
3.40%
1
20.00%
Total
147
100.00%
5
100.00%


unsigned long

hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
		unsigned long len, unsigned long pgoff, unsigned long flags)
{
	struct hstate *h = hstate_file(file);
	struct mm_struct *mm = current->mm;
	struct vm_area_struct *vma;

	if (len & ~huge_page_mask(h))
		return -EINVAL;
	if (len > TASK_SIZE)
		return -ENOMEM;

	if (flags & MAP_FIXED) {
		if (prepare_hugepage_range(file, addr, len))
			return -EINVAL;
		return addr;
	}

	if (addr) {
		addr = ALIGN(addr, huge_page_size(h));
		vma = find_vma(mm, addr);
		if (TASK_SIZE - len >= addr &&
		    (!vma || addr + len <= vma->vm_start))
			return addr;
	}
	if (mm->get_unmapped_area == arch_get_unmapped_area)
		return hugetlb_get_unmapped_area_bottomup(file, addr, len,
				pgoff, flags);
	else
		return hugetlb_get_unmapped_area_topdown(file, addr, len,
				pgoff, flags);
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Yanmin Zhang
145
76.72%
1
25.00%
Benjamin Herrenschmidt
24
12.70%
1
25.00%
Andi Kleen
20
10.58%
2
50.00%
Total
189
100.00%
4
100.00%

#endif /* CONFIG_HUGETLB_PAGE */

#ifdef CONFIG_X86_64


static __init int setup_hugepagesz(char *opt)
{
	unsigned long ps = memparse(opt, &opt);
	if (ps == PMD_SIZE) {
		hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
	} else if (ps == PUD_SIZE && boot_cpu_has(X86_FEATURE_GBPAGES)) {
		hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
	} else {
		hugetlb_bad_size();
		printk(KERN_ERR "hugepagesz: Unsupported page size %lu M\n",
			ps >> 20);
		return 0;
	}
	return 1;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Andi Kleen
74
91.36%
1
33.33%
Borislav Petkov
4
4.94%
1
33.33%
Vaishali Thakkar
3
3.70%
1
33.33%
Total
81
100.00%
3
100.00%

__setup("hugepagesz=", setup_hugepagesz);

#if (defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || defined(CONFIG_CMA)


static __init int gigantic_pages_init(void)
{
	/* With compaction or CMA we can allocate gigantic pages at runtime */
	if (boot_cpu_has(X86_FEATURE_GBPAGES) && !size_to_hstate(1UL << PUD_SHIFT))
		hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
	return 0;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Kirill A. Shutemov
30
85.71%
1
33.33%
Borislav Petkov
4
11.43%
1
33.33%
Vlastimil Babka
1
2.86%
1
33.33%
Total
35
100.00%
3
100.00%


arch_initcall(gigantic_pages_init);
#endif
#endif
Overall Contributors
Person
Tokens
Prop
Commits
CommitProp
Yanmin Zhang
242
35.12%
1
4.35%
Andi Kleen
152
22.06%
4
17.39%
Michel Lespinasse
89
12.92%
1
4.35%
Kirill A. Shutemov
45
6.53%
2
8.70%
Andrew Morton
31
4.50%
4
17.39%
David Gibson
29
4.21%
1
4.35%
Wolfgang Wander
24
3.48%
1
4.35%
Benjamin Herrenschmidt
24
3.48%
1
4.35%
Vlastimil Babka
18
2.61%
1
4.35%
Naoya Horiguchi
13
1.89%
2
8.70%
Borislav Petkov
8
1.16%
1
4.35%
Xiao Guangrong
5
0.73%
1
4.35%
Ingo Molnar
3
0.44%
1
4.35%
Jeremy Fitzhardinge
3
0.44%
1
4.35%
Vaishali Thakkar
3
0.44%
1
4.35%
Total
689
100.00%
23
100.00%
Directory: arch/x86/mm

Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.