Contributors: 61
| Author |
Tokens |
Token Proportion |
Commits |
Commit Proportion |
| Pavel Tatashin |
397 |
23.83% |
6 |
5.36% |
| van der Linden, Frank |
180 |
10.80% |
1 |
0.89% |
| Andy Whitcroft |
166 |
9.96% |
5 |
4.46% |
| Mel Gorman |
125 |
7.50% |
3 |
2.68% |
| Mike Rapoport |
111 |
6.66% |
8 |
7.14% |
| Bob Picco |
97 |
5.82% |
2 |
1.79% |
| Dave Hansen |
80 |
4.80% |
5 |
4.46% |
| Lecopzer Chen |
68 |
4.08% |
2 |
1.79% |
| Christoph Lameter |
54 |
3.24% |
2 |
1.79% |
| Baoquan He |
48 |
2.88% |
1 |
0.89% |
| Yinghai Lu |
41 |
2.46% |
5 |
4.46% |
| Linus Torvalds (pre-git) |
29 |
1.74% |
9 |
8.04% |
| Dan J Williams |
27 |
1.62% |
4 |
3.57% |
| Kiryl Shutsemau |
25 |
1.50% |
1 |
0.89% |
| Wei Yang |
19 |
1.14% |
3 |
2.68% |
| Logan Gunthorpe |
18 |
1.08% |
1 |
0.89% |
| Mike Kravetz |
16 |
0.96% |
2 |
1.79% |
| Fabian Frederick |
13 |
0.78% |
1 |
0.89% |
| Ingo Molnar |
12 |
0.72% |
1 |
0.89% |
| Matthew Wilcox |
10 |
0.60% |
3 |
2.68% |
| Américo Wang |
9 |
0.54% |
1 |
0.89% |
| Michal Hocko |
7 |
0.42% |
3 |
2.68% |
| Joao Martins |
7 |
0.42% |
1 |
0.89% |
| Alastair D'Silva |
6 |
0.36% |
1 |
0.89% |
| Gideon Israel Dsouza |
6 |
0.36% |
1 |
0.89% |
| David Hildenbrand (Red Hat) |
6 |
0.36% |
2 |
1.79% |
| Jeff Xu |
6 |
0.36% |
1 |
0.89% |
| Dev Jain |
5 |
0.30% |
1 |
0.89% |
| Sourav Panda |
5 |
0.30% |
1 |
0.89% |
| Yasunori Goto |
5 |
0.30% |
3 |
2.68% |
| JoonSoo Kim |
5 |
0.30% |
1 |
0.89% |
| Thomas Gleixner |
4 |
0.24% |
1 |
0.89% |
| Santosh Shilimkar |
4 |
0.24% |
1 |
0.89% |
| Kamezawa Hiroyuki |
4 |
0.24% |
1 |
0.89% |
| Christoph Hellwig |
4 |
0.24% |
1 |
0.89% |
| Zhang Yanfei |
3 |
0.18% |
2 |
1.79% |
| Sumanth Korikkar |
3 |
0.18% |
1 |
0.89% |
| Adrian Bunk |
3 |
0.18% |
1 |
0.89% |
| Wang Wensheng |
3 |
0.18% |
1 |
0.89% |
| Song Muchun |
3 |
0.18% |
1 |
0.89% |
| David Rientjes |
3 |
0.18% |
1 |
0.89% |
| David Howells |
3 |
0.18% |
1 |
0.89% |
| Brent Casavant |
3 |
0.18% |
1 |
0.89% |
| Paul Mackerras |
2 |
0.12% |
1 |
0.89% |
| 권오훈 |
2 |
0.12% |
1 |
0.89% |
| Nadia Yvette Chambers |
2 |
0.12% |
1 |
0.89% |
| David Hildenbrand |
2 |
0.12% |
1 |
0.89% |
| Nicholas Piggin |
2 |
0.12% |
1 |
0.89% |
| Wanpeng Li |
1 |
0.06% |
1 |
0.89% |
| Ravikiran G. Thirumalai |
1 |
0.06% |
1 |
0.89% |
| Kirill A. Shutemov |
1 |
0.06% |
1 |
0.89% |
| Wen Congyang |
1 |
0.06% |
1 |
0.89% |
| Linus Torvalds |
1 |
0.06% |
1 |
0.89% |
| Miaohe Lin |
1 |
0.06% |
1 |
0.89% |
| Gavin Shan |
1 |
0.06% |
1 |
0.89% |
| John Hubbard |
1 |
0.06% |
1 |
0.89% |
| Stanislaw Gruszka |
1 |
0.06% |
1 |
0.89% |
| Cyrill V. Gorcunov |
1 |
0.06% |
1 |
0.89% |
| Greg Kroah-Hartman |
1 |
0.06% |
1 |
0.89% |
| Paul Gortmaker |
1 |
0.06% |
1 |
0.89% |
| guoweikang |
1 |
0.06% |
1 |
0.89% |
| Total |
1666 |
|
112 |
|
// SPDX-License-Identifier: GPL-2.0
/*
* sparse memory mappings.
*/
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/mmzone.h>
#include <linux/memblock.h>
#include <linux/compiler.h>
#include <linux/highmem.h>
#include <linux/export.h>
#include <linux/spinlock.h>
#include <linux/vmalloc.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/bootmem_info.h>
#include <linux/vmstat.h>
#include "internal.h"
#include <asm/dma.h>
/*
* Permanent SPARSEMEM data:
*
* 1) mem_section - memory sections, mem_map's for valid memory
*/
#ifdef CONFIG_SPARSEMEM_EXTREME
struct mem_section **mem_section;
#else
struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT]
____cacheline_internodealigned_in_smp;
#endif
EXPORT_SYMBOL(mem_section);
#ifdef NODE_NOT_IN_PAGE_FLAGS
/*
* If we did not store the node number in the page then we have to
* do a lookup in the section_to_node_table in order to find which
* node the page belongs to.
*/
#if MAX_NUMNODES <= 256
static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;
#else
static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;
#endif
int memdesc_nid(memdesc_flags_t mdf)
{
return section_to_node_table[memdesc_section(mdf)];
}
EXPORT_SYMBOL(memdesc_nid);
static void set_section_nid(unsigned long section_nr, int nid)
{
section_to_node_table[section_nr] = nid;
}
#else /* !NODE_NOT_IN_PAGE_FLAGS */
static inline void set_section_nid(unsigned long section_nr, int nid)
{
}
#endif
#ifdef CONFIG_SPARSEMEM_EXTREME
static noinline struct mem_section __ref *sparse_index_alloc(int nid)
{
struct mem_section *section = NULL;
unsigned long array_size = SECTIONS_PER_ROOT *
sizeof(struct mem_section);
if (slab_is_available()) {
section = kzalloc_node(array_size, GFP_KERNEL, nid);
} else {
section = memblock_alloc_node(array_size, SMP_CACHE_BYTES,
nid);
if (!section)
panic("%s: Failed to allocate %lu bytes nid=%d\n",
__func__, array_size, nid);
}
return section;
}
int __meminit sparse_index_init(unsigned long section_nr, int nid)
{
unsigned long root = SECTION_NR_TO_ROOT(section_nr);
struct mem_section *section;
/*
* An existing section is possible in the sub-section hotplug
* case. First hot-add instantiates, follow-on hot-add reuses
* the existing section.
*
* The mem_hotplug_lock resolves the apparent race below.
*/
if (mem_section[root])
return 0;
section = sparse_index_alloc(nid);
if (!section)
return -ENOMEM;
mem_section[root] = section;
return 0;
}
#else /* !SPARSEMEM_EXTREME */
int sparse_index_init(unsigned long section_nr, int nid)
{
return 0;
}
#endif
/*
* During early boot, before section_mem_map is used for an actual
* mem_map, we use section_mem_map to store the section's NUMA
* node. This keeps us from having to use another data structure. The
* node information is cleared just before we store the real mem_map.
*/
static inline unsigned long sparse_encode_early_nid(int nid)
{
return ((unsigned long)nid << SECTION_NID_SHIFT);
}
static inline int sparse_early_nid(struct mem_section *section)
{
return (section->section_mem_map >> SECTION_NID_SHIFT);
}
/* Validate the physical addressing limitations of the model */
static void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn,
unsigned long *end_pfn)
{
unsigned long max_sparsemem_pfn = (DIRECT_MAP_PHYSMEM_END + 1) >> PAGE_SHIFT;
/*
* Sanity checks - do not allow an architecture to pass
* in larger pfns than the maximum scope of sparsemem:
*/
if (*start_pfn > max_sparsemem_pfn) {
mminit_dprintk(MMINIT_WARNING, "pfnvalidation",
"Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
*start_pfn, *end_pfn, max_sparsemem_pfn);
WARN_ON_ONCE(1);
*start_pfn = max_sparsemem_pfn;
*end_pfn = max_sparsemem_pfn;
} else if (*end_pfn > max_sparsemem_pfn) {
mminit_dprintk(MMINIT_WARNING, "pfnvalidation",
"End of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
*start_pfn, *end_pfn, max_sparsemem_pfn);
WARN_ON_ONCE(1);
*end_pfn = max_sparsemem_pfn;
}
}
/*
* There are a number of times that we loop over NR_MEM_SECTIONS,
* looking for section_present() on each. But, when we have very
* large physical address spaces, NR_MEM_SECTIONS can also be
* very large which makes the loops quite long.
*
* Keeping track of this gives us an easy way to break out of
* those loops early.
*/
unsigned long __highest_present_section_nr;
static inline unsigned long first_present_section_nr(void)
{
return next_present_section_nr(-1);
}
/* Record a memory area against a node. */
static void __init memory_present(int nid, unsigned long start, unsigned long end)
{
unsigned long pfn;
start &= PAGE_SECTION_MASK;
mminit_validate_memmodel_limits(&start, &end);
for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) {
unsigned long section_nr = pfn_to_section_nr(pfn);
struct mem_section *ms;
sparse_index_init(section_nr, nid);
set_section_nid(section_nr, nid);
ms = __nr_to_section(section_nr);
if (!ms->section_mem_map) {
ms->section_mem_map = sparse_encode_early_nid(nid) |
SECTION_IS_ONLINE;
__section_mark_present(ms, section_nr);
}
}
}
/*
* Mark all memblocks as present using memory_present().
* This is a convenience function that is useful to mark all of the systems
* memory as present during initialization.
*/
static void __init memblocks_present(void)
{
unsigned long start, end;
int i, nid;
#ifdef CONFIG_SPARSEMEM_EXTREME
if (unlikely(!mem_section)) {
unsigned long size, align;
size = sizeof(struct mem_section *) * NR_SECTION_ROOTS;
align = 1 << (INTERNODE_CACHE_SHIFT);
mem_section = memblock_alloc_or_panic(size, align);
}
#endif
for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid)
memory_present(nid, start, end);
}
static unsigned long usemap_size(void)
{
return BITS_TO_LONGS(SECTION_BLOCKFLAGS_BITS) * sizeof(unsigned long);
}
size_t mem_section_usage_size(void)
{
return sizeof(struct mem_section_usage) + usemap_size();
}
#ifdef CONFIG_SPARSEMEM_VMEMMAP
unsigned long __init section_map_size(void)
{
return ALIGN(sizeof(struct page) * PAGES_PER_SECTION, PMD_SIZE);
}
#else
unsigned long __init section_map_size(void)
{
return PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION);
}
struct page __init *__populate_section_memmap(unsigned long pfn,
unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
struct dev_pagemap *pgmap)
{
unsigned long size = section_map_size();
struct page *map = sparse_buffer_alloc(size);
phys_addr_t addr = __pa(MAX_DMA_ADDRESS);
if (map)
return map;
map = memmap_alloc(size, size, addr, nid, false);
if (!map)
panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa\n",
__func__, size, PAGE_SIZE, nid, &addr);
return map;
}
#endif /* !CONFIG_SPARSEMEM_VMEMMAP */
static void *sparsemap_buf __meminitdata;
static void *sparsemap_buf_end __meminitdata;
static inline void __meminit sparse_buffer_free(unsigned long size)
{
WARN_ON(!sparsemap_buf || size == 0);
memblock_free(sparsemap_buf, size);
}
static void __init sparse_buffer_init(unsigned long size, int nid)
{
phys_addr_t addr = __pa(MAX_DMA_ADDRESS);
WARN_ON(sparsemap_buf); /* forgot to call sparse_buffer_fini()? */
/*
* Pre-allocated buffer is mainly used by __populate_section_memmap
* and we want it to be properly aligned to the section size - this is
* especially the case for VMEMMAP which maps memmap to PMDs
*/
sparsemap_buf = memmap_alloc(size, section_map_size(), addr, nid, true);
sparsemap_buf_end = sparsemap_buf + size;
}
static void __init sparse_buffer_fini(void)
{
unsigned long size = sparsemap_buf_end - sparsemap_buf;
if (sparsemap_buf && size > 0)
sparse_buffer_free(size);
sparsemap_buf = NULL;
}
void * __meminit sparse_buffer_alloc(unsigned long size)
{
void *ptr = NULL;
if (sparsemap_buf) {
ptr = (void *) roundup((unsigned long)sparsemap_buf, size);
if (ptr + size > sparsemap_buf_end)
ptr = NULL;
else {
/* Free redundant aligned space */
if ((unsigned long)(ptr - sparsemap_buf) > 0)
sparse_buffer_free((unsigned long)(ptr - sparsemap_buf));
sparsemap_buf = ptr + size;
}
}
return ptr;
}
void __weak __meminit vmemmap_populate_print_last(void)
{
}
static void *sparse_usagebuf __meminitdata;
static void *sparse_usagebuf_end __meminitdata;
/*
* Helper function that is used for generic section initialization, and
* can also be used by any hooks added above.
*/
void __init sparse_init_early_section(int nid, struct page *map,
unsigned long pnum, unsigned long flags)
{
BUG_ON(!sparse_usagebuf || sparse_usagebuf >= sparse_usagebuf_end);
sparse_init_one_section(__nr_to_section(pnum), pnum, map,
sparse_usagebuf, SECTION_IS_EARLY | flags);
sparse_usagebuf = (void *)sparse_usagebuf + mem_section_usage_size();
}
static int __init sparse_usage_init(int nid, unsigned long map_count)
{
unsigned long size;
size = mem_section_usage_size() * map_count;
sparse_usagebuf = memblock_alloc_node(size, SMP_CACHE_BYTES, nid);
if (!sparse_usagebuf) {
sparse_usagebuf_end = NULL;
return -ENOMEM;
}
sparse_usagebuf_end = sparse_usagebuf + size;
return 0;
}
static void __init sparse_usage_fini(void)
{
sparse_usagebuf = sparse_usagebuf_end = NULL;
}
/*
* Initialize sparse on a specific node. The node spans [pnum_begin, pnum_end)
* And number of present sections in this node is map_count.
*/
static void __init sparse_init_nid(int nid, unsigned long pnum_begin,
unsigned long pnum_end,
unsigned long map_count)
{
unsigned long pnum;
struct page *map;
struct mem_section *ms;
if (sparse_usage_init(nid, map_count)) {
pr_err("%s: node[%d] usemap allocation failed", __func__, nid);
goto failed;
}
sparse_buffer_init(map_count * section_map_size(), nid);
sparse_vmemmap_init_nid_early(nid);
for_each_present_section_nr(pnum_begin, pnum) {
unsigned long pfn = section_nr_to_pfn(pnum);
if (pnum >= pnum_end)
break;
ms = __nr_to_section(pnum);
if (!preinited_vmemmap_section(ms)) {
map = __populate_section_memmap(pfn, PAGES_PER_SECTION,
nid, NULL, NULL);
if (!map) {
pr_err("%s: node[%d] memory map backing failed. Some memory will not be available.",
__func__, nid);
pnum_begin = pnum;
sparse_usage_fini();
sparse_buffer_fini();
goto failed;
}
memmap_boot_pages_add(DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page),
PAGE_SIZE));
sparse_init_early_section(nid, map, pnum, 0);
}
}
sparse_usage_fini();
sparse_buffer_fini();
return;
failed:
/*
* We failed to allocate, mark all the following pnums as not present,
* except the ones already initialized earlier.
*/
for_each_present_section_nr(pnum_begin, pnum) {
if (pnum >= pnum_end)
break;
ms = __nr_to_section(pnum);
if (!preinited_vmemmap_section(ms))
ms->section_mem_map = 0;
}
}
/*
* Allocate the accumulated non-linear sections, allocate a mem_map
* for each and record the physical to section mapping.
*/
void __init sparse_init(void)
{
unsigned long pnum_end, pnum_begin, map_count = 1;
int nid_begin;
/* see include/linux/mmzone.h 'struct mem_section' definition */
BUILD_BUG_ON(!is_power_of_2(sizeof(struct mem_section)));
memblocks_present();
if (compound_info_has_mask()) {
VM_WARN_ON_ONCE(!IS_ALIGNED((unsigned long) pfn_to_page(0),
MAX_FOLIO_VMEMMAP_ALIGN));
}
pnum_begin = first_present_section_nr();
nid_begin = sparse_early_nid(__nr_to_section(pnum_begin));
/* Setup pageblock_order for HUGETLB_PAGE_SIZE_VARIABLE */
set_pageblock_order();
for_each_present_section_nr(pnum_begin + 1, pnum_end) {
int nid = sparse_early_nid(__nr_to_section(pnum_end));
if (nid == nid_begin) {
map_count++;
continue;
}
/* Init node with sections in range [pnum_begin, pnum_end) */
sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count);
nid_begin = nid;
pnum_begin = pnum_end;
map_count = 1;
}
/* cover the last node */
sparse_init_nid(nid_begin, pnum_begin, pnum_end, map_count);
vmemmap_populate_print_last();
}