Release 4.14 drivers/acpi/numa.c

/*
 *  acpi_numa.c - ACPI NUMA support
 *
 *  Copyright (C) 2002 Takayoshi Kochi <t-kochi@bq.jp.nec.com>
 *
 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 *
 */


#define pr_fmt(fmt) "ACPI: " fmt

#include <linux/module.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/acpi.h>
#include <linux/bootmem.h>
#include <linux/memblock.h>
#include <linux/numa.h>
#include <linux/nodemask.h>
#include <linux/topology.h>


static nodemask_t nodes_found_map = NODE_MASK_NONE;

/* maps to convert between proximity domain and logical node ID */

static int pxm_to_node_map[MAX_PXM_DOMAINS]
			= { [0 ... MAX_PXM_DOMAINS - 1] = NUMA_NO_NODE };

static int node_to_pxm_map[MAX_NUMNODES]
			= { [0 ... MAX_NUMNODES - 1] = PXM_INVAL };


unsigned char acpi_srat_revision __initdata;

int acpi_numa __initdata;



int pxm_to_node(int pxm)
{
	if (pxm < 0)
		return NUMA_NO_NODE;
	return pxm_to_node_map[pxm];
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Yasunori Goto
22
95.65%
1
50.00%
David Rientjes
1
4.35%
1
50.00%
Total
23
100.00%
2
100.00%



int node_to_pxm(int node)
{
	if (node < 0)
		return PXM_INVAL;
	return node_to_pxm_map[node];
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Yasunori Goto
23
100.00%
1
100.00%
Total
23
100.00%
1
100.00%



static void __acpi_map_pxm_to_node(int pxm, int node)
{
	if (pxm_to_node_map[pxm] == NUMA_NO_NODE || node < pxm_to_node_map[pxm])
		pxm_to_node_map[pxm] = node;
	if (node_to_pxm_map[node] == PXM_INVAL || pxm < node_to_pxm_map[node])
		node_to_pxm_map[node] = pxm;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
David Rientjes
57
98.28%
2
66.67%
Björn Helgaas
1
1.72%
1
33.33%
Total
58
100.00%
3
100.00%



int acpi_map_pxm_to_node(int pxm)
{
	int node;

	if (pxm < 0 || pxm >= MAX_PXM_DOMAINS || numa_off)
		return NUMA_NO_NODE;

	node = pxm_to_node_map[pxm];

	if (node == NUMA_NO_NODE) {
		if (nodes_weight(nodes_found_map) >= MAX_NUMNODES)
			return NUMA_NO_NODE;
		node = first_unset_node(nodes_found_map);
		__acpi_map_pxm_to_node(pxm, node);
		node_set(node, nodes_found_map);
	}

	return node;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Yasunori Goto
51
66.23%
1
16.67%
Toshi Kani
17
22.08%
1
16.67%
David Rientjes
5
6.49%
2
33.33%
Jianguo Wu
2
2.60%
1
16.67%
Boris Ostrovsky
2
2.60%
1
16.67%
Total
77
100.00%
6
100.00%

/**
 * acpi_map_pxm_to_online_node - Map proximity ID to online node
 * @pxm: ACPI proximity ID
 *
 * This is similar to acpi_map_pxm_to_node(), but always returns an online
 * node.  When the mapped node from a given proximity ID is offline, it
 * looks up the node distance table and returns the nearest online node.
 *
 * ACPI device drivers, which are called after the NUMA initialization has
 * completed in the kernel, can call this interface to obtain their device
 * NUMA topology from ACPI tables.  Such drivers do not have to deal with
 * offline nodes.  A node may be offline when a device proximity ID is
 * unique, SRAT memory entry does not exist, or NUMA is disabled, ex.
 * "numa=off" on x86.
 */


int acpi_map_pxm_to_online_node(int pxm)
{
	int node, n, dist, min_dist;

	node = acpi_map_pxm_to_node(pxm);

	if (node == NUMA_NO_NODE)
		node = 0;

	if (!node_online(node)) {
		min_dist = INT_MAX;
		for_each_online_node(n) {
			dist = node_distance(node, n);
			if (dist < min_dist) {
				min_dist = dist;
				node = n;
			}
		}
	}

	return node;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Toshi Kani
82
100.00%
1
100.00%
Total
82
100.00%
1
100.00%


EXPORT_SYMBOL(acpi_map_pxm_to_online_node);


static void __init

acpi_table_print_srat_entry(struct acpi_subtable_header *header)
{
	switch (header->type) {
	case ACPI_SRAT_TYPE_CPU_AFFINITY:
		{
			struct acpi_srat_cpu_affinity *p =
			    (struct acpi_srat_cpu_affinity *)header;
			pr_debug("SRAT Processor (id[0x%02x] eid[0x%02x]) in proximity domain %d %s\n",
				 p->apic_id, p->local_sapic_eid,
				 p->proximity_domain_lo,
				 (p->flags & ACPI_SRAT_CPU_ENABLED) ?
				 "enabled" : "disabled");
		}
		break;

	case ACPI_SRAT_TYPE_MEMORY_AFFINITY:
		{
			struct acpi_srat_mem_affinity *p =
			    (struct acpi_srat_mem_affinity *)header;
			pr_debug("SRAT Memory (0x%lx length 0x%lx) in proximity domain %d %s%s%s\n",
				 (unsigned long)p->base_address,
				 (unsigned long)p->length,
				 p->proximity_domain,
				 (p->flags & ACPI_SRAT_MEM_ENABLED) ?
				 "enabled" : "disabled",
				 (p->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) ?
				 " hot-pluggable" : "",
				 (p->flags & ACPI_SRAT_MEM_NON_VOLATILE) ?
				 " non-volatile" : "");
		}
		break;

	case ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY:
		{
			struct acpi_srat_x2apic_cpu_affinity *p =
			    (struct acpi_srat_x2apic_cpu_affinity *)header;
			pr_debug("SRAT Processor (x2apicid[0x%08x]) in proximity domain %d %s\n",
				 p->apic_id,
				 p->proximity_domain,
				 (p->flags & ACPI_SRAT_CPU_ENABLED) ?
				 "enabled" : "disabled");
		}
		break;

	case ACPI_SRAT_TYPE_GICC_AFFINITY:
		{
			struct acpi_srat_gicc_affinity *p =
			    (struct acpi_srat_gicc_affinity *)header;
			pr_debug("SRAT Processor (acpi id[0x%04x]) in proximity domain %d %s\n",
				 p->acpi_processor_uid,
				 p->proximity_domain,
				 (p->flags & ACPI_SRAT_GICC_ENABLED) ?
				 "enabled" : "disabled");
		}
		break;

	default:
		pr_warn("Found unsupported SRAT entry (type = 0x%x)\n",
			header->type);
		break;
	}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Hanjun Guo
65
26.75%
3
33.33%
Andy Grover
64
26.34%
1
11.11%
Suresh B. Siddha
38
15.64%
1
11.11%
Alexey Y. Starikovskiy
32
13.17%
1
11.11%
Len Brown
31
12.76%
1
11.11%
Davidlohr Bueso A
12
4.94%
1
11.11%
David Rientjes
1
0.41%
1
11.11%
Total
243
100.00%
9
100.00%

/*
 * A lot of BIOS fill in 10 (= no distance) everywhere. This messes
 * up the NUMA heuristics which wants the local node to have a smaller
 * distance than the others.
 * Do some quick checks here and only use the SLIT if it passes.
 */


static int __init slit_valid(struct acpi_table_slit *slit)
{
	int i, j;
	int d = slit->locality_count;
	for (i = 0; i < d; i++) {
		for (j = 0; j < d; j++)  {
			u8 val = slit->entry[d*i + j];
			if (i == j) {
				if (val != LOCAL_DISTANCE)
					return 0;
			} else if (val <= LOCAL_DISTANCE)
				return 0;
		}
	}
	return 1;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Fenghua Yu
97
98.98%
1
50.00%
Hanjun Guo
1
1.02%
1
50.00%
Total
98
100.00%
2
100.00%



void __init bad_srat(void)
{
	pr_err("SRAT: SRAT not used.\n");
	acpi_numa = -1;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
David Daney
18
100.00%
1
100.00%
Total
18
100.00%
1
100.00%



int __init srat_disabled(void)
{
	return acpi_numa < 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
David Daney
13
100.00%
1
100.00%
Total
13
100.00%
1
100.00%

#if defined(CONFIG_X86) || defined(CONFIG_ARM64)
/*
 * Callback for SLIT parsing.  pxm_to_node() returns NUMA_NO_NODE for
 * I/O localities since SRAT does not list them.  I/O localities are
 * not supported at this point.
 */


void __init acpi_numa_slit_init(struct acpi_table_slit *slit)
{
	int i, j;

	for (i = 0; i < slit->locality_count; i++) {
		const int from_node = pxm_to_node(i);

		if (from_node == NUMA_NO_NODE)
			continue;

		for (j = 0; j < slit->locality_count; j++) {
			const int to_node = pxm_to_node(j);

			if (to_node == NUMA_NO_NODE)
				continue;

			numa_set_distance(from_node, to_node,
				slit->entry[slit->locality_count * i + j]);
		}
	}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Hanjun Guo
102
100.00%
1
100.00%
Total
102
100.00%
1
100.00%

/*
 * Default callback for parsing of the Proximity Domain <-> Memory
 * Area mappings
 */

int __init

acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma)
{
	u64 start, end;
	u32 hotpluggable;
	int node, pxm;

	if (srat_disabled())
		goto out_err;
	if (ma->header.length < sizeof(struct acpi_srat_mem_affinity)) {
		pr_err("SRAT: Unexpected header length: %d\n",
		       ma->header.length);
		goto out_err_bad_srat;
	}
	if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0)
		goto out_err;
	hotpluggable = ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE;
	if (hotpluggable && !IS_ENABLED(CONFIG_MEMORY_HOTPLUG))
		goto out_err;

	start = ma->base_address;
	end = start + ma->length;
	pxm = ma->proximity_domain;
	if (acpi_srat_revision <= 1)
		pxm &= 0xff;

	node = acpi_map_pxm_to_node(pxm);
	if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) {
		pr_err("SRAT: Too many proximity domains.\n");
		goto out_err_bad_srat;
	}

	if (numa_add_memblk(node, start, end) < 0) {
		pr_err("SRAT: Failed to add memblk to node %u [mem %#010Lx-%#010Lx]\n",
		       node, (unsigned long long) start,
		       (unsigned long long) end - 1);
		goto out_err_bad_srat;
	}

	node_set(node, numa_nodes_parsed);

	pr_info("SRAT: Node %u PXM %u [mem %#010Lx-%#010Lx]%s%s\n",
		node, pxm,
		(unsigned long long) start, (unsigned long long) end - 1,
		hotpluggable ? " hotplug" : "",
		ma->flags & ACPI_SRAT_MEM_NON_VOLATILE ? " non-volatile" : "");

	/* Mark hotplug range in memblock. */
	if (hotpluggable && memblock_mark_hotplug(start, ma->length))
		pr_warn("SRAT: Failed to mark hotplug range [mem %#010Lx-%#010Lx] in memblock\n",
			(unsigned long long)start, (unsigned long long)end - 1);

	max_possible_pfn = max(max_possible_pfn, PFN_UP(end - 1));

	return 0;
out_err_bad_srat:
	bad_srat();
out_err:
	return -EINVAL;
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Hanjun Guo
261
85.02%
1
50.00%
David Daney
46
14.98%
1
50.00%
Total
307
100.00%
2
100.00%

#endif /* defined(CONFIG_X86) || defined (CONFIG_ARM64) */



static int __init acpi_parse_slit(struct acpi_table_header *table)
{
	struct acpi_table_slit *slit = (struct acpi_table_slit *)table;

	if (!slit_valid(slit)) {
		pr_info("SLIT table looks invalid. Not used.\n");
		return -EINVAL;
	}
	acpi_numa_slit_init(slit);

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Andy Grover
27
52.94%
1
20.00%
Fenghua Yu
15
29.41%
1
20.00%
Alexey Y. Starikovskiy
5
9.80%
1
20.00%
Hanjun Guo
4
7.84%
2
40.00%
Total
51
100.00%
5
100.00%


void __init __weak

acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa)
{
	pr_warn("Found unsupported x2apic [0x%08x] SRAT entry\n", pa->apic_id);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Suresh B. Siddha
11
52.38%
1
33.33%
Björn Helgaas
5
23.81%
1
33.33%
Hanjun Guo
5
23.81%
1
33.33%
Total
21
100.00%
3
100.00%


static int __init

acpi_parse_x2apic_affinity(struct acpi_subtable_header *header,
			   const unsigned long end)
{
	struct acpi_srat_x2apic_cpu_affinity *processor_affinity;

	processor_affinity = (struct acpi_srat_x2apic_cpu_affinity *)header;
	if (!processor_affinity)
		return -EINVAL;

	acpi_table_print_srat_entry(header);

	/* let architecture-dependent part to do it */
	acpi_numa_x2apic_affinity_init(processor_affinity);

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Suresh B. Siddha
54
100.00%
1
100.00%
Total
54
100.00%
1
100.00%


static int __init

acpi_parse_processor_affinity(struct acpi_subtable_header *header,
			      const unsigned long end)
{
	struct acpi_srat_cpu_affinity *processor_affinity;

	processor_affinity = (struct acpi_srat_cpu_affinity *)header;
	if (!processor_affinity)
		return -EINVAL;

	acpi_table_print_srat_entry(header);

	/* let architecture-dependent part to do it */
	acpi_numa_processor_affinity_init(processor_affinity);

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Andy Grover
45
83.33%
1
33.33%
Len Brown
5
9.26%
1
33.33%
Alexey Y. Starikovskiy
4
7.41%
1
33.33%
Total
54
100.00%
3
100.00%


static int __init

acpi_parse_gicc_affinity(struct acpi_subtable_header *header,
			 const unsigned long end)
{
	struct acpi_srat_gicc_affinity *processor_affinity;

	processor_affinity = (struct acpi_srat_gicc_affinity *)header;
	if (!processor_affinity)
		return -EINVAL;

	acpi_table_print_srat_entry(header);

	/* let architecture-dependent part to do it */
	acpi_numa_gicc_affinity_init(processor_affinity);

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Hanjun Guo
54
100.00%
1
100.00%
Total
54
100.00%
1
100.00%


static int __initdata parsed_numa_memblks;


static int __init

acpi_parse_memory_affinity(struct acpi_subtable_header * header,
			   const unsigned long end)
{
	struct acpi_srat_mem_affinity *memory_affinity;

	memory_affinity = (struct acpi_srat_mem_affinity *)header;
	if (!memory_affinity)
		return -EINVAL;

	acpi_table_print_srat_entry(header);

	/* let architecture-dependent part to do it */
	if (!acpi_numa_memory_affinity_init(memory_affinity))
		parsed_numa_memblks++;
	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Andy Grover
45
75.00%
1
25.00%
Thomas Renninger
6
10.00%
1
25.00%
Len Brown
5
8.33%
1
25.00%
Alexey Y. Starikovskiy
4
6.67%
1
25.00%
Total
60
100.00%
4
100.00%



static int __init acpi_parse_srat(struct acpi_table_header *table)
{
	struct acpi_table_srat *srat = (struct acpi_table_srat *)table;

	acpi_srat_revision = srat->header.revision;

	/* Real work done in acpi_table_parse_srat below. */

	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Kurt Garloff
18
50.00%
1
20.00%
Andy Grover
11
30.56%
1
20.00%
Alexey Y. Starikovskiy
4
11.11%
1
20.00%
Hanjun Guo
2
5.56%
1
20.00%
Andi Kleen
1
2.78%
1
20.00%
Total
36
100.00%
5
100.00%


static int __init

acpi_table_parse_srat(enum acpi_srat_type id,
		      acpi_tbl_entry_handler handler, unsigned int max_entries)
{
	return acpi_table_parse_entries(ACPI_SIG_SRAT,
					    sizeof(struct acpi_table_srat), id,
					    handler, max_entries);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Andy Grover
25
69.44%
1
16.67%
Jes Sorensen
6
16.67%
1
16.67%
Alexey Y. Starikovskiy
2
5.56%
1
16.67%
Len Brown
1
2.78%
1
16.67%
Lv Zheng
1
2.78%
1
16.67%
David Rientjes
1
2.78%
1
16.67%
Total
36
100.00%
6
100.00%



int __init acpi_numa_init(void)
{
	int cnt = 0;

	if (acpi_disabled)
		return -EINVAL;

	/*
         * Should not limit number with cpu num that is from NR_CPUS or nr_cpus=
         * SRAT cpu entries could have different order with that in MADT.
         * So go over all cpu entries in SRAT to get apicid to node mapping.
         */

	/* SRAT: System Resource Affinity Table */
	if (!acpi_table_parse(ACPI_SIG_SRAT, acpi_parse_srat)) {
		struct acpi_subtable_proc srat_proc[3];

		memset(srat_proc, 0, sizeof(srat_proc));
		srat_proc[0].id = ACPI_SRAT_TYPE_CPU_AFFINITY;
		srat_proc[0].handler = acpi_parse_processor_affinity;
		srat_proc[1].id = ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY;
		srat_proc[1].handler = acpi_parse_x2apic_affinity;
		srat_proc[2].id = ACPI_SRAT_TYPE_GICC_AFFINITY;
		srat_proc[2].handler = acpi_parse_gicc_affinity;

		acpi_table_parse_entries_array(ACPI_SIG_SRAT,
					sizeof(struct acpi_table_srat),
					srat_proc, ARRAY_SIZE(srat_proc), 0);

		cnt = acpi_table_parse_srat(ACPI_SRAT_TYPE_MEMORY_AFFINITY,
					    acpi_parse_memory_affinity,
					    NR_NODE_MEMBLKS);
	}

	/* SLIT: System Locality Information Table */
	acpi_table_parse(ACPI_SIG_SLIT, acpi_parse_slit);

	if (cnt < 0)
		return cnt;
	else if (!parsed_numa_memblks)
		return -ENOENT;
	return 0;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Lukasz Anaczkowski
63
37.28%
1
5.88%
Andy Grover
30
17.75%
1
5.88%
Hanjun Guo
27
15.98%
1
5.88%
Yinghai Lu
11
6.51%
2
11.76%
Tejun Heo
9
5.33%
1
5.88%
Thomas Renninger
9
5.33%
2
11.76%
Len Brown
7
4.14%
3
17.65%
Alexey Y. Starikovskiy
4
2.37%
1
5.88%
Suresh B. Siddha
3
1.78%
1
5.88%
Tony Luck
2
1.18%
1
5.88%
Jes Sorensen
2
1.18%
1
5.88%
David Rientjes
1
0.59%
1
5.88%
Ross Zwisler
1
0.59%
1
5.88%
Total
169
100.00%
17
100.00%



static int acpi_get_pxm(acpi_handle h)
{
	unsigned long long pxm;
	acpi_status status;
	acpi_handle handle;
	acpi_handle phandle = h;

	do {
		handle = phandle;
		status = acpi_evaluate_integer(handle, "_PXM", NULL, &pxm);
		if (ACPI_SUCCESS(status))
			return pxm;
		status = acpi_get_parent(handle, &phandle);
	} while (ACPI_SUCCESS(status));
	return -1;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Len Brown
76
97.44%
2
50.00%
Matthew Wilcox
1
1.28%
1
25.00%
Björn Helgaas
1
1.28%
1
25.00%
Total
78
100.00%
4
100.00%



int acpi_get_node(acpi_handle handle)
{
	int pxm;

	pxm = acpi_get_pxm(handle);

	return acpi_map_pxm_to_node(pxm);
}
Contributors
Person
Tokens
Prop
Commits
CommitProp
Yasunori Goto
22
91.67%
1
33.33%
Len Brown
1
4.17%
1
33.33%
Björn Helgaas
1
4.17%
1
33.33%
Total
24
100.00%
3
100.00%


EXPORT_SYMBOL(acpi_get_node);
Overall Contributors
Person
Tokens
Prop
Commits
CommitProp
Hanjun Guo
550
30.40%
7
13.46%
Andy Grover
264
14.59%
2
3.85%
Yasunori Goto
165
9.12%
2
3.85%
Len Brown
129
7.13%
9
17.31%
Fenghua Yu
113
6.25%
1
1.92%
Toshi Kani
111
6.14%
1
1.92%
Suresh B. Siddha
106
5.86%
1
1.92%
David Daney
81
4.48%
2
3.85%
David Rientjes
68
3.76%
5
9.62%
Lukasz Anaczkowski
63
3.48%
1
1.92%
Alexey Y. Starikovskiy
55
3.04%
1
1.92%
Kurt Garloff
23
1.27%
1
1.92%
Thomas Renninger
20
1.11%
2
3.85%
Davidlohr Bueso A
12
0.66%
1
1.92%
Yinghai Lu
11
0.61%
2
3.85%
Tejun Heo
9
0.50%
1
1.92%
Björn Helgaas
8
0.44%
3
5.77%
Jes Sorensen
8
0.44%
1
1.92%
Aaron Durbin
2
0.11%
1
1.92%
Tony Luck
2
0.11%
1
1.92%
Jianguo Wu
2
0.11%
1
1.92%
Boris Ostrovsky
2
0.11%
1
1.92%
Andi Kleen
1
0.06%
1
1.92%
Matthew Wilcox
1
0.06%
1
1.92%
Jarkko Nikula
1
0.06%
1
1.92%
Lv Zheng
1
0.06%
1
1.92%
Ross Zwisler
1
0.06%
1
1.92%
Total
1809
100.00%
52
100.00%
Directory: drivers/acpi

Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.