Contributors: 20
| Author |
Tokens |
Token Proportion |
Commits |
Commit Proportion |
| Martin Schwidefsky |
935 |
52.82% |
28 |
40.00% |
| Claudio Imbrenda |
560 |
31.64% |
3 |
4.29% |
| Gerald Schaefer |
75 |
4.24% |
5 |
7.14% |
| Heiko Carstens |
50 |
2.82% |
8 |
11.43% |
| Christian Bornträger |
30 |
1.69% |
3 |
4.29% |
| David Hildenbrand |
26 |
1.47% |
2 |
2.86% |
| Niklas Schnelle |
20 |
1.13% |
1 |
1.43% |
| Linus Torvalds (pre-git) |
17 |
0.96% |
6 |
8.57% |
| Andrew Morton |
14 |
0.79% |
1 |
1.43% |
| Motohiro Kosaki |
9 |
0.51% |
1 |
1.43% |
| Dominik Dingel |
8 |
0.45% |
1 |
1.43% |
| Aneesh Kumar K.V |
6 |
0.34% |
3 |
4.29% |
| Rusty Russell |
6 |
0.34% |
1 |
1.43% |
| Cornelia Huck |
4 |
0.23% |
1 |
1.43% |
| Tejun Heo |
4 |
0.23% |
1 |
1.43% |
| Konstantin Weitz |
2 |
0.11% |
1 |
1.43% |
| Lorenzo Stoakes |
1 |
0.06% |
1 |
1.43% |
| Jan Glauber |
1 |
0.06% |
1 |
1.43% |
| Greg Kroah-Hartman |
1 |
0.06% |
1 |
1.43% |
| Linus Torvalds |
1 |
0.06% |
1 |
1.43% |
| Total |
1770 |
|
70 |
|
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright IBM Corp. 2007, 2011
* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
*/
#include <linux/cpufeature.h>
#include <linux/export.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/gfp.h>
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/smp.h>
#include <linux/spinlock.h>
#include <linux/rcupdate.h>
#include <linux/slab.h>
#include <linux/leafops.h>
#include <linux/sysctl.h>
#include <linux/ksm.h>
#include <linux/mman.h>
#include <asm/tlbflush.h>
#include <asm/mmu_context.h>
#include <asm/page-states.h>
#include <asm/machine.h>
pgprot_t pgprot_writecombine(pgprot_t prot)
{
/*
* mio_wb_bit_mask may be set on a different CPU, but it is only set
* once at init and only read afterwards.
*/
return __pgprot(pgprot_val(prot) | mio_wb_bit_mask);
}
EXPORT_SYMBOL_GPL(pgprot_writecombine);
static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, int nodat)
{
unsigned long opt, asce;
if (machine_has_tlb_guest()) {
opt = 0;
asce = READ_ONCE(mm->context.gmap_asce);
if (asce == 0UL || nodat)
opt |= IPTE_NODAT;
if (asce != -1UL) {
asce = asce ? : mm->context.asce;
opt |= IPTE_GUEST_ASCE;
}
__ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL);
} else {
__ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL);
}
}
static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, int nodat)
{
unsigned long opt, asce;
if (machine_has_tlb_guest()) {
opt = 0;
asce = READ_ONCE(mm->context.gmap_asce);
if (asce == 0UL || nodat)
opt |= IPTE_NODAT;
if (asce != -1UL) {
asce = asce ? : mm->context.asce;
opt |= IPTE_GUEST_ASCE;
}
__ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL);
} else {
__ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL);
}
}
static inline pte_t ptep_flush_direct(struct mm_struct *mm,
unsigned long addr, pte_t *ptep,
int nodat)
{
pte_t old;
old = *ptep;
if (unlikely(pte_val(old) & _PAGE_INVALID))
return old;
atomic_inc(&mm->context.flush_count);
if (cpu_has_tlb_lc() &&
cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
ptep_ipte_local(mm, addr, ptep, nodat);
else
ptep_ipte_global(mm, addr, ptep, nodat);
atomic_dec(&mm->context.flush_count);
return old;
}
static inline pte_t ptep_flush_lazy(struct mm_struct *mm,
unsigned long addr, pte_t *ptep,
int nodat)
{
pte_t old;
old = *ptep;
if (unlikely(pte_val(old) & _PAGE_INVALID))
return old;
atomic_inc(&mm->context.flush_count);
if (cpumask_equal(&mm->context.cpu_attach_mask,
cpumask_of(smp_processor_id()))) {
set_pte(ptep, set_pte_bit(*ptep, __pgprot(_PAGE_INVALID)));
mm->context.flush_mm = 1;
} else
ptep_ipte_global(mm, addr, ptep, nodat);
atomic_dec(&mm->context.flush_count);
return old;
}
pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t new)
{
pte_t old;
preempt_disable();
old = ptep_flush_direct(mm, addr, ptep, 1);
set_pte(ptep, new);
preempt_enable();
return old;
}
EXPORT_SYMBOL(ptep_xchg_direct);
/*
* Caller must check that new PTE only differs in _PAGE_PROTECT HW bit, so that
* RDP can be used instead of IPTE. See also comments at pte_allow_rdp().
*/
void ptep_reset_dat_prot(struct mm_struct *mm, unsigned long addr, pte_t *ptep,
pte_t new)
{
preempt_disable();
atomic_inc(&mm->context.flush_count);
if (cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
__ptep_rdp(addr, ptep, 1);
else
__ptep_rdp(addr, ptep, 0);
/*
* PTE is not invalidated by RDP, only _PAGE_PROTECT is cleared. That
* means it is still valid and active, and must not be changed according
* to the architecture. But writing a new value that only differs in SW
* bits is allowed.
*/
set_pte(ptep, new);
atomic_dec(&mm->context.flush_count);
preempt_enable();
}
EXPORT_SYMBOL(ptep_reset_dat_prot);
pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t new)
{
pte_t old;
preempt_disable();
old = ptep_flush_lazy(mm, addr, ptep, 1);
set_pte(ptep, new);
preempt_enable();
return old;
}
EXPORT_SYMBOL(ptep_xchg_lazy);
pte_t ptep_modify_prot_start(struct vm_area_struct *vma, unsigned long addr,
pte_t *ptep)
{
return ptep_flush_lazy(vma->vm_mm, addr, ptep, 1);
}
void ptep_modify_prot_commit(struct vm_area_struct *vma, unsigned long addr,
pte_t *ptep, pte_t old_pte, pte_t pte)
{
set_pte(ptep, pte);
}
static inline void pmdp_idte_local(struct mm_struct *mm,
unsigned long addr, pmd_t *pmdp)
{
if (machine_has_tlb_guest())
__pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE, mm->context.asce, IDTE_LOCAL);
else
__pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL);
}
static inline void pmdp_idte_global(struct mm_struct *mm,
unsigned long addr, pmd_t *pmdp)
{
if (machine_has_tlb_guest()) {
__pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
mm->context.asce, IDTE_GLOBAL);
} else {
__pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL);
}
}
static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
unsigned long addr, pmd_t *pmdp)
{
pmd_t old;
old = *pmdp;
if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
return old;
atomic_inc(&mm->context.flush_count);
if (cpu_has_tlb_lc() &&
cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
pmdp_idte_local(mm, addr, pmdp);
else
pmdp_idte_global(mm, addr, pmdp);
atomic_dec(&mm->context.flush_count);
return old;
}
static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm,
unsigned long addr, pmd_t *pmdp)
{
pmd_t old;
old = *pmdp;
if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
return old;
atomic_inc(&mm->context.flush_count);
if (cpumask_equal(&mm->context.cpu_attach_mask,
cpumask_of(smp_processor_id()))) {
set_pmd(pmdp, set_pmd_bit(*pmdp, __pgprot(_SEGMENT_ENTRY_INVALID)));
mm->context.flush_mm = 1;
} else {
pmdp_idte_global(mm, addr, pmdp);
}
atomic_dec(&mm->context.flush_count);
return old;
}
pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr,
pmd_t *pmdp, pmd_t new)
{
pmd_t old;
preempt_disable();
old = pmdp_flush_direct(mm, addr, pmdp);
set_pmd(pmdp, new);
preempt_enable();
return old;
}
EXPORT_SYMBOL(pmdp_xchg_direct);
pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr,
pmd_t *pmdp, pmd_t new)
{
pmd_t old;
preempt_disable();
old = pmdp_flush_lazy(mm, addr, pmdp);
set_pmd(pmdp, new);
preempt_enable();
return old;
}
EXPORT_SYMBOL(pmdp_xchg_lazy);
static inline void pudp_idte_local(struct mm_struct *mm,
unsigned long addr, pud_t *pudp)
{
if (machine_has_tlb_guest())
__pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE,
mm->context.asce, IDTE_LOCAL);
else
__pudp_idte(addr, pudp, 0, 0, IDTE_LOCAL);
}
static inline void pudp_idte_global(struct mm_struct *mm,
unsigned long addr, pud_t *pudp)
{
if (machine_has_tlb_guest())
__pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE,
mm->context.asce, IDTE_GLOBAL);
else
__pudp_idte(addr, pudp, 0, 0, IDTE_GLOBAL);
}
static inline pud_t pudp_flush_direct(struct mm_struct *mm,
unsigned long addr, pud_t *pudp)
{
pud_t old;
old = *pudp;
if (pud_val(old) & _REGION_ENTRY_INVALID)
return old;
atomic_inc(&mm->context.flush_count);
if (cpu_has_tlb_lc() &&
cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
pudp_idte_local(mm, addr, pudp);
else
pudp_idte_global(mm, addr, pudp);
atomic_dec(&mm->context.flush_count);
return old;
}
pud_t pudp_xchg_direct(struct mm_struct *mm, unsigned long addr,
pud_t *pudp, pud_t new)
{
pud_t old;
preempt_disable();
old = pudp_flush_direct(mm, addr, pudp);
set_pud(pudp, new);
preempt_enable();
return old;
}
EXPORT_SYMBOL(pudp_xchg_direct);
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
pgtable_t pgtable)
{
struct list_head *lh = (struct list_head *) pgtable;
assert_spin_locked(pmd_lockptr(mm, pmdp));
/* FIFO */
if (!pmd_huge_pte(mm, pmdp))
INIT_LIST_HEAD(lh);
else
list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
pmd_huge_pte(mm, pmdp) = pgtable;
}
pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
{
struct list_head *lh;
pgtable_t pgtable;
pte_t *ptep;
assert_spin_locked(pmd_lockptr(mm, pmdp));
/* FIFO */
pgtable = pmd_huge_pte(mm, pmdp);
lh = (struct list_head *) pgtable;
if (list_empty(lh))
pmd_huge_pte(mm, pmdp) = NULL;
else {
pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
list_del(lh);
}
ptep = (pte_t *) pgtable;
set_pte(ptep, __pte(_PAGE_INVALID));
ptep++;
set_pte(ptep, __pte(_PAGE_INVALID));
return pgtable;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */