Release 4.7 arch/x86/include/asm/special_insns.h

Directory: arch/x86/include/asm
#ifndef _ASM_X86_SPECIAL_INSNS_H

#define _ASM_X86_SPECIAL_INSNS_H


#ifdef __KERNEL__

#include <asm/nops.h>



static inline void native_clts(void)
{
	asm volatile("clts");
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 12 100.00% 1 100.00%
Total 12 100.00% 1 100.00%

/*
 * Volatile isn't enough to prevent the compiler from reordering the
 * read/write functions for the control registers and messing everything up.
 * A memory clobber would solve the problem, but would prevent reordering of
 * all loads stores around it, which can hurt performance. Solution is to
 * use a variable and mimic reads and writes to it to enforce serialization
 */
extern unsigned long __force_order;



static inline unsigned long native_read_cr0(void)
{
	unsigned long val;
	asm volatile("mov %%cr0,%0\n\t" : "=r" (val), "=m" (__force_order));
	return val;
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%



static inline void native_write_cr0(unsigned long val)
{
	asm volatile("mov %0,%%cr0": : "r" (val), "m" (__force_order));
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 14 100.00% 1 100.00%
Total 14 100.00% 1 100.00%



static inline unsigned long native_read_cr2(void)
{
	unsigned long val;
	asm volatile("mov %%cr2,%0\n\t" : "=r" (val), "=m" (__force_order));
	return val;
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%



static inline void native_write_cr2(unsigned long val)
{
	asm volatile("mov %0,%%cr2": : "r" (val), "m" (__force_order));
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 14 100.00% 1 100.00%
Total 14 100.00% 1 100.00%



static inline unsigned long native_read_cr3(void)
{
	unsigned long val;
	asm volatile("mov %%cr3,%0\n\t" : "=r" (val), "=m" (__force_order));
	return val;
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%



static inline void native_write_cr3(unsigned long val)
{
	asm volatile("mov %0,%%cr3": : "r" (val), "m" (__force_order));
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 14 100.00% 1 100.00%
Total 14 100.00% 1 100.00%



static inline unsigned long native_read_cr4(void)
{
	unsigned long val;
	asm volatile("mov %%cr4,%0\n\t" : "=r" (val), "=m" (__force_order));
	return val;
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%



static inline unsigned long native_read_cr4_safe(void)
{
	unsigned long val;
	/* This could fault if %cr4 does not exist. In x86_64, a cr4 always
         * exists, so it will never fail. */
#ifdef CONFIG_X86_32
	asm volatile("1: mov %%cr4, %0\n"
                     "2:\n"
                     _ASM_EXTABLE(1b, 2b)
                     : "=r" (val), "=m" (__force_order) : "0" (0));
#else
	val = native_read_cr4();
#endif
	return val;
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 33 100.00% 1 100.00%
Total 33 100.00% 1 100.00%



static inline void native_write_cr4(unsigned long val)
{
	asm volatile("mov %0,%%cr4": : "r" (val), "m" (__force_order));
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 14 100.00% 1 100.00%
Total 14 100.00% 1 100.00%

#ifdef CONFIG_X86_64


static inline unsigned long native_read_cr8(void)
{
	unsigned long cr8;
	asm volatile("movq %%cr8,%0" : "=r" (cr8));
	return cr8;
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%



static inline void native_write_cr8(unsigned long val)
{
	asm volatile("movq %0,%%cr8" :: "r" (val) : "memory");
}
Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 14 100.00% 1 100.00%
Total 14 100.00% 1 100.00%

#endif

#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS


static inline u32 __read_pkru(void)
{
	u32 ecx = 0;
	u32 edx, pkru;

	/*
         * "rdpkru" instruction.  Places PKRU contents in to EAX,
         * clears EDX and requires that ecx=0.
         */
	asm volatile(".byte 0x0f,0x01,0xee\n\t"
                     : "=a" (pkru), "=d" (edx)
                     : "c" (ecx));
	return pkru;
}

Contributors
Person Tokens Prop Commits CommitProp
dave hansen dave hansen 26 100.00% 1 100.00%
Total 26 100.00% 1 100.00%



static inline void __write_pkru(u32 pkru)
{
	u32 ecx = 0, edx = 0;

	/*
         * "wrpkru" instruction.  Loads contents in EAX to PKRU,
         * requires that ecx = edx = 0.
         */
	asm volatile(".byte 0x0f,0x01,0xef\n\t"
                     : : "a" (pkru), "c"(ecx), "d"(edx));
}
Contributors
Person Tokens Prop Commits CommitProp
xiao guangrong xiao guangrong 23 100.00% 1 100.00%
Total 23 100.00% 1 100.00%

#else


static inline u32 __read_pkru(void)
{
	return 0;
}

Contributors
Person Tokens Prop Commits CommitProp
dave hansen dave hansen 12 100.00% 1 100.00%
Total 12 100.00% 1 100.00%



static inline void __write_pkru(u32 pkru)
{
}
Contributors
Person Tokens Prop Commits CommitProp
xiao guangrong xiao guangrong 9 100.00% 1 100.00%
Total 9 100.00% 1 100.00%

#endif



static inline void native_wbinvd(void)
{
	asm volatile("wbinvd": : :"memory");
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 12 100.00% 1 100.00%
Total 12 100.00% 1 100.00%

extern asmlinkage void native_load_gs_index(unsigned);

#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#else



static inline unsigned long read_cr0(void)
{
	return native_read_cr0();
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 14 100.00% 1 100.00%
Total 14 100.00% 1 100.00%



static inline void write_cr0(unsigned long x)
{
	native_write_cr0(x);
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 16 100.00% 1 100.00%
Total 16 100.00% 1 100.00%



static inline unsigned long read_cr2(void)
{
	return native_read_cr2();
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 14 100.00% 1 100.00%
Total 14 100.00% 1 100.00%



static inline void write_cr2(unsigned long x)
{
	native_write_cr2(x);
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 16 100.00% 1 100.00%
Total 16 100.00% 1 100.00%



static inline unsigned long read_cr3(void)
{
	return native_read_cr3();
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 14 100.00% 1 100.00%
Total 14 100.00% 1 100.00%



static inline void write_cr3(unsigned long x)
{
	native_write_cr3(x);
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 16 100.00% 1 100.00%
Total 16 100.00% 1 100.00%



static inline unsigned long __read_cr4(void)
{
	return native_read_cr4();
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 13 92.86% 1 50.00%
andy lutomirski andy lutomirski 1 7.14% 1 50.00%
Total 14 100.00% 2 100.00%



static inline unsigned long __read_cr4_safe(void)
{
	return native_read_cr4_safe();
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 13 92.86% 1 50.00%
andy lutomirski andy lutomirski 1 7.14% 1 50.00%
Total 14 100.00% 2 100.00%



static inline void __write_cr4(unsigned long x)
{
	native_write_cr4(x);
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 15 93.75% 1 50.00%
andy lutomirski andy lutomirski 1 6.25% 1 50.00%
Total 16 100.00% 2 100.00%



static inline void wbinvd(void)
{
	native_wbinvd();
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 12 100.00% 1 100.00%
Total 12 100.00% 1 100.00%

#ifdef CONFIG_X86_64



static inline unsigned long read_cr8(void)
{
	return native_read_cr8();
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 14 100.00% 1 100.00%
Total 14 100.00% 1 100.00%



static inline void write_cr8(unsigned long x)
{
	native_write_cr8(x);
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 16 100.00% 1 100.00%
Total 16 100.00% 1 100.00%



static inline void load_gs_index(unsigned selector)
{
	native_load_gs_index(selector);
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 15 100.00% 1 100.00%
Total 15 100.00% 1 100.00%

#endif

/* Clear the 'TS' bit */


static inline void clts(void)
{
	native_clts();
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 12 100.00% 1 100.00%
Total 12 100.00% 1 100.00%

#endif/* CONFIG_PARAVIRT */


#define stts() write_cr0(read_cr0() | X86_CR0_TS)



static inline void clflush(volatile void *__p)
{
	asm volatile("clflush %0" : "+m" (*(volatile char __force *)__p));
}

Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 15 100.00% 1 100.00%
Total 15 100.00% 1 100.00%



static inline void clflushopt(volatile void *__p)
{
	alternative_io(".byte " __stringify(NOP_DS_PREFIX) "; clflush %P0",
		       ".byte 0x66; clflush %P0",
		       X86_FEATURE_CLFLUSHOPT,
		       "+m" (*(volatile char __force *)__p));
}

Contributors
Person Tokens Prop Commits CommitProp
ross zwisler ross zwisler 38 100.00% 1 100.00%
Total 38 100.00% 1 100.00%



static inline void clwb(volatile void *__p)
{
	volatile struct { char x[64]; } *p = __p;

	asm volatile(ALTERNATIVE_2(
                ".byte " __stringify(NOP_DS_PREFIX) "; clflush (%[pax])",
                ".byte 0x66; clflush (%[pax])", /* clflushopt (%%rax) */
		X86_FEATURE_CLFLUSHOPT,
                ".byte 0x66, 0x0f, 0xae, 0x30",  /* clwb (%%rax) */
		X86_FEATURE_CLWB)
                : [p] "+m" (*p)
                : [pax] "a" (p));
}

Contributors
Person Tokens Prop Commits CommitProp
ross zwisler ross zwisler 34 100.00% 1 100.00%
Total 34 100.00% 1 100.00%

/**
 * pcommit_sfence() - persistent commit and fence
 *
 * The PCOMMIT instruction ensures that data that has been flushed from the
 * processor's cache hierarchy with CLWB, CLFLUSHOPT or CLFLUSH is accepted to
 * memory and is durable on the DIMM.  The primary use case for this is
 * persistent memory.
 *
 * This function shows how to properly use CLWB/CLFLUSHOPT/CLFLUSH and PCOMMIT
 * with appropriate fencing.
 *
 * Example:
 * void flush_and_commit_buffer(void *vaddr, unsigned int size)
 * {
 *         unsigned long clflush_mask = boot_cpu_data.x86_clflush_size - 1;
 *         void *vend = vaddr + size;
 *         void *p;
 *
 *         for (p = (void *)((unsigned long)vaddr & ~clflush_mask);
 *              p < vend; p += boot_cpu_data.x86_clflush_size)
 *                 clwb(p);
 *
 *         // SFENCE to order CLWB/CLFLUSHOPT/CLFLUSH cache flushes
 *         // MFENCE via mb() also works
 *         wmb();
 *
 *         // PCOMMIT and the required SFENCE for ordering
 *         pcommit_sfence();
 * }
 *
 * After this function completes the data pointed to by 'vaddr' has been
 * accepted to memory and will be durable if the 'vaddr' points to persistent
 * memory.
 *
 * PCOMMIT must always be ordered by an MFENCE or SFENCE, so to help simplify
 * things we include both the PCOMMIT and the required SFENCE in the
 * alternatives generated by pcommit_sfence().
 */


static inline void pcommit_sfence(void)
{
	alternative(ASM_NOP7,
		    ".byte 0x66, 0x0f, 0xae, 0xf8\n\t" /* pcommit */
		    "sfence",
		    X86_FEATURE_PCOMMIT);
}

Contributors
Person Tokens Prop Commits CommitProp
ross zwisler ross zwisler 20 100.00% 1 100.00%
Total 20 100.00% 1 100.00%


#define nop() asm volatile ("nop")


#endif /* __KERNEL__ */

#endif /* _ASM_X86_SPECIAL_INSNS_H */
Overall Contributors
Person Tokens Prop Commits CommitProp
david howells david howells 501 73.78% 1 10.00%
ross zwisler ross zwisler 96 14.14% 4 40.00%
dave hansen dave hansen 45 6.63% 1 10.00%
xiao guangrong xiao guangrong 32 4.71% 1 10.00%
andy lutomirski andy lutomirski 3 0.44% 1 10.00%
andi kleen andi kleen 1 0.15% 1 10.00%
jan beulich jan beulich 1 0.15% 1 10.00%
Total 679 100.00% 10 100.00%
  Directory: arch/x86/include/asm

Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
	Person	Tokens	Prop	Commits	CommitProp
david howells	david howells	12	100.00%	1	100.00%
	Total	12	100.00%	1	100.00%
cregit-Linux how code gets into the kernel

Release 4.7 arch/x86/include/asm/special_insns.h

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Contributors

Overall Contributors
