Release 4.18 arch/x86/include/asm/uaccess_64.h

/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_UACCESS_64_H

#define _ASM_X86_UACCESS_64_H

/*
 * User space memory access functions
 */
#include <linux/compiler.h>
#include <linux/lockdep.h>
#include <linux/kasan-checks.h>
#include <asm/alternative.h>
#include <asm/cpufeatures.h>
#include <asm/page.h>

/*
 * Copy To/From Userspace
 */

/* Handles exceptions in both to and from, but doesn't do access_ok */
__must_check unsigned long
copy_user_enhanced_fast_string(void *to, const void *from, unsigned len);
__must_check unsigned long
copy_user_generic_string(void *to, const void *from, unsigned len);
__must_check unsigned long
copy_user_generic_unrolled(void *to, const void *from, unsigned len);


static __always_inline __must_check unsigned long

copy_user_generic(void *to, const void *from, unsigned len)
{
	unsigned ret;

	/*
         * If CPU has ERMS feature, use copy_user_enhanced_fast_string.
         * Otherwise, if CPU has rep_good feature, use copy_user_generic_string.
         * Otherwise, use copy_user_generic_unrolled.
         */
	alternative_call_2(copy_user_generic_unrolled,
			 copy_user_generic_string,
			 X86_FEATURE_REP_GOOD,
			 copy_user_enhanced_fast_string,
			 X86_FEATURE_ERMS,
			 ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from),
				     "=d" (len)),
			 "1" (to), "2" (from), "3" (len)
			 : "memory", "rcx", "r8", "r9", "r10", "r11");
	return ret;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Jan Beulich
85
93.41%
1
50.00%
Fenghua Yu
6
6.59%
1
50.00%
Total
91
100.00%
2
100.00%


static __always_inline __must_check unsigned long

copy_to_user_mcsafe(void *to, const void *from, unsigned len)
{
	unsigned long ret;

	__uaccess_begin();
	/*
         * Note, __memcpy_mcsafe() is explicitly used since it can
         * handle exceptions / faults.  memcpy_mcsafe() may fall back to
         * memcpy() which lacks this handling.
         */
	ret = __memcpy_mcsafe(to, from, len);
	__uaccess_end();
	return ret;
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
41
89.13%
2
33.33%
Al Viro
2
4.35%
1
16.67%
Andi Kleen
2
4.35%
2
33.33%
Ingo Molnar
1
2.17%
1
16.67%
Total
46
100.00%
6
100.00%


static __always_inline __must_check unsigned long

raw_copy_from_user(void *dst, const void __user *src, unsigned long size)
{
	int ret = 0;

	if (!__builtin_constant_p(size))
		return copy_user_generic(dst, (__force void *)src, size);
	switch (size) {
	case 1:
		__uaccess_begin_nospec();
		__get_user_asm_nozero(*(u8 *)dst, (u8 __user *)src,
			      ret, "b", "b", "=q", 1);
		__uaccess_end();
		return ret;
	case 2:
		__uaccess_begin_nospec();
		__get_user_asm_nozero(*(u16 *)dst, (u16 __user *)src,
			      ret, "w", "w", "=r", 2);
		__uaccess_end();
		return ret;
	case 4:
		__uaccess_begin_nospec();
		__get_user_asm_nozero(*(u32 *)dst, (u32 __user *)src,
			      ret, "l", "k", "=r", 4);
		__uaccess_end();
		return ret;
	case 8:
		__uaccess_begin_nospec();
		__get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src,
			      ret, "q", "", "=r", 8);
		__uaccess_end();
		return ret;
	case 10:
		__uaccess_begin_nospec();
		__get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src,
			       ret, "q", "", "=r", 10);
		if (likely(!ret))
			__get_user_asm_nozero(*(u16 *)(8 + (char *)dst),
				       (u16 __user *)(8 + (char __user *)src),
				       ret, "w", "w", "=r", 2);
		__uaccess_end();
		return ret;
	case 16:
		__uaccess_begin_nospec();
		__get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src,
			       ret, "q", "", "=r", 16);
		if (likely(!ret))
			__get_user_asm_nozero(*(u64 *)(8 + (char *)dst),
				       (u64 __user *)(8 + (char __user *)src),
				       ret, "q", "", "=r", 8);
		__uaccess_end();
		return ret;
	default:
		return copy_user_generic(dst, (__force void *)src, size);
	}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Andi Kleen
334
81.27%
5
38.46%
Linus Torvalds
34
8.27%
1
7.69%
Al Viro
31
7.54%
4
30.77%
Dan J Williams
11
2.68%
2
15.38%
Hiroshi Shimamoto
1
0.24%
1
7.69%
Total
411
100.00%
13
100.00%


static __always_inline __must_check unsigned long

raw_copy_to_user(void __user *dst, const void *src, unsigned long size)
{
	int ret = 0;

	if (!__builtin_constant_p(size))
		return copy_user_generic((__force void *)dst, src, size);
	switch (size) {
	case 1:
		__uaccess_begin();
		__put_user_asm(*(u8 *)src, (u8 __user *)dst,
			      ret, "b", "b", "iq", 1);
		__uaccess_end();
		return ret;
	case 2:
		__uaccess_begin();
		__put_user_asm(*(u16 *)src, (u16 __user *)dst,
			      ret, "w", "w", "ir", 2);
		__uaccess_end();
		return ret;
	case 4:
		__uaccess_begin();
		__put_user_asm(*(u32 *)src, (u32 __user *)dst,
			      ret, "l", "k", "ir", 4);
		__uaccess_end();
		return ret;
	case 8:
		__uaccess_begin();
		__put_user_asm(*(u64 *)src, (u64 __user *)dst,
			      ret, "q", "", "er", 8);
		__uaccess_end();
		return ret;
	case 10:
		__uaccess_begin();
		__put_user_asm(*(u64 *)src, (u64 __user *)dst,
			       ret, "q", "", "er", 10);
		if (likely(!ret)) {
			asm("":::"memory");
			__put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst,
				       ret, "w", "w", "ir", 2);
		}
		__uaccess_end();
		return ret;
	case 16:
		__uaccess_begin();
		__put_user_asm(*(u64 *)src, (u64 __user *)dst,
			       ret, "q", "", "er", 16);
		if (likely(!ret)) {
			asm("":::"memory");
			__put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst,
				       ret, "q", "", "er", 8);
		}
		__uaccess_end();
		return ret;
	default:
		return copy_user_generic((__force void *)dst, src, size);
	}
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Andi Kleen
320
81.84%
6
50.00%
Linus Torvalds
44
11.25%
1
8.33%
Al Viro
22
5.63%
3
25.00%
Uros Bizjak
4
1.02%
1
8.33%
Ingo Molnar
1
0.26%
1
8.33%
Total
391
100.00%
12
100.00%


static __always_inline __must_check

unsigned long raw_copy_in_user(void __user *dst, const void __user *src, unsigned long size)
{
	return copy_user_generic((__force void *)dst,
				 (__force void *)src, size);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Andi Kleen
28
63.64%
3
42.86%
Al Viro
15
34.09%
3
42.86%
Ingo Molnar
1
2.27%
1
14.29%
Total
44
100.00%
7
100.00%

extern long __copy_user_nocache(void *dst, const void __user *src,
				unsigned size, int zerorest);

extern long __copy_user_flushcache(void *dst, const void __user *src, unsigned size);
extern void memcpy_page_flushcache(char *to, struct page *page, size_t offset,
			   size_t len);


static inline int

__copy_from_user_inatomic_nocache(void *dst, const void __user *src,
				  unsigned size)
{
	kasan_check_write(dst, size);
	return __copy_user_nocache(dst, src, size, 0);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Andi Kleen
32
82.05%
1
50.00%
Andrey Ryabinin
7
17.95%
1
50.00%
Total
39
100.00%
2
100.00%


static inline int

__copy_from_user_flushcache(void *dst, const void __user *src, unsigned size)
{
	kasan_check_write(dst, size);
	return __copy_user_flushcache(dst, src, size);
}

Contributors
Person
Tokens
Prop
Commits
CommitProp
Dan J Williams
37
100.00%
1
100.00%
Total
37
100.00%
1
100.00%

unsigned long
copy_user_handle_tail(char *to, char *from, unsigned len);

unsigned long
mcsafe_handle_tail(char *to, char *from, unsigned len);

#endif /* _ASM_X86_UACCESS_64_H */
Overall Contributors
Person
Tokens
Prop
Commits
CommitProp
Andi Kleen
769
62.27%
12
36.36%
Dan J Williams
143
11.58%
5
15.15%
Jan Beulich
109
8.83%
1
3.03%
Linus Torvalds
78
6.32%
1
3.03%
Al Viro
70
5.67%
4
12.12%
Fenghua Yu
24
1.94%
1
3.03%
Vitaly Mayatskikh
16
1.30%
1
3.03%
Andrey Ryabinin
10
0.81%
1
3.03%
Uros Bizjak
4
0.32%
1
3.03%
Ingo Molnar
3
0.24%
1
3.03%
Nicholas Piggin
3
0.24%
1
3.03%
H. Peter Anvin
3
0.24%
1
3.03%
Hiroshi Shimamoto
1
0.08%
1
3.03%
Greg Kroah-Hartman
1
0.08%
1
3.03%
Borislav Petkov
1
0.08%
1
3.03%
Total
1235
100.00%
33
100.00%
Directory: arch/x86/include/asm

Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.