| Author | Tokens | Token Proportion | Commits | Commit Proportion |
|---|---|---|---|---|
| Paul Mackerras | 932 | 57.35% | 3 | 11.54% |
| Pan Xinhui | 264 | 16.25% | 2 | 7.69% |
| David Howells | 120 | 7.38% | 1 | 3.85% |
| Mathieu Desnoyers | 106 | 6.52% | 2 | 7.69% |
| Nicholas Piggin | 79 | 4.86% | 2 | 7.69% |
| Boqun Feng | 76 | 4.68% | 3 | 11.54% |
| Mark Rutland | 16 | 0.98% | 2 | 7.69% |
| Christophe Leroy | 10 | 0.62% | 1 | 3.85% |
| Michael Ellerman | 7 | 0.43% | 3 | 11.54% |
| Linus Torvalds (pre-git) | 4 | 0.25% | 2 | 7.69% |
| Linus Torvalds | 3 | 0.18% | 1 | 3.85% |
| Becky Bruce | 3 | 0.18% | 1 | 3.85% |
| Benjamin Herrenschmidt | 2 | 0.12% | 1 | 3.85% |
| Andrzej Hajda | 2 | 0.12% | 1 | 3.85% |
| Greg Kroah-Hartman | 1 | 0.06% | 1 | 3.85% |
| Total | 1625 | 26 |
/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_CMPXCHG_H_ #define _ASM_POWERPC_CMPXCHG_H_ #ifdef __KERNEL__ #include <linux/compiler.h> #include <asm/synch.h> #include <linux/bug.h> #ifdef __BIG_ENDIAN #define BITOFF_CAL(size, off) ((sizeof(u32) - size - off) * BITS_PER_BYTE) #else #define BITOFF_CAL(size, off) (off * BITS_PER_BYTE) #endif #define XCHG_GEN(type, sfx, cl) \ static inline u32 __xchg_##type##sfx(volatile void *p, u32 val) \ { \ unsigned int prev, prev_mask, tmp, bitoff, off; \ \ off = (unsigned long)p % sizeof(u32); \ bitoff = BITOFF_CAL(sizeof(type), off); \ p -= off; \ val <<= bitoff; \ prev_mask = (u32)(type)-1 << bitoff; \ \ __asm__ __volatile__( \ "1: lwarx %0,0,%3\n" \ " andc %1,%0,%5\n" \ " or %1,%1,%4\n" \ " stwcx. %1,0,%3\n" \ " bne- 1b\n" \ : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p) \ : "r" (p), "r" (val), "r" (prev_mask) \ : "cc", cl); \ \ return prev >> bitoff; \ } #define CMPXCHG_GEN(type, sfx, br, br2, cl) \ static inline \ u32 __cmpxchg_##type##sfx(volatile void *p, u32 old, u32 new) \ { \ unsigned int prev, prev_mask, tmp, bitoff, off; \ \ off = (unsigned long)p % sizeof(u32); \ bitoff = BITOFF_CAL(sizeof(type), off); \ p -= off; \ old <<= bitoff; \ new <<= bitoff; \ prev_mask = (u32)(type)-1 << bitoff; \ \ __asm__ __volatile__( \ br \ "1: lwarx %0,0,%3\n" \ " and %1,%0,%6\n" \ " cmpw 0,%1,%4\n" \ " bne- 2f\n" \ " andc %1,%0,%6\n" \ " or %1,%1,%5\n" \ " stwcx. %1,0,%3\n" \ " bne- 1b\n" \ br2 \ "\n" \ "2:" \ : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p) \ : "r" (p), "r" (old), "r" (new), "r" (prev_mask) \ : "cc", cl); \ \ return prev >> bitoff; \ } /* * Atomic exchange * * Changes the memory location '*p' to be val and returns * the previous value stored there. */ #ifndef CONFIG_PPC_HAS_LBARX_LHARX XCHG_GEN(u8, _local, "memory"); XCHG_GEN(u8, _relaxed, "cc"); XCHG_GEN(u16, _local, "memory"); XCHG_GEN(u16, _relaxed, "cc"); #else static __always_inline unsigned long __xchg_u8_local(volatile void *p, unsigned long val) { unsigned long prev; __asm__ __volatile__( "1: lbarx %0,0,%2 # __xchg_u8_local\n" " stbcx. %3,0,%2 \n" " bne- 1b" : "=&r" (prev), "+m" (*(volatile unsigned char *)p) : "r" (p), "r" (val) : "cc", "memory"); return prev; } static __always_inline unsigned long __xchg_u8_relaxed(u8 *p, unsigned long val) { unsigned long prev; __asm__ __volatile__( "1: lbarx %0,0,%2 # __xchg_u8_relaxed\n" " stbcx. %3,0,%2\n" " bne- 1b" : "=&r" (prev), "+m" (*p) : "r" (p), "r" (val) : "cc"); return prev; } static __always_inline unsigned long __xchg_u16_local(volatile void *p, unsigned long val) { unsigned long prev; __asm__ __volatile__( "1: lharx %0,0,%2 # __xchg_u16_local\n" " sthcx. %3,0,%2\n" " bne- 1b" : "=&r" (prev), "+m" (*(volatile unsigned short *)p) : "r" (p), "r" (val) : "cc", "memory"); return prev; } static __always_inline unsigned long __xchg_u16_relaxed(u16 *p, unsigned long val) { unsigned long prev; __asm__ __volatile__( "1: lharx %0,0,%2 # __xchg_u16_relaxed\n" " sthcx. %3,0,%2\n" " bne- 1b" : "=&r" (prev), "+m" (*p) : "r" (p), "r" (val) : "cc"); return prev; } #endif static __always_inline unsigned long __xchg_u32_local(volatile void *p, unsigned long val) { unsigned long prev; __asm__ __volatile__( "1: lwarx %0,0,%2 \n" " stwcx. %3,0,%2 \n\ bne- 1b" : "=&r" (prev), "+m" (*(volatile unsigned int *)p) : "r" (p), "r" (val) : "cc", "memory"); return prev; } static __always_inline unsigned long __xchg_u32_relaxed(u32 *p, unsigned long val) { unsigned long prev; __asm__ __volatile__( "1: lwarx %0,0,%2\n" " stwcx. %3,0,%2\n" " bne- 1b" : "=&r" (prev), "+m" (*p) : "r" (p), "r" (val) : "cc"); return prev; } #ifdef CONFIG_PPC64 static __always_inline unsigned long __xchg_u64_local(volatile void *p, unsigned long val) { unsigned long prev; __asm__ __volatile__( "1: ldarx %0,0,%2 \n" " stdcx. %3,0,%2 \n\ bne- 1b" : "=&r" (prev), "+m" (*(volatile unsigned long *)p) : "r" (p), "r" (val) : "cc", "memory"); return prev; } static __always_inline unsigned long __xchg_u64_relaxed(u64 *p, unsigned long val) { unsigned long prev; __asm__ __volatile__( "1: ldarx %0,0,%2\n" " stdcx. %3,0,%2\n" " bne- 1b" : "=&r" (prev), "+m" (*p) : "r" (p), "r" (val) : "cc"); return prev; } #endif static __always_inline unsigned long __xchg_local(void *ptr, unsigned long x, unsigned int size) { switch (size) { case 1: return __xchg_u8_local(ptr, x); case 2: return __xchg_u16_local(ptr, x); case 4: return __xchg_u32_local(ptr, x); #ifdef CONFIG_PPC64 case 8: return __xchg_u64_local(ptr, x); #endif } BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local"); return x; } static __always_inline unsigned long __xchg_relaxed(void *ptr, unsigned long x, unsigned int size) { switch (size) { case 1: return __xchg_u8_relaxed(ptr, x); case 2: return __xchg_u16_relaxed(ptr, x); case 4: return __xchg_u32_relaxed(ptr, x); #ifdef CONFIG_PPC64 case 8: return __xchg_u64_relaxed(ptr, x); #endif } BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_relaxed"); return x; } #define arch_xchg_local(ptr,x) \ ({ \ __typeof__(*(ptr)) _x_ = (x); \ (__typeof__(*(ptr))) __xchg_local((ptr), \ (unsigned long)_x_, sizeof(*(ptr))); \ }) #define arch_xchg_relaxed(ptr, x) \ ({ \ __typeof__(*(ptr)) _x_ = (x); \ (__typeof__(*(ptr))) __xchg_relaxed((ptr), \ (unsigned long)_x_, sizeof(*(ptr))); \ }) /* * Compare and exchange - if *p == old, set it to new, * and return the old value of *p. */ #ifndef CONFIG_PPC_HAS_LBARX_LHARX CMPXCHG_GEN(u8, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory"); CMPXCHG_GEN(u8, _local, , , "memory"); CMPXCHG_GEN(u8, _acquire, , PPC_ACQUIRE_BARRIER, "memory"); CMPXCHG_GEN(u8, _relaxed, , , "cc"); CMPXCHG_GEN(u16, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory"); CMPXCHG_GEN(u16, _local, , , "memory"); CMPXCHG_GEN(u16, _acquire, , PPC_ACQUIRE_BARRIER, "memory"); CMPXCHG_GEN(u16, _relaxed, , , "cc"); #else static __always_inline unsigned long __cmpxchg_u8(volatile unsigned char *p, unsigned long old, unsigned long new) { unsigned int prev; __asm__ __volatile__ ( PPC_ATOMIC_ENTRY_BARRIER "1: lbarx %0,0,%2 # __cmpxchg_u8\n" " cmpw 0,%0,%3\n" " bne- 2f\n" " stbcx. %4,0,%2\n" " bne- 1b" PPC_ATOMIC_EXIT_BARRIER "\n\ 2:" : "=&r" (prev), "+m" (*p) : "r" (p), "r" (old), "r" (new) : "cc", "memory"); return prev; } static __always_inline unsigned long __cmpxchg_u8_local(volatile unsigned char *p, unsigned long old, unsigned long new) { unsigned int prev; __asm__ __volatile__ ( "1: lbarx %0,0,%2 # __cmpxchg_u8_local\n" " cmpw 0,%0,%3\n" " bne- 2f\n" " stbcx. %4,0,%2\n" " bne- 1b\n" "2:" : "=&r" (prev), "+m" (*p) : "r" (p), "r" (old), "r" (new) : "cc", "memory"); return prev; } static __always_inline unsigned long __cmpxchg_u8_relaxed(u8 *p, unsigned long old, unsigned long new) { unsigned long prev; __asm__ __volatile__ ( "1: lbarx %0,0,%2 # __cmpxchg_u8_relaxed\n" " cmpw 0,%0,%3\n" " bne- 2f\n" " stbcx. %4,0,%2\n" " bne- 1b\n" "2:" : "=&r" (prev), "+m" (*p) : "r" (p), "r" (old), "r" (new) : "cc"); return prev; } static __always_inline unsigned long __cmpxchg_u8_acquire(u8 *p, unsigned long old, unsigned long new) { unsigned long prev; __asm__ __volatile__ ( "1: lbarx %0,0,%2 # __cmpxchg_u8_acquire\n" " cmpw 0,%0,%3\n" " bne- 2f\n" " stbcx. %4,0,%2\n" " bne- 1b\n" PPC_ACQUIRE_BARRIER "2:" : "=&r" (prev), "+m" (*p) : "r" (p), "r" (old), "r" (new) : "cc", "memory"); return prev; } static __always_inline unsigned long __cmpxchg_u16(volatile unsigned short *p, unsigned long old, unsigned long new) { unsigned int prev; __asm__ __volatile__ ( PPC_ATOMIC_ENTRY_BARRIER "1: lharx %0,0,%2 # __cmpxchg_u16\n" " cmpw 0,%0,%3\n" " bne- 2f\n" " sthcx. %4,0,%2\n" " bne- 1b\n" PPC_ATOMIC_EXIT_BARRIER "2:" : "=&r" (prev), "+m" (*p) : "r" (p), "r" (old), "r" (new) : "cc", "memory"); return prev; } static __always_inline unsigned long __cmpxchg_u16_local(volatile unsigned short *p, unsigned long old, unsigned long new) { unsigned int prev; __asm__ __volatile__ ( "1: lharx %0,0,%2 # __cmpxchg_u16_local\n" " cmpw 0,%0,%3\n" " bne- 2f\n" " sthcx. %4,0,%2\n" " bne- 1b" "2:" : "=&r" (prev), "+m" (*p) : "r" (p), "r" (old), "r" (new) : "cc", "memory"); return prev; } static __always_inline unsigned long __cmpxchg_u16_relaxed(u16 *p, unsigned long old, unsigned long new) { unsigned long prev; __asm__ __volatile__ ( "1: lharx %0,0,%2 # __cmpxchg_u16_relaxed\n" " cmpw 0,%0,%3\n" " bne- 2f\n" " sthcx. %4,0,%2\n" " bne- 1b\n" "2:" : "=&r" (prev), "+m" (*p) : "r" (p), "r" (old), "r" (new) : "cc"); return prev; } static __always_inline unsigned long __cmpxchg_u16_acquire(u16 *p, unsigned long old, unsigned long new) { unsigned long prev; __asm__ __volatile__ ( "1: lharx %0,0,%2 # __cmpxchg_u16_acquire\n" " cmpw 0,%0,%3\n" " bne- 2f\n" " sthcx. %4,0,%2\n" " bne- 1b\n" PPC_ACQUIRE_BARRIER "2:" : "=&r" (prev), "+m" (*p) : "r" (p), "r" (old), "r" (new) : "cc", "memory"); return prev; } #endif static __always_inline unsigned long __cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new) { unsigned int prev; __asm__ __volatile__ ( PPC_ATOMIC_ENTRY_BARRIER "1: lwarx %0,0,%2 # __cmpxchg_u32\n\ cmpw 0,%0,%3\n\ bne- 2f\n" " stwcx. %4,0,%2\n\ bne- 1b" PPC_ATOMIC_EXIT_BARRIER "\n\ 2:" : "=&r" (prev), "+m" (*p) : "r" (p), "r" (old), "r" (new) : "cc", "memory"); return prev; } static __always_inline unsigned long __cmpxchg_u32_local(volatile unsigned int *p, unsigned long old, unsigned long new) { unsigned int prev; __asm__ __volatile__ ( "1: lwarx %0,0,%2 # __cmpxchg_u32\n\ cmpw 0,%0,%3\n\ bne- 2f\n" " stwcx. %4,0,%2\n\ bne- 1b" "\n\ 2:" : "=&r" (prev), "+m" (*p) : "r" (p), "r" (old), "r" (new) : "cc", "memory"); return prev; } static __always_inline unsigned long __cmpxchg_u32_relaxed(u32 *p, unsigned long old, unsigned long new) { unsigned long prev; __asm__ __volatile__ ( "1: lwarx %0,0,%2 # __cmpxchg_u32_relaxed\n" " cmpw 0,%0,%3\n" " bne- 2f\n" " stwcx. %4,0,%2\n" " bne- 1b\n" "2:" : "=&r" (prev), "+m" (*p) : "r" (p), "r" (old), "r" (new) : "cc"); return prev; } /* * cmpxchg family don't have order guarantee if cmp part fails, therefore we * can avoid superfluous barriers if we use assembly code to implement * cmpxchg() and cmpxchg_acquire(), however we don't do the similar for * cmpxchg_release() because that will result in putting a barrier in the * middle of a ll/sc loop, which is probably a bad idea. For example, this * might cause the conditional store more likely to fail. */ static __always_inline unsigned long __cmpxchg_u32_acquire(u32 *p, unsigned long old, unsigned long new) { unsigned long prev; __asm__ __volatile__ ( "1: lwarx %0,0,%2 # __cmpxchg_u32_acquire\n" " cmpw 0,%0,%3\n" " bne- 2f\n" " stwcx. %4,0,%2\n" " bne- 1b\n" PPC_ACQUIRE_BARRIER "\n" "2:" : "=&r" (prev), "+m" (*p) : "r" (p), "r" (old), "r" (new) : "cc", "memory"); return prev; } #ifdef CONFIG_PPC64 static __always_inline unsigned long __cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new) { unsigned long prev; __asm__ __volatile__ ( PPC_ATOMIC_ENTRY_BARRIER "1: ldarx %0,0,%2 # __cmpxchg_u64\n\ cmpd 0,%0,%3\n\ bne- 2f\n\ stdcx. %4,0,%2\n\ bne- 1b" PPC_ATOMIC_EXIT_BARRIER "\n\ 2:" : "=&r" (prev), "+m" (*p) : "r" (p), "r" (old), "r" (new) : "cc", "memory"); return prev; } static __always_inline unsigned long __cmpxchg_u64_local(volatile unsigned long *p, unsigned long old, unsigned long new) { unsigned long prev; __asm__ __volatile__ ( "1: ldarx %0,0,%2 # __cmpxchg_u64\n\ cmpd 0,%0,%3\n\ bne- 2f\n\ stdcx. %4,0,%2\n\ bne- 1b" "\n\ 2:" : "=&r" (prev), "+m" (*p) : "r" (p), "r" (old), "r" (new) : "cc", "memory"); return prev; } static __always_inline unsigned long __cmpxchg_u64_relaxed(u64 *p, unsigned long old, unsigned long new) { unsigned long prev; __asm__ __volatile__ ( "1: ldarx %0,0,%2 # __cmpxchg_u64_relaxed\n" " cmpd 0,%0,%3\n" " bne- 2f\n" " stdcx. %4,0,%2\n" " bne- 1b\n" "2:" : "=&r" (prev), "+m" (*p) : "r" (p), "r" (old), "r" (new) : "cc"); return prev; } static __always_inline unsigned long __cmpxchg_u64_acquire(u64 *p, unsigned long old, unsigned long new) { unsigned long prev; __asm__ __volatile__ ( "1: ldarx %0,0,%2 # __cmpxchg_u64_acquire\n" " cmpd 0,%0,%3\n" " bne- 2f\n" " stdcx. %4,0,%2\n" " bne- 1b\n" PPC_ACQUIRE_BARRIER "\n" "2:" : "=&r" (prev), "+m" (*p) : "r" (p), "r" (old), "r" (new) : "cc", "memory"); return prev; } #endif static __always_inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, unsigned int size) { switch (size) { case 1: return __cmpxchg_u8(ptr, old, new); case 2: return __cmpxchg_u16(ptr, old, new); case 4: return __cmpxchg_u32(ptr, old, new); #ifdef CONFIG_PPC64 case 8: return __cmpxchg_u64(ptr, old, new); #endif } BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg"); return old; } static __always_inline unsigned long __cmpxchg_local(void *ptr, unsigned long old, unsigned long new, unsigned int size) { switch (size) { case 1: return __cmpxchg_u8_local(ptr, old, new); case 2: return __cmpxchg_u16_local(ptr, old, new); case 4: return __cmpxchg_u32_local(ptr, old, new); #ifdef CONFIG_PPC64 case 8: return __cmpxchg_u64_local(ptr, old, new); #endif } BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_local"); return old; } static __always_inline unsigned long __cmpxchg_relaxed(void *ptr, unsigned long old, unsigned long new, unsigned int size) { switch (size) { case 1: return __cmpxchg_u8_relaxed(ptr, old, new); case 2: return __cmpxchg_u16_relaxed(ptr, old, new); case 4: return __cmpxchg_u32_relaxed(ptr, old, new); #ifdef CONFIG_PPC64 case 8: return __cmpxchg_u64_relaxed(ptr, old, new); #endif } BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_relaxed"); return old; } static __always_inline unsigned long __cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new, unsigned int size) { switch (size) { case 1: return __cmpxchg_u8_acquire(ptr, old, new); case 2: return __cmpxchg_u16_acquire(ptr, old, new); case 4: return __cmpxchg_u32_acquire(ptr, old, new); #ifdef CONFIG_PPC64 case 8: return __cmpxchg_u64_acquire(ptr, old, new); #endif } BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_acquire"); return old; } #define arch_cmpxchg(ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \ (unsigned long)_n_, sizeof(*(ptr))); \ }) #define arch_cmpxchg_local(ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_, \ (unsigned long)_n_, sizeof(*(ptr))); \ }) #define arch_cmpxchg_relaxed(ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \ (unsigned long)_o_, (unsigned long)_n_, \ sizeof(*(ptr))); \ }) #define arch_cmpxchg_acquire(ptr, o, n) \ ({ \ __typeof__(*(ptr)) _o_ = (o); \ __typeof__(*(ptr)) _n_ = (n); \ (__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \ (unsigned long)_o_, (unsigned long)_n_, \ sizeof(*(ptr))); \ }) #ifdef CONFIG_PPC64 #define arch_cmpxchg64(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ arch_cmpxchg((ptr), (o), (n)); \ }) #define arch_cmpxchg64_local(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ arch_cmpxchg_local((ptr), (o), (n)); \ }) #define arch_cmpxchg64_relaxed(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ arch_cmpxchg_relaxed((ptr), (o), (n)); \ }) #define arch_cmpxchg64_acquire(ptr, o, n) \ ({ \ BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ arch_cmpxchg_acquire((ptr), (o), (n)); \ }) #else #include <asm-generic/cmpxchg-local.h> #define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n)) #endif #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_CMPXCHG_H_ */
Information contained on this website is for historical information purposes only and does not indicate or represent copyright ownership.
Created with Cregit http://github.com/cregit/cregit
Version 2.0-RC1