/* SPDX-License-Identifier: GPL-2.0 */ /* * The owner field of the rw_semaphore structure will be set to * RWSEM_READER_OWNED when a reader grabs the lock. A writer will clear * the owner field when it unlocks. A reader, on the other hand, will * not touch the owner field when it unlocks. * * In essence, the owner field now has the following 4 states: * 1) 0 * - lock is free or the owner hasn't set the field yet * 2) RWSEM_READER_OWNED * - lock is currently or previously owned by readers (lock is free * or not set by owner yet) * 3) RWSEM_ANONYMOUSLY_OWNED bit set with some other bits set as well * - lock is owned by an anonymous writer, so spinning on the lock * owner should be disabled. * 4) Other non-zero value * - a writer owns the lock and other writers can spin on the lock owner. */ #define RWSEM_ANONYMOUSLY_OWNED (1UL << 0) #define RWSEM_READER_OWNED ((struct task_struct *)RWSEM_ANONYMOUSLY_OWNED) #ifdef CONFIG_DEBUG_RWSEMS # define DEBUG_RWSEMS_WARN_ON(c) DEBUG_LOCKS_WARN_ON(c) #else # define DEBUG_RWSEMS_WARN_ON(c) #endif #ifdef CONFIG_RWSEM_SPIN_ON_OWNER /* * All writes to owner are protected by WRITE_ONCE() to make sure that * store tearing can't happen as optimistic spinners may read and use * the owner value concurrently without lock. Read from owner, however, * may not need READ_ONCE() as long as the pointer value is only used * for comparison and isn't being dereferenced. */ static inline void rwsem_set_owner(struct rw_semaphore *sem) { WRITE_ONCE(sem->owner, current); } static inline void rwsem_clear_owner(struct rw_semaphore *sem) { WRITE_ONCE(sem->owner, NULL); } static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) { /* * We check the owner value first to make sure that we will only * do a write to the rwsem cacheline when it is really necessary * to minimize cacheline contention. */ if (READ_ONCE(sem->owner) != RWSEM_READER_OWNED) WRITE_ONCE(sem->owner, RWSEM_READER_OWNED); } /* * Return true if the a rwsem waiter can spin on the rwsem's owner * and steal the lock, i.e. the lock is not anonymously owned. * N.B. !owner is considered spinnable. */ static inline bool is_rwsem_owner_spinnable(struct task_struct *owner) { return !((unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED); } /* * Return true if rwsem is owned by an anonymous writer or readers. */ static inline bool rwsem_has_anonymous_owner(struct task_struct *owner) { return (unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED; } #else static inline void rwsem_set_owner(struct rw_semaphore *sem) { } static inline void rwsem_clear_owner(struct rw_semaphore *sem) { } static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) { } #endif