ANDROID: vendor_hook: add hooks to protect locking-tsk in cpu scheduler
Providing vendor hooks to record the start time of holding the lock, which protects rwsem/mutex locking-process from being preemptedfor a short time in some cases. - android_vh_record_mutex_lock_starttime - android_vh_record_rtmutex_lock_starttime - android_vh_record_rwsem_lock_starttime - android_vh_record_pcpu_rwsem_starttime Bug: 241191475 Signed-off-by: Peifeng Li <lipeifeng@oppo.com> Change-Id: I0e967a1e8b77c32a1ad588acd54028fae2f90c4e (cherry picked from commit f7294947672eb6b786f3c16b49e71e6a239101ad)
This commit is contained in:
Peifeng Li
Todd Kjos
parent
33ab98b39d
commit
6c1c1552e6
7 changed files with 65 additions and 4 deletions
|
|
@ -137,6 +137,10 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_commit_creds);
|
|||
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_exit_creds);
|
||||
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_override_creds);
|
||||
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_revert_creds);
|
||||
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_record_mutex_lock_starttime);
|
||||
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_record_rtmutex_lock_starttime);
|
||||
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_record_rwsem_lock_starttime);
|
||||
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_record_pcpu_rwsem_starttime);
|
||||
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_module_core_rw_nx);
|
||||
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_module_init_rw_nx);
|
||||
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_module_permit_before_init);
|
||||
|
|
|
|||
|
|
@ -9,6 +9,9 @@
|
|||
#include <linux/rcu_sync.h>
|
||||
#include <linux/lockdep.h>
|
||||
|
||||
void _trace_android_vh_record_pcpu_rwsem_starttime(
|
||||
struct task_struct *tsk, unsigned long settime);
|
||||
|
||||
struct percpu_rw_semaphore {
|
||||
struct rcu_sync rss;
|
||||
unsigned int __percpu *read_count;
|
||||
|
|
@ -68,6 +71,7 @@ static inline void percpu_down_read(struct percpu_rw_semaphore *sem)
|
|||
* bleeding the critical section out.
|
||||
*/
|
||||
preempt_enable();
|
||||
_trace_android_vh_record_pcpu_rwsem_starttime(current, jiffies);
|
||||
}
|
||||
|
||||
static inline bool percpu_down_read_trylock(struct percpu_rw_semaphore *sem)
|
||||
|
|
@ -88,14 +92,17 @@ static inline bool percpu_down_read_trylock(struct percpu_rw_semaphore *sem)
|
|||
* bleeding the critical section out.
|
||||
*/
|
||||
|
||||
if (ret)
|
||||
if (ret) {
|
||||
_trace_android_vh_record_pcpu_rwsem_starttime(current, jiffies);
|
||||
rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline void percpu_up_read(struct percpu_rw_semaphore *sem)
|
||||
{
|
||||
_trace_android_vh_record_pcpu_rwsem_starttime(current, 0);
|
||||
rwsem_release(&sem->dep_map, _RET_IP_);
|
||||
|
||||
preempt_disable();
|
||||
|
|
|
|||
|
|
@ -47,6 +47,18 @@ DECLARE_HOOK(android_vh_sched_show_task,
|
|||
TP_PROTO(struct task_struct *task),
|
||||
TP_ARGS(task));
|
||||
|
||||
DECLARE_HOOK(android_vh_record_mutex_lock_starttime,
|
||||
TP_PROTO(struct task_struct *tsk, unsigned long settime_jiffies),
|
||||
TP_ARGS(tsk, settime_jiffies));
|
||||
DECLARE_HOOK(android_vh_record_rtmutex_lock_starttime,
|
||||
TP_PROTO(struct task_struct *tsk, unsigned long settime_jiffies),
|
||||
TP_ARGS(tsk, settime_jiffies));
|
||||
DECLARE_HOOK(android_vh_record_rwsem_lock_starttime,
|
||||
TP_PROTO(struct task_struct *tsk, unsigned long settime_jiffies),
|
||||
TP_ARGS(tsk, settime_jiffies));
|
||||
DECLARE_HOOK(android_vh_record_pcpu_rwsem_starttime,
|
||||
TP_PROTO(struct task_struct *tsk, unsigned long settime_jiffies),
|
||||
TP_ARGS(tsk, settime_jiffies));
|
||||
#endif /* _TRACE_HOOK_DTASK_H */
|
||||
|
||||
/* This part must be outside protection */
|
||||
|
|
|
|||
|
|
@ -171,8 +171,10 @@ static __always_inline bool __mutex_trylock_fast(struct mutex *lock)
|
|||
unsigned long curr = (unsigned long)current;
|
||||
unsigned long zero = 0UL;
|
||||
|
||||
if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr))
|
||||
if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr)) {
|
||||
trace_android_vh_record_mutex_lock_starttime(current, jiffies);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
|
@ -539,6 +541,7 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
|
|||
*/
|
||||
void __sched mutex_unlock(struct mutex *lock)
|
||||
{
|
||||
trace_android_vh_record_mutex_lock_starttime(current, 0);
|
||||
#ifndef CONFIG_DEBUG_LOCK_ALLOC
|
||||
if (__mutex_unlock_fast(lock))
|
||||
return;
|
||||
|
|
@ -613,6 +616,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
|
|||
if (ww_ctx)
|
||||
ww_mutex_set_context_fastpath(ww, ww_ctx);
|
||||
trace_contention_end(lock, 0);
|
||||
trace_android_vh_record_mutex_lock_starttime(current, jiffies);
|
||||
preempt_enable();
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -731,6 +735,7 @@ skip_wait:
|
|||
|
||||
raw_spin_unlock(&lock->wait_lock);
|
||||
preempt_enable();
|
||||
trace_android_vh_record_mutex_lock_starttime(current, jiffies);
|
||||
return 0;
|
||||
|
||||
err:
|
||||
|
|
@ -1091,8 +1096,10 @@ int __sched mutex_trylock(struct mutex *lock)
|
|||
MUTEX_WARN_ON(lock->magic != lock);
|
||||
|
||||
locked = __mutex_trylock(lock);
|
||||
if (locked)
|
||||
if (locked) {
|
||||
trace_android_vh_record_mutex_lock_starttime(current, jiffies);
|
||||
mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
|
||||
}
|
||||
|
||||
return locked;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,21 @@
|
|||
#include <linux/errno.h>
|
||||
#include <trace/events/lock.h>
|
||||
|
||||
#include <trace/hooks/dtask.h>
|
||||
|
||||
/*
|
||||
* trace_android_vh_record_pcpu_rwsem_starttime is called in
|
||||
* include/linux/percpu-rwsem.h by including include/hooks/dtask.h, which
|
||||
* will result to build-err. So we create
|
||||
* func:_trace_android_vh_record_pcpu_rwsem_starttime for percpu-rwsem.h to call.
|
||||
*/
|
||||
void _trace_android_vh_record_pcpu_rwsem_starttime(struct task_struct *tsk,
|
||||
unsigned long settime)
|
||||
{
|
||||
trace_android_vh_record_pcpu_rwsem_starttime(tsk, settime);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(_trace_android_vh_record_pcpu_rwsem_starttime);
|
||||
|
||||
int __percpu_init_rwsem(struct percpu_rw_semaphore *sem,
|
||||
const char *name, struct lock_class_key *key)
|
||||
{
|
||||
|
|
@ -248,11 +263,13 @@ void __sched percpu_down_write(struct percpu_rw_semaphore *sem)
|
|||
/* Wait for all active readers to complete. */
|
||||
rcuwait_wait_event(&sem->writer, readers_active_check(sem), TASK_UNINTERRUPTIBLE);
|
||||
trace_contention_end(sem, 0);
|
||||
trace_android_vh_record_pcpu_rwsem_starttime(current, jiffies);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(percpu_down_write);
|
||||
|
||||
void percpu_up_write(struct percpu_rw_semaphore *sem)
|
||||
{
|
||||
trace_android_vh_record_pcpu_rwsem_starttime(current, 0);
|
||||
rwsem_release(&sem->dep_map, _RET_IP_);
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -31,6 +31,8 @@ static __always_inline int __rt_mutex_lock_common(struct rt_mutex *lock,
|
|||
ret = __rt_mutex_lock(&lock->rtmutex, state);
|
||||
if (ret)
|
||||
mutex_release(&lock->dep_map, _RET_IP_);
|
||||
else
|
||||
trace_android_vh_record_rtmutex_lock_starttime(current, jiffies);
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
|
@ -123,8 +125,10 @@ int __sched rt_mutex_trylock(struct rt_mutex *lock)
|
|||
return 0;
|
||||
|
||||
ret = __rt_mutex_trylock(&lock->rtmutex);
|
||||
if (ret)
|
||||
if (ret) {
|
||||
trace_android_vh_record_rtmutex_lock_starttime(current, jiffies);
|
||||
mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
|
@ -137,6 +141,7 @@ EXPORT_SYMBOL_GPL(rt_mutex_trylock);
|
|||
*/
|
||||
void __sched rt_mutex_unlock(struct rt_mutex *lock)
|
||||
{
|
||||
trace_android_vh_record_rtmutex_lock_starttime(current, 0);
|
||||
mutex_release(&lock->dep_map, _RET_IP_);
|
||||
__rt_mutex_unlock(&lock->rtmutex);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -249,6 +249,7 @@ static inline bool rwsem_read_trylock(struct rw_semaphore *sem, long *cntp)
|
|||
|
||||
if (!(*cntp & RWSEM_READ_FAILED_MASK)) {
|
||||
rwsem_set_reader_owned(sem);
|
||||
trace_android_vh_record_rwsem_lock_starttime(current, jiffies);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -262,6 +263,7 @@ static inline bool rwsem_write_trylock(struct rw_semaphore *sem)
|
|||
|
||||
preempt_disable();
|
||||
if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, RWSEM_WRITER_LOCKED)) {
|
||||
trace_android_vh_record_rwsem_lock_starttime(current, jiffies);
|
||||
rwsem_set_owner(sem);
|
||||
ret = true;
|
||||
}
|
||||
|
|
@ -1040,6 +1042,7 @@ rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int stat
|
|||
raw_spin_unlock_irq(&sem->wait_lock);
|
||||
wake_up_q(&wake_q);
|
||||
}
|
||||
trace_android_vh_record_rwsem_lock_starttime(current, jiffies);
|
||||
return sem;
|
||||
}
|
||||
|
||||
|
|
@ -1109,6 +1112,7 @@ queue:
|
|||
trace_android_vh_rwsem_read_wait_finish(sem);
|
||||
lockevent_inc(rwsem_rlock);
|
||||
trace_contention_end(sem, 0);
|
||||
trace_android_vh_record_rwsem_lock_starttime(current, jiffies);
|
||||
return sem;
|
||||
|
||||
out_nolock:
|
||||
|
|
@ -1133,6 +1137,7 @@ rwsem_down_write_slowpath(struct rw_semaphore *sem, int state)
|
|||
/* do optimistic spinning and steal lock if possible */
|
||||
if (rwsem_can_spin_on_owner(sem) && rwsem_optimistic_spin(sem)) {
|
||||
/* rwsem_optimistic_spin() implies ACQUIRE on success */
|
||||
trace_android_vh_record_rwsem_lock_starttime(current, jiffies);
|
||||
return sem;
|
||||
}
|
||||
|
||||
|
|
@ -1216,6 +1221,7 @@ trylock_again:
|
|||
raw_spin_unlock_irq(&sem->wait_lock);
|
||||
lockevent_inc(rwsem_wlock);
|
||||
trace_contention_end(sem, 0);
|
||||
trace_android_vh_record_rwsem_lock_starttime(current, jiffies);
|
||||
return sem;
|
||||
|
||||
out_nolock:
|
||||
|
|
@ -1319,6 +1325,7 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)
|
|||
tmp + RWSEM_READER_BIAS)) {
|
||||
rwsem_set_reader_owned(sem);
|
||||
ret = 1;
|
||||
trace_android_vh_record_rwsem_lock_starttime(current, jiffies);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -1366,6 +1373,7 @@ static inline void __up_read(struct rw_semaphore *sem)
|
|||
DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem);
|
||||
|
||||
preempt_disable();
|
||||
trace_android_vh_record_rwsem_lock_starttime(current, 0);
|
||||
rwsem_clear_reader_owned(sem);
|
||||
tmp = atomic_long_add_return_release(-RWSEM_READER_BIAS, &sem->count);
|
||||
DEBUG_RWSEMS_WARN_ON(tmp < 0, sem);
|
||||
|
|
@ -1393,6 +1401,7 @@ static inline void __up_write(struct rw_semaphore *sem)
|
|||
!rwsem_test_oflags(sem, RWSEM_NONSPINNABLE), sem);
|
||||
|
||||
preempt_disable();
|
||||
trace_android_vh_record_rwsem_lock_starttime(current, 0);
|
||||
rwsem_clear_owner(sem);
|
||||
tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count);
|
||||
preempt_enable();
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue