From 97f2f8a065cbfc2e2706e81276956cbdb5aa65df Mon Sep 17 00:00:00 2001 From: Peifeng Li Date: Thu, 13 Apr 2023 15:53:28 +0800 Subject: [PATCH] ANDROID: uid_sys_stat: split the global lock uid_lock to the fine-grained locks for each hlist in hash_table. 1.Hash_table in uid_sys_stat is protected by a global lock named id_lock, which causes some lock competition issue. Actually, uid_lock can be split to several file-grained locks for each hlist in hash_table, which avoid the unnecessary lock competition when get different-uid process info. 2. Switching rt-mutex to spinlock, in order to operate with read_rcu_lock. Bug: 278138377 Signed-off-by: Peifeng Li (cherry picked from https://android-review.googlesource.com/q/commit:c949fbdce0bc792dea206c709d909094be579c3a) Merged-In: Ib252b65e9aebe3a594e6edf075f7aa01f8e6105d Change-Id: Ib252b65e9aebe3a594e6edf075f7aa01f8e6105d --- drivers/misc/uid_sys_stats.c | 168 ++++++++++++++++++++++++----------- 1 file changed, 115 insertions(+), 53 deletions(-) diff --git a/drivers/misc/uid_sys_stats.c b/drivers/misc/uid_sys_stats.c index 842597bbeb89..f713de2b3a08 100644 --- a/drivers/misc/uid_sys_stats.c +++ b/drivers/misc/uid_sys_stats.c @@ -23,17 +23,20 @@ #include #include #include -#include #include #include #include #include - +#include #define UID_HASH_BITS 10 +#define UID_HASH_NUMS (1 << UID_HASH_BITS) DECLARE_HASHTABLE(hash_table, UID_HASH_BITS); +/* + * uid_lock[bkt] ensure consistency of hash_table[bkt] + */ +spinlock_t uid_lock[UID_HASH_NUMS]; -static DEFINE_RT_MUTEX(uid_lock); static struct proc_dir_entry *cpu_parent; static struct proc_dir_entry *io_parent; static struct proc_dir_entry *proc_parent; @@ -78,6 +81,32 @@ struct uid_entry { #endif }; +static inline int trylock_uid(uid_t uid) +{ + return spin_trylock( + &uid_lock[hash_min(uid, HASH_BITS(hash_table))]); +} + +static inline void lock_uid(uid_t uid) +{ + spin_lock(&uid_lock[hash_min(uid, HASH_BITS(hash_table))]); +} + +static inline void unlock_uid(uid_t uid) +{ + spin_unlock(&uid_lock[hash_min(uid, HASH_BITS(hash_table))]); +} + +static inline void lock_uid_by_bkt(u32 bkt) +{ + spin_lock(&uid_lock[bkt]); +} + +static inline void unlock_uid_by_bkt(u32 bkt) +{ + spin_unlock(&uid_lock[bkt]); +} + static u64 compute_write_bytes(struct task_io_accounting *ioac) { if (ioac->write_bytes <= ioac->cancelled_write_bytes) @@ -333,24 +362,29 @@ static int uid_cputime_show(struct seq_file *m, void *v) struct user_namespace *user_ns = current_user_ns(); u64 utime; u64 stime; - unsigned long bkt; + u32 bkt; uid_t uid; - rt_mutex_lock(&uid_lock); - - hash_for_each(hash_table, bkt, uid_entry, hash) { - uid_entry->active_stime = 0; - uid_entry->active_utime = 0; + for (bkt = 0, uid_entry = NULL; uid_entry == NULL && + bkt < HASH_SIZE(hash_table); bkt++) { + lock_uid_by_bkt(bkt); + hlist_for_each_entry(uid_entry, &hash_table[bkt], hash) { + uid_entry->active_stime = 0; + uid_entry->active_utime = 0; + } + unlock_uid_by_bkt(bkt); } rcu_read_lock(); do_each_thread(temp, task) { uid = from_kuid_munged(user_ns, task_uid(task)); + lock_uid(uid); + if (!uid_entry || uid_entry->uid != uid) uid_entry = find_or_register_uid(uid); if (!uid_entry) { rcu_read_unlock(); - rt_mutex_unlock(&uid_lock); + unlock_uid(uid); pr_err("%s: failed to find the uid_entry for uid %d\n", __func__, uid); return -ENOMEM; @@ -361,19 +395,24 @@ static int uid_cputime_show(struct seq_file *m, void *v) uid_entry->active_utime += utime; uid_entry->active_stime += stime; } + unlock_uid(uid); } while_each_thread(temp, task); rcu_read_unlock(); - hash_for_each(hash_table, bkt, uid_entry, hash) { - u64 total_utime = uid_entry->utime + - uid_entry->active_utime; - u64 total_stime = uid_entry->stime + - uid_entry->active_stime; - seq_printf(m, "%d: %llu %llu\n", uid_entry->uid, - ktime_to_us(total_utime), ktime_to_us(total_stime)); + for (bkt = 0, uid_entry = NULL; uid_entry == NULL && + bkt < HASH_SIZE(hash_table); bkt++) { + lock_uid_by_bkt(bkt); + hlist_for_each_entry(uid_entry, &hash_table[bkt], hash) { + u64 total_utime = uid_entry->utime + + uid_entry->active_utime; + u64 total_stime = uid_entry->stime + + uid_entry->active_stime; + seq_printf(m, "%d: %llu %llu\n", uid_entry->uid, + ktime_to_us(total_utime), ktime_to_us(total_stime)); + } + unlock_uid_by_bkt(bkt); } - rt_mutex_unlock(&uid_lock); return 0; } @@ -421,9 +460,8 @@ static ssize_t uid_remove_write(struct file *file, return -EINVAL; } - rt_mutex_lock(&uid_lock); - for (; uid_start <= uid_end; uid_start++) { + lock_uid(uid_start); hash_for_each_possible_safe(hash_table, uid_entry, tmp, hash, (uid_t)uid_start) { if (uid_start == uid_entry->uid) { @@ -432,9 +470,9 @@ static ssize_t uid_remove_write(struct file *file, kfree(uid_entry); } } + unlock_uid(uid_start); } - rt_mutex_unlock(&uid_lock); return count; } @@ -472,37 +510,51 @@ static void add_uid_io_stats(struct uid_entry *uid_entry, __add_uid_io_stats(uid_entry, &task->ioac, slot); } -static void update_io_stats_all_locked(void) +static void update_io_stats_all(void) { struct uid_entry *uid_entry = NULL; struct task_struct *task, *temp; struct user_namespace *user_ns = current_user_ns(); - unsigned long bkt; + u32 bkt; uid_t uid; - hash_for_each(hash_table, bkt, uid_entry, hash) { - memset(&uid_entry->io[UID_STATE_TOTAL_CURR], 0, - sizeof(struct io_stats)); - set_io_uid_tasks_zero(uid_entry); + for (bkt = 0, uid_entry = NULL; uid_entry == NULL && bkt < HASH_SIZE(hash_table); + bkt++) { + lock_uid_by_bkt(bkt); + hlist_for_each_entry(uid_entry, &hash_table[bkt], hash) { + memset(&uid_entry->io[UID_STATE_TOTAL_CURR], 0, + sizeof(struct io_stats)); + set_io_uid_tasks_zero(uid_entry); + } + unlock_uid_by_bkt(bkt); } rcu_read_lock(); do_each_thread(temp, task) { uid = from_kuid_munged(user_ns, task_uid(task)); + lock_uid(uid); if (!uid_entry || uid_entry->uid != uid) uid_entry = find_or_register_uid(uid); - if (!uid_entry) + if (!uid_entry) { + unlock_uid(uid); continue; + } add_uid_io_stats(uid_entry, task, UID_STATE_TOTAL_CURR); + unlock_uid(uid); } while_each_thread(temp, task); rcu_read_unlock(); - hash_for_each(hash_table, bkt, uid_entry, hash) { - compute_io_bucket_stats(&uid_entry->io[uid_entry->state], - &uid_entry->io[UID_STATE_TOTAL_CURR], - &uid_entry->io[UID_STATE_TOTAL_LAST], - &uid_entry->io[UID_STATE_DEAD_TASKS]); - compute_io_uid_tasks(uid_entry); + for (bkt = 0, uid_entry = NULL; uid_entry == NULL && bkt < HASH_SIZE(hash_table); + bkt++) { + lock_uid_by_bkt(bkt); + hlist_for_each_entry(uid_entry, &hash_table[bkt], hash) { + compute_io_bucket_stats(&uid_entry->io[uid_entry->state], + &uid_entry->io[UID_STATE_TOTAL_CURR], + &uid_entry->io[UID_STATE_TOTAL_LAST], + &uid_entry->io[UID_STATE_DEAD_TASKS]); + compute_io_uid_tasks(uid_entry); + } + unlock_uid_by_bkt(bkt); } } @@ -534,14 +586,15 @@ static void update_io_stats_uid_locked(struct uid_entry *uid_entry) static int uid_io_show(struct seq_file *m, void *v) { struct uid_entry *uid_entry; - unsigned long bkt; + u32 bkt; - rt_mutex_lock(&uid_lock); + update_io_stats_all(); + for (bkt = 0, uid_entry = NULL; uid_entry == NULL && bkt < HASH_SIZE(hash_table); + bkt++) { - update_io_stats_all_locked(); - - hash_for_each(hash_table, bkt, uid_entry, hash) { - seq_printf(m, "%d %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu\n", + lock_uid_by_bkt(bkt); + hlist_for_each_entry(uid_entry, &hash_table[bkt], hash) { + seq_printf(m, "%d %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu\n", uid_entry->uid, uid_entry->io[UID_STATE_FOREGROUND].rchar, uid_entry->io[UID_STATE_FOREGROUND].wchar, @@ -554,10 +607,11 @@ static int uid_io_show(struct seq_file *m, void *v) uid_entry->io[UID_STATE_FOREGROUND].fsync, uid_entry->io[UID_STATE_BACKGROUND].fsync); - show_io_uid_tasks(m, uid_entry); + show_io_uid_tasks(m, uid_entry); + } + unlock_uid_by_bkt(bkt); } - rt_mutex_unlock(&uid_lock); return 0; } @@ -601,16 +655,15 @@ static ssize_t uid_procstat_write(struct file *file, if (state != UID_STATE_BACKGROUND && state != UID_STATE_FOREGROUND) return -EINVAL; - rt_mutex_lock(&uid_lock); - + lock_uid(uid); uid_entry = find_or_register_uid(uid); if (!uid_entry) { - rt_mutex_unlock(&uid_lock); + unlock_uid(uid); return -EINVAL; } if (uid_entry->state == state) { - rt_mutex_unlock(&uid_lock); + unlock_uid(uid); return count; } @@ -618,7 +671,7 @@ static ssize_t uid_procstat_write(struct file *file, uid_entry->state = state; - rt_mutex_unlock(&uid_lock); + unlock_uid(uid); return count; } @@ -649,10 +702,9 @@ static void update_stats_workfn(struct work_struct *work) struct task_entry *task_entry __maybe_unused; struct llist_node *node; - rt_mutex_lock(&uid_lock); - node = llist_del_all(&work_usw); llist_for_each_entry_safe(usw, t, node, node) { + lock_uid(usw->uid); uid_entry = find_uid_entry(usw->uid); if (!uid_entry) goto next; @@ -669,12 +721,13 @@ static void update_stats_workfn(struct work_struct *work) #endif __add_uid_io_stats(uid_entry, &usw->ioac, UID_STATE_DEAD_TASKS); next: + unlock_uid(usw->uid); #ifdef CONFIG_UID_SYS_STATS_DEBUG put_task_struct(usw->task); #endif kfree(usw); } - rt_mutex_unlock(&uid_lock); + } static DECLARE_WORK(update_stats_work, update_stats_workfn); @@ -690,7 +743,7 @@ static int process_notifier(struct notifier_block *self, return NOTIFY_OK; uid = from_kuid_munged(current_user_ns(), task_uid(task)); - if (!rt_mutex_trylock(&uid_lock)) { + if (!trylock_uid(uid)) { struct update_stats_work *usw; usw = kmalloc(sizeof(struct update_stats_work), GFP_KERNEL); @@ -724,7 +777,7 @@ static int process_notifier(struct notifier_block *self, add_uid_io_stats(uid_entry, task, UID_STATE_DEAD_TASKS); exit: - rt_mutex_unlock(&uid_lock); + unlock_uid(uid); return NOTIFY_OK; } @@ -732,9 +785,18 @@ static struct notifier_block process_notifier_block = { .notifier_call = process_notifier, }; +static void init_hash_table_and_lock(void) +{ + int i; + + hash_init(hash_table); + for (i = 0; i < UID_HASH_NUMS; i++) + spin_lock_init(&uid_lock[i]); +} + static int __init proc_uid_sys_stats_init(void) { - hash_init(hash_table); + init_hash_table_and_lock(); cpu_parent = proc_mkdir("uid_cputime", NULL); if (!cpu_parent) {