ANDROID: uid_sys_stat: split the global lock uid_lock to the fine-grained

locks for each hlist in hash_table.

1.Hash_table in uid_sys_stat is protected by a global lock named id_lock,
which causes some lock competition issue. Actually, uid_lock can be split to
several file-grained locks for each hlist in hash_table, which avoid
the unnecessary lock competition when get different-uid process info.

2. Switching rt-mutex to spinlock, in order to operate with read_rcu_lock.

Bug: 278138377
Signed-off-by: Peifeng Li <lipeifeng@oppo.com>
(cherry picked from https://android-review.googlesource.com/q/commit:c949fbdce0bc792dea206c709d909094be579c3a)
Merged-In: Ib252b65e9aebe3a594e6edf075f7aa01f8e6105d
Change-Id: Ib252b65e9aebe3a594e6edf075f7aa01f8e6105d

drivers/misc/uid_sys_stats.c

@@ -23,17 +23,20 @@
 #include <linux/mm.h>
 #include <linux/proc_fs.h>
 #include <linux/profile.h>
-#include <linux/rtmutex.h>
 #include <linux/sched/cputime.h>
 #include <linux/seq_file.h>
 #include <linux/slab.h>
 #include <linux/uaccess.h>
-
+#include <linux/spinlock_types.h>
 
 #define UID_HASH_BITS	10
+#define UID_HASH_NUMS	(1 << UID_HASH_BITS)
 DECLARE_HASHTABLE(hash_table, UID_HASH_BITS);
+/*
+ * uid_lock[bkt] ensure consistency of hash_table[bkt]
+ */
+spinlock_t uid_lock[UID_HASH_NUMS];
 
-static DEFINE_RT_MUTEX(uid_lock);
 static struct proc_dir_entry *cpu_parent;
 static struct proc_dir_entry *io_parent;
 static struct proc_dir_entry *proc_parent;
@@ -78,6 +81,32 @@ struct uid_entry {
 #endif
 };
 
+static inline int trylock_uid(uid_t uid)
+{
+	return spin_trylock(
+		&uid_lock[hash_min(uid, HASH_BITS(hash_table))]);
+}
+
+static inline void lock_uid(uid_t uid)
+{
+	spin_lock(&uid_lock[hash_min(uid, HASH_BITS(hash_table))]);
+}
+
+static inline void unlock_uid(uid_t uid)
+{
+	spin_unlock(&uid_lock[hash_min(uid, HASH_BITS(hash_table))]);
+}
+
+static inline void lock_uid_by_bkt(u32 bkt)
+{
+	spin_lock(&uid_lock[bkt]);
+}
+
+static inline void unlock_uid_by_bkt(u32 bkt)
+{
+	spin_unlock(&uid_lock[bkt]);
+}
+
 static u64 compute_write_bytes(struct task_io_accounting *ioac)
 {
 	if (ioac->write_bytes <= ioac->cancelled_write_bytes)
@@ -333,24 +362,29 @@ static int uid_cputime_show(struct seq_file *m, void *v)
 	struct user_namespace *user_ns = current_user_ns();
 	u64 utime;
 	u64 stime;
-	unsigned long bkt;
+	u32 bkt;
 	uid_t uid;
 
-	rt_mutex_lock(&uid_lock);
+	for (bkt = 0, uid_entry = NULL; uid_entry == NULL &&
-
+		bkt < HASH_SIZE(hash_table); bkt++) {
-	hash_for_each(hash_table, bkt, uid_entry, hash) {
+		lock_uid_by_bkt(bkt);
-		uid_entry->active_stime = 0;
+		hlist_for_each_entry(uid_entry, &hash_table[bkt], hash) {
-		uid_entry->active_utime = 0;
+			uid_entry->active_stime = 0;
+			uid_entry->active_utime = 0;
+		}
+		unlock_uid_by_bkt(bkt);
 	}
 
 	rcu_read_lock();
 	do_each_thread(temp, task) {
 		uid = from_kuid_munged(user_ns, task_uid(task));
+		lock_uid(uid);
+
 		if (!uid_entry || uid_entry->uid != uid)
 			uid_entry = find_or_register_uid(uid);
 		if (!uid_entry) {
 			rcu_read_unlock();
-			rt_mutex_unlock(&uid_lock);
+			unlock_uid(uid);
 			pr_err("%s: failed to find the uid_entry for uid %d\n",
 				__func__, uid);
 			return -ENOMEM;
@@ -361,19 +395,24 @@ static int uid_cputime_show(struct seq_file *m, void *v)
 			uid_entry->active_utime += utime;
 			uid_entry->active_stime += stime;
 		}
+		unlock_uid(uid);
 	} while_each_thread(temp, task);
 	rcu_read_unlock();
 
-	hash_for_each(hash_table, bkt, uid_entry, hash) {
+	for (bkt = 0, uid_entry = NULL; uid_entry == NULL &&
-		u64 total_utime = uid_entry->utime +
+		bkt < HASH_SIZE(hash_table); bkt++) {
-							uid_entry->active_utime;
+		lock_uid_by_bkt(bkt);
-		u64 total_stime = uid_entry->stime +
+		hlist_for_each_entry(uid_entry, &hash_table[bkt], hash) {
-							uid_entry->active_stime;
+			u64 total_utime = uid_entry->utime +
-		seq_printf(m, "%d: %llu %llu\n", uid_entry->uid,
+						uid_entry->active_utime;
-			ktime_to_us(total_utime), ktime_to_us(total_stime));
+			u64 total_stime = uid_entry->stime +
+						uid_entry->active_stime;
+			seq_printf(m, "%d: %llu %llu\n", uid_entry->uid,
+				ktime_to_us(total_utime), ktime_to_us(total_stime));
+		}
+		unlock_uid_by_bkt(bkt);
 	}
 
-	rt_mutex_unlock(&uid_lock);
 	return 0;
 }
 
@@ -421,9 +460,8 @@ static ssize_t uid_remove_write(struct file *file,
 		return -EINVAL;
 	}
 
-	rt_mutex_lock(&uid_lock);
-
 	for (; uid_start <= uid_end; uid_start++) {
+		lock_uid(uid_start);
 		hash_for_each_possible_safe(hash_table, uid_entry, tmp,
 							hash, (uid_t)uid_start) {
 			if (uid_start == uid_entry->uid) {
@@ -432,9 +470,9 @@ static ssize_t uid_remove_write(struct file *file,
 				kfree(uid_entry);
 			}
 		}
+		unlock_uid(uid_start);
 	}
 
-	rt_mutex_unlock(&uid_lock);
 	return count;
 }
 
@@ -472,37 +510,51 @@ static void add_uid_io_stats(struct uid_entry *uid_entry,
 	__add_uid_io_stats(uid_entry, &task->ioac, slot);
 }
 
-static void update_io_stats_all_locked(void)
+static void update_io_stats_all(void)
 {
 	struct uid_entry *uid_entry = NULL;
 	struct task_struct *task, *temp;
 	struct user_namespace *user_ns = current_user_ns();
-	unsigned long bkt;
+	u32 bkt;
 	uid_t uid;
 
-	hash_for_each(hash_table, bkt, uid_entry, hash) {
+	for (bkt = 0, uid_entry = NULL; uid_entry == NULL && bkt < HASH_SIZE(hash_table);
-		memset(&uid_entry->io[UID_STATE_TOTAL_CURR], 0,
+		bkt++) {
-			sizeof(struct io_stats));
+		lock_uid_by_bkt(bkt);
-		set_io_uid_tasks_zero(uid_entry);
+		hlist_for_each_entry(uid_entry, &hash_table[bkt], hash) {
+			memset(&uid_entry->io[UID_STATE_TOTAL_CURR], 0,
+				sizeof(struct io_stats));
+			set_io_uid_tasks_zero(uid_entry);
+		}
+		unlock_uid_by_bkt(bkt);
 	}
 
 	rcu_read_lock();
 	do_each_thread(temp, task) {
 		uid = from_kuid_munged(user_ns, task_uid(task));
+		lock_uid(uid);
 		if (!uid_entry || uid_entry->uid != uid)
 			uid_entry = find_or_register_uid(uid);
-		if (!uid_entry)
+		if (!uid_entry) {
+			unlock_uid(uid);
 			continue;
+		}
 		add_uid_io_stats(uid_entry, task, UID_STATE_TOTAL_CURR);
+		unlock_uid(uid);
 	} while_each_thread(temp, task);
 	rcu_read_unlock();
 
-	hash_for_each(hash_table, bkt, uid_entry, hash) {
+	for (bkt = 0, uid_entry = NULL; uid_entry == NULL && bkt < HASH_SIZE(hash_table);
-		compute_io_bucket_stats(&uid_entry->io[uid_entry->state],
+			bkt++) {
-					&uid_entry->io[UID_STATE_TOTAL_CURR],
+		lock_uid_by_bkt(bkt);
-					&uid_entry->io[UID_STATE_TOTAL_LAST],
+		hlist_for_each_entry(uid_entry, &hash_table[bkt], hash) {
-					&uid_entry->io[UID_STATE_DEAD_TASKS]);
+			compute_io_bucket_stats(&uid_entry->io[uid_entry->state],
-		compute_io_uid_tasks(uid_entry);
+						&uid_entry->io[UID_STATE_TOTAL_CURR],
+						&uid_entry->io[UID_STATE_TOTAL_LAST],
+						&uid_entry->io[UID_STATE_DEAD_TASKS]);
+			compute_io_uid_tasks(uid_entry);
+		}
+		unlock_uid_by_bkt(bkt);
 	}
 }
 
@@ -534,14 +586,15 @@ static void update_io_stats_uid_locked(struct uid_entry *uid_entry)
 static int uid_io_show(struct seq_file *m, void *v)
 {
 	struct uid_entry *uid_entry;
-	unsigned long bkt;
+	u32 bkt;
 
-	rt_mutex_lock(&uid_lock);
+	update_io_stats_all();
+	for (bkt = 0, uid_entry = NULL; uid_entry == NULL && bkt < HASH_SIZE(hash_table);
+			bkt++) {
 
-	update_io_stats_all_locked();
+		lock_uid_by_bkt(bkt);
-
+		hlist_for_each_entry(uid_entry, &hash_table[bkt], hash) {
-	hash_for_each(hash_table, bkt, uid_entry, hash) {
+			seq_printf(m, "%d %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu\n",
-		seq_printf(m, "%d %llu %llu %llu %llu %llu %llu %llu %llu %llu %llu\n",
 				uid_entry->uid,
 				uid_entry->io[UID_STATE_FOREGROUND].rchar,
 				uid_entry->io[UID_STATE_FOREGROUND].wchar,
@@ -554,10 +607,11 @@ static int uid_io_show(struct seq_file *m, void *v)
 				uid_entry->io[UID_STATE_FOREGROUND].fsync,
 				uid_entry->io[UID_STATE_BACKGROUND].fsync);
 
-		show_io_uid_tasks(m, uid_entry);
+			show_io_uid_tasks(m, uid_entry);
+		}
+		unlock_uid_by_bkt(bkt);
 	}
 
-	rt_mutex_unlock(&uid_lock);
 	return 0;
 }
 
@@ -601,16 +655,15 @@ static ssize_t uid_procstat_write(struct file *file,
 	if (state != UID_STATE_BACKGROUND && state != UID_STATE_FOREGROUND)
 		return -EINVAL;
 
-	rt_mutex_lock(&uid_lock);
+	lock_uid(uid);
-
 	uid_entry = find_or_register_uid(uid);
 	if (!uid_entry) {
-		rt_mutex_unlock(&uid_lock);
+		unlock_uid(uid);
 		return -EINVAL;
 	}
 
 	if (uid_entry->state == state) {
-		rt_mutex_unlock(&uid_lock);
+		unlock_uid(uid);
 		return count;
 	}
 
@@ -618,7 +671,7 @@ static ssize_t uid_procstat_write(struct file *file,
 
 	uid_entry->state = state;
 
-	rt_mutex_unlock(&uid_lock);
+	unlock_uid(uid);
 
 	return count;
 }
@@ -649,10 +702,9 @@ static void update_stats_workfn(struct work_struct *work)
 	struct task_entry *task_entry __maybe_unused;
 	struct llist_node *node;
 
-	rt_mutex_lock(&uid_lock);
-
 	node = llist_del_all(&work_usw);
 	llist_for_each_entry_safe(usw, t, node, node) {
+		lock_uid(usw->uid);
 		uid_entry = find_uid_entry(usw->uid);
 		if (!uid_entry)
 			goto next;
@@ -669,12 +721,13 @@ static void update_stats_workfn(struct work_struct *work)
 #endif
 		__add_uid_io_stats(uid_entry, &usw->ioac, UID_STATE_DEAD_TASKS);
 next:
+		unlock_uid(usw->uid);
 #ifdef CONFIG_UID_SYS_STATS_DEBUG
 		put_task_struct(usw->task);
 #endif
 		kfree(usw);
 	}
-	rt_mutex_unlock(&uid_lock);
+
 }
 static DECLARE_WORK(update_stats_work, update_stats_workfn);
 
@@ -690,7 +743,7 @@ static int process_notifier(struct notifier_block *self,
 		return NOTIFY_OK;
 
 	uid = from_kuid_munged(current_user_ns(), task_uid(task));
-	if (!rt_mutex_trylock(&uid_lock)) {
+	if (!trylock_uid(uid)) {
 		struct update_stats_work *usw;
 
 		usw = kmalloc(sizeof(struct update_stats_work), GFP_KERNEL);
@@ -724,7 +777,7 @@ static int process_notifier(struct notifier_block *self,
 	add_uid_io_stats(uid_entry, task, UID_STATE_DEAD_TASKS);
 
 exit:
-	rt_mutex_unlock(&uid_lock);
+	unlock_uid(uid);
 	return NOTIFY_OK;
 }
 
@@ -732,9 +785,18 @@ static struct notifier_block process_notifier_block = {
 	.notifier_call	= process_notifier,
 };
 
+static void init_hash_table_and_lock(void)
+{
+	int i;
+
+	hash_init(hash_table);
+	for (i = 0; i < UID_HASH_NUMS; i++)
+		spin_lock_init(&uid_lock[i]);
+}
+
 static int __init proc_uid_sys_stats_init(void)
 {
-	hash_init(hash_table);
+	init_hash_table_and_lock();
 
 	cpu_parent = proc_mkdir("uid_cputime", NULL);
 	if (!cpu_parent) {