From daac8818641173416a4f38e1161f56d8b830917b Mon Sep 17 00:00:00 2001
From: Hyeonggon Yoo <42.hyeyoo@gmail.com>
Date: Fri, 23 Apr 2021 12:37:39 +0900
Subject: [PATCH] kernel/locking: Introduce rwlock_t to wait_list of semaphore

there's many readers that check if their waiter.up is true and one writer that currently holding the lock.

so introduced rwlock_t for optimization. it can help for contended case.

Signed-off-by: Hyeonggon Yoo <42.hyeyoo@gmail.com>
---
 include/linux/semaphore.h  |  3 ++
 kernel/locking/semaphore.c | 66 +++++++++++++++++++++++---------------
 2 files changed, 43 insertions(+), 26 deletions(-)

diff --git a/include/linux/semaphore.h b/include/linux/semaphore.h
index 6694d0019a68..7fb4c3811c44 100644
--- a/include/linux/semaphore.h
+++ b/include/linux/semaphore.h
@@ -10,12 +10,14 @@
 
 #include <linux/list.h>
 #include <linux/spinlock.h>
+#include <linux/rwlock_types.h>
 
 /* Please don't access any members of this structure directly */
 struct semaphore {
 	raw_spinlock_t		lock;
 	unsigned int		count;
 	struct list_head	wait_list;
+	rwlock_t		wait_list_lock;
 };
 
 #define __SEMAPHORE_INITIALIZER(name, n)				\
@@ -23,6 +25,7 @@ struct semaphore {
 	.lock		= __RAW_SPIN_LOCK_UNLOCKED((name).lock),	\
 	.count		= n,						\
 	.wait_list	= LIST_HEAD_INIT((name).wait_list),		\
+	.wait_list_lock = __RW_LOCK_UNLOCKED((name).wait_list_lock),	\
 }
 
 #define DEFINE_SEMAPHORE(name)	\
diff --git a/kernel/locking/semaphore.c b/kernel/locking/semaphore.c
index 9aa855a96c4a..887cffed8516 100644
--- a/kernel/locking/semaphore.c
+++ b/kernel/locking/semaphore.c
@@ -32,6 +32,7 @@
 #include <linux/semaphore.h>
 #include <linux/spinlock.h>
 #include <linux/ftrace.h>
+#include <linux/rwlock_types.h>
 
 static noinline void __down(struct semaphore *sem);
 static noinline int __down_interruptible(struct semaphore *sem);
@@ -55,11 +56,13 @@ void down(struct semaphore *sem)
 	unsigned long flags;
 
 	raw_spin_lock_irqsave(&sem->lock, flags);
-	if (likely(sem->count > 0))
+	if (likely(sem->count > 0)) {
 		sem->count--;
-	else
+		raw_spin_unlock_irqrestore(&sem->lock, flags);
+	} else {
+		raw_spin_unlock_irqrestore(&sem->lock, flags);
 		__down(sem);
-	raw_spin_unlock_irqrestore(&sem->lock, flags);
+	}
 }
 EXPORT_SYMBOL(down);
 
@@ -75,16 +78,16 @@ EXPORT_SYMBOL(down);
 int down_interruptible(struct semaphore *sem)
 {
 	unsigned long flags;
-	int result = 0;
 
 	raw_spin_lock_irqsave(&sem->lock, flags);
-	if (likely(sem->count > 0))
+	if (likely(sem->count > 0)) {
 		sem->count--;
-	else
-		result = __down_interruptible(sem);
-	raw_spin_unlock_irqrestore(&sem->lock, flags);
-
-	return result;
+		raw_spin_unlock_irqrestore(&sem->lock, flags);
+		return 0;
+	} else {
+		raw_spin_unlock_irqrestore(&sem->lock, flags);
+		return __down_interruptible(sem);
+	}
 }
 EXPORT_SYMBOL(down_interruptible);
 
@@ -101,16 +104,16 @@ EXPORT_SYMBOL(down_interruptible);
 int down_killable(struct semaphore *sem)
 {
 	unsigned long flags;
-	int result = 0;
 
 	raw_spin_lock_irqsave(&sem->lock, flags);
-	if (likely(sem->count > 0))
+	if (likely(sem->count > 0)) {
 		sem->count--;
-	else
-		result = __down_killable(sem);
-	raw_spin_unlock_irqrestore(&sem->lock, flags);
-
-	return result;
+		raw_spin_unlock_irqrestore(&sem->lock, flags);
+		return 0;
+	} else {
+		raw_spin_unlock_irqrestore(&sem->lock, flags);
+		return __down_killable(sem);
+	}
 }
 EXPORT_SYMBOL(down_killable);
 
@@ -158,11 +161,13 @@ int down_timeout(struct semaphore *sem, long timeout)
 	int result = 0;
 
 	raw_spin_lock_irqsave(&sem->lock, flags);
-	if (likely(sem->count > 0))
+	if (likely(sem->count > 0)) {
 		sem->count--;
-	else
+		raw_spin_unlock_irqrestore(&sem->lock, flags);
+	} else {
+		raw_spin_unlock_irqrestore(&sem->lock, flags);
 		result = __down_timeout(sem, timeout);
-	raw_spin_unlock_irqrestore(&sem->lock, flags);
+	}
 
 	return result;
 }
@@ -180,11 +185,13 @@ void up(struct semaphore *sem)
 	unsigned long flags;
 
 	raw_spin_lock_irqsave(&sem->lock, flags);
-	if (likely(list_empty(&sem->wait_list)))
+	if (likely(list_empty(&sem->wait_list))) {
 		sem->count++;
-	else
+		raw_spin_unlock_irqrestore(&sem->lock, flags);
+	} else {
+		raw_spin_unlock_irqrestore(&sem->lock, flags);
 		__up(sem);
-	raw_spin_unlock_irqrestore(&sem->lock, flags);
+	}
 }
 EXPORT_SYMBOL(up);
 
@@ -206,9 +213,11 @@ static inline int __sched __down_common(struct semaphore *sem, long state,
 {
 	struct semaphore_waiter waiter;
 
+	write_lock_irq(&sem->wait_list_lock);
 	list_add_tail(&waiter.list, &sem->wait_list);
 	waiter.task = current;
 	waiter.up = false;
+	write_unlock_irq(&sem->wait_list_lock);
 
 	for (;;) {
 		if (signal_pending_state(state, current))
@@ -216,11 +225,11 @@ static inline int __sched __down_common(struct semaphore *sem, long state,
 		if (unlikely(timeout <= 0))
 			goto timed_out;
 		__set_current_state(state);
-		raw_spin_unlock_irq(&sem->lock);
 		timeout = schedule_timeout(timeout);
-		raw_spin_lock_irq(&sem->lock);
+		read_lock_irq(&sem->wait_list_lock);
 		if (waiter.up)
-			return 0;
+			goto acquired;
+		read_unlock_irq(&sem->wait_list_lock);
 	}
 
  timed_out:
@@ -230,6 +239,11 @@ static inline int __sched __down_common(struct semaphore *sem, long state,
  interrupted:
 	list_del(&waiter.list);
 	return -EINTR;
+
+ acquired:
+	list_del(&waiter.list);
+	read_unlock_irq(&sem->wait_list_lock);
+	return 0;
 }
 
 static noinline void __sched __down(struct semaphore *sem)
-- 
2.25.1

