From: Willy Tarreau <w@1wt.eu>
Date: Wed, 16 Aug 2023 20:40:05 +0000 (+0200)
Subject: IMPORT: lorw: support inlining the wait call
X-Git-Tag: v2.9-dev4~46
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=e56275378fa0d303a58dac69b2c75a5d6ab04f2b;p=thirdparty%2Fhaproxy.git

IMPORT: lorw: support inlining the wait call

Now when PLOCK_LORW_INLINE_WAIT is defined, the pl_wait_unlock_long()
calls in pl_lorw_rdlock() and pl_lorw_wrlock() will be inlined so that
all the CPU time is accounted for in the calling function.

This is plock upstream commit c993f81d581732a6eb8fe3033f21970420d21e5e.
---

diff --git a/include/import/plock.h b/include/import/plock.h
index 07350009f8..7f2179491e 100644
--- a/include/import/plock.h
+++ b/include/import/plock.h
@@ -1318,14 +1318,22 @@ static inline void pl_lorw_rdlock(unsigned long *lock)
 	 * lock to be empty of visitors.
 	 */
 	if (lk & PLOCK_LORW_WRQ_MASK)
+#if defined(PLOCK_LORW_INLINE_WAIT)
+		lk = __pl_wait_unlock_long(lock, PLOCK_LORW_WRQ_MASK);
+#else
 		lk = pl_wait_unlock_long(lock, PLOCK_LORW_WRQ_MASK);
+#endif
 
 	/* count us as visitor among others */
 	lk = pl_ldadd_acq(lock, PLOCK_LORW_SHR_BASE);
 
 	/* wait for end of exclusive access if any */
 	if (lk & PLOCK_LORW_EXC_MASK)
+#if defined(PLOCK_LORW_INLINE_WAIT)
+		lk = __pl_wait_unlock_long(lock, PLOCK_LORW_EXC_MASK);
+#else
 		lk = pl_wait_unlock_long(lock, PLOCK_LORW_EXC_MASK);
+#endif
 }
 
 
@@ -1341,7 +1349,11 @@ static inline void pl_lorw_wrlock(unsigned long *lock)
 	 */
 	lk = pl_deref_long(lock);
 	if (__builtin_expect(lk & PLOCK_LORW_WRQ_MASK, 1))
+#if defined(PLOCK_LORW_INLINE_WAIT)
+		lk = __pl_wait_unlock_long(lock, PLOCK_LORW_WRQ_MASK);
+#else
 		lk = pl_wait_unlock_long(lock, PLOCK_LORW_WRQ_MASK);
+#endif
 
 	do {
 		/* let's check for the two sources of contention at once */
@@ -1354,12 +1366,20 @@ static inline void pl_lorw_wrlock(unsigned long *lock)
 				/* note below, an OR is significantly cheaper than BTS or XADD */
 				if (!(lk & PLOCK_LORW_WRQ_MASK))
 					pl_or_noret(lock, PLOCK_LORW_WRQ_BASE);
+#if defined(PLOCK_LORW_INLINE_WAIT)
+				lk = __pl_wait_unlock_long(lock, PLOCK_LORW_SHR_MASK);
+#else
 				lk = pl_wait_unlock_long(lock, PLOCK_LORW_SHR_MASK);
+#endif
 			}
 
 			/* And also wait for a previous writer to finish. */
 			if (lk & PLOCK_LORW_EXC_MASK)
+#if defined(PLOCK_LORW_INLINE_WAIT)
+				lk = __pl_wait_unlock_long(lock, PLOCK_LORW_EXC_MASK);
+#else
 				lk = pl_wait_unlock_long(lock, PLOCK_LORW_EXC_MASK);
+#endif
 		}
 
 		/* A fresh new reader may appear right now if there were none