path: root/debian/patches/0001-ipc-rework-the-kernel-message-buffer-cache.patch

From e23a941eaf5aabe087f50700213c05c6d269d63b Mon Sep 17 00:00:00 2001
From: Justus Winter <4winter@informatik.uni-hamburg.de>
Date: Wed, 17 Dec 2014 14:21:15 +0100
Subject: [PATCH gnumach 1/3] ipc: rework the kernel message buffer cache

Keep a per-processor stack of free kernel messages buffers.

* ipc/ipc_kmsg.h (IKM_CACHE_SIZE): New macro.
(struct ipc_mksg_cpu_cache): New type.
(ipc_kmsg_cache): Use the new type for the cache.
(ikm_cache): Drop macro.
(ikm_cache_get, ikm_cache_put): New functions.
(ikm_free): Return buffers instead of freeing them.
(_ikm_free): New version of `ikm_free' that only frees buffers.
* ipc/ipc_kmsg.c (ipc_kmsg_cache): Use new type.
(ipc_kmsg_get, ipc_kmsg_put): Use new functions.
* ipc/mach_msg.c (mach_msg_trap): Likewise.
* kern/exception.c (exception_raise): Likewise.
(exception_parse_reply): Likewise.
* kern/ipc_kobject.c (ipc_kobject_server): Likewise.
---
 ipc/ipc_kmsg.c     | 29 ++++---------------
 ipc/ipc_kmsg.h     | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++++--
 ipc/mach_msg.c     | 14 ++++-----
 kern/exception.c   | 28 ++++--------------
 kern/ipc_kobject.c |  6 +---
 5 files changed, 97 insertions(+), 63 deletions(-)

diff --git a/ipc/ipc_kmsg.c b/ipc/ipc_kmsg.c
index 66643fd..04cdbc9 100644
--- a/ipc/ipc_kmsg.c
+++ b/ipc/ipc_kmsg.c
@@ -73,7 +73,7 @@
 #define ptr_align(x)	\
 	( ( ((vm_offset_t)(x)) + (sizeof(vm_offset_t)-1) ) & ~(sizeof(vm_offset_t)-1) )
 
-ipc_kmsg_t ipc_kmsg_cache[NCPUS];
+struct ipc_kmsg_cpu_cache ipc_kmsg_cache[NCPUS];
 
 /*
  *	Routine:	ipc_kmsg_enqueue
@@ -506,23 +506,9 @@ ipc_kmsg_get(
 	if ((size < sizeof(mach_msg_header_t)) || (size & 3))
 		return MACH_SEND_MSG_TOO_SMALL;
 
-	if (size <= IKM_SAVED_MSG_SIZE) {
-		kmsg = ikm_cache();
-		if (kmsg != IKM_NULL) {
-			ikm_cache() = IKM_NULL;
-			ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE);
-		} else {
-			kmsg = ikm_alloc(IKM_SAVED_MSG_SIZE);
-			if (kmsg == IKM_NULL)
-				return MACH_SEND_NO_BUFFER;
-			ikm_init(kmsg, IKM_SAVED_MSG_SIZE);
-		}
-	} else {
-		kmsg = ikm_alloc(size);
-		if (kmsg == IKM_NULL)
-			return MACH_SEND_NO_BUFFER;
-		ikm_init(kmsg, size);
-	}
+	kmsg = ikm_cache_get (size);
+	if (kmsg == IKM_NULL)
+		return MACH_SEND_NO_BUFFER;
 
 	if (copyinmsg(msg, &kmsg->ikm_header, size)) {
 		ikm_free(kmsg);
@@ -599,12 +585,7 @@ ipc_kmsg_put(
 	else
 		mr = MACH_MSG_SUCCESS;
 
-	if ((kmsg->ikm_size == IKM_SAVED_KMSG_SIZE) &&
-	    (ikm_cache() == IKM_NULL))
-		ikm_cache() = kmsg;
-	else
-		ikm_free(kmsg);
-
+	ikm_cache_put (kmsg);
 	return mr;
 }
 
diff --git a/ipc/ipc_kmsg.h b/ipc/ipc_kmsg.h
index 620785b..c676996 100644
--- a/ipc/ipc_kmsg.h
+++ b/ipc/ipc_kmsg.h
@@ -96,11 +96,36 @@ MACRO_END
  *	The per-processor cache seems to miss less than a per-thread cache,
  *	and it also uses less memory.  Access to the cache doesn't
  *	require locking.
+ *
+ *	The per-processor cache is a stack containing unused kernel
+ *	message buffers.  We choose IKM_CACHE_SIZE so that the size of
+ *	the struct ipc_kmsg_cpu_cache is a multiple of a cache line,
+ *	to prevent it bouncing between per-cpu caches.
+ *
+ *	A kernel message buffer can be allocated using
+ *	`ikm_cache_get', and returned using `ikm_cache_put'.
  */
 
-extern ipc_kmsg_t	ipc_kmsg_cache[NCPUS];
+#define IKM_CACHE_SIZE	(((1 << CPU_L1_SHIFT) - sizeof (size_t))	\
+			 / sizeof (ipc_kmsg_t))
+
+struct ipc_kmsg_cpu_cache
+{
+	size_t		count;
+	ipc_kmsg_t	buffers[IKM_CACHE_SIZE];
+};
+
+extern struct ipc_kmsg_cpu_cache ipc_kmsg_cache[NCPUS];
+
+/* Return a kernel message buffer of at least SIZE size, preferably
+   from the cache.  This functions is defined below.  */
+static inline ipc_kmsg_t ikm_cache_get (mach_msg_size_t size)
+	__attribute__ ((always_inline));
 
-#define ikm_cache()     ipc_kmsg_cache[cpu_number()]
+/* Return a kernel message buffer to the cache, or free it.  This
+   functions is defined below.  */
+static inline void ikm_cache_put (ipc_kmsg_t kmsg)
+	__attribute__ ((always_inline));
 
 /*
  *	The size of the kernel message buffers that will be cached.
@@ -150,11 +175,18 @@ MACRO_BEGIN								\
 	register vm_size_t _size = (kmsg)->ikm_size;			\
 									\
 	if ((integer_t)_size > 0)					\
-		kfree((vm_offset_t) (kmsg), _size);			\
+		ikm_cache_put (kmsg);					\
 	else								\
 		ipc_kmsg_free(kmsg);					\
 MACRO_END
 
+#define	_ikm_free(kmsg)							\
+MACRO_BEGIN								\
+	vm_size_t _size = (kmsg)->ikm_size;				\
+	assert (_size != IKM_SIZE_NETWORK);				\
+	kfree((vm_offset_t) (kmsg), _size);				\
+MACRO_END
+
 /*
  *	struct ipc_kmsg_queue is defined in ipc/ipc_kmsg_queue.h
  */
@@ -280,5 +312,50 @@ ipc_kmsg_copyout_pseudo(ipc_kmsg_t, ipc_space_t, vm_map_t);
 
 extern void
 ipc_kmsg_copyout_dest(ipc_kmsg_t, ipc_space_t);
+
+static inline ipc_kmsg_t
+ikm_cache_get (mach_msg_size_t size)
+{
+	ipc_kmsg_t kmsg;
+	struct ipc_kmsg_cpu_cache *c = &ipc_kmsg_cache[cpu_number ()];
+
+	if (size < IKM_SAVED_MSG_SIZE)
+		size = IKM_SAVED_MSG_SIZE;
+
+	if (size > IKM_SAVED_MSG_SIZE
+	    || c->count == 0) {
+		kmsg = ikm_alloc (size);
+		if (kmsg)
+			ikm_init (kmsg, size);
+		return kmsg;
+	}
+
+	c->count -= 1;
+	kmsg = c->buffers[c->count];
+
+#if	MACH_IPC_TEST
+	ikm_check_initialized (kmsg, IKM_SAVED_KMSG_SIZE);
+#endif	/* MACH_IPC_TEST */
+	return kmsg;
+}
+
+static inline void
+ikm_cache_put (ipc_kmsg_t kmsg)
+{
+	struct ipc_kmsg_cpu_cache *c = &ipc_kmsg_cache[cpu_number ()];
+
+	if (kmsg->ikm_size != IKM_SAVED_KMSG_SIZE
+	    || c->count == IKM_CACHE_SIZE) {
+		_ikm_free (kmsg);
+		return;
+	}
+
+	c->buffers[c->count] = kmsg;
+	c->count += 1;
+
+#if	MACH_IPC_TEST
+	ikm_check_initialized (kmsg, IKM_SAVED_KMSG_SIZE);
+#endif	/* MACH_IPC_TEST */
+}
 
 #endif	/* _IPC_IPC_KMSG_H_ */
diff --git a/ipc/mach_msg.c b/ipc/mach_msg.c
index 1e122c7..cdf16ea 100644
--- a/ipc/mach_msg.c
+++ b/ipc/mach_msg.c
@@ -451,16 +451,14 @@ mach_msg_trap(
 
 		if ((send_size > IKM_SAVED_MSG_SIZE) ||
 		    (send_size < sizeof(mach_msg_header_t)) ||
-		    (send_size & 3) ||
-		    ((kmsg = ikm_cache()) == IKM_NULL))
+		    (send_size & 3))
 			goto slow_get;
 
-		ikm_cache() = IKM_NULL;
-		ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE);
+		kmsg = ikm_cache_get (IKM_SAVED_MSG_SIZE);
 
 		if (copyinmsg(msg, &kmsg->ikm_header,
 			      send_size)) {
-			ikm_free(kmsg);
+			ikm_cache_put (kmsg);
 			goto slow_get;
 		}
 
@@ -1263,18 +1261,16 @@ mach_msg_trap(
 		 *	We have the reply message data in kmsg,
 		 *	and the reply message size in reply_size.
 		 *	Just need to copy it out to the user and free kmsg.
-		 *	We must check ikm_cache after copyoutmsg.
 		 */
 
 		ikm_check_initialized(kmsg, kmsg->ikm_size);
 
 		if ((kmsg->ikm_size != IKM_SAVED_KMSG_SIZE) ||
 		    copyoutmsg(&kmsg->ikm_header, msg,
-			       reply_size) ||
-		    (ikm_cache() != IKM_NULL))
+			       reply_size))
 			goto slow_put;
 
-		ikm_cache() = kmsg;
+		ikm_cache_put (kmsg);
 		thread_syscall_return(MACH_MSG_SUCCESS);
 		/*NOTREACHED*/
 		return MACH_MSG_SUCCESS; /* help for the compiler */
diff --git a/kern/exception.c b/kern/exception.c
index 7954fba..72a4f29 100644
--- a/kern/exception.c
+++ b/kern/exception.c
@@ -348,16 +348,9 @@ exception_raise(
 	 *	and it will give the buffer back with its reply.
 	 */
 
-	kmsg = ikm_cache();
-	if (kmsg != IKM_NULL) {
-		ikm_cache() = IKM_NULL;
-		ikm_check_initialized(kmsg, IKM_SAVED_KMSG_SIZE);
-	} else {
-		kmsg = ikm_alloc(IKM_SAVED_MSG_SIZE);
-		if (kmsg == IKM_NULL)
-			panic("exception_raise");
-		ikm_init(kmsg, IKM_SAVED_MSG_SIZE);
-	}
+	kmsg = ikm_cache_get (IKM_SAVED_MSG_SIZE);
+	if (kmsg == IKM_NULL)
+		panic("exception_raise");
 
 	/*
 	 *	We need a reply port for the RPC.
@@ -681,22 +674,19 @@ exception_raise(
 
 	/*
 	 *	Optimized version of ipc_kmsg_put.
-	 *	We must check ikm_cache after copyoutmsg.
 	 */
-
 	ikm_check_initialized(kmsg, kmsg->ikm_size);
 	assert(kmsg->ikm_size == IKM_SAVED_KMSG_SIZE);
 
 	if (copyoutmsg(&kmsg->ikm_header, receiver->ith_msg,
-		       sizeof(struct mach_exception)) ||
-	    (ikm_cache() != IKM_NULL)) {
+		       sizeof(struct mach_exception))) {
 		mr = ipc_kmsg_put(receiver->ith_msg, kmsg,
 				  kmsg->ikm_header.msgh_size);
 		thread_syscall_return(mr);
 		/*NOTREACHED*/
 	}
 
-	ikm_cache() = kmsg;
+	ikm_cache_put (kmsg);
 	thread_syscall_return(MACH_MSG_SUCCESS);
 	/*NOTREACHED*/
 #ifndef	__GNUC__
@@ -809,13 +799,7 @@ exception_parse_reply(ipc_kmsg_t kmsg)
 	}
 
 	kr = msg->RetCode;
-
-	if ((kmsg->ikm_size == IKM_SAVED_KMSG_SIZE) &&
-	    (ikm_cache() == IKM_NULL))
-		ikm_cache() = kmsg;
-	else
-		ikm_free(kmsg);
-
+	ikm_cache_put (kmsg);
 	return kr;
 }
 
diff --git a/kern/ipc_kobject.c b/kern/ipc_kobject.c
index bf22028..27535b0 100644
--- a/kern/ipc_kobject.c
+++ b/kern/ipc_kobject.c
@@ -236,11 +236,7 @@ ipc_kobject_server(request)
 		/* like ipc_kmsg_put, but without the copyout */
 
 		ikm_check_initialized(request, request->ikm_size);
-		if ((request->ikm_size == IKM_SAVED_KMSG_SIZE) &&
-		    (ikm_cache() == IKM_NULL))
-			ikm_cache() = request;
-		else
-			ikm_free(request);
+		ikm_cache_put (request);
 	} else {
 		/*
 		 *	The message contents of the request are intact.
-- 
2.1.3