path: root/debian/patches/0005-ipc-replace-the-IPC-table-with-a-radix-tree.patch

From 44fe5f9150654752aec324c8cd71d7914748cd45 Mon Sep 17 00:00:00 2001
From: Justus Winter <4winter@informatik.uni-hamburg.de>
Date: Fri, 20 Mar 2015 00:21:14 +0100
Subject: [PATCH gnumach 5/8] ipc: replace the IPC table with a radix tree

Currently, the port names are mapped to an IPC object (e.g. a port)
using a table.  This, however, requires large chunks of continuous
memory, and leads to scalability problems as virtual kernel memory is
a scarce resource.  To avoid excessive overhead, non-contiguous port
names are spilled into a splay tree.

Replace the IPC table with a radix tree.  As the radix tree is able to
store non-contiguous names with reasonable overhead, we can drop the
splay tree as well.

* ipc/ipc_entry.c (ipc_entry_cache): New variable.
(ipc_entry_lookup): Replace with a radix tree lookup.
(ipc_entry_get): The free list handling is changed a little.  Adopt
accordingly.
(ipc_entry_free_name): New function.
(ipc_entry_alloc): Adopt accordingly.
(ipc_entry_alloc_name): Likewise.
(ipc_entry_dealloc): Likewise.
* ipc/ipc_entry.h (struct ipc_entry): Update comment, add field for
free list.
(ipc_entry_cache, ie_alloc, ie_free): New declarations.
* ipc/ipc_hash.c: We no longer use splay trees, so we do not need the
global reverse hash table.
* ipc/ipc_init.c (ipc_bootstrap): Initialize `ipc_entry' cache.
* ipc/ipc_kmsg.c (ipc_kmsg_copyout_header): Use `ipc_entry_alloc'
instead of re-coding it.
* ipc/ipc_object.c (ipc_object_copyout): Likewise.
(ipc_object_copyout_multiname): Likewise.
* ipc/ipc_space.c (ipc_space_create): Initialize radix tree and free list.
(ipc_space_destroy): Free ipc entries and radix tree.
* ipc/ipc_space.h (struct ipc_space): Add radix tree, free list, and size.
---
 ipc/ipc_entry.c  | 372 ++++++++++++++++---------------------------------------
 ipc/ipc_entry.h  |  11 +-
 ipc/ipc_hash.c   |  23 +---
 ipc/ipc_init.c   |   3 +
 ipc/ipc_kmsg.c   |  24 ++--
 ipc/ipc_object.c |  22 +---
 ipc/ipc_space.c  |  42 +++----
 ipc/ipc_space.h  |   8 +-
 8 files changed, 162 insertions(+), 343 deletions(-)

diff --git a/ipc/ipc_entry.c b/ipc/ipc_entry.c
index 5b9fd98..58d453b 100644
--- a/ipc/ipc_entry.c
+++ b/ipc/ipc_entry.c
@@ -51,6 +51,8 @@
 #include <ipc/ipc_table.h>
 #include <ipc/ipc_object.h>
 
+struct kmem_cache ipc_entry_cache;
+
 struct kmem_cache ipc_tree_entry_cache;
 
 /*
@@ -69,6 +71,7 @@ ipc_entry_tree_collision(
 	ipc_space_t	space,
 	mach_port_t	name)
 {
+	assert(!"reached");
 	mach_port_index_t index;
 	mach_port_t lower, upper;
 
@@ -100,29 +103,13 @@ ipc_entry_lookup(
 	ipc_space_t space,
 	mach_port_t name)
 {
-	mach_port_index_t index;
 	ipc_entry_t entry;
 
 	assert(space->is_active);
-
-	index = MACH_PORT_INDEX(name);
-	if (index < space->is_table_size) {
-		entry = &space->is_table[index];
-		if (IE_BITS_GEN(entry->ie_bits) != MACH_PORT_GEN(name))
-			if (entry->ie_bits & IE_BITS_COLLISION) {
-				assert(space->is_tree_total > 0);
-				goto tree_lookup;
-			} else
-				entry = IE_NULL;
-		else if (IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE)
-			entry = IE_NULL;
-	} else if (space->is_tree_total == 0)
-		entry = IE_NULL;
-	else
-	    tree_lookup:
-		entry = (ipc_entry_t)
-				ipc_splay_tree_lookup(&space->is_tree, name);
-
+	entry = rdxtree_lookup(&space->is_map, name);
+	if (entry != IE_NULL
+	    && IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE)
+		entry = NULL;
 	assert((entry == IE_NULL) || IE_BITS_TYPE(entry->ie_bits));
 	return entry;
 }
@@ -145,21 +132,18 @@ ipc_entry_get(
 	mach_port_t *namep,
 	ipc_entry_t *entryp)
 {
-	ipc_entry_t table;
-	mach_port_index_t first_free;
 	mach_port_t new_name;
 	ipc_entry_t free_entry;
 
 	assert(space->is_active);
 
-	table = space->is_table;
-	first_free = table->ie_next;
-
-	if (first_free == 0)
+	/* Get entry from the free list.  */
+	free_entry = space->is_free_list;
+	if (free_entry == IE_NULL)
 		return KERN_NO_SPACE;
 
-	free_entry = &table[first_free];
-	table->ie_next = free_entry->ie_next;
+	space->is_free_list = free_entry->ie_next_free;
+	space->is_free_list_size -= 1;
 
 	/*
 	 *	Initialize the new entry.  We need only
@@ -173,7 +157,7 @@ ipc_entry_get(
 	gen = free_entry->ie_bits + IE_BITS_GEN_ONE;
 	free_entry->ie_bits = gen;
 	free_entry->ie_request = 0;
-	new_name = MACH_PORT_MAKE(first_free, gen);
+	new_name = MACH_PORT_MAKE(free_entry->ie_name, gen);
     }
 
 	/*
@@ -186,6 +170,7 @@ ipc_entry_get(
 	assert(MACH_PORT_VALID(new_name));
 	assert(free_entry->ie_object == IO_NULL);
 
+	space->is_size += 1;
 	*namep = new_name;
 	*entryp = free_entry;
 	return KERN_SUCCESS;
@@ -212,23 +197,44 @@ ipc_entry_alloc(
 	ipc_entry_t	*entryp)
 {
 	kern_return_t kr;
+	ipc_entry_t entry;
+	unsigned long long key;
 
 	is_write_lock(space);
 
-	for (;;) {
-		if (!space->is_active) {
-			is_write_unlock(space);
-			return KERN_INVALID_TASK;
-		}
+	if (!space->is_active) {
+		is_write_unlock(space);
+		return KERN_INVALID_TASK;
+	}
+
+	kr = ipc_entry_get(space, namep, entryp);
+	if (kr == KERN_SUCCESS)
+		/* Success.  Space is write-locked.  */
+		return kr;
 
-		kr = ipc_entry_get(space, namep, entryp);
-		if (kr == KERN_SUCCESS)
-			return kr;
+	entry = ie_alloc();
+	if (entry == IE_NULL) {
+		is_write_unlock(space);
+		return KERN_RESOURCE_SHORTAGE;
+	}
 
-		kr = ipc_entry_grow_table(space);
-		if (kr != KERN_SUCCESS)
-			return kr; /* space is unlocked */
+	kr = rdxtree_insert_alloc(&space->is_map, entry, &key);
+	if (kr) {
+		is_write_unlock(space);
+		ie_free(entry);
+		return kr;
 	}
+	space->is_size += 1;
+
+	entry->ie_bits = 0;
+	entry->ie_object = IO_NULL;
+	entry->ie_request = 0;
+	entry->ie_name = (mach_port_t) key;
+
+	*entryp = entry;
+	*namep = (mach_port_t) key;
+	/* Success.  Space is write-locked.  */
+	return KERN_SUCCESS;
 }
 
 /*
@@ -252,166 +258,79 @@ ipc_entry_alloc_name(
 	mach_port_t	name,
 	ipc_entry_t	*entryp)
 {
+	kern_return_t kr;
 	mach_port_index_t index = MACH_PORT_INDEX(name);
 	mach_port_gen_t gen = MACH_PORT_GEN(name);
-	ipc_tree_entry_t tree_entry = ITE_NULL;
 
+	ipc_entry_t entry, e, *prevp;
+	void **slot;
 	assert(MACH_PORT_VALID(name));
 
-
 	is_write_lock(space);
 
-	for (;;) {
-		ipc_entry_t entry;
-		ipc_tree_entry_t tentry;
-		ipc_table_size_t its;
-
-		if (!space->is_active) {
-			is_write_unlock(space);
-			if (tree_entry) ite_free(tree_entry);
-			return KERN_INVALID_TASK;
-		}
-
-		/*
-		 *	If we are under the table cutoff,
-		 *	there are three cases:
-		 *		1) The entry is inuse, for the same name
-		 *		2) The entry is inuse, for a different name
-		 *		3) The entry is free
-		 */
-
-		if ((0 < index) && (index < space->is_table_size)) {
-			ipc_entry_t table = space->is_table;
-
-			entry = &table[index];
-
-			if (IE_BITS_TYPE(entry->ie_bits)) {
-				if (IE_BITS_GEN(entry->ie_bits) == gen) {
-					*entryp = entry;
-					if (tree_entry) ite_free(tree_entry);
-					return KERN_SUCCESS;
-				}
-			} else {
-				mach_port_index_t free_index, next_index;
-
-				/*
-				 *	Rip the entry out of the free list.
-				 */
-
-				for (free_index = 0;
-				     (next_index = table[free_index].ie_next)
-							!= index;
-				     free_index = next_index)
-					continue;
-
-				table[free_index].ie_next =
-					table[next_index].ie_next;
-
-				entry->ie_bits = gen;
-				assert(entry->ie_object == IO_NULL);
-				entry->ie_request = 0;
-
-				*entryp = entry;
-				if (tree_entry) ite_free(tree_entry);
-				return KERN_SUCCESS;
-			}
-		}
-
-		/*
-		 *	Before trying to allocate any memory,
-		 *	check if the entry already exists in the tree.
-		 *	This avoids spurious resource errors.
-		 *	The splay tree makes a subsequent lookup/insert
-		 *	of the same name cheap, so this costs little.
-		 */
+	if (!space->is_active) {
+		is_write_unlock(space);
+		return KERN_INVALID_TASK;
+	}
 
-		if ((space->is_tree_total > 0) &&
-		    ((tentry = ipc_splay_tree_lookup(&space->is_tree, name))
-							!= ITE_NULL)) {
-			assert(tentry->ite_space == space);
-			assert(IE_BITS_TYPE(tentry->ite_bits));
+	slot = rdxtree_lookup_slot(&space->is_map, name);
+	if (slot != NULL)
+		entry = *(ipc_entry_t *) slot;
 
-			*entryp = &tentry->ite_entry;
-			if (tree_entry) ite_free(tree_entry);
-			return KERN_SUCCESS;
+	if (slot == NULL || entry == IE_NULL) {
+		entry = ie_alloc();
+		if (entry == IE_NULL) {
+			is_write_unlock(space);
+			return KERN_RESOURCE_SHORTAGE;
 		}
 
-		its = space->is_table_next;
+		entry->ie_bits = 0;
+		entry->ie_object = IO_NULL;
+		entry->ie_request = 0;
+		entry->ie_name = name;
 
-		/*
-		 *	Check if the table should be grown.
-		 *
-		 *	Note that if space->is_table_size == its->its_size,
-		 *	then we won't ever try to grow the table.
-		 *
-		 *	Note that we are optimistically assuming that name
-		 *	doesn't collide with any existing names.  (So if
-		 *	it were entered into the tree, is_tree_small would
-		 *	be incremented.)  This is OK, because even in that
-		 *	case, we don't lose memory by growing the table.
-		 */
-
-		if ((space->is_table_size <= index) &&
-		    (index < its->its_size) &&
-		    (((its->its_size - space->is_table_size) *
-		      sizeof(struct ipc_entry)) <
-		     ((space->is_tree_small + 1) *
-		      sizeof(struct ipc_tree_entry)))) {
-			kern_return_t kr;
-
-			/*
-			 *	Can save space by growing the table.
-			 *	Because the space will be unlocked,
-			 *	we must restart.
-			 */
-
-			kr = ipc_entry_grow_table(space);
-			assert(kr != KERN_NO_SPACE);
+		if (slot != NULL)
+			rdxtree_replace_slot(slot, entry);
+		else {
+			kr = rdxtree_insert(&space->is_map, name, entry);
 			if (kr != KERN_SUCCESS) {
-				/* space is unlocked */
-				if (tree_entry) ite_free(tree_entry);
+				is_write_unlock(space);
+				ie_free(entry);
 				return kr;
 			}
-
-			continue;
 		}
+		space->is_size += 1;
 
-		/*
-		 *	If a splay-tree entry was allocated previously,
-		 *	go ahead and insert it into the tree.
-		 */
+		*entryp = entry;
+		/* Success.  Space is write-locked.  */
+		return KERN_SUCCESS;
+	}
 
-		if (tree_entry != ITE_NULL) {
-			space->is_tree_total++;
+	if (IE_BITS_TYPE(entry->ie_bits)) {
+		/* Used entry.  */
+		*entryp = entry;
+		/* Success.  Space is write-locked.  */
+		return KERN_SUCCESS;
+	}
 
-			if (index < space->is_table_size)
-				space->is_table[index].ie_bits |=
-					IE_BITS_COLLISION;
-			else if ((index < its->its_size) &&
-				 !ipc_entry_tree_collision(space, name))
-				space->is_tree_small++;
+	/* Free entry.  Rip the entry out of the free list.  */
+	for (prevp = &space->is_free_list, e = space->is_free_list;
+	     e != entry;
+	     ({ prevp = &e->ie_next_free; e = e->ie_next_free; }))
+		continue;
 
-			ipc_splay_tree_insert(&space->is_tree,
-					      name, tree_entry);
+	*prevp = entry->ie_next_free;
+	space->is_free_list_size -= 1;
 
-			tree_entry->ite_bits = 0;
-			tree_entry->ite_object = IO_NULL;
-			tree_entry->ite_request = 0;
-			tree_entry->ite_space = space;
-			*entryp = &tree_entry->ite_entry;
-			return KERN_SUCCESS;
-		}
-
-		/*
-		 *	Allocate a tree entry and try again.
-		 */
+	entry->ie_bits = gen;
+	assert(entry->ie_object == IO_NULL);
+	assert(entry->ie_name == name);
+	entry->ie_request = 0;
 
-		is_write_unlock(space);
-		tree_entry = ite_alloc();
-		if (tree_entry == ITE_NULL)
-			return KERN_RESOURCE_SHORTAGE;
-		is_write_lock(space);
-	}
+	space->is_size += 1;
+	*entryp = entry;
+	/* Success.  Space is write-locked.  */
+	return KERN_SUCCESS;
 }
 
 /*
@@ -429,100 +348,20 @@ ipc_entry_dealloc(
 	mach_port_t	name,
 	ipc_entry_t	entry)
 {
-	ipc_entry_t table;
-	ipc_entry_num_t size;
-	mach_port_index_t index;
-
 	assert(space->is_active);
 	assert(entry->ie_object == IO_NULL);
 	assert(entry->ie_request == 0);
 
-	index = MACH_PORT_INDEX(name);
-	table = space->is_table;
-	size = space->is_table_size;
-
-	if ((index < size) && (entry == &table[index])) {
-		assert(IE_BITS_GEN(entry->ie_bits) == MACH_PORT_GEN(name));
-
-		if (entry->ie_bits & IE_BITS_COLLISION) {
-			struct ipc_splay_tree small, collisions;
-			ipc_tree_entry_t tentry;
-			mach_port_t tname;
-			boolean_t pick;
-			ipc_entry_bits_t bits;
-			ipc_object_t obj;
-
-			/* must move an entry from tree to table */
-
-			ipc_splay_tree_split(&space->is_tree,
-					     MACH_PORT_MAKE(index+1, 0),
-					     &collisions);
-			ipc_splay_tree_split(&collisions,
-					     MACH_PORT_MAKE(index, 0),
-					     &small);
-
-			pick = ipc_splay_tree_pick(&collisions,
-						   &tname, &tentry);
-			assert(pick);
-			assert(MACH_PORT_INDEX(tname) == index);
-
-			bits = tentry->ite_bits;
-			entry->ie_bits = bits | MACH_PORT_GEN(tname);
-			entry->ie_object = obj = tentry->ite_object;
-			entry->ie_request = tentry->ite_request;
-			assert(tentry->ite_space == space);
-
-			if (IE_BITS_TYPE(bits) == MACH_PORT_TYPE_SEND) {
-				ipc_hash_global_delete(space, obj,
-						       tname, tentry);
-				ipc_hash_local_insert(space, obj,
-						      index, entry);
-			}
-
-			ipc_splay_tree_delete(&collisions, tname, tentry);
-
-			assert(space->is_tree_total > 0);
-			space->is_tree_total--;
-
-			/* check if collision bit should still be on */
-
-			pick = ipc_splay_tree_pick(&collisions,
-						   &tname, &tentry);
-			if (pick) {
-				entry->ie_bits |= IE_BITS_COLLISION;
-				ipc_splay_tree_join(&space->is_tree,
-						    &collisions);
-			}
-
-			ipc_splay_tree_join(&space->is_tree, &small);
-		} else {
-			entry->ie_bits &= IE_BITS_GEN_MASK;
-			entry->ie_next = table->ie_next;
-			table->ie_next = index;
-		}
+	if (space->is_free_list_size < IS_FREE_LIST_SIZE_LIMIT) {
+		space->is_free_list_size += 1;
+		entry->ie_bits &= IE_BITS_GEN_MASK;
+		entry->ie_next_free = space->is_free_list;
+		space->is_free_list = entry;
 	} else {
-		ipc_tree_entry_t tentry = (ipc_tree_entry_t) entry;
-
-		assert(tentry->ite_space == space);
-
-		ipc_splay_tree_delete(&space->is_tree, name, tentry);
-
-		assert(space->is_tree_total > 0);
-		space->is_tree_total--;
-
-		if (index < size) {
-			ipc_entry_t ientry = &table[index];
-
-			assert(ientry->ie_bits & IE_BITS_COLLISION);
-
-			if (!ipc_entry_tree_collision(space, name))
-				ientry->ie_bits &= ~IE_BITS_COLLISION;
-		} else if ((index < space->is_table_next->its_size) &&
-			   !ipc_entry_tree_collision(space, name)) {
-			assert(space->is_tree_small > 0);
-			space->is_tree_small--;
-		}
+		rdxtree_remove(&space->is_map, (unsigned long long) name);
+		ie_free(entry);
 	}
+	space->is_size -= 1;
 }
 
 /*
@@ -544,6 +383,7 @@ ipc_entry_dealloc(
 kern_return_t
 ipc_entry_grow_table(ipc_space_t space)
 {
+	assert(!"reached");
 	ipc_entry_num_t osize, size, nsize;
 
 	do {
diff --git a/ipc/ipc_entry.h b/ipc/ipc_entry.h
index 0caa70b..09e1223 100644
--- a/ipc/ipc_entry.h
+++ b/ipc/ipc_entry.h
@@ -56,10 +56,7 @@
  *
  *	Free (unallocated) entries in the table have null ie_object
  *	fields.  The ie_bits field is zero except for IE_BITS_GEN.
- *	The ie_next (ie_request) field links free entries into a free list.
- *
- *	The first entry in the table (index 0) is always free.
- *	It is used as the head of the free list.
+ *	The ie_next_free field links free entries into a free list.
  */
 
 typedef unsigned int ipc_entry_bits_t;
@@ -69,6 +66,7 @@ typedef struct ipc_entry {
 	ipc_entry_bits_t ie_bits;
 	struct ipc_object *ie_object;
 	union {
+		struct ipc_entry *next_free;
 		mach_port_index_t next;
 		/*XXX ipc_port_request_index_t request;*/
 		unsigned int request;
@@ -84,6 +82,8 @@ typedef struct ipc_entry {
 #define	ie_request	index.request
 #define	ie_next		index.next
 #define	ie_index	hash.table
+#define	ie_name		hash.table
+#define	ie_next_free	index.next_free
 
 #define	IE_BITS_UREFS_MASK	0x0000ffff	/* 16 bits of user-reference */
 #define	IE_BITS_UREFS(bits)	((bits) & IE_BITS_UREFS_MASK)
@@ -129,6 +129,9 @@ extern struct kmem_cache ipc_tree_entry_cache;
 #define ite_alloc()	((ipc_tree_entry_t) kmem_cache_alloc(&ipc_tree_entry_cache))
 #define	ite_free(ite)	kmem_cache_free(&ipc_tree_entry_cache, (vm_offset_t) (ite))
 
+extern struct kmem_cache ipc_entry_cache;
+#define ie_alloc()	((ipc_entry_t) kmem_cache_alloc(&ipc_entry_cache))
+#define	ie_free(e)	kmem_cache_free(&ipc_entry_cache, (vm_offset_t) (e))
 
 extern ipc_entry_t
 ipc_entry_lookup(ipc_space_t space, mach_port_t name);
diff --git a/ipc/ipc_hash.c b/ipc/ipc_hash.c
index 87952a7..682b854 100644
--- a/ipc/ipc_hash.c
+++ b/ipc/ipc_hash.c
@@ -70,10 +70,7 @@ ipc_hash_lookup(
 	mach_port_t *namep,
 	ipc_entry_t *entryp)
 {
-	return (ipc_hash_local_lookup(space, obj, namep, entryp) ||
-		((space->is_tree_hash > 0) &&
-		 ipc_hash_global_lookup(space, obj, namep,
-					(ipc_tree_entry_t *) entryp)));
+	return ipc_hash_local_lookup(space, obj, namep, entryp);
 }
 
 /*
@@ -95,12 +92,7 @@ ipc_hash_insert(
 	mach_port_index_t index;
 
 	index = MACH_PORT_INDEX(name);
-	if ((index < space->is_table_size) &&
-	    (entry == &space->is_table[index]))
-		ipc_hash_local_insert(space, obj, index, entry);
-	else
-		ipc_hash_global_insert(space, obj, name,
-				       (ipc_tree_entry_t) entry);
+	ipc_hash_local_insert(space, obj, index, entry);
 }
 
 /*
@@ -121,12 +113,7 @@ ipc_hash_delete(
 	mach_port_index_t index;
 
 	index = MACH_PORT_INDEX(name);
-	if ((index < space->is_table_size) &&
-	    (entry == &space->is_table[index]))
-		ipc_hash_local_delete(space, obj, index, entry);
-	else
-		ipc_hash_global_delete(space, obj, name,
-				       (ipc_tree_entry_t) entry);
+	ipc_hash_local_delete(space, obj, index, entry);
 }
 
 /*
@@ -174,6 +161,7 @@ ipc_hash_global_lookup(
 	mach_port_t		*namep,
 	ipc_tree_entry_t	*entryp)
 {
+	assert(!"reached");
 	ipc_hash_global_bucket_t bucket;
 	ipc_tree_entry_t this, *last;
 
@@ -227,6 +215,7 @@ ipc_hash_global_insert(
 	mach_port_t		name,
 	ipc_tree_entry_t	entry)
 {
+	assert(!"reached");
 	ipc_hash_global_bucket_t bucket;
 
 
@@ -265,6 +254,7 @@ ipc_hash_global_delete(
 	mach_port_t		name,
 	ipc_tree_entry_t	entry)
 {
+	assert(!"reached");
 	ipc_hash_global_bucket_t bucket;
 	ipc_tree_entry_t this, *last;
 
@@ -307,7 +297,6 @@ ipc_hash_global_delete(
 	       IE_BITS_TYPE((E)->ie_bits) == MACH_PORT_TYPE_NONE);	\
 	assert((E)->ie_object == (O));					\
 	assert((E)->ie_index == (N));					\
-	assert(&(S)->is_table[N] == (E));				\
 	MACRO_END
 
 /*
diff --git a/ipc/ipc_init.c b/ipc/ipc_init.c
index debda47..096e0fb 100644
--- a/ipc/ipc_init.c
+++ b/ipc/ipc_init.c
@@ -79,6 +79,9 @@ ipc_bootstrap(void)
 	kmem_cache_init(&ipc_tree_entry_cache, "ipc_tree_entry",
 			sizeof(struct ipc_tree_entry), 0, NULL, NULL, NULL, 0);
 
+	kmem_cache_init(&ipc_entry_cache, "ipc_entry",
+			sizeof(struct ipc_entry), 0, NULL, NULL, NULL, 0);
+
 	kmem_cache_init(&ipc_object_caches[IOT_PORT], "ipc_port",
 			sizeof(struct ipc_port), 0, NULL, NULL, NULL, 0);
 
diff --git a/ipc/ipc_kmsg.c b/ipc/ipc_kmsg.c
index a4a3b42..c484a5d 100644
--- a/ipc/ipc_kmsg.c
+++ b/ipc/ipc_kmsg.c
@@ -1998,28 +1998,20 @@ ipc_kmsg_copyout_header(
 				goto copyout_dest;
 			}
 
-			kr = ipc_entry_get(space, &reply_name, &entry);
+			kr = ipc_entry_alloc(space, &reply_name, &entry);
 			if (kr != KERN_SUCCESS) {
 				ip_unlock(reply);
 
 				if (notify_port != IP_NULL)
 					ipc_port_release_sonce(notify_port);
 
-				/* space is locked */
-				kr = ipc_entry_grow_table(space);
-				if (kr != KERN_SUCCESS) {
-					/* space is unlocked */
-
-					if (kr == KERN_RESOURCE_SHORTAGE)
-						return (MACH_RCV_HEADER_ERROR|
-							MACH_MSG_IPC_KERNEL);
-					else
-						return (MACH_RCV_HEADER_ERROR|
-							MACH_MSG_IPC_SPACE);
-				}
-				/* space is locked again; start over */
-
-				continue;
+				is_write_unlock(space);
+				if (kr == KERN_RESOURCE_SHORTAGE)
+					return (MACH_RCV_HEADER_ERROR|
+					        MACH_MSG_IPC_KERNEL);
+				else
+					return (MACH_RCV_HEADER_ERROR|
+					        MACH_MSG_IPC_SPACE);
 			}
 
 			assert(IE_BITS_TYPE(entry->ie_bits)
diff --git a/ipc/ipc_object.c b/ipc/ipc_object.c
index db6ef01..ec40062 100644
--- a/ipc/ipc_object.c
+++ b/ipc/ipc_object.c
@@ -630,15 +630,10 @@ ipc_object_copyout(
 			break;
 		}
 
-		kr = ipc_entry_get(space, &name, &entry);
+		kr = ipc_entry_alloc(space, &name, &entry);
 		if (kr != KERN_SUCCESS) {
-			/* unlocks/locks space, so must start again */
-
-			kr = ipc_entry_grow_table(space);
-			if (kr != KERN_SUCCESS)
-				return kr; /* space is unlocked */
-
-			continue;
+			is_write_unlock(space);
+			return kr;
 		}
 
 		assert(IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE);
@@ -691,15 +686,10 @@ ipc_object_copyout_multiname(space, object, namep)
 			return KERN_INVALID_TASK;
 		}
 
-		kr = ipc_entry_get(space, &name, &entry);
+		kr = ipc_entry_alloc(space, &name, &entry);
 		if (kr != KERN_SUCCESS) {
-			/* unlocks/locks space, so must start again */
-
-			kr = ipc_entry_grow_table(space);
-			if (kr != KERN_SUCCESS)
-				return kr; /* space is unlocked */
-
-			continue;
+			is_write_unlock(space);
+			return kr; /* space is unlocked */
 		}
 
 		assert(IE_BITS_TYPE(entry->ie_bits) == MACH_PORT_TYPE_NONE);
diff --git a/ipc/ipc_space.c b/ipc/ipc_space.c
index dcc96b3..5f939bb 100644
--- a/ipc/ipc_space.c
+++ b/ipc/ipc_space.c
@@ -82,6 +82,9 @@ ipc_space_release(
 	ipc_space_release_macro(space);
 }
 
+/* A place-holder object for the zeroth entry.  */
+struct ipc_entry zero_entry;
+
 /*
  *	Routine:	ipc_space_create
  *	Purpose:
@@ -148,7 +151,13 @@ ipc_space_create(
 	space->is_tree_total = 0;
 	space->is_tree_small = 0;
 	space->is_tree_hash = 0;
+	rdxtree_init(&space->is_map);
 	rdxtree_init(&space->is_reverse_map);
+	/* The zeroth entry is reserved.  */
+	rdxtree_insert(&space->is_map, 0, &zero_entry);
+	space->is_size = 1;
+	space->is_free_list = NULL;
+	space->is_free_list_size = 0;
 
 	*spacep = space;
 	return KERN_SUCCESS;
@@ -218,30 +227,12 @@ ipc_space_destroy(
 	if (!active)
 		return;
 
-	/*
-	 *	If somebody is trying to grow the table,
-	 *	we must wait until they finish and figure
-	 *	out the space died.
-	 */
-
-	is_read_lock(space);
-	while (space->is_growing) {
-		assert_wait((event_t) space, FALSE);
-		is_read_unlock(space);
-		thread_block((void (*)(void)) 0);
-		is_read_lock(space);
-	}
-	is_read_unlock(space);
-
-	/*
-	 *	Now we can futz with it	without having it locked.
-	 */
+	ipc_entry_t entry;
+	struct rdxtree_iter iter;
+	rdxtree_for_each(&space->is_map, &iter, entry) {
+		if (entry->ie_name == MACH_PORT_NULL)
+			continue;
 
-	table = space->is_table;
-	size = space->is_table_size;
-
-	for (index = 0; index < size; index++) {
-		ipc_entry_t entry = &table[index];
 		mach_port_type_t type = IE_BITS_TYPE(entry->ie_bits);
 
 		if (type != MACH_PORT_TYPE_NONE) {
@@ -272,6 +263,11 @@ ipc_space_destroy(
 	}
 	ipc_splay_traverse_finish(&space->is_tree);
 
+	ipc_entry_t entry;
+	struct rdxtree_iter iter;
+	rdxtree_for_each(&space->is_map, &iter, entry)
+		ie_free(entry);
+	rdxtree_remove_all(&space->is_map);
 	rdxtree_remove_all(&space->is_reverse_map);
 
 	/*
diff --git a/ipc/ipc_space.h b/ipc/ipc_space.h
index 04eb0dd..17d5b75 100644
--- a/ipc/ipc_space.h
+++ b/ipc/ipc_space.h
@@ -81,10 +81,16 @@ struct ipc_space {
 	ipc_entry_num_t is_tree_total;	/* number of entries in the tree */
 	ipc_entry_num_t is_tree_small;	/* # of small entries in the tree */
 	ipc_entry_num_t is_tree_hash;	/* # of hashed entries in the tree */
+	struct rdxtree is_map;		/* a map of entries */
+	size_t is_size;			/* number of entries */
 	struct rdxtree is_reverse_map;	/* maps objects to entries */
-
+	ipc_entry_t is_free_list;	/* a linked list of free entries */
+	size_t is_free_list_size;	/* number of free entries */
+#define IS_FREE_LIST_SIZE_LIMIT	64	/* maximum number of entries
+					   in the free list */
 };
 
+
 #define	IS_NULL			((ipc_space_t) 0)
 
 extern struct kmem_cache ipc_space_cache;
-- 
2.1.4