ipc: undo manual inlining of `ipc_entry_X' functions

Today we can rely on the compiler to inline functions.  Undoing this
manual optimization is a first step to replace the IPC tables.

* ipc/mach_msg.c (mach_msg_trap): Undo the manual inlining of
`ipc_entry_lookup', `ipc_entry_dealloc', and `ipc_entry_get'.
* ipc/ipc_kmsg.c (ipc_kmsg_copyin_header, ipc_kmsg_copyout_header): Likewise.
* kern/exception.c (exception_raise): Likewise.
* kern/ipc_mig.c (fast_send_right_lookup): Likewise.

2 files changed, 46 insertions, 200 deletions

diff --git a/ipc/ipc_kmsg.c b/ipc/ipc_kmsg.c
index c0f07dd..cae700f 100644
--- a/ipc/ipc_kmsg.c
+++ b/ipc/ipc_kmsg.c
@@ -698,24 +698,14 @@ ipc_kmsg_copyin_header(
 		if (!space->is_active)
 			goto abort_async;
 
-		/* optimized ipc_entry_lookup */
-
-	    {
-		mach_port_index_t index = MACH_PORT_INDEX(dest_name);
-		mach_port_gen_t gen = MACH_PORT_GEN(dest_name);
-
-		if (index >= space->is_table_size)
+		entry = ipc_entry_lookup (space, dest_name);
+		if (entry == IE_NULL)
 			goto abort_async;
-
-		entry = &space->is_table[index];
 		bits = entry->ie_bits;
 
-		/* check generation number and type bit */
-
-		if ((bits & (IE_BITS_GEN_MASK|MACH_PORT_TYPE_SEND)) !=
-		    (gen | MACH_PORT_TYPE_SEND))
+		/* check type bits */
+		if (IE_BITS_TYPE (bits) != MACH_PORT_TYPE_SEND)
 			goto abort_async;
-	    }
 
 		/* optimized ipc_right_copyin */
 
@@ -750,8 +740,6 @@ ipc_kmsg_copyin_header(
 
 	    case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND,
 				MACH_MSG_TYPE_MAKE_SEND_ONCE): {
-		ipc_entry_num_t size;
-		ipc_entry_t table;
 		ipc_entry_t entry;
 		ipc_entry_bits_t bits;
 		ipc_port_t dest_port, reply_port;
@@ -762,51 +750,28 @@ ipc_kmsg_copyin_header(
 		if (!space->is_active)
 			goto abort_request;
 
-		size = space->is_table_size;
-		table = space->is_table;
-
-		/* optimized ipc_entry_lookup of dest_name */
-
-	    {
-		mach_port_index_t index = MACH_PORT_INDEX(dest_name);
-		mach_port_gen_t gen = MACH_PORT_GEN(dest_name);
-
-		if (index >= size)
+		entry = ipc_entry_lookup (space, dest_name);
+		if (entry == IE_NULL)
 			goto abort_request;
-
-		entry = &table[index];
 		bits = entry->ie_bits;
 
-		/* check generation number and type bit */
-
-		if ((bits & (IE_BITS_GEN_MASK|MACH_PORT_TYPE_SEND)) !=
-		    (gen | MACH_PORT_TYPE_SEND))
+		/* check type bits */
+		if (IE_BITS_TYPE (bits) != MACH_PORT_TYPE_SEND)
 			goto abort_request;
-	    }
 
 		assert(IE_BITS_UREFS(bits) > 0);
 
 		dest_port = (ipc_port_t) entry->ie_object;
 		assert(dest_port != IP_NULL);
 
-		/* optimized ipc_entry_lookup of reply_name */
-
-	    {
-		mach_port_index_t index = MACH_PORT_INDEX(reply_name);
-		mach_port_gen_t gen = MACH_PORT_GEN(reply_name);
-
-		if (index >= size)
+		entry = ipc_entry_lookup (space, reply_name);
+		if (entry == IE_NULL)
 			goto abort_request;
-
-		entry = &table[index];
 		bits = entry->ie_bits;
 
-		/* check generation number and type bit */
-
-		if ((bits & (IE_BITS_GEN_MASK|MACH_PORT_TYPE_RECEIVE)) !=
-		    (gen | MACH_PORT_TYPE_RECEIVE))
+		/* check type bits */
+		if (IE_BITS_TYPE (bits) != MACH_PORT_TYPE_RECEIVE)
 			goto abort_request;
-	    }
 
 		reply_port = (ipc_port_t) entry->ie_object;
 		assert(reply_port != IP_NULL);
@@ -853,9 +818,6 @@ ipc_kmsg_copyin_header(
 	    }
 
 	    case MACH_MSGH_BITS(MACH_MSG_TYPE_MOVE_SEND_ONCE, 0): {
-		mach_port_index_t index;
-		mach_port_gen_t gen;
-		ipc_entry_t table;
 		ipc_entry_t entry;
 		ipc_entry_bits_t bits;
 		ipc_port_t dest_port;
@@ -869,24 +831,13 @@ ipc_kmsg_copyin_header(
 		if (!space->is_active)
 			goto abort_reply;
 
-		/* optimized ipc_entry_lookup */
-
-		table = space->is_table;
-
-		index = MACH_PORT_INDEX(dest_name);
-		gen = MACH_PORT_GEN(dest_name);
-
-		if (index >= space->is_table_size)
+		entry = ipc_entry_lookup (space, dest_name);
+		if (entry == IE_NULL)
 			goto abort_reply;
-
-		entry = &table[index];
 		bits = entry->ie_bits;
 
-		/* check generation number, collision bit, and type bit */
-
-		if ((bits & (IE_BITS_GEN_MASK|IE_BITS_COLLISION|
-			     MACH_PORT_TYPE_SEND_ONCE)) !=
-		    (gen | MACH_PORT_TYPE_SEND_ONCE))
+		/* check and type bits */
+		if (IE_BITS_TYPE (bits) != MACH_PORT_TYPE_SEND_ONCE)
 			goto abort_reply;
 
 		/* optimized ipc_right_copyin */
@@ -910,12 +861,8 @@ ipc_kmsg_copyin_header(
 		assert(dest_port->ip_sorights > 0);
 		ip_unlock(dest_port);
 
-		/* optimized ipc_entry_dealloc */
-
-		entry->ie_next = table->ie_next;
-		table->ie_next = index;
-		entry->ie_bits = gen;
 		entry->ie_object = IO_NULL;
+		ipc_entry_dealloc (space, dest_name, entry);
 		is_write_unlock(space);
 
 		msg->msgh_bits = (MACH_MSGH_BITS_OTHER(mbits) |
@@ -1815,8 +1762,6 @@ ipc_kmsg_copyout_header(
 
 	    case MACH_MSGH_BITS(MACH_MSG_TYPE_PORT_SEND,
 				MACH_MSG_TYPE_PORT_SEND_ONCE): {
-		ipc_entry_t table;
-		mach_port_index_t index;
 		ipc_entry_t entry;
 		ipc_port_t reply = (ipc_port_t) msg->msgh_local_port;
 		mach_port_t dest_name, reply_name;
@@ -1829,8 +1774,7 @@ ipc_kmsg_copyout_header(
 			break;
 
 		is_write_lock(space);
-		if (!space->is_active ||
-		    ((index = (table = space->is_table)->ie_next) == 0)) {
+		if (!space->is_active || space->is_table->ie_next == 0) {
 			is_write_unlock(space);
 			break;
 		}
@@ -1860,11 +1804,14 @@ ipc_kmsg_copyout_header(
 		assert(reply->ip_sorights > 0);
 		ip_unlock(reply);
 
-		/* optimized ipc_entry_get */
-
-		entry = &table[index];
-		table->ie_next = entry->ie_next;
-		entry->ie_request = 0;
+		kern_return_t kr;
+		kr = ipc_entry_get (space, &reply_name, &entry);
+		if (kr) {
+			ip_unlock(reply);
+			ip_unlock(dest);
+			is_write_unlock(space);
+			break;
+		}
 
 	    {
 		mach_port_gen_t gen;
@@ -1872,8 +1819,6 @@ ipc_kmsg_copyout_header(
 		assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
 		gen = entry->ie_bits + IE_BITS_GEN_ONE;
 
-		reply_name = MACH_PORT_MAKE(index, gen);
-
 		/* optimized ipc_right_copyout */
 
 		entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);
diff --git a/ipc/mach_msg.c b/ipc/mach_msg.c
index aecfcd4..fe0c43e 100644
--- a/ipc/mach_msg.c
+++ b/ipc/mach_msg.c
@@ -482,16 +482,7 @@ mach_msg_trap(
 		switch (kmsg->ikm_header.msgh_bits) {
 		    case MACH_MSGH_BITS(MACH_MSG_TYPE_COPY_SEND,
 					MACH_MSG_TYPE_MAKE_SEND_ONCE): {
-			ipc_entry_t table;
-			ipc_entry_num_t size;
 			ipc_port_t reply_port;
-
-			/* sending a request message */
-
-		    {
-			mach_port_index_t index;
-			mach_port_gen_t gen;
-
 		    {
 			mach_port_t reply_name =
 				kmsg->ikm_header.msgh_local_port;
@@ -499,68 +490,30 @@ mach_msg_trap(
 			if (reply_name != rcv_name)
 				goto slow_copyin;
 
-			/* optimized ipc_entry_lookup of reply_name */
-
-			index = MACH_PORT_INDEX(reply_name);
-			gen = MACH_PORT_GEN(reply_name);
-		    }
-
 			is_read_lock(space);
 			assert(space->is_active);
 
-			size = space->is_table_size;
-			table = space->is_table;
-
-			if (index >= size)
-				goto abort_request_copyin;
-
-		    {
 			ipc_entry_t entry;
-			ipc_entry_bits_t bits;
-
-			entry = &table[index];
-			bits = entry->ie_bits;
-
-			/* check generation number and type bit */
-
-			if ((bits & (IE_BITS_GEN_MASK|
-				     MACH_PORT_TYPE_RECEIVE)) !=
-			    (gen | MACH_PORT_TYPE_RECEIVE))
+			entry = ipc_entry_lookup (space, reply_name);
+			if (entry == IE_NULL)
 				goto abort_request_copyin;
-
 			reply_port = (ipc_port_t) entry->ie_object;
 			assert(reply_port != IP_NULL);
 		    }
-		    }
-
-			/* optimized ipc_entry_lookup of dest_name */
-
-		    {
-			mach_port_index_t index;
-			mach_port_gen_t gen;
 
 		    {
 			mach_port_t dest_name =
 				kmsg->ikm_header.msgh_remote_port;
 
-			index = MACH_PORT_INDEX(dest_name);
-			gen = MACH_PORT_GEN(dest_name);
-		    }
-
-			if (index >= size)
-				goto abort_request_copyin;
-
-		    {
 			ipc_entry_t entry;
 			ipc_entry_bits_t bits;
-
-			entry = &table[index];
+			entry = ipc_entry_lookup (space, dest_name);
+			if (entry == IE_NULL)
+				goto abort_request_copyin;
 			bits = entry->ie_bits;
 
-			/* check generation number and type bit */
-
-			if ((bits & (IE_BITS_GEN_MASK|MACH_PORT_TYPE_SEND)) !=
-			    (gen | MACH_PORT_TYPE_SEND))
+			/* check type bits */
+			if (IE_BITS_TYPE (bits) != MACH_PORT_TYPE_SEND)
 				goto abort_request_copyin;
 
 			assert(IE_BITS_UREFS(bits) > 0);
@@ -568,7 +521,6 @@ mach_msg_trap(
 			dest_port = (ipc_port_t) entry->ie_object;
 			assert(dest_port != IP_NULL);
 		    }
-		    }
 
 			/*
 			 *	To do an atomic copyin, need simultaneous
@@ -649,9 +601,6 @@ mach_msg_trap(
 		    }
 
 		    case MACH_MSGH_BITS(MACH_MSG_TYPE_MOVE_SEND_ONCE, 0): {
-			ipc_entry_num_t size;
-			ipc_entry_t table;
-
 			/* sending a reply message */
 
 		    {
@@ -665,35 +614,18 @@ mach_msg_trap(
 			is_write_lock(space);
 			assert(space->is_active);
 
-			/* optimized ipc_entry_lookup */
-
-			size = space->is_table_size;
-			table = space->is_table;
-
 		    {
 			ipc_entry_t entry;
-			mach_port_gen_t gen;
-			mach_port_index_t index;
-
-		    {
 			mach_port_t dest_name =
 				kmsg->ikm_header.msgh_remote_port;
 
-			index = MACH_PORT_INDEX(dest_name);
-			gen = MACH_PORT_GEN(dest_name);
-		    }
-
-			if (index >= size)
+			entry = ipc_entry_lookup (space, dest_name);
+			if (entry == IE_NULL)
 				goto abort_reply_dest_copyin;
 
-			entry = &table[index];
-
-			/* check generation, collision bit, and type bit */
-
-			if ((entry->ie_bits & (IE_BITS_GEN_MASK|
-					       IE_BITS_COLLISION|
-					       MACH_PORT_TYPE_SEND_ONCE)) !=
-			    (gen | MACH_PORT_TYPE_SEND_ONCE))
+			/* check type bits */
+			if (IE_BITS_TYPE (entry->ie_bits) !=
+			    MACH_PORT_TYPE_SEND_ONCE)
 				goto abort_reply_dest_copyin;
 
 			/* optimized ipc_right_copyin */
@@ -716,13 +648,8 @@ mach_msg_trap(
 			}
 
 			assert(dest_port->ip_sorights > 0);
-
-			/* optimized ipc_entry_dealloc */
-
-			entry->ie_next = table->ie_next;
-			table->ie_next = index;
-			entry->ie_bits = gen;
 			entry->ie_object = IO_NULL;
+			ipc_entry_dealloc (space, dest_name, entry);
 		    }
 
 			kmsg->ikm_header.msgh_bits =
@@ -735,31 +662,16 @@ mach_msg_trap(
 
 			assert(dest_port->ip_receiver != ipc_space_kernel);
 
-			/* optimized ipc_entry_lookup/ipc_mqueue_copyin */
+			/* optimized ipc_mqueue_copyin */
 
 		    {
 			ipc_entry_t entry;
 			ipc_entry_bits_t bits;
-
-		    {
-			mach_port_index_t index;
-			mach_port_gen_t gen;
-
-			index = MACH_PORT_INDEX(rcv_name);
-			gen = MACH_PORT_GEN(rcv_name);
-
-			if (index >= size)
+			entry = ipc_entry_lookup (space, rcv_name);
+			if (entry == IE_NULL)
 				goto abort_reply_rcv_copyin;
-
-			entry = &table[index];
 			bits = entry->ie_bits;
 
-			/* check generation number */
-
-			if ((bits & IE_BITS_GEN_MASK) != gen)
-				goto abort_reply_rcv_copyin;
-		    }
-
 			/* check type bits; looking for receive or set */
 
 			if (bits & MACH_PORT_TYPE_PORT_SET) {
@@ -1073,21 +985,12 @@ mach_msg_trap(
 			ip_unlock(reply_port);
 
 		    {
-			ipc_entry_t table;
 			ipc_entry_t entry;
-			mach_port_index_t index;
-
-			/* optimized ipc_entry_get */
-
-			table = space->is_table;
-			index = table->ie_next;
-
-			if (index == 0)
+			kern_return_t kr;
+			kr = ipc_entry_get (space, &reply_name, &entry);
+			if (kr)
 				goto abort_request_copyout;
-
-			entry = &table[index];
-			table->ie_next = entry->ie_next;
-			entry->ie_request = 0;
+			assert (entry != NULL);
 
 		    {
 			mach_port_gen_t gen;
@@ -1095,8 +998,6 @@ mach_msg_trap(
 			assert((entry->ie_bits &~ IE_BITS_GEN_MASK) == 0);
 			gen = entry->ie_bits + IE_BITS_GEN_ONE;
 
-			reply_name = MACH_PORT_MAKE(index, gen);
-
 			/* optimized ipc_right_copyout */
 
 			entry->ie_bits = gen | (MACH_PORT_TYPE_SEND_ONCE | 1);