Initial source

1 files changed, 2182 insertions, 0 deletions

diff --git a/vm/vm_fault.c b/vm/vm_fault.c
new file mode 100644
index 0000000..e45687c
--- /dev/null
+++ b/vm/vm_fault.c
@@ -0,0 +1,2182 @@
+/*
+ * Mach Operating System
+ * Copyright (c) 1994,1990,1989,1988,1987 Carnegie Mellon University.
+ * Copyright (c) 1993,1994 The University of Utah and
+ * the Computer Systems Laboratory (CSL).
+ * All rights reserved.
+ *
+ * Permission to use, copy, modify and distribute this software and its
+ * documentation is hereby granted, provided that both the copyright
+ * notice and this permission notice appear in all copies of the
+ * software, derivative works or modified versions, and any portions
+ * thereof, and that both notices appear in supporting documentation.
+ *
+ * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
+ * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
+ * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
+ * THIS SOFTWARE.
+ *
+ * Carnegie Mellon requests users of this software to return to
+ *
+ *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
+ *  School of Computer Science
+ *  Carnegie Mellon University
+ *  Pittsburgh PA 15213-3890
+ *
+ * any improvements or extensions that they make and grant Carnegie Mellon
+ * the rights to redistribute these changes.
+ */
+/*
+ *	File:	vm_fault.c
+ *	Author:	Avadis Tevanian, Jr., Michael Wayne Young
+ *
+ *	Page fault handling module.
+ */
+#include <mach_pagemap.h>
+#include <mach_kdb.h>
+#include <mach_pcsample.h>
+
+
+#include <vm/vm_fault.h>
+#include <mach/kern_return.h>
+#include <mach/message.h>	/* for error codes */
+#include <kern/counters.h>
+#include <kern/thread.h>
+#include <kern/sched_prim.h>
+#include <vm/vm_map.h>
+#include <vm/vm_object.h>
+#include <vm/vm_page.h>
+#include <vm/pmap.h>
+#include <mach/vm_statistics.h>
+#include <vm/vm_pageout.h>
+#include <mach/vm_param.h>
+#include <mach/memory_object.h>
+#include "memory_object_user.h"
+				/* For memory_object_data_{request,unlock} */
+#include <kern/mach_param.h>
+#include <kern/macro_help.h>
+#include <kern/zalloc.h>
+
+#if	MACH_PCSAMPLE
+#include <kern/pc_sample.h>
+#endif
+
+
+
+/*
+ *	State needed by vm_fault_continue.
+ *	This is a little hefty to drop directly
+ *	into the thread structure.
+ */
+typedef struct vm_fault_state {
+	struct vm_map *vmf_map;
+	vm_offset_t vmf_vaddr;
+	vm_prot_t vmf_fault_type;
+	boolean_t vmf_change_wiring;
+	void (*vmf_continuation)();
+	vm_map_version_t vmf_version;
+	boolean_t vmf_wired;
+	struct vm_object *vmf_object;
+	vm_offset_t vmf_offset;
+	vm_prot_t vmf_prot;
+
+	boolean_t vmfp_backoff;
+	struct vm_object *vmfp_object;
+	vm_offset_t vmfp_offset;
+	struct vm_page *vmfp_first_m;
+	vm_prot_t vmfp_access;
+} vm_fault_state_t;
+
+zone_t		vm_fault_state_zone = 0;
+
+int		vm_object_absent_max = 50;
+
+int		vm_fault_debug = 0;
+
+boolean_t	vm_fault_dirty_handling = FALSE;
+boolean_t	vm_fault_interruptible = TRUE;
+
+boolean_t	software_reference_bits = TRUE;
+
+#if	MACH_KDB
+extern struct db_watchpoint *db_watchpoint_list;
+#endif	MACH_KDB
+
+/*
+ *	Routine:	vm_fault_init
+ *	Purpose:
+ *		Initialize our private data structures.
+ */
+void vm_fault_init()
+{
+	vm_fault_state_zone = zinit(sizeof(vm_fault_state_t),
+				    THREAD_MAX * sizeof(vm_fault_state_t),
+				    sizeof(vm_fault_state_t),
+				    0, "vm fault state");
+}
+
+/*
+ *	Routine:	vm_fault_cleanup
+ *	Purpose:
+ *		Clean up the result of vm_fault_page.
+ *	Results:
+ *		The paging reference for "object" is released.
+ *		"object" is unlocked.
+ *		If "top_page" is not null,  "top_page" is
+ *		freed and the paging reference for the object
+ *		containing it is released.
+ *
+ *	In/out conditions:
+ *		"object" must be locked.
+ */
+void
+vm_fault_cleanup(object, top_page)
+	register vm_object_t	object;
+	register vm_page_t	top_page;
+{
+	vm_object_paging_end(object);
+	vm_object_unlock(object);
+
+	if (top_page != VM_PAGE_NULL) {
+	    object = top_page->object;
+	    vm_object_lock(object);
+	    VM_PAGE_FREE(top_page);
+	    vm_object_paging_end(object);
+	    vm_object_unlock(object);
+	}
+}
+
+
+#if	MACH_PCSAMPLE
+/*
+ *	Do PC sampling on current thread, assuming
+ *	that it is the thread taking this page fault.
+ *
+ *	Must check for THREAD_NULL, since faults
+ *	can occur before threads are running.
+ */
+
+#define	vm_stat_sample(flavor) \
+    MACRO_BEGIN \
+      thread_t _thread_ = current_thread(); \
+ \
+      if (_thread_ != THREAD_NULL) \
+	  take_pc_sample_macro(_thread_, (flavor)); \
+    MACRO_END
+
+#else
+#define	vm_stat_sample(x)
+#endif	/* MACH_PCSAMPLE */
+
+
+
+/*
+ *	Routine:	vm_fault_page
+ *	Purpose:
+ *		Find the resident page for the virtual memory
+ *		specified by the given virtual memory object
+ *		and offset.
+ *	Additional arguments:
+ *		The required permissions for the page is given
+ *		in "fault_type".  Desired permissions are included
+ *		in "protection".
+ *
+ *		If the desired page is known to be resident (for
+ *		example, because it was previously wired down), asserting
+ *		the "unwiring" parameter will speed the search.
+ *
+ *		If the operation can be interrupted (by thread_abort
+ *		or thread_terminate), then the "interruptible"
+ *		parameter should be asserted.
+ *
+ *	Results:
+ *		The page containing the proper data is returned
+ *		in "result_page".
+ *
+ *	In/out conditions:
+ *		The source object must be locked and referenced,
+ *		and must donate one paging reference.  The reference
+ *		is not affected.  The paging reference and lock are
+ *		consumed.
+ *
+ *		If the call succeeds, the object in which "result_page"
+ *		resides is left locked and holding a paging reference.
+ *		If this is not the original object, a busy page in the
+ *		original object is returned in "top_page", to prevent other
+ *		callers from pursuing this same data, along with a paging
+ *		reference for the original object.  The "top_page" should
+ *		be destroyed when this guarantee is no longer required.
+ *		The "result_page" is also left busy.  It is not removed
+ *		from the pageout queues.
+ */
+vm_fault_return_t vm_fault_page(first_object, first_offset,
+				fault_type, must_be_resident, interruptible,
+				protection,
+				result_page, top_page,
+				resume, continuation)
+ /* Arguments: */
+	vm_object_t	first_object;	/* Object to begin search */
+	vm_offset_t	first_offset;	/* Offset into object */
+	vm_prot_t	fault_type;	/* What access is requested */
+	boolean_t	must_be_resident;/* Must page be resident? */
+	boolean_t	interruptible;	/* May fault be interrupted? */
+ /* Modifies in place: */
+	vm_prot_t	*protection;	/* Protection for mapping */
+ /* Returns: */
+	vm_page_t	*result_page;	/* Page found, if successful */
+	vm_page_t	*top_page;	/* Page in top object, if
+					 * not result_page.
+					 */
+ /* More arguments: */
+	boolean_t	resume;		/* We are restarting. */
+	void		(*continuation)(); /* Continuation for blocking. */
+{
+	register
+	vm_page_t	m;
+	register
+	vm_object_t	object;
+	register
+	vm_offset_t	offset;
+	vm_page_t	first_m;
+	vm_object_t	next_object;
+	vm_object_t	copy_object;
+	boolean_t	look_for_page;
+	vm_prot_t	access_required;
+
+#ifdef CONTINUATIONS
+	if (resume) {
+		register vm_fault_state_t *state =
+			(vm_fault_state_t *) current_thread()->ith_other;
+
+		if (state->vmfp_backoff)
+			goto after_block_and_backoff;
+
+		object = state->vmfp_object;
+		offset = state->vmfp_offset;
+		first_m = state->vmfp_first_m;
+		access_required = state->vmfp_access;
+		goto after_thread_block;
+	}
+#else /* not CONTINUATIONS */
+	assert(continuation == 0);
+	assert(!resume);
+#endif /* not CONTINUATIONS */
+
+	vm_stat_sample(SAMPLED_PC_VM_FAULTS_ANY);
+	vm_stat.faults++;		/* needs lock XXX */
+
+/*
+ *	Recovery actions
+ */
+#define RELEASE_PAGE(m)					\
+	MACRO_BEGIN					\
+	PAGE_WAKEUP_DONE(m);				\
+	vm_page_lock_queues();				\
+	if (!m->active && !m->inactive)			\
+		vm_page_activate(m);			\
+	vm_page_unlock_queues();			\
+	MACRO_END
+
+	if (vm_fault_dirty_handling
+#if	MACH_KDB
+		/*
+		 *	If there are watchpoints set, then
+		 *	we don't want to give away write permission
+		 *	on a read fault.  Make the task write fault,
+		 *	so that the watchpoint code notices the access.
+		 */
+	    || db_watchpoint_list
+#endif	MACH_KDB
+	    ) {
+		/*
+		 *	If we aren't asking for write permission,
+		 *	then don't give it away.  We're using write
+		 *	faults to set the dirty bit.
+		 */
+		if (!(fault_type & VM_PROT_WRITE))
+			*protection &= ~VM_PROT_WRITE;
+	}
+
+	if (!vm_fault_interruptible)
+		interruptible = FALSE;
+
+	/*
+	 *	INVARIANTS (through entire routine):
+	 *
+	 *	1)	At all times, we must either have the object
+	 *		lock or a busy page in some object to prevent
+	 *		some other thread from trying to bring in
+	 *		the same page.
+	 *
+	 *		Note that we cannot hold any locks during the
+	 *		pager access or when waiting for memory, so
+	 *		we use a busy page then.
+	 *
+	 *		Note also that we aren't as concerned about more than
+	 *		one thread attempting to memory_object_data_unlock
+	 *		the same page at once, so we don't hold the page
+	 *		as busy then, but do record the highest unlock
+	 *		value so far.  [Unlock requests may also be delivered
+	 *		out of order.]
+	 *
+	 *	2)	To prevent another thread from racing us down the
+	 *		shadow chain and entering a new page in the top
+	 *		object before we do, we must keep a busy page in
+	 *		the top object while following the shadow chain.
+	 *
+	 *	3)	We must increment paging_in_progress on any object
+	 *		for which we have a busy page, to prevent
+	 *		vm_object_collapse from removing the busy page
+	 *		without our noticing.
+	 *
+	 *	4)	We leave busy pages on the pageout queues.
+	 *		If the pageout daemon comes across a busy page,
+	 *		it will remove the page from the pageout queues.
+	 */
+
+	/*
+	 *	Search for the page at object/offset.
+	 */
+
+	object = first_object;
+	offset = first_offset;
+	first_m = VM_PAGE_NULL;
+	access_required = fault_type;
+
+	/*
+	 *	See whether this page is resident
+	 */
+
+	while (TRUE) {
+		m = vm_page_lookup(object, offset);
+		if (m != VM_PAGE_NULL) {
+			/*
+			 *	If the page is being brought in,
+			 *	wait for it and then retry.
+			 *
+			 *	A possible optimization: if the page
+			 *	is known to be resident, we can ignore
+			 *	pages that are absent (regardless of
+			 *	whether they're busy).
+			 */
+
+			if (m->busy) {
+				kern_return_t	wait_result;
+
+				PAGE_ASSERT_WAIT(m, interruptible);
+				vm_object_unlock(object);
+#ifdef CONTINUATIONS
+				if (continuation != (void (*)()) 0) {
+					register vm_fault_state_t *state =
+						(vm_fault_state_t *) current_thread()->ith_other;
+
+					/*
+					 *	Save variables in case
+					 *	thread_block discards
+					 *	our kernel stack.
+					 */
+
+					state->vmfp_backoff = FALSE;
+					state->vmfp_object = object;
+					state->vmfp_offset = offset;
+					state->vmfp_first_m = first_m;
+					state->vmfp_access =
+						access_required;
+					state->vmf_prot = *protection;
+
+					counter(c_vm_fault_page_block_busy_user++);
+					thread_block(continuation);
+				} else
+#endif /* CONTINUATIONS */
+				{
+					counter(c_vm_fault_page_block_busy_kernel++);
+					thread_block((void (*)()) 0);
+				}
+			    after_thread_block:
+				wait_result = current_thread()->wait_result;
+				vm_object_lock(object);
+				if (wait_result != THREAD_AWAKENED) {
+					vm_fault_cleanup(object, first_m);
+					if (wait_result == THREAD_RESTART)
+						return(VM_FAULT_RETRY);
+					else
+						return(VM_FAULT_INTERRUPTED);
+				}
+				continue;
+			}
+
+			/*
+			 *	If the page is in error, give up now.
+			 */
+
+			if (m->error) {
+				VM_PAGE_FREE(m);
+				vm_fault_cleanup(object, first_m);
+				return(VM_FAULT_MEMORY_ERROR);
+			}
+
+			/*
+			 *	If the page isn't busy, but is absent,
+			 *	then it was deemed "unavailable".
+			 */
+
+			if (m->absent) {
+				/*
+				 * Remove the non-existent page (unless it's
+				 * in the top object) and move on down to the
+				 * next object (if there is one).
+				 */
+
+				offset += object->shadow_offset;
+				access_required = VM_PROT_READ;
+				next_object = object->shadow;
+				if (next_object == VM_OBJECT_NULL) {
+					vm_page_t real_m;
+
+					assert(!must_be_resident);
+
+					/*
+					 * Absent page at bottom of shadow
+					 * chain; zero fill the page we left
+					 * busy in the first object, and flush
+					 * the absent page.  But first we
+					 * need to allocate a real page.
+					 */
+
+					real_m = vm_page_grab();
+					if (real_m == VM_PAGE_NULL) {
+						vm_fault_cleanup(object, first_m);
+						return(VM_FAULT_MEMORY_SHORTAGE);
+					}
+
+					if (object != first_object) {
+						VM_PAGE_FREE(m);
+						vm_object_paging_end(object);
+						vm_object_unlock(object);
+						object = first_object;
+						offset = first_offset;
+						m = first_m;
+						first_m = VM_PAGE_NULL;
+						vm_object_lock(object);
+					}
+
+					VM_PAGE_FREE(m);
+					assert(real_m->busy);
+					vm_page_lock_queues();
+					vm_page_insert(real_m, object, offset);
+					vm_page_unlock_queues();
+					m = real_m;
+
+					/*
+					 *  Drop the lock while zero filling
+					 *  page.  Then break because this
+					 *  is the page we wanted.  Checking
+					 *  the page lock is a waste of time;
+					 *  this page was either absent or
+					 *  newly allocated -- in both cases
+					 *  it can't be page locked by a pager.
+					 */
+					vm_object_unlock(object);
+
+					vm_page_zero_fill(m);
+
+					vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS);
+					
+					vm_stat.zero_fill_count++;
+					vm_object_lock(object);
+					pmap_clear_modify(m->phys_addr);
+					break;
+				} else {
+					if (must_be_resident) {
+						vm_object_paging_end(object);
+					} else if (object != first_object) {
+						vm_object_paging_end(object);
+						VM_PAGE_FREE(m);
+					} else {
+						first_m = m;
+						m->absent = FALSE;
+						vm_object_absent_release(object);
+						m->busy = TRUE;
+
+						vm_page_lock_queues();
+						VM_PAGE_QUEUES_REMOVE(m);
+						vm_page_unlock_queues();
+					}
+					vm_object_lock(next_object);
+					vm_object_unlock(object);
+					object = next_object;
+					vm_object_paging_begin(object);
+					continue;
+				}
+			}
+
+			/*
+			 *	If the desired access to this page has
+			 *	been locked out, request that it be unlocked.
+			 */
+
+			if (access_required & m->page_lock) {
+				if ((access_required & m->unlock_request) != access_required) {
+					vm_prot_t	new_unlock_request;
+					kern_return_t	rc;
+					
+					if (!object->pager_ready) {
+						vm_object_assert_wait(object,
+							VM_OBJECT_EVENT_PAGER_READY,
+							interruptible);
+						goto block_and_backoff;
+					}
+
+					new_unlock_request = m->unlock_request =
+						(access_required | m->unlock_request);
+					vm_object_unlock(object);
+					if ((rc = memory_object_data_unlock(
+						object->pager,
+						object->pager_request,
+						offset + object->paging_offset,
+						PAGE_SIZE,
+						new_unlock_request))
+					     != KERN_SUCCESS) {
+					     	printf("vm_fault: memory_object_data_unlock failed\n");
+						vm_object_lock(object);
+						vm_fault_cleanup(object, first_m);
+						return((rc == MACH_SEND_INTERRUPTED) ?
+							VM_FAULT_INTERRUPTED :
+							VM_FAULT_MEMORY_ERROR);
+					}
+					vm_object_lock(object);
+					continue;
+				}
+
+				PAGE_ASSERT_WAIT(m, interruptible);
+				goto block_and_backoff;
+			}
+
+			/*
+			 *	We mark the page busy and leave it on
+			 *	the pageout queues.  If the pageout
+			 *	deamon comes across it, then it will
+			 *	remove the page.
+			 */
+
+			if (!software_reference_bits) {
+				vm_page_lock_queues();
+				if (m->inactive)  {
+				    	vm_stat_sample(SAMPLED_PC_VM_REACTIVATION_FAULTS);
+					vm_stat.reactivations++;
+				}
+
+				VM_PAGE_QUEUES_REMOVE(m);
+				vm_page_unlock_queues();
+			}
+
+			assert(!m->busy);
+			m->busy = TRUE;
+			assert(!m->absent);
+			break;
+		}
+
+		look_for_page =
+			(object->pager_created)
+#if	MACH_PAGEMAP
+			&& (vm_external_state_get(object->existence_info, offset + object->paging_offset) !=
+			 VM_EXTERNAL_STATE_ABSENT)
+#endif	MACH_PAGEMAP
+			 ;
+
+		if ((look_for_page || (object == first_object))
+				 && !must_be_resident) {
+			/*
+			 *	Allocate a new page for this object/offset
+			 *	pair.
+			 */
+
+			m = vm_page_grab_fictitious();
+			if (m == VM_PAGE_NULL) {
+				vm_fault_cleanup(object, first_m);
+				return(VM_FAULT_FICTITIOUS_SHORTAGE);
+			}
+
+			vm_page_lock_queues();
+			vm_page_insert(m, object, offset);
+			vm_page_unlock_queues();
+		}
+
+		if (look_for_page && !must_be_resident) {
+			kern_return_t	rc;
+
+			/*
+			 *	If the memory manager is not ready, we
+			 *	cannot make requests.
+			 */
+			if (!object->pager_ready) {
+				vm_object_assert_wait(object,
+					VM_OBJECT_EVENT_PAGER_READY,
+					interruptible);
+				VM_PAGE_FREE(m);
+				goto block_and_backoff;
+			}
+
+			if (object->internal) {
+				/*
+				 *	Requests to the default pager
+				 *	must reserve a real page in advance,
+				 *	because the pager's data-provided
+				 *	won't block for pages.
+				 */
+
+				if (m->fictitious && !vm_page_convert(m)) {
+					VM_PAGE_FREE(m);
+					vm_fault_cleanup(object, first_m);
+					return(VM_FAULT_MEMORY_SHORTAGE);
+				}
+			} else if (object->absent_count >
+						vm_object_absent_max) {
+				/*
+				 *	If there are too many outstanding page
+				 *	requests pending on this object, we
+				 *	wait for them to be resolved now.
+				 */
+
+				vm_object_absent_assert_wait(object, interruptible);
+				VM_PAGE_FREE(m);
+				goto block_and_backoff;
+			}
+
+			/*
+			 *	Indicate that the page is waiting for data
+			 *	from the memory manager.
+			 */
+
+			m->absent = TRUE;
+			object->absent_count++;
+
+			/*
+			 *	We have a busy page, so we can
+			 *	release the object lock.
+			 */
+			vm_object_unlock(object);
+
+			/*
+			 *	Call the memory manager to retrieve the data.
+			 */
+
+			vm_stat.pageins++;
+		    	vm_stat_sample(SAMPLED_PC_VM_PAGEIN_FAULTS);
+
+			if ((rc = memory_object_data_request(object->pager, 
+				object->pager_request,
+				m->offset + object->paging_offset, 
+				PAGE_SIZE, access_required)) != KERN_SUCCESS) {
+				if (rc != MACH_SEND_INTERRUPTED)
+					printf("%s(0x%x, 0x%x, 0x%x, 0x%x, 0x%x) failed, %d\n",
+						"memory_object_data_request",
+						object->pager,
+						object->pager_request,
+						m->offset + object->paging_offset, 
+						PAGE_SIZE, access_required, rc);
+				/*
+				 *	Don't want to leave a busy page around,
+				 *	but the data request may have blocked,
+				 *	so check if it's still there and busy.
+				 */
+				vm_object_lock(object);
+				if (m == vm_page_lookup(object,offset) &&
+				    m->absent && m->busy)
+					VM_PAGE_FREE(m);
+				vm_fault_cleanup(object, first_m);
+				return((rc == MACH_SEND_INTERRUPTED) ?
+					VM_FAULT_INTERRUPTED :
+					VM_FAULT_MEMORY_ERROR);
+			}
+			
+			/*
+			 * Retry with same object/offset, since new data may
+			 * be in a different page (i.e., m is meaningless at
+			 * this point).
+			 */
+			vm_object_lock(object);
+			continue;
+		}
+
+		/*
+		 * For the XP system, the only case in which we get here is if
+		 * object has no pager (or unwiring).  If the pager doesn't
+		 * have the page this is handled in the m->absent case above
+		 * (and if you change things here you should look above).
+		 */
+		if (object == first_object)
+			first_m = m;
+		else
+		{
+			assert(m == VM_PAGE_NULL);
+		}
+
+		/*
+		 *	Move on to the next object.  Lock the next
+		 *	object before unlocking the current one.
+		 */
+		access_required = VM_PROT_READ;
+
+		offset += object->shadow_offset;
+		next_object = object->shadow;
+		if (next_object == VM_OBJECT_NULL) {
+			assert(!must_be_resident);
+
+			/*
+			 *	If there's no object left, fill the page
+			 *	in the top object with zeros.  But first we
+			 *	need to allocate a real page.
+			 */
+
+			if (object != first_object) {
+				vm_object_paging_end(object);
+				vm_object_unlock(object);
+
+				object = first_object;
+				offset = first_offset;
+				vm_object_lock(object);
+			}
+
+			m = first_m;
+			assert(m->object == object);
+			first_m = VM_PAGE_NULL;
+
+			if (m->fictitious && !vm_page_convert(m)) {
+				VM_PAGE_FREE(m);
+				vm_fault_cleanup(object, VM_PAGE_NULL);
+				return(VM_FAULT_MEMORY_SHORTAGE);
+			}
+
+			vm_object_unlock(object);
+			vm_page_zero_fill(m);
+			vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS);
+			vm_stat.zero_fill_count++;
+			vm_object_lock(object);
+			pmap_clear_modify(m->phys_addr);
+			break;
+		}
+		else {
+			vm_object_lock(next_object);
+			if ((object != first_object) || must_be_resident)
+				vm_object_paging_end(object);
+			vm_object_unlock(object);
+			object = next_object;
+			vm_object_paging_begin(object);
+		}
+	}
+
+	/*
+	 *	PAGE HAS BEEN FOUND.
+	 *
+	 *	This page (m) is:
+	 *		busy, so that we can play with it;
+	 *		not absent, so that nobody else will fill it;
+	 *		possibly eligible for pageout;
+	 *
+	 *	The top-level page (first_m) is:
+	 *		VM_PAGE_NULL if the page was found in the
+	 *		 top-level object;
+	 *		busy, not absent, and ineligible for pageout.
+	 *
+	 *	The current object (object) is locked.  A paging
+	 *	reference is held for the current and top-level
+	 *	objects.
+	 */
+
+#if	EXTRA_ASSERTIONS
+	assert(m->busy && !m->absent);
+	assert((first_m == VM_PAGE_NULL) ||
+		(first_m->busy && !first_m->absent &&
+		 !first_m->active && !first_m->inactive));
+#endif	EXTRA_ASSERTIONS
+
+	/*
+	 *	If the page is being written, but isn't
+	 *	already owned by the top-level object,
+	 *	we have to copy it into a new page owned
+	 *	by the top-level object.
+	 */
+
+	if (object != first_object) {
+	    	/*
+		 *	We only really need to copy if we
+		 *	want to write it.
+		 */
+
+	    	if (fault_type & VM_PROT_WRITE) {
+			vm_page_t copy_m;
+
+			assert(!must_be_resident);
+
+			/*
+			 *	If we try to collapse first_object at this
+			 *	point, we may deadlock when we try to get
+			 *	the lock on an intermediate object (since we
+			 *	have the bottom object locked).  We can't
+			 *	unlock the bottom object, because the page
+			 *	we found may move (by collapse) if we do.
+			 *
+			 *	Instead, we first copy the page.  Then, when
+			 *	we have no more use for the bottom object,
+			 *	we unlock it and try to collapse.
+			 *
+			 *	Note that we copy the page even if we didn't
+			 *	need to... that's the breaks.
+			 */
+
+			/*
+			 *	Allocate a page for the copy
+			 */
+			copy_m = vm_page_grab();
+			if (copy_m == VM_PAGE_NULL) {
+				RELEASE_PAGE(m);
+				vm_fault_cleanup(object, first_m);
+				return(VM_FAULT_MEMORY_SHORTAGE);
+			}
+
+			vm_object_unlock(object);
+			vm_page_copy(m, copy_m);
+			vm_object_lock(object);
+
+			/*
+			 *	If another map is truly sharing this
+			 *	page with us, we have to flush all
+			 *	uses of the original page, since we
+			 *	can't distinguish those which want the
+			 *	original from those which need the
+			 *	new copy.
+			 *
+			 *	XXXO If we know that only one map has
+			 *	access to this page, then we could
+			 *	avoid the pmap_page_protect() call.
+			 */
+
+			vm_page_lock_queues();
+			vm_page_deactivate(m);
+			pmap_page_protect(m->phys_addr, VM_PROT_NONE);
+			vm_page_unlock_queues();
+
+			/*
+			 *	We no longer need the old page or object.
+			 */
+
+			PAGE_WAKEUP_DONE(m);
+			vm_object_paging_end(object);
+			vm_object_unlock(object);
+
+			vm_stat.cow_faults++;
+			vm_stat_sample(SAMPLED_PC_VM_COW_FAULTS);
+			object = first_object;
+			offset = first_offset;
+
+			vm_object_lock(object);
+			VM_PAGE_FREE(first_m);
+			first_m = VM_PAGE_NULL;
+			assert(copy_m->busy);
+			vm_page_lock_queues();
+			vm_page_insert(copy_m, object, offset);
+			vm_page_unlock_queues();
+			m = copy_m;
+
+			/*
+			 *	Now that we've gotten the copy out of the
+			 *	way, let's try to collapse the top object.
+			 *	But we have to play ugly games with
+			 *	paging_in_progress to do that...
+			 */
+
+			vm_object_paging_end(object);
+			vm_object_collapse(object);
+			vm_object_paging_begin(object);
+		}
+		else {
+		    	*protection &= (~VM_PROT_WRITE);
+		}
+	}
+
+	/*
+	 *	Now check whether the page needs to be pushed into the
+	 *	copy object.  The use of asymmetric copy on write for
+	 *	shared temporary objects means that we may do two copies to
+	 *	satisfy the fault; one above to get the page from a
+	 *	shadowed object, and one here to push it into the copy.
+	 */
+
+	while ((copy_object = first_object->copy) != VM_OBJECT_NULL) {
+		vm_offset_t	copy_offset;
+		vm_page_t	copy_m;
+
+		/*
+		 *	If the page is being written, but hasn't been
+		 *	copied to the copy-object, we have to copy it there.
+		 */
+
+		if ((fault_type & VM_PROT_WRITE) == 0) {
+			*protection &= ~VM_PROT_WRITE;
+			break;
+		}
+
+		/*
+		 *	If the page was guaranteed to be resident,
+		 *	we must have already performed the copy.
+		 */
+
+		if (must_be_resident)
+			break;
+
+		/*
+		 *	Try to get the lock on the copy_object.
+		 */
+		if (!vm_object_lock_try(copy_object)) {
+			vm_object_unlock(object);
+
+			simple_lock_pause();	/* wait a bit */
+
+			vm_object_lock(object);
+			continue;
+		}
+
+		/*
+		 *	Make another reference to the copy-object,
+		 *	to keep it from disappearing during the
+		 *	copy.
+		 */
+		assert(copy_object->ref_count > 0);
+		copy_object->ref_count++;
+
+		/*
+		 *	Does the page exist in the copy?
+		 */
+		copy_offset = first_offset - copy_object->shadow_offset;
+		copy_m = vm_page_lookup(copy_object, copy_offset);
+		if (copy_m != VM_PAGE_NULL) {
+			if (copy_m->busy) {
+				/*
+				 *	If the page is being brought
+				 *	in, wait for it and then retry.
+				 */
+				PAGE_ASSERT_WAIT(copy_m, interruptible);
+				RELEASE_PAGE(m);
+				copy_object->ref_count--;
+				assert(copy_object->ref_count > 0);
+				vm_object_unlock(copy_object);
+				goto block_and_backoff;
+			}
+		}
+		else {
+			/*
+			 *	Allocate a page for the copy
+			 */
+			copy_m = vm_page_alloc(copy_object, copy_offset);
+			if (copy_m == VM_PAGE_NULL) {
+				RELEASE_PAGE(m);
+				copy_object->ref_count--;
+				assert(copy_object->ref_count > 0);
+				vm_object_unlock(copy_object);
+				vm_fault_cleanup(object, first_m);
+				return(VM_FAULT_MEMORY_SHORTAGE);
+			}
+
+			/*
+			 *	Must copy page into copy-object.
+			 */
+
+			vm_page_copy(m, copy_m);
+			
+			/*
+			 *	If the old page was in use by any users
+			 *	of the copy-object, it must be removed
+			 *	from all pmaps.  (We can't know which
+			 *	pmaps use it.)
+			 */
+
+			vm_page_lock_queues();
+			pmap_page_protect(m->phys_addr, VM_PROT_NONE);
+			copy_m->dirty = TRUE;
+			vm_page_unlock_queues();
+
+			/*
+			 *	If there's a pager, then immediately
+			 *	page out this page, using the "initialize"
+			 *	option.  Else, we use the copy.
+			 */
+
+		 	if (!copy_object->pager_created) {
+				vm_page_lock_queues();
+				vm_page_activate(copy_m);
+				vm_page_unlock_queues();
+				PAGE_WAKEUP_DONE(copy_m);
+			} else {
+				/*
+				 *	The page is already ready for pageout:
+				 *	not on pageout queues and busy.
+				 *	Unlock everything except the
+				 *	copy_object itself.
+				 */
+
+				vm_object_unlock(object);
+
+				/*
+				 *	Write the page to the copy-object,
+				 *	flushing it from the kernel.
+				 */
+
+				vm_pageout_page(copy_m, TRUE, TRUE);
+
+				/*
+				 *	Since the pageout may have
+				 *	temporarily dropped the
+				 *	copy_object's lock, we
+				 *	check whether we'll have
+				 *	to deallocate the hard way.
+				 */
+
+				if ((copy_object->shadow != object) ||
+				    (copy_object->ref_count == 1)) {
+					vm_object_unlock(copy_object);
+					vm_object_deallocate(copy_object);
+					vm_object_lock(object);
+					continue;
+				}
+
+				/*
+				 *	Pick back up the old object's
+				 *	lock.  [It is safe to do so,
+				 *	since it must be deeper in the
+				 *	object tree.]
+				 */
+
+				vm_object_lock(object);
+			}
+
+			/*
+			 *	Because we're pushing a page upward
+			 *	in the object tree, we must restart
+			 *	any faults that are waiting here.
+			 *	[Note that this is an expansion of
+			 *	PAGE_WAKEUP that uses the THREAD_RESTART
+			 *	wait result].  Can't turn off the page's
+			 *	busy bit because we're not done with it.
+			 */
+			 
+			if (m->wanted) {
+				m->wanted = FALSE;
+				thread_wakeup_with_result((event_t) m,
+					THREAD_RESTART);
+			}
+		}
+
+		/*
+		 *	The reference count on copy_object must be
+		 *	at least 2: one for our extra reference,
+		 *	and at least one from the outside world
+		 *	(we checked that when we last locked
+		 *	copy_object).
+		 */
+		copy_object->ref_count--;
+		assert(copy_object->ref_count > 0);
+		vm_object_unlock(copy_object);
+
+		break;
+	}
+
+	*result_page = m;
+	*top_page = first_m;
+
+	/*
+	 *	If the page can be written, assume that it will be.
+	 *	[Earlier, we restrict the permission to allow write
+	 *	access only if the fault so required, so we don't
+	 *	mark read-only data as dirty.]
+	 */
+
+	if (vm_fault_dirty_handling && (*protection & VM_PROT_WRITE))
+		m->dirty = TRUE;
+
+	return(VM_FAULT_SUCCESS);
+
+    block_and_backoff:
+	vm_fault_cleanup(object, first_m);
+
+#ifdef CONTINUATIONS
+	if (continuation != (void (*)()) 0) {
+		register vm_fault_state_t *state =
+			(vm_fault_state_t *) current_thread()->ith_other;
+
+		/*
+		 *	Save variables in case we must restart.
+		 */
+
+		state->vmfp_backoff = TRUE;
+		state->vmf_prot = *protection;
+
+		counter(c_vm_fault_page_block_backoff_user++);
+		thread_block(continuation);
+	} else
+#endif /* CONTINUATIONS */
+	{
+		counter(c_vm_fault_page_block_backoff_kernel++);
+		thread_block((void (*)()) 0);
+	}
+    after_block_and_backoff:
+	if (current_thread()->wait_result == THREAD_AWAKENED)
+		return VM_FAULT_RETRY;
+	else
+		return VM_FAULT_INTERRUPTED;
+
+#undef	RELEASE_PAGE
+}
+
+/*
+ *	Routine:	vm_fault
+ *	Purpose:
+ *		Handle page faults, including pseudo-faults
+ *		used to change the wiring status of pages.
+ *	Returns:
+ *		If an explicit (expression) continuation is supplied,
+ *		then we call the continuation instead of returning.
+ *	Implementation:
+ *		Explicit continuations make this a little icky,
+ *		because it hasn't been rewritten to embrace CPS.
+ *		Instead, we have resume arguments for vm_fault and
+ *		vm_fault_page, to let continue the fault computation.
+ *
+ *		vm_fault and vm_fault_page save mucho state
+ *		in the moral equivalent of a closure.  The state
+ *		structure is allocated when first entering vm_fault
+ *		and deallocated when leaving vm_fault.
+ */
+
+#ifdef CONTINUATIONS
+void
+vm_fault_continue()
+{
+	register vm_fault_state_t *state =
+		(vm_fault_state_t *) current_thread()->ith_other;
+
+	(void) vm_fault(state->vmf_map,
+			state->vmf_vaddr,
+			state->vmf_fault_type,
+			state->vmf_change_wiring,
+			TRUE, state->vmf_continuation);
+	/*NOTREACHED*/
+}
+#endif /* CONTINUATIONS */
+
+kern_return_t vm_fault(map, vaddr, fault_type, change_wiring,
+		       resume, continuation)
+	vm_map_t	map;
+	vm_offset_t	vaddr;
+	vm_prot_t	fault_type;
+	boolean_t	change_wiring;
+	boolean_t	resume;
+	void		(*continuation)();
+{
+	vm_map_version_t	version;	/* Map version for verificiation */
+	boolean_t		wired;		/* Should mapping be wired down? */
+	vm_object_t		object;		/* Top-level object */
+	vm_offset_t		offset;		/* Top-level offset */
+	vm_prot_t		prot;		/* Protection for mapping */
+	vm_object_t		old_copy_object; /* Saved copy object */
+	vm_page_t		result_page;	/* Result of vm_fault_page */
+	vm_page_t		top_page;	/* Placeholder page */
+	kern_return_t		kr;
+
+	register
+	vm_page_t		m;	/* Fast access to result_page */
+
+#ifdef CONTINUATIONS
+	if (resume) {
+		register vm_fault_state_t *state =
+			(vm_fault_state_t *) current_thread()->ith_other;
+
+		/*
+		 *	Retrieve cached variables and
+		 *	continue vm_fault_page.
+		 */
+
+		object = state->vmf_object;
+		if (object == VM_OBJECT_NULL)
+			goto RetryFault;
+		version = state->vmf_version;
+		wired = state->vmf_wired;
+		offset = state->vmf_offset;
+		prot = state->vmf_prot;
+
+		kr = vm_fault_page(object, offset, fault_type,
+				(change_wiring && !wired), !change_wiring,
+				&prot, &result_page, &top_page,
+				TRUE, vm_fault_continue);
+		goto after_vm_fault_page;
+	}
+
+	if (continuation != (void (*)()) 0) {
+		/*
+		 *	We will probably need to save state.
+		 */
+
+		char *	state;
+
+		/*
+		 * if this assignment stmt is written as
+		 * 'active_threads[cpu_number()] = zalloc()',
+		 * cpu_number may be evaluated before zalloc;
+		 * if zalloc blocks, cpu_number will be wrong
+		 */
+
+		state = (char *) zalloc(vm_fault_state_zone);
+		current_thread()->ith_other = state;
+
+	}
+#else /* not CONTINUATIONS */
+	assert(continuation == 0);
+	assert(!resume);
+#endif /* not CONTINUATIONS */
+
+    RetryFault: ;
+
+	/*
+	 *	Find the backing store object and offset into
+	 *	it to begin the search.
+	 */
+
+	if ((kr = vm_map_lookup(&map, vaddr, fault_type, &version,
+				&object, &offset,
+				&prot, &wired)) != KERN_SUCCESS) {
+		goto done;
+	}
+
+	/*
+	 *	If the page is wired, we must fault for the current protection
+	 *	value, to avoid further faults.
+	 */
+
+	if (wired)
+		fault_type = prot;
+
+   	/*
+	 *	Make a reference to this object to
+	 *	prevent its disposal while we are messing with
+	 *	it.  Once we have the reference, the map is free
+	 *	to be diddled.  Since objects reference their
+	 *	shadows (and copies), they will stay around as well.
+	 */
+
+	assert(object->ref_count > 0);
+	object->ref_count++;
+	vm_object_paging_begin(object);
+
+#ifdef CONTINUATIONS
+	if (continuation != (void (*)()) 0) {
+		register vm_fault_state_t *state =
+			(vm_fault_state_t *) current_thread()->ith_other;
+
+		/*
+		 *	Save variables, in case vm_fault_page discards
+		 *	our kernel stack and we have to restart.
+		 */
+
+		state->vmf_map = map;
+		state->vmf_vaddr = vaddr;
+		state->vmf_fault_type = fault_type;
+		state->vmf_change_wiring = change_wiring;
+		state->vmf_continuation = continuation;
+
+		state->vmf_version = version;
+		state->vmf_wired = wired;
+		state->vmf_object = object;
+		state->vmf_offset = offset;
+		state->vmf_prot = prot;
+
+		kr = vm_fault_page(object, offset, fault_type,
+				   (change_wiring && !wired), !change_wiring,
+				   &prot, &result_page, &top_page,
+				   FALSE, vm_fault_continue);
+	} else
+#endif /* CONTINUATIONS */
+	{
+		kr = vm_fault_page(object, offset, fault_type,
+				   (change_wiring && !wired), !change_wiring,
+				   &prot, &result_page, &top_page,
+				   FALSE, (void (*)()) 0);
+	}
+    after_vm_fault_page:
+
+	/*
+	 *	If we didn't succeed, lose the object reference immediately.
+	 */
+
+	if (kr != VM_FAULT_SUCCESS)
+		vm_object_deallocate(object);
+
+	/*
+	 *	See why we failed, and take corrective action.
+	 */
+
+	switch (kr) {
+		case VM_FAULT_SUCCESS:
+			break;
+		case VM_FAULT_RETRY:
+			goto RetryFault;
+		case VM_FAULT_INTERRUPTED:
+			kr = KERN_SUCCESS;
+			goto done;
+		case VM_FAULT_MEMORY_SHORTAGE:
+#ifdef CONTINUATIONS
+			if (continuation != (void (*)()) 0) {
+				register vm_fault_state_t *state =
+					(vm_fault_state_t *) current_thread()->ith_other;
+
+				/*
+				 *	Save variables in case VM_PAGE_WAIT
+				 *	discards our kernel stack.
+				 */
+
+				state->vmf_map = map;
+				state->vmf_vaddr = vaddr;
+				state->vmf_fault_type = fault_type;
+				state->vmf_change_wiring = change_wiring;
+				state->vmf_continuation = continuation;
+				state->vmf_object = VM_OBJECT_NULL;
+
+				VM_PAGE_WAIT(vm_fault_continue);
+			} else
+#endif /* CONTINUATIONS */
+				VM_PAGE_WAIT((void (*)()) 0);
+			goto RetryFault;
+		case VM_FAULT_FICTITIOUS_SHORTAGE:
+			vm_page_more_fictitious();
+			goto RetryFault;
+		case VM_FAULT_MEMORY_ERROR:
+			kr = KERN_MEMORY_ERROR;
+			goto done;
+	}
+
+	m = result_page;
+
+	assert((change_wiring && !wired) ?
+	       (top_page == VM_PAGE_NULL) :
+	       ((top_page == VM_PAGE_NULL) == (m->object == object)));
+
+	/*
+	 *	How to clean up the result of vm_fault_page.  This
+	 *	happens whether the mapping is entered or not.
+	 */
+
+#define UNLOCK_AND_DEALLOCATE				\
+	MACRO_BEGIN					\
+	vm_fault_cleanup(m->object, top_page);		\
+	vm_object_deallocate(object);			\
+	MACRO_END
+
+	/*
+	 *	What to do with the resulting page from vm_fault_page
+	 *	if it doesn't get entered into the physical map:
+	 */
+
+#define RELEASE_PAGE(m)					\
+	MACRO_BEGIN					\
+	PAGE_WAKEUP_DONE(m);				\
+	vm_page_lock_queues();				\
+	if (!m->active && !m->inactive)			\
+		vm_page_activate(m);			\
+	vm_page_unlock_queues();			\
+	MACRO_END
+
+	/*
+	 *	We must verify that the maps have not changed
+	 *	since our last lookup.
+	 */
+
+	old_copy_object = m->object->copy;
+
+	vm_object_unlock(m->object);
+	while (!vm_map_verify(map, &version)) {
+		vm_object_t	retry_object;
+		vm_offset_t	retry_offset;
+		vm_prot_t	retry_prot;
+
+		/*
+		 *	To avoid trying to write_lock the map while another
+		 *	thread has it read_locked (in vm_map_pageable), we
+		 *	do not try for write permission.  If the page is
+		 *	still writable, we will get write permission.  If it
+		 *	is not, or has been marked needs_copy, we enter the
+		 *	mapping without write permission, and will merely
+		 *	take another fault.
+		 */
+		kr = vm_map_lookup(&map, vaddr,
+				   fault_type & ~VM_PROT_WRITE, &version,
+				   &retry_object, &retry_offset, &retry_prot,
+				   &wired);
+
+		if (kr != KERN_SUCCESS) {
+			vm_object_lock(m->object);
+			RELEASE_PAGE(m);
+			UNLOCK_AND_DEALLOCATE;
+			goto done;
+		}
+
+		vm_object_unlock(retry_object);
+		vm_object_lock(m->object);
+
+		if ((retry_object != object) ||
+		    (retry_offset != offset)) {
+			RELEASE_PAGE(m);
+			UNLOCK_AND_DEALLOCATE;
+			goto RetryFault;
+		}
+
+		/*
+		 *	Check whether the protection has changed or the object
+		 *	has been copied while we left the map unlocked.
+		 */
+		prot &= retry_prot;
+		vm_object_unlock(m->object);
+	}
+	vm_object_lock(m->object);
+
+	/*
+	 *	If the copy object changed while the top-level object
+	 *	was unlocked, then we must take away write permission.
+	 */
+
+	if (m->object->copy != old_copy_object)
+		prot &= ~VM_PROT_WRITE;
+
+	/*
+	 *	If we want to wire down this page, but no longer have
+	 *	adequate permissions, we must start all over.
+	 */
+
+	if (wired && (prot != fault_type)) {
+		vm_map_verify_done(map, &version);
+		RELEASE_PAGE(m);
+		UNLOCK_AND_DEALLOCATE;
+		goto RetryFault;
+	}
+
+	/*
+	 *	It's critically important that a wired-down page be faulted
+	 *	only once in each map for which it is wired.
+	 */
+
+	vm_object_unlock(m->object);
+
+	/*
+	 *	Put this page into the physical map.
+	 *	We had to do the unlock above because pmap_enter
+	 *	may cause other faults.  The page may be on
+	 *	the pageout queues.  If the pageout daemon comes
+	 *	across the page, it will remove it from the queues.
+	 */
+
+	PMAP_ENTER(map->pmap, vaddr, m, prot, wired);
+
+	/*
+	 *	If the page is not wired down and isn't already
+	 *	on a pageout queue, then put it where the
+	 *	pageout daemon can find it.
+	 */
+	vm_object_lock(m->object);
+	vm_page_lock_queues();
+	if (change_wiring) {
+		if (wired)
+			vm_page_wire(m);
+		else
+			vm_page_unwire(m);
+	} else if (software_reference_bits) {
+		if (!m->active && !m->inactive)
+			vm_page_activate(m);
+		m->reference = TRUE;
+	} else {
+		vm_page_activate(m);
+	}
+	vm_page_unlock_queues();
+
+	/*
+	 *	Unlock everything, and return
+	 */
+
+	vm_map_verify_done(map, &version);
+	PAGE_WAKEUP_DONE(m);
+	kr = KERN_SUCCESS;
+	UNLOCK_AND_DEALLOCATE;
+
+#undef	UNLOCK_AND_DEALLOCATE
+#undef	RELEASE_PAGE
+
+    done:
+#ifdef CONTINUATIONS
+	if (continuation != (void (*)()) 0) {
+		register vm_fault_state_t *state =
+			(vm_fault_state_t *) current_thread()->ith_other;
+
+		zfree(vm_fault_state_zone, (vm_offset_t) state);
+		(*continuation)(kr);
+		/*NOTREACHED*/
+	}
+#endif /* CONTINUATIONS */
+
+	return(kr);
+}
+
+kern_return_t	vm_fault_wire_fast();
+
+/*
+ *	vm_fault_wire:
+ *
+ *	Wire down a range of virtual addresses in a map.
+ */
+void vm_fault_wire(map, entry)
+	vm_map_t	map;
+	vm_map_entry_t	entry;
+{
+
+	register vm_offset_t	va;
+	register pmap_t		pmap;
+	register vm_offset_t	end_addr = entry->vme_end;
+
+	pmap = vm_map_pmap(map);
+
+	/*
+	 *	Inform the physical mapping system that the
+	 *	range of addresses may not fault, so that
+	 *	page tables and such can be locked down as well.
+	 */
+
+	pmap_pageable(pmap, entry->vme_start, end_addr, FALSE);
+
+	/*
+	 *	We simulate a fault to get the page and enter it
+	 *	in the physical map.
+	 */
+
+	for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) {
+		if (vm_fault_wire_fast(map, va, entry) != KERN_SUCCESS)
+			(void) vm_fault(map, va, VM_PROT_NONE, TRUE,
+					FALSE, (void (*)()) 0);
+	}
+}
+
+/*
+ *	vm_fault_unwire:
+ *
+ *	Unwire a range of virtual addresses in a map.
+ */
+void vm_fault_unwire(map, entry)
+	vm_map_t	map;
+	vm_map_entry_t	entry;
+{
+	register vm_offset_t	va;
+	register pmap_t		pmap;
+	register vm_offset_t	end_addr = entry->vme_end;
+	vm_object_t		object;
+
+	pmap = vm_map_pmap(map);
+
+	object = (entry->is_sub_map)
+			? VM_OBJECT_NULL : entry->object.vm_object;
+
+	/*
+	 *	Since the pages are wired down, we must be able to
+	 *	get their mappings from the physical map system.
+	 */
+
+	for (va = entry->vme_start; va < end_addr; va += PAGE_SIZE) {
+		pmap_change_wiring(pmap, va, FALSE);
+
+		if (object == VM_OBJECT_NULL) {
+			vm_map_lock_set_recursive(map);
+			(void) vm_fault(map, va, VM_PROT_NONE, TRUE,
+					FALSE, (void (*)()) 0);
+			vm_map_lock_clear_recursive(map);
+		} else {
+		 	vm_prot_t	prot;
+			vm_page_t	result_page;
+			vm_page_t	top_page;
+			vm_fault_return_t result;
+
+			do {
+				prot = VM_PROT_NONE;
+
+				vm_object_lock(object);
+				vm_object_paging_begin(object);
+			 	result = vm_fault_page(object,
+						entry->offset +
+						  (va - entry->vme_start),
+						VM_PROT_NONE, TRUE,
+						FALSE, &prot,
+						&result_page,
+						&top_page,
+						FALSE, (void (*)()) 0);
+			} while (result == VM_FAULT_RETRY);
+
+			if (result != VM_FAULT_SUCCESS)
+				panic("vm_fault_unwire: failure");
+
+			vm_page_lock_queues();
+			vm_page_unwire(result_page);
+			vm_page_unlock_queues();
+			PAGE_WAKEUP_DONE(result_page);
+
+			vm_fault_cleanup(result_page->object, top_page);
+		}
+	}
+
+	/*
+	 *	Inform the physical mapping system that the range
+	 *	of addresses may fault, so that page tables and
+	 *	such may be unwired themselves.
+	 */
+
+	pmap_pageable(pmap, entry->vme_start, end_addr, TRUE);
+}
+
+/*
+ *	vm_fault_wire_fast:
+ *
+ *	Handle common case of a wire down page fault at the given address.
+ *	If successful, the page is inserted into the associated physical map.
+ *	The map entry is passed in to avoid the overhead of a map lookup.
+ *
+ *	NOTE: the given address should be truncated to the
+ *	proper page address.
+ *
+ *	KERN_SUCCESS is returned if the page fault is handled; otherwise,
+ *	a standard error specifying why the fault is fatal is returned.
+ *
+ *	The map in question must be referenced, and remains so.
+ *	Caller has a read lock on the map.
+ *
+ *	This is a stripped version of vm_fault() for wiring pages.  Anything
+ *	other than the common case will return KERN_FAILURE, and the caller
+ *	is expected to call vm_fault().
+ */
+kern_return_t vm_fault_wire_fast(map, va, entry)
+	vm_map_t	map;
+	vm_offset_t	va;
+	vm_map_entry_t	entry;
+{
+	vm_object_t		object;
+	vm_offset_t		offset;
+	register vm_page_t	m;
+	vm_prot_t		prot;
+
+	vm_stat.faults++;		/* needs lock XXX */
+/*
+ *	Recovery actions
+ */
+
+#undef	RELEASE_PAGE
+#define RELEASE_PAGE(m)	{				\
+	PAGE_WAKEUP_DONE(m);				\
+	vm_page_lock_queues();				\
+	vm_page_unwire(m);				\
+	vm_page_unlock_queues();			\
+}
+
+
+#undef	UNLOCK_THINGS
+#define UNLOCK_THINGS	{				\
+	object->paging_in_progress--;			\
+	vm_object_unlock(object);			\
+}
+
+#undef	UNLOCK_AND_DEALLOCATE
+#define UNLOCK_AND_DEALLOCATE	{			\
+	UNLOCK_THINGS;					\
+	vm_object_deallocate(object);			\
+}
+/*
+ *	Give up and have caller do things the hard way.
+ */
+
+#define GIVE_UP {					\
+	UNLOCK_AND_DEALLOCATE;				\
+	return(KERN_FAILURE);				\
+}
+
+
+	/*
+	 *	If this entry is not directly to a vm_object, bail out.
+	 */
+	if (entry->is_sub_map)
+		return(KERN_FAILURE);
+
+	/*
+	 *	Find the backing store object and offset into it.
+	 */
+
+	object = entry->object.vm_object;
+	offset = (va - entry->vme_start) + entry->offset;
+	prot = entry->protection;
+
+   	/*
+	 *	Make a reference to this object to prevent its
+	 *	disposal while we are messing with it.
+	 */
+
+	vm_object_lock(object);
+	assert(object->ref_count > 0);
+	object->ref_count++;
+	object->paging_in_progress++;
+
+	/*
+	 *	INVARIANTS (through entire routine):
+	 *
+	 *	1)	At all times, we must either have the object
+	 *		lock or a busy page in some object to prevent
+	 *		some other thread from trying to bring in
+	 *		the same page.
+	 *
+	 *	2)	Once we have a busy page, we must remove it from
+	 *		the pageout queues, so that the pageout daemon
+	 *		will not grab it away.
+	 *
+	 */
+
+	/*
+	 *	Look for page in top-level object.  If it's not there or
+	 *	there's something going on, give up.
+	 */
+	m = vm_page_lookup(object, offset);
+	if ((m == VM_PAGE_NULL) || (m->error) ||
+	    (m->busy) || (m->absent) || (prot & m->page_lock)) {
+		GIVE_UP;
+	}
+
+	/*
+	 *	Wire the page down now.  All bail outs beyond this
+	 *	point must unwire the page.  
+	 */
+
+	vm_page_lock_queues();
+	vm_page_wire(m);
+	vm_page_unlock_queues();
+
+	/*
+	 *	Mark page busy for other threads.
+	 */
+	assert(!m->busy);
+	m->busy = TRUE;
+	assert(!m->absent);
+
+	/*
+	 *	Give up if the page is being written and there's a copy object
+	 */
+	if ((object->copy != VM_OBJECT_NULL) && (prot & VM_PROT_WRITE)) {
+		RELEASE_PAGE(m);
+		GIVE_UP;
+	}
+
+	/*
+	 *	Put this page into the physical map.
+	 *	We have to unlock the object because pmap_enter
+	 *	may cause other faults.   
+	 */
+	vm_object_unlock(object);
+
+	PMAP_ENTER(map->pmap, va, m, prot, TRUE);
+
+	/*
+	 *	Must relock object so that paging_in_progress can be cleared.
+	 */
+	vm_object_lock(object);
+
+	/*
+	 *	Unlock everything, and return
+	 */
+
+	PAGE_WAKEUP_DONE(m);
+	UNLOCK_AND_DEALLOCATE;
+
+	return(KERN_SUCCESS);
+
+}
+
+/*
+ *	Routine:	vm_fault_copy_cleanup
+ *	Purpose:
+ *		Release a page used by vm_fault_copy.
+ */
+
+void	vm_fault_copy_cleanup(page, top_page)
+	vm_page_t	page;
+	vm_page_t	top_page;
+{
+	vm_object_t	object = page->object;
+
+	vm_object_lock(object);
+	PAGE_WAKEUP_DONE(page);
+	vm_page_lock_queues();
+	if (!page->active && !page->inactive)
+		vm_page_activate(page);
+	vm_page_unlock_queues();
+	vm_fault_cleanup(object, top_page);
+}
+
+/*
+ *	Routine:	vm_fault_copy
+ *
+ *	Purpose:
+ *		Copy pages from one virtual memory object to another --
+ *		neither the source nor destination pages need be resident.
+ *
+ *		Before actually copying a page, the version associated with
+ *		the destination address map wil be verified.
+ *
+ *	In/out conditions:
+ *		The caller must hold a reference, but not a lock, to
+ *		each of the source and destination objects and to the
+ *		destination map.
+ *
+ *	Results:
+ *		Returns KERN_SUCCESS if no errors were encountered in
+ *		reading or writing the data.  Returns KERN_INTERRUPTED if
+ *		the operation was interrupted (only possible if the
+ *		"interruptible" argument is asserted).  Other return values
+ *		indicate a permanent error in copying the data.
+ *
+ *		The actual amount of data copied will be returned in the
+ *		"copy_size" argument.  In the event that the destination map
+ *		verification failed, this amount may be less than the amount
+ *		requested.
+ */
+kern_return_t	vm_fault_copy(
+			src_object,
+			src_offset,
+			src_size,
+			dst_object,
+			dst_offset,
+			dst_map,
+			dst_version,
+			interruptible
+			)
+	vm_object_t	src_object;
+	vm_offset_t	src_offset;
+	vm_size_t	*src_size;		/* INOUT */
+	vm_object_t	dst_object;
+	vm_offset_t	dst_offset;
+	vm_map_t	dst_map;
+	vm_map_version_t *dst_version;
+	boolean_t	interruptible;
+{
+	vm_page_t		result_page;
+	vm_prot_t		prot;
+	
+	vm_page_t		src_page;
+	vm_page_t		src_top_page;
+
+	vm_page_t		dst_page;
+	vm_page_t		dst_top_page;
+
+	vm_size_t		amount_done;
+	vm_object_t		old_copy_object;
+
+#define	RETURN(x)					\
+	MACRO_BEGIN					\
+	*src_size = amount_done;			\
+	MACRO_RETURN(x);				\
+	MACRO_END
+
+	amount_done = 0;
+	do { /* while (amount_done != *src_size) */
+
+	    RetrySourceFault: ;
+
+		if (src_object == VM_OBJECT_NULL) {
+			/*
+			 *	No source object.  We will just
+			 *	zero-fill the page in dst_object.
+			 */
+
+			src_page = VM_PAGE_NULL;
+		} else {
+			prot = VM_PROT_READ;
+
+			vm_object_lock(src_object);
+			vm_object_paging_begin(src_object);
+
+			switch (vm_fault_page(src_object, src_offset,
+					VM_PROT_READ, FALSE, interruptible,
+					&prot, &result_page, &src_top_page,
+					FALSE, (void (*)()) 0)) {
+
+				case VM_FAULT_SUCCESS:
+					break;
+				case VM_FAULT_RETRY:
+					goto RetrySourceFault;
+				case VM_FAULT_INTERRUPTED:
+					RETURN(MACH_SEND_INTERRUPTED);
+				case VM_FAULT_MEMORY_SHORTAGE:
+					VM_PAGE_WAIT((void (*)()) 0);
+					goto RetrySourceFault;
+				case VM_FAULT_FICTITIOUS_SHORTAGE:
+					vm_page_more_fictitious();
+					goto RetrySourceFault;
+				case VM_FAULT_MEMORY_ERROR:
+					return(KERN_MEMORY_ERROR);
+			}
+
+			src_page = result_page;
+
+			assert((src_top_page == VM_PAGE_NULL) ==
+					(src_page->object == src_object));
+
+			assert ((prot & VM_PROT_READ) != VM_PROT_NONE);
+
+			vm_object_unlock(src_page->object);
+		}
+
+	    RetryDestinationFault: ;
+
+		prot = VM_PROT_WRITE;
+
+		vm_object_lock(dst_object);
+		vm_object_paging_begin(dst_object);
+
+		switch (vm_fault_page(dst_object, dst_offset, VM_PROT_WRITE,
+				FALSE, FALSE /* interruptible */,
+				&prot, &result_page, &dst_top_page,
+				FALSE, (void (*)()) 0)) {
+
+			case VM_FAULT_SUCCESS:
+				break;
+			case VM_FAULT_RETRY:
+				goto RetryDestinationFault;
+			case VM_FAULT_INTERRUPTED:
+				if (src_page != VM_PAGE_NULL)
+					vm_fault_copy_cleanup(src_page,
+							      src_top_page);
+				RETURN(MACH_SEND_INTERRUPTED);
+			case VM_FAULT_MEMORY_SHORTAGE:
+				VM_PAGE_WAIT((void (*)()) 0);
+				goto RetryDestinationFault;
+			case VM_FAULT_FICTITIOUS_SHORTAGE:
+				vm_page_more_fictitious();
+				goto RetryDestinationFault;
+			case VM_FAULT_MEMORY_ERROR:
+				if (src_page != VM_PAGE_NULL)
+					vm_fault_copy_cleanup(src_page,
+							      src_top_page);
+				return(KERN_MEMORY_ERROR);
+		}
+		assert ((prot & VM_PROT_WRITE) != VM_PROT_NONE);
+
+		dst_page = result_page;
+
+		old_copy_object = dst_page->object->copy;
+
+		vm_object_unlock(dst_page->object);
+
+		if (!vm_map_verify(dst_map, dst_version)) {
+
+		 BailOut: ;
+
+			if (src_page != VM_PAGE_NULL)
+				vm_fault_copy_cleanup(src_page, src_top_page);
+			vm_fault_copy_cleanup(dst_page, dst_top_page);
+			break;
+		}
+
+
+		vm_object_lock(dst_page->object);
+		if (dst_page->object->copy != old_copy_object) {
+			vm_object_unlock(dst_page->object);
+			vm_map_verify_done(dst_map, dst_version);
+			goto BailOut;
+		}
+		vm_object_unlock(dst_page->object);
+
+		/*
+		 *	Copy the page, and note that it is dirty
+		 *	immediately.
+		 */
+
+		if (src_page == VM_PAGE_NULL)
+			vm_page_zero_fill(dst_page);
+		else
+			vm_page_copy(src_page, dst_page);
+		dst_page->dirty = TRUE;
+
+		/*
+		 *	Unlock everything, and return
+		 */
+
+		vm_map_verify_done(dst_map, dst_version);
+
+		if (src_page != VM_PAGE_NULL)
+			vm_fault_copy_cleanup(src_page, src_top_page);
+		vm_fault_copy_cleanup(dst_page, dst_top_page);
+
+		amount_done += PAGE_SIZE;
+		src_offset += PAGE_SIZE;
+		dst_offset += PAGE_SIZE;
+
+	} while (amount_done != *src_size);
+
+	RETURN(KERN_SUCCESS);
+#undef	RETURN
+
+	/*NOTREACHED*/	
+}
+
+
+
+
+
+#ifdef	notdef
+
+/*
+ *	Routine:	vm_fault_page_overwrite
+ *
+ *	Description:
+ *		A form of vm_fault_page that assumes that the
+ *		resulting page will be overwritten in its entirety,
+ *		making it unnecessary to obtain the correct *contents*
+ *		of the page.
+ *
+ *	Implementation:
+ *		XXX Untested.  Also unused.  Eventually, this technology
+ *		could be used in vm_fault_copy() to advantage.
+ */
+vm_fault_return_t vm_fault_page_overwrite(dst_object, dst_offset, result_page)
+	register
+	vm_object_t	dst_object;
+	vm_offset_t	dst_offset;
+	vm_page_t	*result_page;	/* OUT */
+{
+	register
+	vm_page_t	dst_page;
+
+#define	interruptible	FALSE	/* XXX */
+
+	while (TRUE) {
+		/*
+		 *	Look for a page at this offset
+		 */
+
+		while ((dst_page = vm_page_lookup(dst_object, dst_offset))
+				 == VM_PAGE_NULL) {
+			/*
+			 *	No page, no problem... just allocate one.
+			 */
+
+			dst_page = vm_page_alloc(dst_object, dst_offset);
+			if (dst_page == VM_PAGE_NULL) {
+				vm_object_unlock(dst_object);
+				VM_PAGE_WAIT((void (*)()) 0);
+				vm_object_lock(dst_object);
+				continue;
+			}
+
+			/*
+			 *	Pretend that the memory manager
+			 *	write-protected the page.
+			 *
+			 *	Note that we will be asking for write
+			 *	permission without asking for the data
+			 *	first.
+			 */
+
+			dst_page->overwriting = TRUE;
+			dst_page->page_lock = VM_PROT_WRITE;
+			dst_page->absent = TRUE;
+			dst_object->absent_count++;
+
+			break;
+
+			/*
+			 *	When we bail out, we might have to throw
+			 *	away the page created here.
+			 */
+
+#define	DISCARD_PAGE						\
+	MACRO_BEGIN						\
+	vm_object_lock(dst_object);				\
+	dst_page = vm_page_lookup(dst_object, dst_offset);	\
+	if ((dst_page != VM_PAGE_NULL) && dst_page->overwriting) \
+	   	VM_PAGE_FREE(dst_page);				\
+	vm_object_unlock(dst_object);				\
+	MACRO_END
+		}
+
+		/*
+		 *	If the page is write-protected...
+		 */
+
+		if (dst_page->page_lock & VM_PROT_WRITE) {
+			/*
+			 *	... and an unlock request hasn't been sent
+			 */
+
+			if ( ! (dst_page->unlock_request & VM_PROT_WRITE)) {
+				vm_prot_t	u;
+				kern_return_t	rc;
+
+				/*
+				 *	... then send one now.
+				 */
+
+				if (!dst_object->pager_ready) {
+					vm_object_assert_wait(dst_object,
+						VM_OBJECT_EVENT_PAGER_READY,
+						interruptible);
+					vm_object_unlock(dst_object);
+					thread_block((void (*)()) 0);
+					if (current_thread()->wait_result !=
+					    THREAD_AWAKENED) {
+						DISCARD_PAGE;
+						return(VM_FAULT_INTERRUPTED);
+					}
+					continue;
+				}
+
+				u = dst_page->unlock_request |= VM_PROT_WRITE;
+				vm_object_unlock(dst_object);
+
+				if ((rc = memory_object_data_unlock(
+						dst_object->pager,
+						dst_object->pager_request,
+						dst_offset + dst_object->paging_offset,
+						PAGE_SIZE,
+						u)) != KERN_SUCCESS) {
+				     	printf("vm_object_overwrite: memory_object_data_unlock failed\n");
+					DISCARD_PAGE;
+					return((rc == MACH_SEND_INTERRUPTED) ?
+						VM_FAULT_INTERRUPTED :
+						VM_FAULT_MEMORY_ERROR);
+				}
+				vm_object_lock(dst_object);
+				continue;
+			}
+
+			/* ... fall through to wait below */
+		} else {
+			/*
+			 *	If the page isn't being used for other
+			 *	purposes, then we're done.
+			 */
+			if ( ! (dst_page->busy || dst_page->absent || dst_page->error) )
+				break;
+		}
+
+		PAGE_ASSERT_WAIT(dst_page, interruptible);
+		vm_object_unlock(dst_object);
+		thread_block((void (*)()) 0);
+		if (current_thread()->wait_result != THREAD_AWAKENED) {
+			DISCARD_PAGE;
+			return(VM_FAULT_INTERRUPTED);
+		}
+	}
+
+	*result_page = dst_page;
+	return(VM_FAULT_SUCCESS);
+
+#undef	interruptible
+#undef	DISCARD_PAGE
+}
+
+#endif	notdef