../vm/vm_fault.c

Bug Summary

File:	obj-scan-build/../vm/vm_fault.c
Location:	line 729, column 5
Description:	Value stored to 'offset' is never read

Annotated Source Code

1	/*
2	* Mach Operating System
3	* Copyright (c) 1994,1990,1989,1988,1987 Carnegie Mellon University.
4	* Copyright (c) 1993,1994 The University of Utah and
5	* the Computer Systems Laboratory (CSL).
6	* All rights reserved.
7	*
8	* Permission to use, copy, modify and distribute this software and its
9	* documentation is hereby granted, provided that both the copyright
10	* notice and this permission notice appear in all copies of the
11	* software, derivative works or modified versions, and any portions
12	* thereof, and that both notices appear in supporting documentation.
13	*
14	* CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15	* THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16	* OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17	* THIS SOFTWARE.
18	*
19	* Carnegie Mellon requests users of this software to return to
20	*
21	* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
22	* School of Computer Science
23	* Carnegie Mellon University
24	* Pittsburgh PA 15213-3890
25	*
26	* any improvements or extensions that they make and grant Carnegie Mellon
27	* the rights to redistribute these changes.
28	*/
29	/*
30	* File: vm_fault.c
31	* Author: Avadis Tevanian, Jr., Michael Wayne Young
32	*
33	* Page fault handling module.
34	*/
35
36	#include <kern/printf.h>
37	#include <vm/vm_fault.h>
38	#include <mach/kern_return.h>
39	#include <mach/message.h> /* for error codes */
40	#include <kern/counters.h>
41	#include <kern/debug.h>
42	#include <kern/thread.h>
43	#include <kern/sched_prim.h>
44	#include <vm/vm_map.h>
45	#include <vm/vm_object.h>
46	#include <vm/vm_page.h>
47	#include <vm/pmap.h>
48	#include <mach/vm_statistics.h>
49	#include <vm/vm_pageout.h>
50	#include <mach/vm_param.h>
51	#include <mach/memory_object.h>
52	#include <vm/memory_object_user.user.h>
53	/* For memory_object_data_{request,unlock} */
54	#include <kern/macro_help.h>
55	#include <kern/slab.h>
56
57	#if MACH_PCSAMPLE1
58	#include <kern/pc_sample.h>
59	#endif
60
61
62
63	/*
64	* State needed by vm_fault_continue.
65	* This is a little hefty to drop directly
66	* into the thread structure.
67	*/
68	typedef struct vm_fault_state {
69	struct vm_map *vmf_map;
70	vm_offset_t vmf_vaddr;
71	vm_prot_t vmf_fault_type;
72	boolean_t vmf_change_wiring;
73	void (*vmf_continuation)();
74	vm_map_version_t vmf_version;
75	boolean_t vmf_wired;
76	struct vm_object *vmf_object;
77	vm_offset_t vmf_offset;
78	vm_prot_t vmf_prot;
79
80	boolean_t vmfp_backoff;
81	struct vm_object *vmfp_object;
82	vm_offset_t vmfp_offset;
83	struct vm_page *vmfp_first_m;
84	vm_prot_t vmfp_access;
85	} vm_fault_state_t;
86
87	struct kmem_cache vm_fault_state_cache;
88
89	int vm_object_absent_max = 50;
90
91	int vm_fault_debug = 0;
92
93	boolean_t vm_fault_dirty_handling = FALSE((boolean_t) 0);
94	boolean_t vm_fault_interruptible = TRUE((boolean_t) 1);
95
96	boolean_t software_reference_bits = TRUE((boolean_t) 1);
97
98	#if MACH_KDB0
99	extern struct db_watchpoint *db_watchpoint_list;
100	#endif /* MACH_KDB */
101
102	/*
103	* Routine: vm_fault_init
104	* Purpose:
105	* Initialize our private data structures.
106	*/
107	void vm_fault_init(void)
108	{
109	kmem_cache_init(&vm_fault_state_cache, "vm_fault_state",
110	sizeof(vm_fault_state_t), 0, NULL((void ) 0), NULL((void ) 0), NULL((void *) 0), 0);
111	}
112
113	/*
114	* Routine: vm_fault_cleanup
115	* Purpose:
116	* Clean up the result of vm_fault_page.
117	* Results:
118	* The paging reference for "object" is released.
119	* "object" is unlocked.
120	* If "top_page" is not null, "top_page" is
121	* freed and the paging reference for the object
122	* containing it is released.
123	*
124	* In/out conditions:
125	* "object" must be locked.
126	*/
127	void
128	vm_fault_cleanup(object, top_page)
129	register vm_object_t object;
130	register vm_page_t top_page;
131	{
132	vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0" , "../vm/vm_fault.c", 132); }); if (--(object)->paging_in_progress == 0) { ({ if ((object)->all_wanted & (1 << (2) )) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2) )), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 << (2)); }); } });
133	vm_object_unlock(object);
134
135	if (top_page != VM_PAGE_NULL((vm_page_t) 0)) {
136	object = top_page->object;
137	vm_object_lock(object);
138	VM_PAGE_FREE(top_page)({ ; vm_page_free(top_page); ; });
139	vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0" , "../vm/vm_fault.c", 139); }); if (--(object)->paging_in_progress == 0) { ({ if ((object)->all_wanted & (1 << (2) )) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2) )), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 << (2)); }); } });
140	vm_object_unlock(object);
141	}
142	}
143
144
145	#if MACH_PCSAMPLE1
146	/*
147	* Do PC sampling on current thread, assuming
148	* that it is the thread taking this page fault.
149	*
150	* Must check for THREAD_NULL, since faults
151	* can occur before threads are running.
152	*/
153
154	#define vm_stat_sample(flavor)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ != ((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample .sampletypes & ((flavor))) take_pc_sample((_thread_), & (_thread_)->pc_sample, ((flavor))); task = (_thread_)-> task; if (task->pc_sample.sampletypes & ((flavor))) take_pc_sample ((_thread_), &task->pc_sample, ((flavor))); }); }) \
155	MACRO_BEGIN({ \
156	thread_t _thread_ = current_thread()(active_threads[(0)]); \
157	\
158	if (_thread_ != THREAD_NULL((thread_t) 0)) \
159	take_pc_sample_macro(_thread_, (flavor))({ task_t task; if ((_thread_)->pc_sample.sampletypes & ((flavor))) take_pc_sample((_thread_), &(_thread_)->pc_sample , ((flavor))); task = (_thread_)->task; if (task->pc_sample .sampletypes & ((flavor))) take_pc_sample((_thread_), & task->pc_sample, ((flavor))); }); \
160	MACRO_END})
161
162	#else
163	#define vm_stat_sample(x)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ != ((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample .sampletypes & ((x))) take_pc_sample((_thread_), &(_thread_ )->pc_sample, ((x))); task = (_thread_)->task; if (task ->pc_sample.sampletypes & ((x))) take_pc_sample((_thread_ ), &task->pc_sample, ((x))); }); })
164	#endif /* MACH_PCSAMPLE */
165
166
167
168	/*
169	* Routine: vm_fault_page
170	* Purpose:
171	* Find the resident page for the virtual memory
172	* specified by the given virtual memory object
173	* and offset.
174	* Additional arguments:
175	* The required permissions for the page is given
176	* in "fault_type". Desired permissions are included
177	* in "protection".
178	*
179	* If the desired page is known to be resident (for
180	* example, because it was previously wired down), asserting
181	* the "unwiring" parameter will speed the search.
182	*
183	* If the operation can be interrupted (by thread_abort
184	* or thread_terminate), then the "interruptible"
185	* parameter should be asserted.
186	*
187	* Results:
188	* The page containing the proper data is returned
189	* in "result_page".
190	*
191	* In/out conditions:
192	* The source object must be locked and referenced,
193	* and must donate one paging reference. The reference
194	* is not affected. The paging reference and lock are
195	* consumed.
196	*
197	* If the call succeeds, the object in which "result_page"
198	* resides is left locked and holding a paging reference.
199	* If this is not the original object, a busy page in the
200	* original object is returned in "top_page", to prevent other
201	* callers from pursuing this same data, along with a paging
202	* reference for the original object. The "top_page" should
203	* be destroyed when this guarantee is no longer required.
204	* The "result_page" is also left busy. It is not removed
205	* from the pageout queues.
206	*/
207	vm_fault_return_t vm_fault_page(first_object, first_offset,
208	fault_type, must_be_resident, interruptible,
209	protection,
210	result_page, top_page,
211	resume, continuation)
212	/* Arguments: */
213	vm_object_t first_object; /* Object to begin search */
214	vm_offset_t first_offset; /* Offset into object */
215	vm_prot_t fault_type; /* What access is requested */
216	boolean_t must_be_resident;/* Must page be resident? */
217	boolean_t interruptible; /* May fault be interrupted? */
218	/* Modifies in place: */
219	vm_prot_t protection; / Protection for mapping */
220	/* Returns: */
221	vm_page_t result_page; / Page found, if successful */
222	vm_page_t top_page; / Page in top object, if
223	* not result_page.
224	*/
225	/* More arguments: */
226	boolean_t resume; /* We are restarting. */
227	void (continuation)(); / Continuation for blocking. */
228	{
229	register
230	vm_page_t m;
231	register
232	vm_object_t object;
233	register
234	vm_offset_t offset;
235	vm_page_t first_m;
236	vm_object_t next_object;
237	vm_object_t copy_object;
238	boolean_t look_for_page;
239	vm_prot_t access_required;
240
241	if (resume) {
242	register vm_fault_state_t *state =
243	(vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
244
245	if (state->vmfp_backoff)
246	goto after_block_and_backoff;
247
248	object = state->vmfp_object;
249	offset = state->vmfp_offset;
250	first_m = state->vmfp_first_m;
251	access_required = state->vmfp_access;
252	goto after_thread_block;
253	}
254
255	vm_stat_sample(SAMPLED_PC_VM_FAULTS_ANY)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ != ((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample .sampletypes & ((0x100))) take_pc_sample((_thread_), & (_thread_)->pc_sample, ((0x100))); task = (_thread_)->task ; if (task->pc_sample.sampletypes & ((0x100))) take_pc_sample ((_thread_), &task->pc_sample, ((0x100))); }); });
256	vm_stat.faults++; /* needs lock XXX */
257	current_task()((active_threads[(0)])->task)->faults++;
258
259	/*
260	* Recovery actions
261	*/
262	#define RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m )->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t ) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ; } \
263	MACRO_BEGIN({ \
264	PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)-> wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)), ((boolean_t) 0), 0); } }); \
265	vm_page_lock_queues(); \
266	if (!m->active && !m->inactive) \
267	vm_page_activate(m); \
268	vm_page_unlock_queues(); \
269	MACRO_END})
270
271	if (vm_fault_dirty_handling
272	#if MACH_KDB0
273	/*
274	* If there are watchpoints set, then
275	* we don't want to give away write permission
276	* on a read fault. Make the task write fault,
277	* so that the watchpoint code notices the access.
278	*/
279	\|\| db_watchpoint_list
280	#endif /* MACH_KDB */
281	) {
282	/*
283	* If we aren't asking for write permission,
284	* then don't give it away. We're using write
285	* faults to set the dirty bit.
286	*/
287	if (!(fault_type & VM_PROT_WRITE((vm_prot_t) 0x02)))
288	*protection &= ~VM_PROT_WRITE((vm_prot_t) 0x02);
289	}
290
291	if (!vm_fault_interruptible)
292	interruptible = FALSE((boolean_t) 0);
293
294	/*
295	* INVARIANTS (through entire routine):
296	*
297	* 1) At all times, we must either have the object
298	* lock or a busy page in some object to prevent
299	* some other thread from trying to bring in
300	* the same page.
301	*
302	* Note that we cannot hold any locks during the
303	* pager access or when waiting for memory, so
304	* we use a busy page then.
305	*
306	* Note also that we aren't as concerned about more than
307	* one thread attempting to memory_object_data_unlock
308	* the same page at once, so we don't hold the page
309	* as busy then, but do record the highest unlock
310	* value so far. [Unlock requests may also be delivered
311	* out of order.]
312	*
313	* 2) To prevent another thread from racing us down the
314	* shadow chain and entering a new page in the top
315	* object before we do, we must keep a busy page in
316	* the top object while following the shadow chain.
317	*
318	* 3) We must increment paging_in_progress on any object
319	* for which we have a busy page, to prevent
320	* vm_object_collapse from removing the busy page
321	* without our noticing.
322	*
323	* 4) We leave busy pages on the pageout queues.
324	* If the pageout daemon comes across a busy page,
325	* it will remove the page from the pageout queues.
326	*/
327
328	/*
329	* Search for the page at object/offset.
330	*/
331
332	object = first_object;
333	offset = first_offset;
334	first_m = VM_PAGE_NULL((vm_page_t) 0);
335	access_required = fault_type;
336
337	/*
338	* See whether this page is resident
339	*/
340
341	while (TRUE((boolean_t) 1)) {
342	m = vm_page_lookup(object, offset);
343	if (m != VM_PAGE_NULL((vm_page_t) 0)) {
344	/*
345	* If the page is being brought in,
346	* wait for it and then retry.
347	*
348	* A possible optimization: if the page
349	* is known to be resident, we can ignore
350	* pages that are absent (regardless of
351	* whether they're busy).
352	*/
353
354	if (m->busy) {
355	kern_return_t wait_result;
356
357	PAGE_ASSERT_WAIT(m, interruptible)({ (m)->wanted = ((boolean_t) 1); assert_wait((event_t) (m ), (interruptible)); });
358	vm_object_unlock(object);
359	if (continuation != (void (*)()) 0) {
360	register vm_fault_state_t *state =
361	(vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
362
363	/*
364	* Save variables in case
365	* thread_block discards
366	* our kernel stack.
367	*/
368
369	state->vmfp_backoff = FALSE((boolean_t) 0);
370	state->vmfp_object = object;
371	state->vmfp_offset = offset;
372	state->vmfp_first_m = first_m;
373	state->vmfp_access =
374	access_required;
375	state->vmf_prot = *protection;
376
377	counter(c_vm_fault_page_block_busy_user++);
378	thread_block(continuation);
379	} else
380	{
381	counter(c_vm_fault_page_block_busy_kernel++);
382	thread_block((void (*)()) 0);
383	}
384	after_thread_block:
385	wait_result = current_thread()(active_threads[(0)])->wait_result;
386	vm_object_lock(object);
387	if (wait_result != THREAD_AWAKENED0) {
388	vm_fault_cleanup(object, first_m);
389	if (wait_result == THREAD_RESTART3)
390	return(VM_FAULT_RETRY1);
391	else
392	return(VM_FAULT_INTERRUPTED2);
393	}
394	continue;
395	}
396
397	/*
398	* If the page is in error, give up now.
399	*/
400
401	if (m->error) {
402	VM_PAGE_FREE(m)({ ; vm_page_free(m); ; });
403	vm_fault_cleanup(object, first_m);
404	return(VM_FAULT_MEMORY_ERROR5);
405	}
406
407	/*
408	* If the page isn't busy, but is absent,
409	* then it was deemed "unavailable".
410	*/
411
412	if (m->absent) {
413	/*
414	* Remove the non-existent page (unless it's
415	* in the top object) and move on down to the
416	* next object (if there is one).
417	*/
418
419	offset += object->shadow_offset;
420	access_required = VM_PROT_READ((vm_prot_t) 0x01);
421	next_object = object->shadow;
422	if (next_object == VM_OBJECT_NULL((vm_object_t) 0)) {
423	vm_page_t real_m;
424
425	assert(!must_be_resident)({ if (!(!must_be_resident)) Assert("!must_be_resident", "../vm/vm_fault.c" , 425); });
426
427	/*
428	* Absent page at bottom of shadow
429	* chain; zero fill the page we left
430	* busy in the first object, and flush
431	* the absent page. But first we
432	* need to allocate a real page.
433	*/
434
435	real_m = vm_page_grab(!object->internal);
436	if (real_m == VM_PAGE_NULL((vm_page_t) 0)) {
437	vm_fault_cleanup(object, first_m);
438	return(VM_FAULT_MEMORY_SHORTAGE3);
439	}
440
441	if (object != first_object) {
442	VM_PAGE_FREE(m)({ ; vm_page_free(m); ; });
443	vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0" , "../vm/vm_fault.c", 443); }); if (--(object)->paging_in_progress == 0) { ({ if ((object)->all_wanted & (1 << (2) )) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2) )), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 << (2)); }); } });
444	vm_object_unlock(object);
445	object = first_object;
446	offset = first_offset;
447	m = first_m;
448	first_m = VM_PAGE_NULL((vm_page_t) 0);
449	vm_object_lock(object);
450	}
451
452	VM_PAGE_FREE(m)({ ; vm_page_free(m); ; });
453	assert(real_m->busy)({ if (!(real_m->busy)) Assert("real_m->busy", "../vm/vm_fault.c" , 453); });
454	vm_page_lock_queues();
455	vm_page_insert(real_m, object, offset);
456	vm_page_unlock_queues();
457	m = real_m;
458
459	/*
460	* Drop the lock while zero filling
461	* page. Then break because this
462	* is the page we wanted. Checking
463	* the page lock is a waste of time;
464	* this page was either absent or
465	* newly allocated -- in both cases
466	* it can't be page locked by a pager.
467	*/
468	vm_object_unlock(object);
469
470	vm_page_zero_fill(m);
471
472	vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ != ((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample .sampletypes & ((0x10))) take_pc_sample((_thread_), & (_thread_)->pc_sample, ((0x10))); task = (_thread_)->task ; if (task->pc_sample.sampletypes & ((0x10))) take_pc_sample ((_thread_), &task->pc_sample, ((0x10))); }); });
473
474	vm_stat.zero_fill_count++;
475	current_task()((active_threads[(0)])->task)->zero_fills++;
476	vm_object_lock(object);
477	pmap_clear_modify(m->phys_addr);
478	break;
479	} else {
480	if (must_be_resident) {
481	vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0" , "../vm/vm_fault.c", 481); }); if (--(object)->paging_in_progress == 0) { ({ if ((object)->all_wanted & (1 << (2) )) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2) )), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 << (2)); }); } });
482	} else if (object != first_object) {
483	vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0" , "../vm/vm_fault.c", 483); }); if (--(object)->paging_in_progress == 0) { ({ if ((object)->all_wanted & (1 << (2) )) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2) )), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 << (2)); }); } });
484	VM_PAGE_FREE(m)({ ; vm_page_free(m); ; });
485	} else {
486	first_m = m;
487	m->absent = FALSE((boolean_t) 0);
488	vm_object_absent_release(object)({ (object)->absent_count--; ({ if (((object))->all_wanted & (1 << (3))) thread_wakeup_prim(((event_t)(((vm_offset_t ) (object)) + (3))), ((boolean_t) 0), 0); ((object))->all_wanted &= ~(1 << (3)); }); });
489	m->busy = TRUE((boolean_t) 1);
490
491	vm_page_lock_queues();
492	VM_PAGE_QUEUES_REMOVE(m)({ if (m->active) { { register queue_entry_t next, prev; next = (m)->pageq.next; prev = (m)->pageq.prev; if ((&vm_page_queue_active ) == next) (&vm_page_queue_active)->prev = prev; else ( (vm_page_t)next)->pageq.prev = prev; if ((&vm_page_queue_active ) == prev) (&vm_page_queue_active)->next = next; else ( (vm_page_t)prev)->pageq.next = next; }; m->active = ((boolean_t ) 0); vm_page_active_count--; } if (m->inactive) { { register queue_entry_t next, prev; next = (m)->pageq.next; prev = ( m)->pageq.prev; if ((&vm_page_queue_inactive) == next) (&vm_page_queue_inactive)->prev = prev; else ((vm_page_t )next)->pageq.prev = prev; if ((&vm_page_queue_inactive ) == prev) (&vm_page_queue_inactive)->next = next; else ((vm_page_t)prev)->pageq.next = next; }; m->inactive = ((boolean_t) 0); vm_page_inactive_count--; } });
493	vm_page_unlock_queues();
494	}
495	vm_object_lock(next_object);
496	vm_object_unlock(object);
497	object = next_object;
498	vm_object_paging_begin(object)((object)->paging_in_progress++);
499	continue;
500	}
501	}
502
503	/*
504	* If the desired access to this page has
505	* been locked out, request that it be unlocked.
506	*/
507
508	if (access_required & m->page_lock) {
509	if ((access_required & m->unlock_request) != access_required) {
510	vm_prot_t new_unlock_request;
511	kern_return_t rc;
512
513	if (!object->pager_ready) {
514	vm_object_assert_wait(object,({ (object)->all_wanted \|= 1 << (1); assert_wait((event_t )(((vm_offset_t) object) + (1)), (interruptible)); })
515	VM_OBJECT_EVENT_PAGER_READY,({ (object)->all_wanted \|= 1 << (1); assert_wait((event_t )(((vm_offset_t) object) + (1)), (interruptible)); })
516	interruptible)({ (object)->all_wanted \|= 1 << (1); assert_wait((event_t )(((vm_offset_t) object) + (1)), (interruptible)); });
517	goto block_and_backoff;
518	}
519
520	new_unlock_request = m->unlock_request =
521	(access_required \| m->unlock_request);
522	vm_object_unlock(object);
523	if ((rc = memory_object_data_unlock(
524	object->pager,
525	object->pager_request,
526	offset + object->paging_offset,
527	PAGE_SIZE(1 << 12),
528	new_unlock_request))
529	!= KERN_SUCCESS0) {
530	printf("vm_fault: memory_object_data_unlock failed\n");
531	vm_object_lock(object);
532	vm_fault_cleanup(object, first_m);
533	return((rc == MACH_SEND_INTERRUPTED0x10000007) ?
534	VM_FAULT_INTERRUPTED2 :
535	VM_FAULT_MEMORY_ERROR5);
536	}
537	vm_object_lock(object);
538	continue;
539	}
540
541	PAGE_ASSERT_WAIT(m, interruptible)({ (m)->wanted = ((boolean_t) 1); assert_wait((event_t) (m ), (interruptible)); });
542	goto block_and_backoff;
543	}
544
545	/*
546	* We mark the page busy and leave it on
547	* the pageout queues. If the pageout
548	* deamon comes across it, then it will
549	* remove the page.
550	*/
551
552	if (!software_reference_bits) {
553	vm_page_lock_queues();
554	if (m->inactive) {
555	vm_stat_sample(SAMPLED_PC_VM_REACTIVATION_FAULTS)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ != ((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample .sampletypes & ((0x20))) take_pc_sample((_thread_), & (_thread_)->pc_sample, ((0x20))); task = (_thread_)->task ; if (task->pc_sample.sampletypes & ((0x20))) take_pc_sample ((_thread_), &task->pc_sample, ((0x20))); }); });
556	vm_stat.reactivations++;
557	current_task()((active_threads[(0)])->task)->reactivations++;
558	}
559
560	VM_PAGE_QUEUES_REMOVE(m)({ if (m->active) { { register queue_entry_t next, prev; next = (m)->pageq.next; prev = (m)->pageq.prev; if ((&vm_page_queue_active ) == next) (&vm_page_queue_active)->prev = prev; else ( (vm_page_t)next)->pageq.prev = prev; if ((&vm_page_queue_active ) == prev) (&vm_page_queue_active)->next = next; else ( (vm_page_t)prev)->pageq.next = next; }; m->active = ((boolean_t ) 0); vm_page_active_count--; } if (m->inactive) { { register queue_entry_t next, prev; next = (m)->pageq.next; prev = ( m)->pageq.prev; if ((&vm_page_queue_inactive) == next) (&vm_page_queue_inactive)->prev = prev; else ((vm_page_t )next)->pageq.prev = prev; if ((&vm_page_queue_inactive ) == prev) (&vm_page_queue_inactive)->next = next; else ((vm_page_t)prev)->pageq.next = next; }; m->inactive = ((boolean_t) 0); vm_page_inactive_count--; } });
561	vm_page_unlock_queues();
562	}
563
564	assert(!m->busy)({ if (!(!m->busy)) Assert("!m->busy", "../vm/vm_fault.c" , 564); });
565	m->busy = TRUE((boolean_t) 1);
566	assert(!m->absent)({ if (!(!m->absent)) Assert("!m->absent", "../vm/vm_fault.c" , 566); });
567	break;
568	}
569
570	look_for_page =
571	(object->pager_created)
572	#if MACH_PAGEMAP1
573	&& (vm_external_state_get(object->existence_info, offset + object->paging_offset)(((object->existence_info) != ((vm_external_t) 0)) ? _vm_external_state_get (object->existence_info, offset + object->paging_offset ) : 2) !=
574	VM_EXTERNAL_STATE_ABSENT3)
575	#endif /* MACH_PAGEMAP */
576	;
577
578	if ((look_for_page \|\| (object == first_object))
579	&& !must_be_resident) {
580	/*
581	* Allocate a new page for this object/offset
582	* pair.
583	*/
584
585	m = vm_page_grab_fictitious();
586	if (m == VM_PAGE_NULL((vm_page_t) 0)) {
587	vm_fault_cleanup(object, first_m);
588	return(VM_FAULT_FICTITIOUS_SHORTAGE4);
589	}
590
591	vm_page_lock_queues();
592	vm_page_insert(m, object, offset);
593	vm_page_unlock_queues();
594	}
595
596	if (look_for_page && !must_be_resident) {
597	kern_return_t rc;
598
599	/*
600	* If the memory manager is not ready, we
601	* cannot make requests.
602	*/
603	if (!object->pager_ready) {
604	vm_object_assert_wait(object,({ (object)->all_wanted \|= 1 << (1); assert_wait((event_t )(((vm_offset_t) object) + (1)), (interruptible)); })
605	VM_OBJECT_EVENT_PAGER_READY,({ (object)->all_wanted \|= 1 << (1); assert_wait((event_t )(((vm_offset_t) object) + (1)), (interruptible)); })
606	interruptible)({ (object)->all_wanted \|= 1 << (1); assert_wait((event_t )(((vm_offset_t) object) + (1)), (interruptible)); });
607	VM_PAGE_FREE(m)({ ; vm_page_free(m); ; });
608	goto block_and_backoff;
609	}
610
611	if (object->internal) {
612	/*
613	* Requests to the default pager
614	* must reserve a real page in advance,
615	* because the pager's data-provided
616	* won't block for pages.
617	*/
618
619	if (m->fictitious && !vm_page_convert(m, FALSE((boolean_t) 0))) {
620	VM_PAGE_FREE(m)({ ; vm_page_free(m); ; });
621	vm_fault_cleanup(object, first_m);
622	return(VM_FAULT_MEMORY_SHORTAGE3);
623	}
624	} else if (object->absent_count >
625	vm_object_absent_max) {
626	/*
627	* If there are too many outstanding page
628	* requests pending on this object, we
629	* wait for them to be resolved now.
630	*/
631
632	vm_object_absent_assert_wait(object, interruptible)({ ({ ((object))->all_wanted \|= 1 << (3); assert_wait ((event_t)(((vm_offset_t) (object)) + (3)), ((interruptible)) ); }); });
633	VM_PAGE_FREE(m)({ ; vm_page_free(m); ; });
634	goto block_and_backoff;
635	}
636
637	/*
638	* Indicate that the page is waiting for data
639	* from the memory manager.
640	*/
641
642	m->absent = TRUE((boolean_t) 1);
643	object->absent_count++;
644
645	/*
646	* We have a busy page, so we can
647	* release the object lock.
648	*/
649	vm_object_unlock(object);
650
651	/*
652	* Call the memory manager to retrieve the data.
653	*/
654
655	vm_stat.pageins++;
656	vm_stat_sample(SAMPLED_PC_VM_PAGEIN_FAULTS)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ != ((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample .sampletypes & ((0x40))) take_pc_sample((_thread_), & (_thread_)->pc_sample, ((0x40))); task = (_thread_)->task ; if (task->pc_sample.sampletypes & ((0x40))) take_pc_sample ((_thread_), &task->pc_sample, ((0x40))); }); });
657	current_task()((active_threads[(0)])->task)->pageins++;
658
659	if ((rc = memory_object_data_request(object->pager,
660	object->pager_request,
661	m->offset + object->paging_offset,
662	PAGE_SIZE(1 << 12), access_required)) != KERN_SUCCESS0) {
663	if (rc != MACH_SEND_INTERRUPTED0x10000007)
664	printf("%s(0x%p, 0x%p, 0x%lx, 0x%x, 0x%x) failed, %x\n",
665	"memory_object_data_request",
666	object->pager,
667	object->pager_request,
668	m->offset + object->paging_offset,
669	PAGE_SIZE(1 << 12), access_required, rc);
670	/*
671	* Don't want to leave a busy page around,
672	* but the data request may have blocked,
673	* so check if it's still there and busy.
674	*/
675	vm_object_lock(object);
676	if (m == vm_page_lookup(object,offset) &&
677	m->absent && m->busy)
678	VM_PAGE_FREE(m)({ ; vm_page_free(m); ; });
679	vm_fault_cleanup(object, first_m);
680	return((rc == MACH_SEND_INTERRUPTED0x10000007) ?
681	VM_FAULT_INTERRUPTED2 :
682	VM_FAULT_MEMORY_ERROR5);
683	}
684
685	/*
686	* Retry with same object/offset, since new data may
687	* be in a different page (i.e., m is meaningless at
688	* this point).
689	*/
690	vm_object_lock(object);
691	continue;
692	}
693
694	/*
695	* For the XP system, the only case in which we get here is if
696	* object has no pager (or unwiring). If the pager doesn't
697	* have the page this is handled in the m->absent case above
698	* (and if you change things here you should look above).
699	*/
700	if (object == first_object)
701	first_m = m;
702	else
703	{
704	assert(m == VM_PAGE_NULL)({ if (!(m == ((vm_page_t) 0))) Assert("m == VM_PAGE_NULL", "../vm/vm_fault.c" , 704); });
705	}
706
707	/*
708	* Move on to the next object. Lock the next
709	* object before unlocking the current one.
710	*/
711	access_required = VM_PROT_READ((vm_prot_t) 0x01);
712
713	offset += object->shadow_offset;
714	next_object = object->shadow;
715	if (next_object == VM_OBJECT_NULL((vm_object_t) 0)) {
716	assert(!must_be_resident)({ if (!(!must_be_resident)) Assert("!must_be_resident", "../vm/vm_fault.c" , 716); });
717
718	/*
719	* If there's no object left, fill the page
720	* in the top object with zeros. But first we
721	* need to allocate a real page.
722	*/
723
724	if (object != first_object) {
725	vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0" , "../vm/vm_fault.c", 725); }); if (--(object)->paging_in_progress == 0) { ({ if ((object)->all_wanted & (1 << (2) )) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2) )), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 << (2)); }); } });
726	vm_object_unlock(object);
727
728	object = first_object;
729	offset = first_offset;
	Value stored to 'offset' is never read
730	vm_object_lock(object);
731	}
732
733	m = first_m;
734	assert(m->object == object)({ if (!(m->object == object)) Assert("m->object == object" , "../vm/vm_fault.c", 734); });
735	first_m = VM_PAGE_NULL((vm_page_t) 0);
736
737	if (m->fictitious && !vm_page_convert(m, !object->internal)) {
738	VM_PAGE_FREE(m)({ ; vm_page_free(m); ; });
739	vm_fault_cleanup(object, VM_PAGE_NULL((vm_page_t) 0));
740	return(VM_FAULT_MEMORY_SHORTAGE3);
741	}
742
743	vm_object_unlock(object);
744	vm_page_zero_fill(m);
745	vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ != ((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample .sampletypes & ((0x10))) take_pc_sample((_thread_), & (_thread_)->pc_sample, ((0x10))); task = (_thread_)->task ; if (task->pc_sample.sampletypes & ((0x10))) take_pc_sample ((_thread_), &task->pc_sample, ((0x10))); }); });
746	vm_stat.zero_fill_count++;
747	current_task()((active_threads[(0)])->task)->zero_fills++;
748	vm_object_lock(object);
749	pmap_clear_modify(m->phys_addr);
750	break;
751	}
752	else {
753	vm_object_lock(next_object);
754	if ((object != first_object) \|\| must_be_resident)
755	vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0" , "../vm/vm_fault.c", 755); }); if (--(object)->paging_in_progress == 0) { ({ if ((object)->all_wanted & (1 << (2) )) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2) )), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 << (2)); }); } });
756	vm_object_unlock(object);
757	object = next_object;
758	vm_object_paging_begin(object)((object)->paging_in_progress++);
759	}
760	}
761
762	/*
763	* PAGE HAS BEEN FOUND.
764	*
765	* This page (m) is:
766	* busy, so that we can play with it;
767	* not absent, so that nobody else will fill it;
768	* possibly eligible for pageout;
769	*
770	* The top-level page (first_m) is:
771	* VM_PAGE_NULL if the page was found in the
772	* top-level object;
773	* busy, not absent, and ineligible for pageout.
774	*
775	* The current object (object) is locked. A paging
776	* reference is held for the current and top-level
777	* objects.
778	*/
779
780	#if EXTRA_ASSERTIONS
781	assert(m->busy && !m->absent)({ if (!(m->busy && !m->absent)) Assert("m->busy && !m->absent" , "../vm/vm_fault.c", 781); });
782	assert((first_m == VM_PAGE_NULL) \|\|({ if (!((first_m == ((vm_page_t) 0)) \|\| (first_m->busy && !first_m->absent && !first_m->active && !first_m->inactive))) Assert("(first_m == VM_PAGE_NULL) \|\| (first_m->busy && !first_m->absent && !first_m->active && !first_m->inactive)" , "../vm/vm_fault.c", 784); })
783	(first_m->busy && !first_m->absent &&({ if (!((first_m == ((vm_page_t) 0)) \|\| (first_m->busy && !first_m->absent && !first_m->active && !first_m->inactive))) Assert("(first_m == VM_PAGE_NULL) \|\| (first_m->busy && !first_m->absent && !first_m->active && !first_m->inactive)" , "../vm/vm_fault.c", 784); })
784	!first_m->active && !first_m->inactive))({ if (!((first_m == ((vm_page_t) 0)) \|\| (first_m->busy && !first_m->absent && !first_m->active && !first_m->inactive))) Assert("(first_m == VM_PAGE_NULL) \|\| (first_m->busy && !first_m->absent && !first_m->active && !first_m->inactive)" , "../vm/vm_fault.c", 784); });
785	#endif /* EXTRA_ASSERTIONS */
786
787	/*
788	* If the page is being written, but isn't
789	* already owned by the top-level object,
790	* we have to copy it into a new page owned
791	* by the top-level object.
792	*/
793
794	if (object != first_object) {
795	/*
796	* We only really need to copy if we
797	* want to write it.
798	*/
799
800	if (fault_type & VM_PROT_WRITE((vm_prot_t) 0x02)) {
801	vm_page_t copy_m;
802
803	assert(!must_be_resident)({ if (!(!must_be_resident)) Assert("!must_be_resident", "../vm/vm_fault.c" , 803); });
804
805	/*
806	* If we try to collapse first_object at this
807	* point, we may deadlock when we try to get
808	* the lock on an intermediate object (since we
809	* have the bottom object locked). We can't
810	* unlock the bottom object, because the page
811	* we found may move (by collapse) if we do.
812	*
813	* Instead, we first copy the page. Then, when
814	* we have no more use for the bottom object,
815	* we unlock it and try to collapse.
816	*
817	* Note that we copy the page even if we didn't
818	* need to... that's the breaks.
819	*/
820
821	/*
822	* Allocate a page for the copy
823	*/
824	copy_m = vm_page_grab(!first_object->internal);
825	if (copy_m == VM_PAGE_NULL((vm_page_t) 0)) {
826	RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m )->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t ) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ; };
827	vm_fault_cleanup(object, first_m);
828	return(VM_FAULT_MEMORY_SHORTAGE3);
829	}
830
831	vm_object_unlock(object);
832	vm_page_copy(m, copy_m);
833	vm_object_lock(object);
834
835	/*
836	* If another map is truly sharing this
837	* page with us, we have to flush all
838	* uses of the original page, since we
839	* can't distinguish those which want the
840	* original from those which need the
841	* new copy.
842	*
843	* XXXO If we know that only one map has
844	* access to this page, then we could
845	* avoid the pmap_page_protect() call.
846	*/
847
848	vm_page_lock_queues();
849	vm_page_deactivate(m);
850	pmap_page_protect(m->phys_addr, VM_PROT_NONE((vm_prot_t) 0x00));
851	vm_page_unlock_queues();
852
853	/*
854	* We no longer need the old page or object.
855	*/
856
857	PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)-> wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)), ((boolean_t) 0), 0); } });
858	vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0" , "../vm/vm_fault.c", 858); }); if (--(object)->paging_in_progress == 0) { ({ if ((object)->all_wanted & (1 << (2) )) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2) )), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 << (2)); }); } });
859	vm_object_unlock(object);
860
861	vm_stat.cow_faults++;
862	vm_stat_sample(SAMPLED_PC_VM_COW_FAULTS)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ != ((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample .sampletypes & ((0x80))) take_pc_sample((_thread_), & (_thread_)->pc_sample, ((0x80))); task = (_thread_)->task ; if (task->pc_sample.sampletypes & ((0x80))) take_pc_sample ((_thread_), &task->pc_sample, ((0x80))); }); });
863	current_task()((active_threads[(0)])->task)->cow_faults++;
864	object = first_object;
865	offset = first_offset;
866
867	vm_object_lock(object);
868	VM_PAGE_FREE(first_m)({ ; vm_page_free(first_m); ; });
869	first_m = VM_PAGE_NULL((vm_page_t) 0);
870	assert(copy_m->busy)({ if (!(copy_m->busy)) Assert("copy_m->busy", "../vm/vm_fault.c" , 870); });
871	vm_page_lock_queues();
872	vm_page_insert(copy_m, object, offset);
873	vm_page_unlock_queues();
874	m = copy_m;
875
876	/*
877	* Now that we've gotten the copy out of the
878	* way, let's try to collapse the top object.
879	* But we have to play ugly games with
880	* paging_in_progress to do that...
881	*/
882
883	vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0" , "../vm/vm_fault.c", 883); }); if (--(object)->paging_in_progress == 0) { ({ if ((object)->all_wanted & (1 << (2) )) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2) )), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 << (2)); }); } });
884	vm_object_collapse(object);
885	vm_object_paging_begin(object)((object)->paging_in_progress++);
886	}
887	else {
888	*protection &= (~VM_PROT_WRITE((vm_prot_t) 0x02));
889	}
890	}
891
892	/*
893	* Now check whether the page needs to be pushed into the
894	* copy object. The use of asymmetric copy on write for
895	* shared temporary objects means that we may do two copies to
896	* satisfy the fault; one above to get the page from a
897	* shadowed object, and one here to push it into the copy.
898	*/
899
900	while ((copy_object = first_object->copy) != VM_OBJECT_NULL((vm_object_t) 0)) {
901	vm_offset_t copy_offset;
902	vm_page_t copy_m;
903
904	/*
905	* If the page is being written, but hasn't been
906	* copied to the copy-object, we have to copy it there.
907	*/
908
909	if ((fault_type & VM_PROT_WRITE((vm_prot_t) 0x02)) == 0) {
910	*protection &= ~VM_PROT_WRITE((vm_prot_t) 0x02);
911	break;
912	}
913
914	/*
915	* If the page was guaranteed to be resident,
916	* we must have already performed the copy.
917	*/
918
919	if (must_be_resident)
920	break;
921
922	/*
923	* Try to get the lock on the copy_object.
924	*/
925	if (!vm_object_lock_try(copy_object)(((boolean_t) 1))) {
926	vm_object_unlock(object);
927
928	simple_lock_pause(); /* wait a bit */
929
930	vm_object_lock(object);
931	continue;
932	}
933
934	/*
935	* Make another reference to the copy-object,
936	* to keep it from disappearing during the
937	* copy.
938	*/
939	assert(copy_object->ref_count > 0)({ if (!(copy_object->ref_count > 0)) Assert("copy_object->ref_count > 0" , "../vm/vm_fault.c", 939); });
940	copy_object->ref_count++;
941
942	/*
943	* Does the page exist in the copy?
944	*/
945	copy_offset = first_offset - copy_object->shadow_offset;
946	copy_m = vm_page_lookup(copy_object, copy_offset);
947	if (copy_m != VM_PAGE_NULL((vm_page_t) 0)) {
948	if (copy_m->busy) {
949	/*
950	* If the page is being brought
951	* in, wait for it and then retry.
952	*/
953	PAGE_ASSERT_WAIT(copy_m, interruptible)({ (copy_m)->wanted = ((boolean_t) 1); assert_wait((event_t ) (copy_m), (interruptible)); });
954	RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m )->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t ) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ; };
955	copy_object->ref_count--;
956	assert(copy_object->ref_count > 0)({ if (!(copy_object->ref_count > 0)) Assert("copy_object->ref_count > 0" , "../vm/vm_fault.c", 956); });
957	vm_object_unlock(copy_object);
958	goto block_and_backoff;
959	}
960	}
961	else {
962	/*
963	* Allocate a page for the copy
964	*/
965	copy_m = vm_page_alloc(copy_object, copy_offset);
966	if (copy_m == VM_PAGE_NULL((vm_page_t) 0)) {
967	RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m )->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t ) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ; };
968	copy_object->ref_count--;
969	assert(copy_object->ref_count > 0)({ if (!(copy_object->ref_count > 0)) Assert("copy_object->ref_count > 0" , "../vm/vm_fault.c", 969); });
970	vm_object_unlock(copy_object);
971	vm_fault_cleanup(object, first_m);
972	return(VM_FAULT_MEMORY_SHORTAGE3);
973	}
974
975	/*
976	* Must copy page into copy-object.
977	*/
978
979	vm_page_copy(m, copy_m);
980
981	/*
982	* If the old page was in use by any users
983	* of the copy-object, it must be removed
984	* from all pmaps. (We can't know which
985	* pmaps use it.)
986	*/
987
988	vm_page_lock_queues();
989	pmap_page_protect(m->phys_addr, VM_PROT_NONE((vm_prot_t) 0x00));
990	copy_m->dirty = TRUE((boolean_t) 1);
991	vm_page_unlock_queues();
992
993	/*
994	* If there's a pager, then immediately
995	* page out this page, using the "initialize"
996	* option. Else, we use the copy.
997	*/
998
999	if (!copy_object->pager_created) {
1000	vm_page_lock_queues();
1001	vm_page_activate(copy_m);
1002	vm_page_unlock_queues();
1003	PAGE_WAKEUP_DONE(copy_m)({ (copy_m)->busy = ((boolean_t) 0); if ((copy_m)->wanted ) { (copy_m)->wanted = ((boolean_t) 0); thread_wakeup_prim ((((event_t) copy_m)), ((boolean_t) 0), 0); } });
1004	} else {
1005	/*
1006	* The page is already ready for pageout:
1007	* not on pageout queues and busy.
1008	* Unlock everything except the
1009	* copy_object itself.
1010	*/
1011
1012	vm_object_unlock(object);
1013
1014	/*
1015	* Write the page to the copy-object,
1016	* flushing it from the kernel.
1017	*/
1018
1019	vm_pageout_page(copy_m, TRUE((boolean_t) 1), TRUE((boolean_t) 1));
1020
1021	/*
1022	* Since the pageout may have
1023	* temporarily dropped the
1024	* copy_object's lock, we
1025	* check whether we'll have
1026	* to deallocate the hard way.
1027	*/
1028
1029	if ((copy_object->shadow != object) \|\|
1030	(copy_object->ref_count == 1)) {
1031	vm_object_unlock(copy_object);
1032	vm_object_deallocate(copy_object);
1033	vm_object_lock(object);
1034	continue;
1035	}
1036
1037	/*
1038	* Pick back up the old object's
1039	* lock. [It is safe to do so,
1040	* since it must be deeper in the
1041	* object tree.]
1042	*/
1043
1044	vm_object_lock(object);
1045	}
1046
1047	/*
1048	* Because we're pushing a page upward
1049	* in the object tree, we must restart
1050	* any faults that are waiting here.
1051	* [Note that this is an expansion of
1052	* PAGE_WAKEUP that uses the THREAD_RESTART
1053	* wait result]. Can't turn off the page's
1054	* busy bit because we're not done with it.
1055	*/
1056
1057	if (m->wanted) {
1058	m->wanted = FALSE((boolean_t) 0);
1059	thread_wakeup_with_result((event_t) m,thread_wakeup_prim(((event_t) m), ((boolean_t) 0), (3))
1060	THREAD_RESTART)thread_wakeup_prim(((event_t) m), ((boolean_t) 0), (3));
1061	}
1062	}
1063
1064	/*
1065	* The reference count on copy_object must be
1066	* at least 2: one for our extra reference,
1067	* and at least one from the outside world
1068	* (we checked that when we last locked
1069	* copy_object).
1070	*/
1071	copy_object->ref_count--;
1072	assert(copy_object->ref_count > 0)({ if (!(copy_object->ref_count > 0)) Assert("copy_object->ref_count > 0" , "../vm/vm_fault.c", 1072); });
1073	vm_object_unlock(copy_object);
1074
1075	break;
1076	}
1077
1078	*result_page = m;
1079	*top_page = first_m;
1080
1081	/*
1082	* If the page can be written, assume that it will be.
1083	* [Earlier, we restrict the permission to allow write
1084	* access only if the fault so required, so we don't
1085	* mark read-only data as dirty.]
1086	*/
1087
1088	if (vm_fault_dirty_handling && (*protection & VM_PROT_WRITE((vm_prot_t) 0x02)))
1089	m->dirty = TRUE((boolean_t) 1);
1090
1091	return(VM_FAULT_SUCCESS0);
1092
1093	block_and_backoff:
1094	vm_fault_cleanup(object, first_m);
1095
1096	if (continuation != (void (*)()) 0) {
1097	register vm_fault_state_t *state =
1098	(vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1099
1100	/*
1101	* Save variables in case we must restart.
1102	*/
1103
1104	state->vmfp_backoff = TRUE((boolean_t) 1);
1105	state->vmf_prot = *protection;
1106
1107	counter(c_vm_fault_page_block_backoff_user++);
1108	thread_block(continuation);
1109	} else
1110	{
1111	counter(c_vm_fault_page_block_backoff_kernel++);
1112	thread_block((void (*)()) 0);
1113	}
1114	after_block_and_backoff:
1115	if (current_thread()(active_threads[(0)])->wait_result == THREAD_AWAKENED0)
1116	return VM_FAULT_RETRY1;
1117	else
1118	return VM_FAULT_INTERRUPTED2;
1119
1120	#undef RELEASE_PAGE
1121	}
1122
1123	/*
1124	* Routine: vm_fault
1125	* Purpose:
1126	* Handle page faults, including pseudo-faults
1127	* used to change the wiring status of pages.
1128	* Returns:
1129	* If an explicit (expression) continuation is supplied,
1130	* then we call the continuation instead of returning.
1131	* Implementation:
1132	* Explicit continuations make this a little icky,
1133	* because it hasn't been rewritten to embrace CPS.
1134	* Instead, we have resume arguments for vm_fault and
1135	* vm_fault_page, to let continue the fault computation.
1136	*
1137	* vm_fault and vm_fault_page save mucho state
1138	* in the moral equivalent of a closure. The state
1139	* structure is allocated when first entering vm_fault
1140	* and deallocated when leaving vm_fault.
1141	*/
1142
1143	void
1144	vm_fault_continue()
1145	{
1146	register vm_fault_state_t *state =
1147	(vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1148
1149	(void) vm_fault(state->vmf_map,
1150	state->vmf_vaddr,
1151	state->vmf_fault_type,
1152	state->vmf_change_wiring,
1153	TRUE((boolean_t) 1), state->vmf_continuation);
1154	/NOTREACHED/
1155	}
1156
1157	kern_return_t vm_fault(map, vaddr, fault_type, change_wiring,
1158	resume, continuation)
1159	vm_map_t map;
1160	vm_offset_t vaddr;
1161	vm_prot_t fault_type;
1162	boolean_t change_wiring;
1163	boolean_t resume;
1164	void (*continuation)();
1165	{
1166	vm_map_version_t version; /* Map version for verificiation */
1167	boolean_t wired; /* Should mapping be wired down? */
1168	vm_object_t object; /* Top-level object */
1169	vm_offset_t offset; /* Top-level offset */
1170	vm_prot_t prot; /* Protection for mapping */
1171	vm_object_t old_copy_object; /* Saved copy object */
1172	vm_page_t result_page; /* Result of vm_fault_page */
1173	vm_page_t top_page; /* Placeholder page */
1174	kern_return_t kr;
1175
1176	register
1177	vm_page_t m; /* Fast access to result_page */
1178
1179	if (resume) {
1180	register vm_fault_state_t *state =
1181	(vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1182
1183	/*
1184	* Retrieve cached variables and
1185	* continue vm_fault_page.
1186	*/
1187
1188	object = state->vmf_object;
1189	if (object == VM_OBJECT_NULL((vm_object_t) 0))
1190	goto RetryFault;
1191	version = state->vmf_version;
1192	wired = state->vmf_wired;
1193	offset = state->vmf_offset;
1194	prot = state->vmf_prot;
1195
1196	kr = vm_fault_page(object, offset, fault_type,
1197	(change_wiring && !wired), !change_wiring,
1198	&prot, &result_page, &top_page,
1199	TRUE((boolean_t) 1), vm_fault_continue);
1200	goto after_vm_fault_page;
1201	}
1202
1203	if (continuation != (void (*)()) 0) {
1204	/*
1205	* We will probably need to save state.
1206	*/
1207
1208	char * state;
1209
1210	/*
1211	* if this assignment stmt is written as
1212	* 'active_threads[cpu_number()] = kmem_cache_alloc()',
1213	* cpu_number may be evaluated before kmem_cache_alloc;
1214	* if kmem_cache_alloc blocks, cpu_number will be wrong
1215	*/
1216
1217	state = (char *) kmem_cache_alloc(&vm_fault_state_cache);
1218	current_thread()(active_threads[(0)])->ith_othersaved.other = state;
1219
1220	}
1221
1222	RetryFault: ;
1223
1224	/*
1225	* Find the backing store object and offset into
1226	* it to begin the search.
1227	*/
1228
1229	if ((kr = vm_map_lookup(&map, vaddr, fault_type, &version,
1230	&object, &offset,
1231	&prot, &wired)) != KERN_SUCCESS0) {
1232	goto done;
1233	}
1234
1235	/*
1236	* If the page is wired, we must fault for the current protection
1237	* value, to avoid further faults.
1238	*/
1239
1240	if (wired)
1241	fault_type = prot;
1242
1243	/*
1244	* Make a reference to this object to
1245	* prevent its disposal while we are messing with
1246	* it. Once we have the reference, the map is free
1247	* to be diddled. Since objects reference their
1248	* shadows (and copies), they will stay around as well.
1249	*/
1250
1251	assert(object->ref_count > 0)({ if (!(object->ref_count > 0)) Assert("object->ref_count > 0" , "../vm/vm_fault.c", 1251); });
1252	object->ref_count++;
1253	vm_object_paging_begin(object)((object)->paging_in_progress++);
1254
1255	if (continuation != (void (*)()) 0) {
1256	register vm_fault_state_t *state =
1257	(vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1258
1259	/*
1260	* Save variables, in case vm_fault_page discards
1261	* our kernel stack and we have to restart.
1262	*/
1263
1264	state->vmf_map = map;
1265	state->vmf_vaddr = vaddr;
1266	state->vmf_fault_type = fault_type;
1267	state->vmf_change_wiring = change_wiring;
1268	state->vmf_continuation = continuation;
1269
1270	state->vmf_version = version;
1271	state->vmf_wired = wired;
1272	state->vmf_object = object;
1273	state->vmf_offset = offset;
1274	state->vmf_prot = prot;
1275
1276	kr = vm_fault_page(object, offset, fault_type,
1277	(change_wiring && !wired), !change_wiring,
1278	&prot, &result_page, &top_page,
1279	FALSE((boolean_t) 0), vm_fault_continue);
1280	} else
1281	{
1282	kr = vm_fault_page(object, offset, fault_type,
1283	(change_wiring && !wired), !change_wiring,
1284	&prot, &result_page, &top_page,
1285	FALSE((boolean_t) 0), (void (*)()) 0);
1286	}
1287	after_vm_fault_page:
1288
1289	/*
1290	* If we didn't succeed, lose the object reference immediately.
1291	*/
1292
1293	if (kr != VM_FAULT_SUCCESS0)
1294	vm_object_deallocate(object);
1295
1296	/*
1297	* See why we failed, and take corrective action.
1298	*/
1299
1300	switch (kr) {
1301	case VM_FAULT_SUCCESS0:
1302	break;
1303	case VM_FAULT_RETRY1:
1304	goto RetryFault;
1305	case VM_FAULT_INTERRUPTED2:
1306	kr = KERN_SUCCESS0;
1307	goto done;
1308	case VM_FAULT_MEMORY_SHORTAGE3:
1309	if (continuation != (void (*)()) 0) {
1310	register vm_fault_state_t *state =
1311	(vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1312
1313	/*
1314	* Save variables in case VM_PAGE_WAIT
1315	* discards our kernel stack.
1316	*/
1317
1318	state->vmf_map = map;
1319	state->vmf_vaddr = vaddr;
1320	state->vmf_fault_type = fault_type;
1321	state->vmf_change_wiring = change_wiring;
1322	state->vmf_continuation = continuation;
1323	state->vmf_object = VM_OBJECT_NULL((vm_object_t) 0);
1324
1325	VM_PAGE_WAIT(vm_fault_continue)vm_page_wait(vm_fault_continue);
1326	} else
1327	VM_PAGE_WAIT((void ()()) 0)vm_page_wait((void ()()) 0);
1328	goto RetryFault;
1329	case VM_FAULT_FICTITIOUS_SHORTAGE4:
1330	vm_page_more_fictitious();
1331	goto RetryFault;
1332	case VM_FAULT_MEMORY_ERROR5:
1333	kr = KERN_MEMORY_ERROR10;
1334	goto done;
1335	}
1336
1337	m = result_page;
1338
1339	assert((change_wiring && !wired) ?({ if (!((change_wiring && !wired) ? (top_page == ((vm_page_t ) 0)) : ((top_page == ((vm_page_t) 0)) == (m->object == object )))) Assert("(change_wiring && !wired) ? (top_page == VM_PAGE_NULL) : ((top_page == VM_PAGE_NULL) == (m->object == object))" , "../vm/vm_fault.c", 1341); })
1340	(top_page == VM_PAGE_NULL) :({ if (!((change_wiring && !wired) ? (top_page == ((vm_page_t ) 0)) : ((top_page == ((vm_page_t) 0)) == (m->object == object )))) Assert("(change_wiring && !wired) ? (top_page == VM_PAGE_NULL) : ((top_page == VM_PAGE_NULL) == (m->object == object))" , "../vm/vm_fault.c", 1341); })
1341	((top_page == VM_PAGE_NULL) == (m->object == object)))({ if (!((change_wiring && !wired) ? (top_page == ((vm_page_t ) 0)) : ((top_page == ((vm_page_t) 0)) == (m->object == object )))) Assert("(change_wiring && !wired) ? (top_page == VM_PAGE_NULL) : ((top_page == VM_PAGE_NULL) == (m->object == object))" , "../vm/vm_fault.c", 1341); });
1342
1343	/*
1344	* How to clean up the result of vm_fault_page. This
1345	* happens whether the mapping is entered or not.
1346	*/
1347
1348	#define UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ; }; vm_object_deallocate (object); } \
1349	MACRO_BEGIN({ \
1350	vm_fault_cleanup(m->object, top_page); \
1351	vm_object_deallocate(object); \
1352	MACRO_END})
1353
1354	/*
1355	* What to do with the resulting page from vm_fault_page
1356	* if it doesn't get entered into the physical map:
1357	*/
1358
1359	#define RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m )->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t ) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ; } \
1360	MACRO_BEGIN({ \
1361	PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)-> wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)), ((boolean_t) 0), 0); } }); \
1362	vm_page_lock_queues(); \
1363	if (!m->active && !m->inactive) \
1364	vm_page_activate(m); \
1365	vm_page_unlock_queues(); \
1366	MACRO_END})
1367
1368	/*
1369	* We must verify that the maps have not changed
1370	* since our last lookup.
1371	*/
1372
1373	old_copy_object = m->object->copy;
1374
1375	vm_object_unlock(m->object);
1376	while (!vm_map_verify(map, &version)) {
1377	vm_object_t retry_object;
1378	vm_offset_t retry_offset;
1379	vm_prot_t retry_prot;
1380
1381	/*
1382	* To avoid trying to write_lock the map while another
1383	* thread has it read_locked (in vm_map_pageable), we
1384	* do not try for write permission. If the page is
1385	* still writable, we will get write permission. If it
1386	* is not, or has been marked needs_copy, we enter the
1387	* mapping without write permission, and will merely
1388	* take another fault.
1389	*/
1390	kr = vm_map_lookup(&map, vaddr,
1391	fault_type & ~VM_PROT_WRITE((vm_prot_t) 0x02), &version,
1392	&retry_object, &retry_offset, &retry_prot,
1393	&wired);
1394
1395	if (kr != KERN_SUCCESS0) {
1396	vm_object_lock(m->object);
1397	RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m )->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t ) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ; };
1398	UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ; }; vm_object_deallocate (object); };
1399	goto done;
1400	}
1401
1402	vm_object_unlock(retry_object);
1403	vm_object_lock(m->object);
1404
1405	if ((retry_object != object) \|\|
1406	(retry_offset != offset)) {
1407	RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m )->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t ) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ; };
1408	UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ; }; vm_object_deallocate (object); };
1409	goto RetryFault;
1410	}
1411
1412	/*
1413	* Check whether the protection has changed or the object
1414	* has been copied while we left the map unlocked.
1415	*/
1416	prot &= retry_prot;
1417	vm_object_unlock(m->object);
1418	}
1419	vm_object_lock(m->object);
1420
1421	/*
1422	* If the copy object changed while the top-level object
1423	* was unlocked, then we must take away write permission.
1424	*/
1425
1426	if (m->object->copy != old_copy_object)
1427	prot &= ~VM_PROT_WRITE((vm_prot_t) 0x02);
1428
1429	/*
1430	* If we want to wire down this page, but no longer have
1431	* adequate permissions, we must start all over.
1432	*/
1433
1434	if (wired && (prot != fault_type)) {
1435	vm_map_verify_done(map, &version)(lock_done(&(map)->lock));
1436	RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m )->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t ) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ; };
1437	UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ; }; vm_object_deallocate (object); };
1438	goto RetryFault;
1439	}
1440
1441	/*
1442	* It's critically important that a wired-down page be faulted
1443	* only once in each map for which it is wired.
1444	*/
1445
1446	vm_object_unlock(m->object);
1447
1448	/*
1449	* Put this page into the physical map.
1450	* We had to do the unlock above because pmap_enter
1451	* may cause other faults. The page may be on
1452	* the pageout queues. If the pageout daemon comes
1453	* across the page, it will remove it from the queues.
1454	*/
1455
1456	PMAP_ENTER(map->pmap, vaddr, m, prot, wired)({ pmap_enter( (map->pmap), (vaddr), (m)->phys_addr, (prot ) & ~(m)->page_lock, (wired) ); });
1457
1458	/*
1459	* If the page is not wired down and isn't already
1460	* on a pageout queue, then put it where the
1461	* pageout daemon can find it.
1462	*/
1463	vm_object_lock(m->object);
1464	vm_page_lock_queues();
1465	if (change_wiring) {
1466	if (wired)
1467	vm_page_wire(m);
1468	else
1469	vm_page_unwire(m);
1470	} else if (software_reference_bits) {
1471	if (!m->active && !m->inactive)
1472	vm_page_activate(m);
1473	m->reference = TRUE((boolean_t) 1);
1474	} else {
1475	vm_page_activate(m);
1476	}
1477	vm_page_unlock_queues();
1478
1479	/*
1480	* Unlock everything, and return
1481	*/
1482
1483	vm_map_verify_done(map, &version)(lock_done(&(map)->lock));
1484	PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)-> wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)), ((boolean_t) 0), 0); } });
1485	kr = KERN_SUCCESS0;
1486	UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ; }; vm_object_deallocate (object); };
1487
1488	#undef UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ; }; vm_object_deallocate (object); }
1489	#undef RELEASE_PAGE
1490
1491	done:
1492	if (continuation != (void (*)()) 0) {
1493	register vm_fault_state_t *state =
1494	(vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1495
1496	kmem_cache_free(&vm_fault_state_cache, (vm_offset_t) state);
1497	(*continuation)(kr);
1498	/NOTREACHED/
1499	}
1500
1501	return(kr);
1502	}
1503
1504	kern_return_t vm_fault_wire_fast();
1505
1506	/*
1507	* vm_fault_wire:
1508	*
1509	* Wire down a range of virtual addresses in a map.
1510	*/
1511	void vm_fault_wire(map, entry)
1512	vm_map_t map;
1513	vm_map_entry_t entry;
1514	{
1515
1516	register vm_offset_t va;
1517	register pmap_t pmap;
1518	register vm_offset_t end_addr = entry->vme_endlinks.end;
1519
1520	pmap = vm_map_pmap(map)((map)->pmap);
1521
1522	/*
1523	* Inform the physical mapping system that the
1524	* range of addresses may not fault, so that
1525	* page tables and such can be locked down as well.
1526	*/
1527
1528	pmap_pageable(pmap, entry->vme_startlinks.start, end_addr, FALSE((boolean_t) 0));
1529
1530	/*
1531	* We simulate a fault to get the page and enter it
1532	* in the physical map.
1533	*/
1534
1535	for (va = entry->vme_startlinks.start; va < end_addr; va += PAGE_SIZE(1 << 12)) {
1536	if (vm_fault_wire_fast(map, va, entry) != KERN_SUCCESS0)
1537	(void) vm_fault(map, va, VM_PROT_NONE((vm_prot_t) 0x00), TRUE((boolean_t) 1),
1538	FALSE((boolean_t) 0), (void (*)()) 0);
1539	}
1540	}
1541
1542	/*
1543	* vm_fault_unwire:
1544	*
1545	* Unwire a range of virtual addresses in a map.
1546	*/
1547	void vm_fault_unwire(map, entry)
1548	vm_map_t map;
1549	vm_map_entry_t entry;
1550	{
1551	register vm_offset_t va;
1552	register pmap_t pmap;
1553	register vm_offset_t end_addr = entry->vme_endlinks.end;
1554	vm_object_t object;
1555
1556	pmap = vm_map_pmap(map)((map)->pmap);
1557
1558	object = (entry->is_sub_map)
1559	? VM_OBJECT_NULL((vm_object_t) 0) : entry->object.vm_object;
1560
1561	/*
1562	* Since the pages are wired down, we must be able to
1563	* get their mappings from the physical map system.
1564	*/
1565
1566	for (va = entry->vme_startlinks.start; va < end_addr; va += PAGE_SIZE(1 << 12)) {
1567	pmap_change_wiring(pmap, va, FALSE((boolean_t) 0));
1568
1569	if (object == VM_OBJECT_NULL((vm_object_t) 0)) {
1570	vm_map_lock_set_recursive(map)lock_set_recursive(&(map)->lock);
1571	(void) vm_fault(map, va, VM_PROT_NONE((vm_prot_t) 0x00), TRUE((boolean_t) 1),
1572	FALSE((boolean_t) 0), (void (*)()) 0);
1573	vm_map_lock_clear_recursive(map)lock_clear_recursive(&(map)->lock);
1574	} else {
1575	vm_prot_t prot;
1576	vm_page_t result_page;
1577	vm_page_t top_page;
1578	vm_fault_return_t result;
1579
1580	do {
1581	prot = VM_PROT_NONE((vm_prot_t) 0x00);
1582
1583	vm_object_lock(object);
1584	vm_object_paging_begin(object)((object)->paging_in_progress++);
1585	result = vm_fault_page(object,
1586	entry->offset +
1587	(va - entry->vme_startlinks.start),
1588	VM_PROT_NONE((vm_prot_t) 0x00), TRUE((boolean_t) 1),
1589	FALSE((boolean_t) 0), &prot,
1590	&result_page,
1591	&top_page,
1592	FALSE((boolean_t) 0), (void (*)()) 0);
1593	} while (result == VM_FAULT_RETRY1);
1594
1595	if (result != VM_FAULT_SUCCESS0)
1596	panic("vm_fault_unwire: failure");
1597
1598	vm_page_lock_queues();
1599	vm_page_unwire(result_page);
1600	vm_page_unlock_queues();
1601	PAGE_WAKEUP_DONE(result_page)({ (result_page)->busy = ((boolean_t) 0); if ((result_page )->wanted) { (result_page)->wanted = ((boolean_t) 0); thread_wakeup_prim ((((event_t) result_page)), ((boolean_t) 0), 0); } });
1602
1603	vm_fault_cleanup(result_page->object, top_page);
1604	}
1605	}
1606
1607	/*
1608	* Inform the physical mapping system that the range
1609	* of addresses may fault, so that page tables and
1610	* such may be unwired themselves.
1611	*/
1612
1613	pmap_pageable(pmap, entry->vme_startlinks.start, end_addr, TRUE((boolean_t) 1));
1614	}
1615
1616	/*
1617	* vm_fault_wire_fast:
1618	*
1619	* Handle common case of a wire down page fault at the given address.
1620	* If successful, the page is inserted into the associated physical map.
1621	* The map entry is passed in to avoid the overhead of a map lookup.
1622	*
1623	* NOTE: the given address should be truncated to the
1624	* proper page address.
1625	*
1626	* KERN_SUCCESS is returned if the page fault is handled; otherwise,
1627	* a standard error specifying why the fault is fatal is returned.
1628	*
1629	* The map in question must be referenced, and remains so.
1630	* Caller has a read lock on the map.
1631	*
1632	* This is a stripped version of vm_fault() for wiring pages. Anything
1633	* other than the common case will return KERN_FAILURE, and the caller
1634	* is expected to call vm_fault().
1635	*/
1636	kern_return_t vm_fault_wire_fast(map, va, entry)
1637	vm_map_t map;
1638	vm_offset_t va;
1639	vm_map_entry_t entry;
1640	{
1641	vm_object_t object;
1642	vm_offset_t offset;
1643	register vm_page_t m;
1644	vm_prot_t prot;
1645
1646	vm_stat.faults++; /* needs lock XXX */
1647	current_task()((active_threads[(0)])->task)->faults++;
1648	/*
1649	* Recovery actions
1650	*/
1651
1652	#undef RELEASE_PAGE
1653	#define RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m )->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t ) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ; } { \
1654	PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)-> wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)), ((boolean_t) 0), 0); } }); \
1655	vm_page_lock_queues(); \
1656	vm_page_unwire(m); \
1657	vm_page_unlock_queues(); \
1658	}
1659
1660
1661	#undef UNLOCK_THINGS{ object->paging_in_progress--; ; }
1662	#define UNLOCK_THINGS{ object->paging_in_progress--; ; } { \
1663	object->paging_in_progress--; \
1664	vm_object_unlock(object); \
1665	}
1666
1667	#undef UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ; }; vm_object_deallocate (object); }
1668	#define UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ; }; vm_object_deallocate (object); } { \
1669	UNLOCK_THINGS{ object->paging_in_progress--; ; }; \
1670	vm_object_deallocate(object); \
1671	}
1672	/*
1673	* Give up and have caller do things the hard way.
1674	*/
1675
1676	#define GIVE_UP{ { { object->paging_in_progress--; ; }; vm_object_deallocate (object); }; return(5); } { \
1677	UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ; }; vm_object_deallocate (object); }; \
1678	return(KERN_FAILURE5); \
1679	}
1680
1681
1682	/*
1683	* If this entry is not directly to a vm_object, bail out.
1684	*/
1685	if (entry->is_sub_map)
1686	return(KERN_FAILURE5);
1687
1688	/*
1689	* Find the backing store object and offset into it.
1690	*/
1691
1692	object = entry->object.vm_object;
1693	offset = (va - entry->vme_startlinks.start) + entry->offset;
1694	prot = entry->protection;
1695
1696	/*
1697	* Make a reference to this object to prevent its
1698	* disposal while we are messing with it.
1699	*/
1700
1701	vm_object_lock(object);
1702	assert(object->ref_count > 0)({ if (!(object->ref_count > 0)) Assert("object->ref_count > 0" , "../vm/vm_fault.c", 1702); });
1703	object->ref_count++;
1704	object->paging_in_progress++;
1705
1706	/*
1707	* INVARIANTS (through entire routine):
1708	*
1709	* 1) At all times, we must either have the object
1710	* lock or a busy page in some object to prevent
1711	* some other thread from trying to bring in
1712	* the same page.
1713	*
1714	* 2) Once we have a busy page, we must remove it from
1715	* the pageout queues, so that the pageout daemon
1716	* will not grab it away.
1717	*
1718	*/
1719
1720	/*
1721	* Look for page in top-level object. If it's not there or
1722	* there's something going on, give up.
1723	*/
1724	m = vm_page_lookup(object, offset);
1725	if ((m == VM_PAGE_NULL((vm_page_t) 0)) \|\| (m->error) \|\|
1726	(m->busy) \|\| (m->absent) \|\| (prot & m->page_lock)) {
1727	GIVE_UP{ { { object->paging_in_progress--; ; }; vm_object_deallocate (object); }; return(5); };
1728	}
1729
1730	/*
1731	* Wire the page down now. All bail outs beyond this
1732	* point must unwire the page.
1733	*/
1734
1735	vm_page_lock_queues();
1736	vm_page_wire(m);
1737	vm_page_unlock_queues();
1738
1739	/*
1740	* Mark page busy for other threads.
1741	*/
1742	assert(!m->busy)({ if (!(!m->busy)) Assert("!m->busy", "../vm/vm_fault.c" , 1742); });
1743	m->busy = TRUE((boolean_t) 1);
1744	assert(!m->absent)({ if (!(!m->absent)) Assert("!m->absent", "../vm/vm_fault.c" , 1744); });
1745
1746	/*
1747	* Give up if the page is being written and there's a copy object
1748	*/
1749	if ((object->copy != VM_OBJECT_NULL((vm_object_t) 0)) && (prot & VM_PROT_WRITE((vm_prot_t) 0x02))) {
1750	RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m )->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t ) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ; };
1751	GIVE_UP{ { { object->paging_in_progress--; ; }; vm_object_deallocate (object); }; return(5); };
1752	}
1753
1754	/*
1755	* Put this page into the physical map.
1756	* We have to unlock the object because pmap_enter
1757	* may cause other faults.
1758	*/
1759	vm_object_unlock(object);
1760
1761	PMAP_ENTER(map->pmap, va, m, prot, TRUE)({ pmap_enter( (map->pmap), (va), (m)->phys_addr, (prot ) & ~(m)->page_lock, (((boolean_t) 1)) ); });
1762
1763	/*
1764	* Must relock object so that paging_in_progress can be cleared.
1765	*/
1766	vm_object_lock(object);
1767
1768	/*
1769	* Unlock everything, and return
1770	*/
1771
1772	PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)-> wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)), ((boolean_t) 0), 0); } });
1773	UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ; }; vm_object_deallocate (object); };
1774
1775	return(KERN_SUCCESS0);
1776
1777	}
1778
1779	/*
1780	* Routine: vm_fault_copy_cleanup
1781	* Purpose:
1782	* Release a page used by vm_fault_copy.
1783	*/
1784
1785	void vm_fault_copy_cleanup(page, top_page)
1786	vm_page_t page;
1787	vm_page_t top_page;
1788	{
1789	vm_object_t object = page->object;
1790
1791	vm_object_lock(object);
1792	PAGE_WAKEUP_DONE(page)({ (page)->busy = ((boolean_t) 0); if ((page)->wanted) { (page)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t ) page)), ((boolean_t) 0), 0); } });
1793	vm_page_lock_queues();
1794	if (!page->active && !page->inactive)
1795	vm_page_activate(page);
1796	vm_page_unlock_queues();
1797	vm_fault_cleanup(object, top_page);
1798	}
1799
1800	/*
1801	* Routine: vm_fault_copy
1802	*
1803	* Purpose:
1804	* Copy pages from one virtual memory object to another --
1805	* neither the source nor destination pages need be resident.
1806	*
1807	* Before actually copying a page, the version associated with
1808	* the destination address map wil be verified.
1809	*
1810	* In/out conditions:
1811	* The caller must hold a reference, but not a lock, to
1812	* each of the source and destination objects and to the
1813	* destination map.
1814	*
1815	* Results:
1816	* Returns KERN_SUCCESS if no errors were encountered in
1817	* reading or writing the data. Returns KERN_INTERRUPTED if
1818	* the operation was interrupted (only possible if the
1819	* "interruptible" argument is asserted). Other return values
1820	* indicate a permanent error in copying the data.
1821	*
1822	* The actual amount of data copied will be returned in the
1823	* "copy_size" argument. In the event that the destination map
1824	* verification failed, this amount may be less than the amount
1825	* requested.
1826	*/
1827	kern_return_t vm_fault_copy(
1828	src_object,
1829	src_offset,
1830	src_size,
1831	dst_object,
1832	dst_offset,
1833	dst_map,
1834	dst_version,
1835	interruptible
1836	)
1837	vm_object_t src_object;
1838	vm_offset_t src_offset;
1839	vm_size_t src_size; / INOUT */
1840	vm_object_t dst_object;
1841	vm_offset_t dst_offset;
1842	vm_map_t dst_map;
1843	vm_map_version_t *dst_version;
1844	boolean_t interruptible;
1845	{
1846	vm_page_t result_page;
1847	vm_prot_t prot;
1848
1849	vm_page_t src_page;
1850	vm_page_t src_top_page;
1851
1852	vm_page_t dst_page;
1853	vm_page_t dst_top_page;
1854
1855	vm_size_t amount_done;
1856	vm_object_t old_copy_object;
1857
1858	#define RETURN(x) \
1859	MACRO_BEGIN({ \
1860	*src_size = amount_done; \
1861	MACRO_RETURNif (((boolean_t) 1)) return(x); \
1862	MACRO_END})
1863
1864	amount_done = 0;
1865	do { /* while (amount_done != src_size) /
1866
1867	RetrySourceFault: ;
1868
1869	if (src_object == VM_OBJECT_NULL((vm_object_t) 0)) {
1870	/*
1871	* No source object. We will just
1872	* zero-fill the page in dst_object.
1873	*/
1874
1875	src_page = VM_PAGE_NULL((vm_page_t) 0);
1876	} else {
1877	prot = VM_PROT_READ((vm_prot_t) 0x01);
1878
1879	vm_object_lock(src_object);
1880	vm_object_paging_begin(src_object)((src_object)->paging_in_progress++);
1881
1882	switch (vm_fault_page(src_object, src_offset,
1883	VM_PROT_READ((vm_prot_t) 0x01), FALSE((boolean_t) 0), interruptible,
1884	&prot, &result_page, &src_top_page,
1885	FALSE((boolean_t) 0), (void (*)()) 0)) {
1886
1887	case VM_FAULT_SUCCESS0:
1888	break;
1889	case VM_FAULT_RETRY1:
1890	goto RetrySourceFault;
1891	case VM_FAULT_INTERRUPTED2:
1892	RETURN(MACH_SEND_INTERRUPTED0x10000007);
1893	case VM_FAULT_MEMORY_SHORTAGE3:
1894	VM_PAGE_WAIT((void ()()) 0)vm_page_wait((void ()()) 0);
1895	goto RetrySourceFault;
1896	case VM_FAULT_FICTITIOUS_SHORTAGE4:
1897	vm_page_more_fictitious();
1898	goto RetrySourceFault;
1899	case VM_FAULT_MEMORY_ERROR5:
1900	return(KERN_MEMORY_ERROR10);
1901	}
1902
1903	src_page = result_page;
1904
1905	assert((src_top_page == VM_PAGE_NULL) ==({ if (!((src_top_page == ((vm_page_t) 0)) == (src_page->object == src_object))) Assert("(src_top_page == VM_PAGE_NULL) == (src_page->object == src_object)" , "../vm/vm_fault.c", 1906); })
1906	(src_page->object == src_object))({ if (!((src_top_page == ((vm_page_t) 0)) == (src_page->object == src_object))) Assert("(src_top_page == VM_PAGE_NULL) == (src_page->object == src_object)" , "../vm/vm_fault.c", 1906); });
1907
1908	assert ((prot & VM_PROT_READ) != VM_PROT_NONE)({ if (!((prot & ((vm_prot_t) 0x01)) != ((vm_prot_t) 0x00 ))) Assert("(prot & VM_PROT_READ) != VM_PROT_NONE", "../vm/vm_fault.c" , 1908); });
1909
1910	vm_object_unlock(src_page->object);
1911	}
1912
1913	RetryDestinationFault: ;
1914
1915	prot = VM_PROT_WRITE((vm_prot_t) 0x02);
1916
1917	vm_object_lock(dst_object);
1918	vm_object_paging_begin(dst_object)((dst_object)->paging_in_progress++);
1919
1920	switch (vm_fault_page(dst_object, dst_offset, VM_PROT_WRITE((vm_prot_t) 0x02),
1921	FALSE((boolean_t) 0), FALSE((boolean_t) 0) /* interruptible */,
1922	&prot, &result_page, &dst_top_page,
1923	FALSE((boolean_t) 0), (void (*)()) 0)) {
1924
1925	case VM_FAULT_SUCCESS0:
1926	break;
1927	case VM_FAULT_RETRY1:
1928	goto RetryDestinationFault;
1929	case VM_FAULT_INTERRUPTED2:
1930	if (src_page != VM_PAGE_NULL((vm_page_t) 0))
1931	vm_fault_copy_cleanup(src_page,
1932	src_top_page);
1933	RETURN(MACH_SEND_INTERRUPTED0x10000007);
1934	case VM_FAULT_MEMORY_SHORTAGE3:
1935	VM_PAGE_WAIT((void ()()) 0)vm_page_wait((void ()()) 0);
1936	goto RetryDestinationFault;
1937	case VM_FAULT_FICTITIOUS_SHORTAGE4:
1938	vm_page_more_fictitious();
1939	goto RetryDestinationFault;
1940	case VM_FAULT_MEMORY_ERROR5:
1941	if (src_page != VM_PAGE_NULL((vm_page_t) 0))
1942	vm_fault_copy_cleanup(src_page,
1943	src_top_page);
1944	return(KERN_MEMORY_ERROR10);
1945	}
1946	assert ((prot & VM_PROT_WRITE) != VM_PROT_NONE)({ if (!((prot & ((vm_prot_t) 0x02)) != ((vm_prot_t) 0x00 ))) Assert("(prot & VM_PROT_WRITE) != VM_PROT_NONE", "../vm/vm_fault.c" , 1946); });
1947
1948	dst_page = result_page;
1949
1950	old_copy_object = dst_page->object->copy;
1951
1952	vm_object_unlock(dst_page->object);
1953
1954	if (!vm_map_verify(dst_map, dst_version)) {
1955
1956	BailOut: ;
1957
1958	if (src_page != VM_PAGE_NULL((vm_page_t) 0))
1959	vm_fault_copy_cleanup(src_page, src_top_page);
1960	vm_fault_copy_cleanup(dst_page, dst_top_page);
1961	break;
1962	}
1963
1964
1965	vm_object_lock(dst_page->object);
1966	if (dst_page->object->copy != old_copy_object) {
1967	vm_object_unlock(dst_page->object);
1968	vm_map_verify_done(dst_map, dst_version)(lock_done(&(dst_map)->lock));
1969	goto BailOut;
1970	}
1971	vm_object_unlock(dst_page->object);
1972
1973	/*
1974	* Copy the page, and note that it is dirty
1975	* immediately.
1976	*/
1977
1978	if (src_page == VM_PAGE_NULL((vm_page_t) 0))
1979	vm_page_zero_fill(dst_page);
1980	else
1981	vm_page_copy(src_page, dst_page);
1982	dst_page->dirty = TRUE((boolean_t) 1);
1983
1984	/*
1985	* Unlock everything, and return
1986	*/
1987
1988	vm_map_verify_done(dst_map, dst_version)(lock_done(&(dst_map)->lock));
1989
1990	if (src_page != VM_PAGE_NULL((vm_page_t) 0))
1991	vm_fault_copy_cleanup(src_page, src_top_page);
1992	vm_fault_copy_cleanup(dst_page, dst_top_page);
1993
1994	amount_done += PAGE_SIZE(1 << 12);
1995	src_offset += PAGE_SIZE(1 << 12);
1996	dst_offset += PAGE_SIZE(1 << 12);
1997
1998	} while (amount_done != *src_size);
1999
2000	RETURN(KERN_SUCCESS0);
2001	#undef RETURN
2002
2003	/NOTREACHED/
2004	}
2005
2006
2007
2008
2009
2010	#ifdef notdef
2011
2012	/*
2013	* Routine: vm_fault_page_overwrite
2014	*
2015	* Description:
2016	* A form of vm_fault_page that assumes that the
2017	* resulting page will be overwritten in its entirety,
2018	* making it unnecessary to obtain the correct contents
2019	* of the page.
2020	*
2021	* Implementation:
2022	* XXX Untested. Also unused. Eventually, this technology
2023	* could be used in vm_fault_copy() to advantage.
2024	*/
2025	vm_fault_return_t vm_fault_page_overwrite(dst_object, dst_offset, result_page)
2026	register
2027	vm_object_t dst_object;
2028	vm_offset_t dst_offset;
2029	vm_page_t result_page; / OUT */
2030	{
2031	register
2032	vm_page_t dst_page;
2033
2034	#define interruptible FALSE((boolean_t) 0) /* XXX */
2035
2036	while (TRUE((boolean_t) 1)) {
2037	/*
2038	* Look for a page at this offset
2039	*/
2040
2041	while ((dst_page = vm_page_lookup(dst_object, dst_offset))
2042	== VM_PAGE_NULL((vm_page_t) 0)) {
2043	/*
2044	* No page, no problem... just allocate one.
2045	*/
2046
2047	dst_page = vm_page_alloc(dst_object, dst_offset);
2048	if (dst_page == VM_PAGE_NULL((vm_page_t) 0)) {
2049	vm_object_unlock(dst_object);
2050	VM_PAGE_WAIT((void ()()) 0)vm_page_wait((void ()()) 0);
2051	vm_object_lock(dst_object);
2052	continue;
2053	}
2054
2055	/*
2056	* Pretend that the memory manager
2057	* write-protected the page.
2058	*
2059	* Note that we will be asking for write
2060	* permission without asking for the data
2061	* first.
2062	*/
2063
2064	dst_page->overwriting = TRUE((boolean_t) 1);
2065	dst_page->page_lock = VM_PROT_WRITE((vm_prot_t) 0x02);
2066	dst_page->absent = TRUE((boolean_t) 1);
2067	dst_object->absent_count++;
2068
2069	break;
2070
2071	/*
2072	* When we bail out, we might have to throw
2073	* away the page created here.
2074	*/
2075
2076	#define DISCARD_PAGE \
2077	MACRO_BEGIN({ \
2078	vm_object_lock(dst_object); \
2079	dst_page = vm_page_lookup(dst_object, dst_offset); \
2080	if ((dst_page != VM_PAGE_NULL((vm_page_t) 0)) && dst_page->overwriting) \
2081	VM_PAGE_FREE(dst_page)({ ; vm_page_free(dst_page); ; }); \
2082	vm_object_unlock(dst_object); \
2083	MACRO_END})
2084	}
2085
2086	/*
2087	* If the page is write-protected...
2088	*/
2089
2090	if (dst_page->page_lock & VM_PROT_WRITE((vm_prot_t) 0x02)) {
2091	/*
2092	* ... and an unlock request hasn't been sent
2093	*/
2094
2095	if ( ! (dst_page->unlock_request & VM_PROT_WRITE((vm_prot_t) 0x02))) {
2096	vm_prot_t u;
2097	kern_return_t rc;
2098
2099	/*
2100	* ... then send one now.
2101	*/
2102
2103	if (!dst_object->pager_ready) {
2104	vm_object_assert_wait(dst_object,({ (dst_object)->all_wanted \|= 1 << (1); assert_wait ((event_t)(((vm_offset_t) dst_object) + (1)), (interruptible) ); })
2105	VM_OBJECT_EVENT_PAGER_READY,({ (dst_object)->all_wanted \|= 1 << (1); assert_wait ((event_t)(((vm_offset_t) dst_object) + (1)), (interruptible) ); })
2106	interruptible)({ (dst_object)->all_wanted \|= 1 << (1); assert_wait ((event_t)(((vm_offset_t) dst_object) + (1)), (interruptible) ); });
2107	vm_object_unlock(dst_object);
2108	thread_block((void (*)()) 0);
2109	if (current_thread()(active_threads[(0)])->wait_result !=
2110	THREAD_AWAKENED0) {
2111	DISCARD_PAGE;
2112	return(VM_FAULT_INTERRUPTED2);
2113	}
2114	continue;
2115	}
2116
2117	u = dst_page->unlock_request \|= VM_PROT_WRITE((vm_prot_t) 0x02);
2118	vm_object_unlock(dst_object);
2119
2120	if ((rc = memory_object_data_unlock(
2121	dst_object->pager,
2122	dst_object->pager_request,
2123	dst_offset + dst_object->paging_offset,
2124	PAGE_SIZE(1 << 12),
2125	u)) != KERN_SUCCESS0) {
2126	printf("vm_object_overwrite: memory_object_data_unlock failed\n");
2127	DISCARD_PAGE;
2128	return((rc == MACH_SEND_INTERRUPTED0x10000007) ?
2129	VM_FAULT_INTERRUPTED2 :
2130	VM_FAULT_MEMORY_ERROR5);
2131	}
2132	vm_object_lock(dst_object);
2133	continue;
2134	}
2135
2136	/* ... fall through to wait below */
2137	} else {
2138	/*
2139	* If the page isn't being used for other
2140	* purposes, then we're done.
2141	*/
2142	if ( ! (dst_page->busy \|\| dst_page->absent \|\| dst_page->error) )
2143	break;
2144	}
2145
2146	PAGE_ASSERT_WAIT(dst_page, interruptible)({ (dst_page)->wanted = ((boolean_t) 1); assert_wait((event_t ) (dst_page), (interruptible)); });
2147	vm_object_unlock(dst_object);
2148	thread_block((void (*)()) 0);
2149	if (current_thread()(active_threads[(0)])->wait_result != THREAD_AWAKENED0) {
2150	DISCARD_PAGE;
2151	return(VM_FAULT_INTERRUPTED2);
2152	}
2153	}
2154
2155	*result_page = dst_page;
2156	return(VM_FAULT_SUCCESS0);
2157
2158	#undef interruptible
2159	#undef DISCARD_PAGE
2160	}
2161
2162	#endif /* notdef */