Bug Summary

File:obj-scan-build/../vm/vm_fault.c
Location:line 548, column 6
Description:Access to field 'task' results in a dereference of a null pointer

Annotated Source Code

1/*
2 * Mach Operating System
3 * Copyright (c) 1994,1990,1989,1988,1987 Carnegie Mellon University.
4 * Copyright (c) 1993,1994 The University of Utah and
5 * the Computer Systems Laboratory (CSL).
6 * All rights reserved.
7 *
8 * Permission to use, copy, modify and distribute this software and its
9 * documentation is hereby granted, provided that both the copyright
10 * notice and this permission notice appear in all copies of the
11 * software, derivative works or modified versions, and any portions
12 * thereof, and that both notices appear in supporting documentation.
13 *
14 * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15 * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16 * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17 * THIS SOFTWARE.
18 *
19 * Carnegie Mellon requests users of this software to return to
20 *
21 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
22 * School of Computer Science
23 * Carnegie Mellon University
24 * Pittsburgh PA 15213-3890
25 *
26 * any improvements or extensions that they make and grant Carnegie Mellon
27 * the rights to redistribute these changes.
28 */
29/*
30 * File: vm_fault.c
31 * Author: Avadis Tevanian, Jr., Michael Wayne Young
32 *
33 * Page fault handling module.
34 */
35
36#include <kern/printf.h>
37#include <vm/vm_fault.h>
38#include <mach/kern_return.h>
39#include <mach/message.h> /* for error codes */
40#include <kern/counters.h>
41#include <kern/debug.h>
42#include <kern/thread.h>
43#include <kern/sched_prim.h>
44#include <vm/vm_map.h>
45#include <vm/vm_object.h>
46#include <vm/vm_page.h>
47#include <vm/pmap.h>
48#include <mach/vm_statistics.h>
49#include <vm/vm_pageout.h>
50#include <mach/vm_param.h>
51#include <mach/memory_object.h>
52#include <vm/memory_object_user.user.h>
53 /* For memory_object_data_{request,unlock} */
54#include <kern/macro_help.h>
55#include <kern/slab.h>
56
57#if MACH_PCSAMPLE1
58#include <kern/pc_sample.h>
59#endif
60
61
62
63/*
64 * State needed by vm_fault_continue.
65 * This is a little hefty to drop directly
66 * into the thread structure.
67 */
68typedef struct vm_fault_state {
69 struct vm_map *vmf_map;
70 vm_offset_t vmf_vaddr;
71 vm_prot_t vmf_fault_type;
72 boolean_t vmf_change_wiring;
73 void (*vmf_continuation)();
74 vm_map_version_t vmf_version;
75 boolean_t vmf_wired;
76 struct vm_object *vmf_object;
77 vm_offset_t vmf_offset;
78 vm_prot_t vmf_prot;
79
80 boolean_t vmfp_backoff;
81 struct vm_object *vmfp_object;
82 vm_offset_t vmfp_offset;
83 struct vm_page *vmfp_first_m;
84 vm_prot_t vmfp_access;
85} vm_fault_state_t;
86
87struct kmem_cache vm_fault_state_cache;
88
89int vm_object_absent_max = 50;
90
91boolean_t vm_fault_dirty_handling = FALSE((boolean_t) 0);
92boolean_t vm_fault_interruptible = TRUE((boolean_t) 1);
93
94boolean_t software_reference_bits = TRUE((boolean_t) 1);
95
96#if MACH_KDB1
97extern struct db_watchpoint *db_watchpoint_list;
98#endif /* MACH_KDB */
99
100/*
101 * Routine: vm_fault_init
102 * Purpose:
103 * Initialize our private data structures.
104 */
105void vm_fault_init(void)
106{
107 kmem_cache_init(&vm_fault_state_cache, "vm_fault_state",
108 sizeof(vm_fault_state_t), 0, NULL((void *) 0), NULL((void *) 0), NULL((void *) 0), 0);
109}
110
111/*
112 * Routine: vm_fault_cleanup
113 * Purpose:
114 * Clean up the result of vm_fault_page.
115 * Results:
116 * The paging reference for "object" is released.
117 * "object" is unlocked.
118 * If "top_page" is not null, "top_page" is
119 * freed and the paging reference for the object
120 * containing it is released.
121 *
122 * In/out conditions:
123 * "object" must be locked.
124 */
125void
126vm_fault_cleanup(
127 vm_object_t object,
128 vm_page_t top_page)
129{
130 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 130); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } })
;
131 vm_object_unlock(object)((void)(&(object)->Lock));
132
133 if (top_page != VM_PAGE_NULL((vm_page_t) 0)) {
134 object = top_page->object;
135 vm_object_lock(object);
136 VM_PAGE_FREE(top_page)({ ; vm_page_free(top_page); ((void)(&vm_page_queue_lock)
); })
;
137 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 137); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } })
;
138 vm_object_unlock(object)((void)(&(object)->Lock));
139 }
140}
141
142
143#if MACH_PCSAMPLE1
144/*
145 * Do PC sampling on current thread, assuming
146 * that it is the thread taking this page fault.
147 *
148 * Must check for THREAD_NULL, since faults
149 * can occur before threads are running.
150 */
151
152#define vm_stat_sample(flavor)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((flavor))) take_pc_sample((_thread_), &
(_thread_)->pc_sample, ((flavor))); task = (_thread_)->
task; if (task->pc_sample.sampletypes & ((flavor))) take_pc_sample
((_thread_), &task->pc_sample, ((flavor))); }); })
\
153 MACRO_BEGIN({ \
154 thread_t _thread_ = current_thread()(active_threads[(0)]); \
155 \
156 if (_thread_ != THREAD_NULL((thread_t) 0)) \
157 take_pc_sample_macro(_thread_, (flavor))({ task_t task; if ((_thread_)->pc_sample.sampletypes &
((flavor))) take_pc_sample((_thread_), &(_thread_)->pc_sample
, ((flavor))); task = (_thread_)->task; if (task->pc_sample
.sampletypes & ((flavor))) take_pc_sample((_thread_), &
task->pc_sample, ((flavor))); })
; \
158 MACRO_END})
159
160#else
161#define vm_stat_sample(x)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((x))) take_pc_sample((_thread_), &(_thread_
)->pc_sample, ((x))); task = (_thread_)->task; if (task
->pc_sample.sampletypes & ((x))) take_pc_sample((_thread_
), &task->pc_sample, ((x))); }); })
162#endif /* MACH_PCSAMPLE */
163
164
165
166/*
167 * Routine: vm_fault_page
168 * Purpose:
169 * Find the resident page for the virtual memory
170 * specified by the given virtual memory object
171 * and offset.
172 * Additional arguments:
173 * The required permissions for the page is given
174 * in "fault_type". Desired permissions are included
175 * in "protection".
176 *
177 * If the desired page is known to be resident (for
178 * example, because it was previously wired down), asserting
179 * the "unwiring" parameter will speed the search.
180 *
181 * If the operation can be interrupted (by thread_abort
182 * or thread_terminate), then the "interruptible"
183 * parameter should be asserted.
184 *
185 * Results:
186 * The page containing the proper data is returned
187 * in "result_page".
188 *
189 * In/out conditions:
190 * The source object must be locked and referenced,
191 * and must donate one paging reference. The reference
192 * is not affected. The paging reference and lock are
193 * consumed.
194 *
195 * If the call succeeds, the object in which "result_page"
196 * resides is left locked and holding a paging reference.
197 * If this is not the original object, a busy page in the
198 * original object is returned in "top_page", to prevent other
199 * callers from pursuing this same data, along with a paging
200 * reference for the original object. The "top_page" should
201 * be destroyed when this guarantee is no longer required.
202 * The "result_page" is also left busy. It is not removed
203 * from the pageout queues.
204 */
205vm_fault_return_t vm_fault_page(
206 /* Arguments: */
207 vm_object_t first_object, /* Object to begin search */
208 vm_offset_t first_offset, /* Offset into object */
209 vm_prot_t fault_type, /* What access is requested */
210 boolean_t must_be_resident,/* Must page be resident? */
211 boolean_t interruptible, /* May fault be interrupted? */
212 /* Modifies in place: */
213 vm_prot_t *protection, /* Protection for mapping */
214 /* Returns: */
215 vm_page_t *result_page, /* Page found, if successful */
216 vm_page_t *top_page, /* Page in top object, if
217 * not result_page.
218 */
219 /* More arguments: */
220 boolean_t resume, /* We are restarting. */
221 void (*continuation)()) /* Continuation for blocking. */
222{
223 vm_page_t m;
224 vm_object_t object;
225 vm_offset_t offset;
226 vm_page_t first_m;
227 vm_object_t next_object;
228 vm_object_t copy_object;
229 boolean_t look_for_page;
230 vm_prot_t access_required;
231
232 if (resume) {
1
Assuming 'resume' is 0
2
Taking false branch
233 vm_fault_state_t *state =
234 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
235
236 if (state->vmfp_backoff)
237 goto after_block_and_backoff;
238
239 object = state->vmfp_object;
240 offset = state->vmfp_offset;
241 first_m = state->vmfp_first_m;
242 access_required = state->vmfp_access;
243 goto after_thread_block;
244 }
245
246 vm_stat_sample(SAMPLED_PC_VM_FAULTS_ANY)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((0x100))) take_pc_sample((_thread_), &
(_thread_)->pc_sample, ((0x100))); task = (_thread_)->task
; if (task->pc_sample.sampletypes & ((0x100))) take_pc_sample
((_thread_), &task->pc_sample, ((0x100))); }); })
;
247 vm_stat.faults++; /* needs lock XXX */
248 current_task()((active_threads[(0)])->task)->faults++;
249
250/*
251 * Recovery actions
252 */
253#define RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ((void
)(&vm_page_queue_lock)); }
\
254 MACRO_BEGIN({ \
255 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } })
; \
256 vm_page_lock_queues(); \
257 if (!m->active && !m->inactive) \
258 vm_page_activate(m); \
259 vm_page_unlock_queues()((void)(&vm_page_queue_lock)); \
260 MACRO_END})
261
262 if (vm_fault_dirty_handling
3
Taking false branch
263#if MACH_KDB1
264 /*
265 * If there are watchpoints set, then
266 * we don't want to give away write permission
267 * on a read fault. Make the task write fault,
268 * so that the watchpoint code notices the access.
269 */
270 || db_watchpoint_list
271#endif /* MACH_KDB */
272 ) {
273 /*
274 * If we aren't asking for write permission,
275 * then don't give it away. We're using write
276 * faults to set the dirty bit.
277 */
278 if (!(fault_type & VM_PROT_WRITE((vm_prot_t) 0x02)))
279 *protection &= ~VM_PROT_WRITE((vm_prot_t) 0x02);
280 }
281
282 if (!vm_fault_interruptible)
4
Assuming 'vm_fault_interruptible' is not equal to 0
5
Taking false branch
283 interruptible = FALSE((boolean_t) 0);
284
285 /*
286 * INVARIANTS (through entire routine):
287 *
288 * 1) At all times, we must either have the object
289 * lock or a busy page in some object to prevent
290 * some other thread from trying to bring in
291 * the same page.
292 *
293 * Note that we cannot hold any locks during the
294 * pager access or when waiting for memory, so
295 * we use a busy page then.
296 *
297 * Note also that we aren't as concerned about more than
298 * one thread attempting to memory_object_data_unlock
299 * the same page at once, so we don't hold the page
300 * as busy then, but do record the highest unlock
301 * value so far. [Unlock requests may also be delivered
302 * out of order.]
303 *
304 * 2) To prevent another thread from racing us down the
305 * shadow chain and entering a new page in the top
306 * object before we do, we must keep a busy page in
307 * the top object while following the shadow chain.
308 *
309 * 3) We must increment paging_in_progress on any object
310 * for which we have a busy page, to prevent
311 * vm_object_collapse from removing the busy page
312 * without our noticing.
313 *
314 * 4) We leave busy pages on the pageout queues.
315 * If the pageout daemon comes across a busy page,
316 * it will remove the page from the pageout queues.
317 */
318
319 /*
320 * Search for the page at object/offset.
321 */
322
323 object = first_object;
324 offset = first_offset;
325 first_m = VM_PAGE_NULL((vm_page_t) 0);
326 access_required = fault_type;
327
328 /*
329 * See whether this page is resident
330 */
331
332 while (TRUE((boolean_t) 1)) {
6
Loop condition is true. Entering loop body
15
Loop condition is true. Entering loop body
22
Loop condition is true. Entering loop body
28
Loop condition is true. Entering loop body
333 m = vm_page_lookup(object, offset);
334 if (m != VM_PAGE_NULL((vm_page_t) 0)) {
7
Assuming 'm' is equal to null
8
Taking false branch
16
Assuming 'm' is equal to null
17
Taking false branch
23
Assuming 'm' is equal to null
24
Taking false branch
29
Assuming 'm' is not equal to null
30
Taking true branch
335 /*
336 * If the page is being brought in,
337 * wait for it and then retry.
338 *
339 * A possible optimization: if the page
340 * is known to be resident, we can ignore
341 * pages that are absent (regardless of
342 * whether they're busy).
343 */
344
345 if (m->busy) {
31
Taking false branch
346 kern_return_t wait_result;
347
348 PAGE_ASSERT_WAIT(m, interruptible)({ (m)->wanted = ((boolean_t) 1); assert_wait((event_t) (m
), (interruptible)); })
;
349 vm_object_unlock(object)((void)(&(object)->Lock));
350 if (continuation != (void (*)()) 0) {
351 vm_fault_state_t *state =
352 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
353
354 /*
355 * Save variables in case
356 * thread_block discards
357 * our kernel stack.
358 */
359
360 state->vmfp_backoff = FALSE((boolean_t) 0);
361 state->vmfp_object = object;
362 state->vmfp_offset = offset;
363 state->vmfp_first_m = first_m;
364 state->vmfp_access =
365 access_required;
366 state->vmf_prot = *protection;
367
368 counter(c_vm_fault_page_block_busy_user++);
369 thread_block(continuation);
370 } else
371 {
372 counter(c_vm_fault_page_block_busy_kernel++);
373 thread_block((void (*)()) 0);
374 }
375 after_thread_block:
376 wait_result = current_thread()(active_threads[(0)])->wait_result;
377 vm_object_lock(object);
378 if (wait_result != THREAD_AWAKENED0) {
379 vm_fault_cleanup(object, first_m);
380 if (wait_result == THREAD_RESTART3)
381 return(VM_FAULT_RETRY1);
382 else
383 return(VM_FAULT_INTERRUPTED2);
384 }
385 continue;
386 }
387
388 /*
389 * If the page is in error, give up now.
390 */
391
392 if (m->error) {
32
Taking false branch
393 VM_PAGE_FREE(m)({ ; vm_page_free(m); ((void)(&vm_page_queue_lock)); });
394 vm_fault_cleanup(object, first_m);
395 return(VM_FAULT_MEMORY_ERROR5);
396 }
397
398 /*
399 * If the page isn't busy, but is absent,
400 * then it was deemed "unavailable".
401 */
402
403 if (m->absent) {
33
Taking false branch
404 /*
405 * Remove the non-existent page (unless it's
406 * in the top object) and move on down to the
407 * next object (if there is one).
408 */
409
410 offset += object->shadow_offset;
411 access_required = VM_PROT_READ((vm_prot_t) 0x01);
412 next_object = object->shadow;
413 if (next_object == VM_OBJECT_NULL((vm_object_t) 0)) {
414 vm_page_t real_m;
415
416 assert(!must_be_resident)({ if (!(!must_be_resident)) Assert("!must_be_resident", "../vm/vm_fault.c"
, 416); })
;
417
418 /*
419 * Absent page at bottom of shadow
420 * chain; zero fill the page we left
421 * busy in the first object, and flush
422 * the absent page. But first we
423 * need to allocate a real page.
424 */
425
426 real_m = vm_page_grab(!object->internal);
427 if (real_m == VM_PAGE_NULL((vm_page_t) 0)) {
428 vm_fault_cleanup(object, first_m);
429 return(VM_FAULT_MEMORY_SHORTAGE3);
430 }
431
432 if (object != first_object) {
433 VM_PAGE_FREE(m)({ ; vm_page_free(m); ((void)(&vm_page_queue_lock)); });
434 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 434); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } })
;
435 vm_object_unlock(object)((void)(&(object)->Lock));
436 object = first_object;
437 offset = first_offset;
438 m = first_m;
439 first_m = VM_PAGE_NULL((vm_page_t) 0);
440 vm_object_lock(object);
441 }
442
443 VM_PAGE_FREE(m)({ ; vm_page_free(m); ((void)(&vm_page_queue_lock)); });
444 assert(real_m->busy)({ if (!(real_m->busy)) Assert("real_m->busy", "../vm/vm_fault.c"
, 444); })
;
445 vm_page_lock_queues();
446 vm_page_insert(real_m, object, offset);
447 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
448 m = real_m;
449
450 /*
451 * Drop the lock while zero filling
452 * page. Then break because this
453 * is the page we wanted. Checking
454 * the page lock is a waste of time;
455 * this page was either absent or
456 * newly allocated -- in both cases
457 * it can't be page locked by a pager.
458 */
459 vm_object_unlock(object)((void)(&(object)->Lock));
460
461 vm_page_zero_fill(m);
462
463 vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((0x10))) take_pc_sample((_thread_), &
(_thread_)->pc_sample, ((0x10))); task = (_thread_)->task
; if (task->pc_sample.sampletypes & ((0x10))) take_pc_sample
((_thread_), &task->pc_sample, ((0x10))); }); })
;
464
465 vm_stat.zero_fill_count++;
466 current_task()((active_threads[(0)])->task)->zero_fills++;
467 vm_object_lock(object);
468 pmap_clear_modify(m->phys_addr);
469 break;
470 } else {
471 if (must_be_resident) {
472 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 472); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } })
;
473 } else if (object != first_object) {
474 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 474); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } })
;
475 VM_PAGE_FREE(m)({ ; vm_page_free(m); ((void)(&vm_page_queue_lock)); });
476 } else {
477 first_m = m;
478 m->absent = FALSE((boolean_t) 0);
479 vm_object_absent_release(object)({ (object)->absent_count--; ({ if (((object))->all_wanted
& (1 << (3))) thread_wakeup_prim(((event_t)(((vm_offset_t
) (object)) + (3))), ((boolean_t) 0), 0); ((object))->all_wanted
&= ~(1 << (3)); }); })
;
480 m->busy = TRUE((boolean_t) 1);
481
482 vm_page_lock_queues();
483 VM_PAGE_QUEUES_REMOVE(m)({ if (m->active) { { queue_entry_t next, prev; next = (m)
->pageq.next; prev = (m)->pageq.prev; if ((&vm_page_queue_active
) == next) (&vm_page_queue_active)->prev = prev; else (
(vm_page_t)next)->pageq.prev = prev; if ((&vm_page_queue_active
) == prev) (&vm_page_queue_active)->next = next; else (
(vm_page_t)prev)->pageq.next = next; }; m->active = ((boolean_t
) 0); vm_page_active_count--; } if (m->inactive) { { queue_entry_t
next, prev; next = (m)->pageq.next; prev = (m)->pageq.
prev; if ((&vm_page_queue_inactive) == next) (&vm_page_queue_inactive
)->prev = prev; else ((vm_page_t)next)->pageq.prev = prev
; if ((&vm_page_queue_inactive) == prev) (&vm_page_queue_inactive
)->next = next; else ((vm_page_t)prev)->pageq.next = next
; }; m->inactive = ((boolean_t) 0); vm_page_inactive_count
--; } })
;
484 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
485 }
486 vm_object_lock(next_object);
487 vm_object_unlock(object)((void)(&(object)->Lock));
488 object = next_object;
489 vm_object_paging_begin(object)((object)->paging_in_progress++);
490 continue;
491 }
492 }
493
494 /*
495 * If the desired access to this page has
496 * been locked out, request that it be unlocked.
497 */
498
499 if (access_required & m->page_lock) {
34
Taking false branch
500 if ((access_required & m->unlock_request) != access_required) {
501 vm_prot_t new_unlock_request;
502 kern_return_t rc;
503
504 if (!object->pager_ready) {
505 vm_object_assert_wait(object,({ (object)->all_wanted |= 1 << (1); assert_wait((event_t
)(((vm_offset_t) object) + (1)), (interruptible)); })
506 VM_OBJECT_EVENT_PAGER_READY,({ (object)->all_wanted |= 1 << (1); assert_wait((event_t
)(((vm_offset_t) object) + (1)), (interruptible)); })
507 interruptible)({ (object)->all_wanted |= 1 << (1); assert_wait((event_t
)(((vm_offset_t) object) + (1)), (interruptible)); })
;
508 goto block_and_backoff;
509 }
510
511 new_unlock_request = m->unlock_request =
512 (access_required | m->unlock_request);
513 vm_object_unlock(object)((void)(&(object)->Lock));
514 if ((rc = memory_object_data_unlock(
515 object->pager,
516 object->pager_request,
517 offset + object->paging_offset,
518 PAGE_SIZE(1 << 12),
519 new_unlock_request))
520 != KERN_SUCCESS0) {
521 printf("vm_fault: memory_object_data_unlock failed\n");
522 vm_object_lock(object);
523 vm_fault_cleanup(object, first_m);
524 return((rc == MACH_SEND_INTERRUPTED0x10000007) ?
525 VM_FAULT_INTERRUPTED2 :
526 VM_FAULT_MEMORY_ERROR5);
527 }
528 vm_object_lock(object);
529 continue;
530 }
531
532 PAGE_ASSERT_WAIT(m, interruptible)({ (m)->wanted = ((boolean_t) 1); assert_wait((event_t) (m
), (interruptible)); })
;
533 goto block_and_backoff;
534 }
535
536 /*
537 * We mark the page busy and leave it on
538 * the pageout queues. If the pageout
539 * deamon comes across it, then it will
540 * remove the page.
541 */
542
543 if (!software_reference_bits) {
35
Assuming 'software_reference_bits' is 0
36
Taking true branch
544 vm_page_lock_queues();
545 if (m->inactive) {
37
Taking true branch
546 vm_stat_sample(SAMPLED_PC_VM_REACTIVATION_FAULTS)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((0x20))) take_pc_sample((_thread_), &
(_thread_)->pc_sample, ((0x20))); task = (_thread_)->task
; if (task->pc_sample.sampletypes & ((0x20))) take_pc_sample
((_thread_), &task->pc_sample, ((0x20))); }); })
;
38
Within the expansion of the macro 'vm_stat_sample':
a
Assuming '_thread_' is equal to null
547 vm_stat.reactivations++;
548 current_task()((active_threads[(0)])->task)->reactivations++;
39
Within the expansion of the macro 'current_task':
a
Access to field 'task' results in a dereference of a null pointer
549 }
550
551 VM_PAGE_QUEUES_REMOVE(m)({ if (m->active) { { queue_entry_t next, prev; next = (m)
->pageq.next; prev = (m)->pageq.prev; if ((&vm_page_queue_active
) == next) (&vm_page_queue_active)->prev = prev; else (
(vm_page_t)next)->pageq.prev = prev; if ((&vm_page_queue_active
) == prev) (&vm_page_queue_active)->next = next; else (
(vm_page_t)prev)->pageq.next = next; }; m->active = ((boolean_t
) 0); vm_page_active_count--; } if (m->inactive) { { queue_entry_t
next, prev; next = (m)->pageq.next; prev = (m)->pageq.
prev; if ((&vm_page_queue_inactive) == next) (&vm_page_queue_inactive
)->prev = prev; else ((vm_page_t)next)->pageq.prev = prev
; if ((&vm_page_queue_inactive) == prev) (&vm_page_queue_inactive
)->next = next; else ((vm_page_t)prev)->pageq.next = next
; }; m->inactive = ((boolean_t) 0); vm_page_inactive_count
--; } })
;
552 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
553 }
554
555 assert(!m->busy)({ if (!(!m->busy)) Assert("!m->busy", "../vm/vm_fault.c"
, 555); })
;
556 m->busy = TRUE((boolean_t) 1);
557 assert(!m->absent)({ if (!(!m->absent)) Assert("!m->absent", "../vm/vm_fault.c"
, 557); })
;
558 break;
559 }
560
561 look_for_page =
562 (object->pager_created)
563#if MACH_PAGEMAP1
564 && (vm_external_state_get(object->existence_info, offset + object->paging_offset)(((object->existence_info) != ((vm_external_t) 0)) ? _vm_external_state_get
(object->existence_info, offset + object->paging_offset
) : 2)
!=
565 VM_EXTERNAL_STATE_ABSENT3)
566#endif /* MACH_PAGEMAP */
567 ;
568
569 if ((look_for_page || (object == first_object))
10
Taking false branch
570 && !must_be_resident) {
9
Assuming 'must_be_resident' is not equal to 0
571 /*
572 * Allocate a new page for this object/offset
573 * pair.
574 */
575
576 m = vm_page_grab_fictitious();
577 if (m == VM_PAGE_NULL((vm_page_t) 0)) {
578 vm_fault_cleanup(object, first_m);
579 return(VM_FAULT_FICTITIOUS_SHORTAGE4);
580 }
581
582 vm_page_lock_queues();
583 vm_page_insert(m, object, offset);
584 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
585 }
586
587 if (look_for_page && !must_be_resident) {
588 kern_return_t rc;
589
590 /*
591 * If the memory manager is not ready, we
592 * cannot make requests.
593 */
594 if (!object->pager_ready) {
595 vm_object_assert_wait(object,({ (object)->all_wanted |= 1 << (1); assert_wait((event_t
)(((vm_offset_t) object) + (1)), (interruptible)); })
596 VM_OBJECT_EVENT_PAGER_READY,({ (object)->all_wanted |= 1 << (1); assert_wait((event_t
)(((vm_offset_t) object) + (1)), (interruptible)); })
597 interruptible)({ (object)->all_wanted |= 1 << (1); assert_wait((event_t
)(((vm_offset_t) object) + (1)), (interruptible)); })
;
598 VM_PAGE_FREE(m)({ ; vm_page_free(m); ((void)(&vm_page_queue_lock)); });
599 goto block_and_backoff;
600 }
601
602 if (object->internal) {
603 /*
604 * Requests to the default pager
605 * must reserve a real page in advance,
606 * because the pager's data-provided
607 * won't block for pages.
608 */
609
610 if (m->fictitious && !vm_page_convert(m, FALSE((boolean_t) 0))) {
611 VM_PAGE_FREE(m)({ ; vm_page_free(m); ((void)(&vm_page_queue_lock)); });
612 vm_fault_cleanup(object, first_m);
613 return(VM_FAULT_MEMORY_SHORTAGE3);
614 }
615 } else if (object->absent_count >
616 vm_object_absent_max) {
617 /*
618 * If there are too many outstanding page
619 * requests pending on this object, we
620 * wait for them to be resolved now.
621 */
622
623 vm_object_absent_assert_wait(object, interruptible)({ ({ ((object))->all_wanted |= 1 << (3); assert_wait
((event_t)(((vm_offset_t) (object)) + (3)), ((interruptible))
); }); })
;
624 VM_PAGE_FREE(m)({ ; vm_page_free(m); ((void)(&vm_page_queue_lock)); });
625 goto block_and_backoff;
626 }
627
628 /*
629 * Indicate that the page is waiting for data
630 * from the memory manager.
631 */
632
633 m->absent = TRUE((boolean_t) 1);
634 object->absent_count++;
635
636 /*
637 * We have a busy page, so we can
638 * release the object lock.
639 */
640 vm_object_unlock(object)((void)(&(object)->Lock));
641
642 /*
643 * Call the memory manager to retrieve the data.
644 */
645
646 vm_stat.pageins++;
647 vm_stat_sample(SAMPLED_PC_VM_PAGEIN_FAULTS)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((0x40))) take_pc_sample((_thread_), &
(_thread_)->pc_sample, ((0x40))); task = (_thread_)->task
; if (task->pc_sample.sampletypes & ((0x40))) take_pc_sample
((_thread_), &task->pc_sample, ((0x40))); }); })
;
648 current_task()((active_threads[(0)])->task)->pageins++;
649
650 if ((rc = memory_object_data_request(object->pager,
651 object->pager_request,
652 m->offset + object->paging_offset,
653 PAGE_SIZE(1 << 12), access_required)) != KERN_SUCCESS0) {
654 if (rc != MACH_SEND_INTERRUPTED0x10000007)
655 printf("%s(0x%p, 0x%p, 0x%lx, 0x%x, 0x%x) failed, %x\n",
656 "memory_object_data_request",
657 object->pager,
658 object->pager_request,
659 m->offset + object->paging_offset,
660 PAGE_SIZE(1 << 12), access_required, rc);
661 /*
662 * Don't want to leave a busy page around,
663 * but the data request may have blocked,
664 * so check if it's still there and busy.
665 */
666 vm_object_lock(object);
667 if (m == vm_page_lookup(object,offset) &&
668 m->absent && m->busy)
669 VM_PAGE_FREE(m)({ ; vm_page_free(m); ((void)(&vm_page_queue_lock)); });
670 vm_fault_cleanup(object, first_m);
671 return((rc == MACH_SEND_INTERRUPTED0x10000007) ?
672 VM_FAULT_INTERRUPTED2 :
673 VM_FAULT_MEMORY_ERROR5);
674 }
675
676 /*
677 * Retry with same object/offset, since new data may
678 * be in a different page (i.e., m is meaningless at
679 * this point).
680 */
681 vm_object_lock(object);
682 continue;
683 }
684
685 /*
686 * For the XP system, the only case in which we get here is if
687 * object has no pager (or unwiring). If the pager doesn't
688 * have the page this is handled in the m->absent case above
689 * (and if you change things here you should look above).
690 */
691 if (object == first_object)
11
Taking true branch
18
Taking false branch
25
Taking false branch
692 first_m = m;
693 else
694 {
695 assert(m == VM_PAGE_NULL)({ if (!(m == ((vm_page_t) 0))) Assert("m == VM_PAGE_NULL", "../vm/vm_fault.c"
, 695); })
;
696 }
697
698 /*
699 * Move on to the next object. Lock the next
700 * object before unlocking the current one.
701 */
702 access_required = VM_PROT_READ((vm_prot_t) 0x01);
703
704 offset += object->shadow_offset;
705 next_object = object->shadow;
706 if (next_object == VM_OBJECT_NULL((vm_object_t) 0)) {
12
Assuming 'next_object' is not equal to null
13
Taking false branch
19
Assuming 'next_object' is not equal to null
20
Taking false branch
26
Assuming 'next_object' is not equal to null
27
Taking false branch
707 assert(!must_be_resident)({ if (!(!must_be_resident)) Assert("!must_be_resident", "../vm/vm_fault.c"
, 707); })
;
708
709 /*
710 * If there's no object left, fill the page
711 * in the top object with zeros. But first we
712 * need to allocate a real page.
713 */
714
715 if (object != first_object) {
716 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 716); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } })
;
717 vm_object_unlock(object)((void)(&(object)->Lock));
718
719 object = first_object;
720 offset = first_offset;
721 vm_object_lock(object);
722 }
723
724 m = first_m;
725 assert(m->object == object)({ if (!(m->object == object)) Assert("m->object == object"
, "../vm/vm_fault.c", 725); })
;
726 first_m = VM_PAGE_NULL((vm_page_t) 0);
727
728 if (m->fictitious && !vm_page_convert(m, !object->internal)) {
729 VM_PAGE_FREE(m)({ ; vm_page_free(m); ((void)(&vm_page_queue_lock)); });
730 vm_fault_cleanup(object, VM_PAGE_NULL((vm_page_t) 0));
731 return(VM_FAULT_MEMORY_SHORTAGE3);
732 }
733
734 vm_object_unlock(object)((void)(&(object)->Lock));
735 vm_page_zero_fill(m);
736 vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((0x10))) take_pc_sample((_thread_), &
(_thread_)->pc_sample, ((0x10))); task = (_thread_)->task
; if (task->pc_sample.sampletypes & ((0x10))) take_pc_sample
((_thread_), &task->pc_sample, ((0x10))); }); })
;
737 vm_stat.zero_fill_count++;
738 current_task()((active_threads[(0)])->task)->zero_fills++;
739 vm_object_lock(object);
740 pmap_clear_modify(m->phys_addr);
741 break;
742 }
743 else {
744 vm_object_lock(next_object);
745 if ((object != first_object) || must_be_resident)
14
Taking true branch
21
Taking true branch
746 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 746); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } })
;
747 vm_object_unlock(object)((void)(&(object)->Lock));
748 object = next_object;
749 vm_object_paging_begin(object)((object)->paging_in_progress++);
750 }
751 }
752
753 /*
754 * PAGE HAS BEEN FOUND.
755 *
756 * This page (m) is:
757 * busy, so that we can play with it;
758 * not absent, so that nobody else will fill it;
759 * possibly eligible for pageout;
760 *
761 * The top-level page (first_m) is:
762 * VM_PAGE_NULL if the page was found in the
763 * top-level object;
764 * busy, not absent, and ineligible for pageout.
765 *
766 * The current object (object) is locked. A paging
767 * reference is held for the current and top-level
768 * objects.
769 */
770
771#if EXTRA_ASSERTIONS
772 assert(m->busy && !m->absent)({ if (!(m->busy && !m->absent)) Assert("m->busy && !m->absent"
, "../vm/vm_fault.c", 772); })
;
773 assert((first_m == VM_PAGE_NULL) ||({ if (!((first_m == ((vm_page_t) 0)) || (first_m->busy &&
!first_m->absent && !first_m->active &&
!first_m->inactive))) Assert("(first_m == VM_PAGE_NULL) || (first_m->busy && !first_m->absent && !first_m->active && !first_m->inactive)"
, "../vm/vm_fault.c", 775); })
774 (first_m->busy && !first_m->absent &&({ if (!((first_m == ((vm_page_t) 0)) || (first_m->busy &&
!first_m->absent && !first_m->active &&
!first_m->inactive))) Assert("(first_m == VM_PAGE_NULL) || (first_m->busy && !first_m->absent && !first_m->active && !first_m->inactive)"
, "../vm/vm_fault.c", 775); })
775 !first_m->active && !first_m->inactive))({ if (!((first_m == ((vm_page_t) 0)) || (first_m->busy &&
!first_m->absent && !first_m->active &&
!first_m->inactive))) Assert("(first_m == VM_PAGE_NULL) || (first_m->busy && !first_m->absent && !first_m->active && !first_m->inactive)"
, "../vm/vm_fault.c", 775); })
;
776#endif /* EXTRA_ASSERTIONS */
777
778 /*
779 * If the page is being written, but isn't
780 * already owned by the top-level object,
781 * we have to copy it into a new page owned
782 * by the top-level object.
783 */
784
785 if (object != first_object) {
786 /*
787 * We only really need to copy if we
788 * want to write it.
789 */
790
791 if (fault_type & VM_PROT_WRITE((vm_prot_t) 0x02)) {
792 vm_page_t copy_m;
793
794 assert(!must_be_resident)({ if (!(!must_be_resident)) Assert("!must_be_resident", "../vm/vm_fault.c"
, 794); })
;
795
796 /*
797 * If we try to collapse first_object at this
798 * point, we may deadlock when we try to get
799 * the lock on an intermediate object (since we
800 * have the bottom object locked). We can't
801 * unlock the bottom object, because the page
802 * we found may move (by collapse) if we do.
803 *
804 * Instead, we first copy the page. Then, when
805 * we have no more use for the bottom object,
806 * we unlock it and try to collapse.
807 *
808 * Note that we copy the page even if we didn't
809 * need to... that's the breaks.
810 */
811
812 /*
813 * Allocate a page for the copy
814 */
815 copy_m = vm_page_grab(!first_object->internal);
816 if (copy_m == VM_PAGE_NULL((vm_page_t) 0)) {
817 RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ((void
)(&vm_page_queue_lock)); }
;
818 vm_fault_cleanup(object, first_m);
819 return(VM_FAULT_MEMORY_SHORTAGE3);
820 }
821
822 vm_object_unlock(object)((void)(&(object)->Lock));
823 vm_page_copy(m, copy_m);
824 vm_object_lock(object);
825
826 /*
827 * If another map is truly sharing this
828 * page with us, we have to flush all
829 * uses of the original page, since we
830 * can't distinguish those which want the
831 * original from those which need the
832 * new copy.
833 *
834 * XXXO If we know that only one map has
835 * access to this page, then we could
836 * avoid the pmap_page_protect() call.
837 */
838
839 vm_page_lock_queues();
840 vm_page_deactivate(m);
841 pmap_page_protect(m->phys_addr, VM_PROT_NONE((vm_prot_t) 0x00));
842 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
843
844 /*
845 * We no longer need the old page or object.
846 */
847
848 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } })
;
849 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 849); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } })
;
850 vm_object_unlock(object)((void)(&(object)->Lock));
851
852 vm_stat.cow_faults++;
853 vm_stat_sample(SAMPLED_PC_VM_COW_FAULTS)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((0x80))) take_pc_sample((_thread_), &
(_thread_)->pc_sample, ((0x80))); task = (_thread_)->task
; if (task->pc_sample.sampletypes & ((0x80))) take_pc_sample
((_thread_), &task->pc_sample, ((0x80))); }); })
;
854 current_task()((active_threads[(0)])->task)->cow_faults++;
855 object = first_object;
856 offset = first_offset;
857
858 vm_object_lock(object);
859 VM_PAGE_FREE(first_m)({ ; vm_page_free(first_m); ((void)(&vm_page_queue_lock))
; })
;
860 first_m = VM_PAGE_NULL((vm_page_t) 0);
861 assert(copy_m->busy)({ if (!(copy_m->busy)) Assert("copy_m->busy", "../vm/vm_fault.c"
, 861); })
;
862 vm_page_lock_queues();
863 vm_page_insert(copy_m, object, offset);
864 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
865 m = copy_m;
866
867 /*
868 * Now that we've gotten the copy out of the
869 * way, let's try to collapse the top object.
870 * But we have to play ugly games with
871 * paging_in_progress to do that...
872 */
873
874 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 874); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } })
;
875 vm_object_collapse(object);
876 vm_object_paging_begin(object)((object)->paging_in_progress++);
877 }
878 else {
879 *protection &= (~VM_PROT_WRITE((vm_prot_t) 0x02));
880 }
881 }
882
883 /*
884 * Now check whether the page needs to be pushed into the
885 * copy object. The use of asymmetric copy on write for
886 * shared temporary objects means that we may do two copies to
887 * satisfy the fault; one above to get the page from a
888 * shadowed object, and one here to push it into the copy.
889 */
890
891 while ((copy_object = first_object->copy) != VM_OBJECT_NULL((vm_object_t) 0)) {
892 vm_offset_t copy_offset;
893 vm_page_t copy_m;
894
895 /*
896 * If the page is being written, but hasn't been
897 * copied to the copy-object, we have to copy it there.
898 */
899
900 if ((fault_type & VM_PROT_WRITE((vm_prot_t) 0x02)) == 0) {
901 *protection &= ~VM_PROT_WRITE((vm_prot_t) 0x02);
902 break;
903 }
904
905 /*
906 * If the page was guaranteed to be resident,
907 * we must have already performed the copy.
908 */
909
910 if (must_be_resident)
911 break;
912
913 /*
914 * Try to get the lock on the copy_object.
915 */
916 if (!vm_object_lock_try(copy_object)(((boolean_t) 1))) {
917 vm_object_unlock(object)((void)(&(object)->Lock));
918
919 simple_lock_pause(); /* wait a bit */
920
921 vm_object_lock(object);
922 continue;
923 }
924
925 /*
926 * Make another reference to the copy-object,
927 * to keep it from disappearing during the
928 * copy.
929 */
930 assert(copy_object->ref_count > 0)({ if (!(copy_object->ref_count > 0)) Assert("copy_object->ref_count > 0"
, "../vm/vm_fault.c", 930); })
;
931 copy_object->ref_count++;
932
933 /*
934 * Does the page exist in the copy?
935 */
936 copy_offset = first_offset - copy_object->shadow_offset;
937 copy_m = vm_page_lookup(copy_object, copy_offset);
938 if (copy_m != VM_PAGE_NULL((vm_page_t) 0)) {
939 if (copy_m->busy) {
940 /*
941 * If the page is being brought
942 * in, wait for it and then retry.
943 */
944 PAGE_ASSERT_WAIT(copy_m, interruptible)({ (copy_m)->wanted = ((boolean_t) 1); assert_wait((event_t
) (copy_m), (interruptible)); })
;
945 RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ((void
)(&vm_page_queue_lock)); }
;
946 copy_object->ref_count--;
947 assert(copy_object->ref_count > 0)({ if (!(copy_object->ref_count > 0)) Assert("copy_object->ref_count > 0"
, "../vm/vm_fault.c", 947); })
;
948 vm_object_unlock(copy_object)((void)(&(copy_object)->Lock));
949 goto block_and_backoff;
950 }
951 }
952 else {
953 /*
954 * Allocate a page for the copy
955 */
956 copy_m = vm_page_alloc(copy_object, copy_offset);
957 if (copy_m == VM_PAGE_NULL((vm_page_t) 0)) {
958 RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ((void
)(&vm_page_queue_lock)); }
;
959 copy_object->ref_count--;
960 assert(copy_object->ref_count > 0)({ if (!(copy_object->ref_count > 0)) Assert("copy_object->ref_count > 0"
, "../vm/vm_fault.c", 960); })
;
961 vm_object_unlock(copy_object)((void)(&(copy_object)->Lock));
962 vm_fault_cleanup(object, first_m);
963 return(VM_FAULT_MEMORY_SHORTAGE3);
964 }
965
966 /*
967 * Must copy page into copy-object.
968 */
969
970 vm_page_copy(m, copy_m);
971
972 /*
973 * If the old page was in use by any users
974 * of the copy-object, it must be removed
975 * from all pmaps. (We can't know which
976 * pmaps use it.)
977 */
978
979 vm_page_lock_queues();
980 pmap_page_protect(m->phys_addr, VM_PROT_NONE((vm_prot_t) 0x00));
981 copy_m->dirty = TRUE((boolean_t) 1);
982 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
983
984 /*
985 * If there's a pager, then immediately
986 * page out this page, using the "initialize"
987 * option. Else, we use the copy.
988 */
989
990 if (!copy_object->pager_created) {
991 vm_page_lock_queues();
992 vm_page_activate(copy_m);
993 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
994 PAGE_WAKEUP_DONE(copy_m)({ (copy_m)->busy = ((boolean_t) 0); if ((copy_m)->wanted
) { (copy_m)->wanted = ((boolean_t) 0); thread_wakeup_prim
((((event_t) copy_m)), ((boolean_t) 0), 0); } })
;
995 } else {
996 /*
997 * The page is already ready for pageout:
998 * not on pageout queues and busy.
999 * Unlock everything except the
1000 * copy_object itself.
1001 */
1002
1003 vm_object_unlock(object)((void)(&(object)->Lock));
1004
1005 /*
1006 * Write the page to the copy-object,
1007 * flushing it from the kernel.
1008 */
1009
1010 vm_pageout_page(copy_m, TRUE((boolean_t) 1), TRUE((boolean_t) 1));
1011
1012 /*
1013 * Since the pageout may have
1014 * temporarily dropped the
1015 * copy_object's lock, we
1016 * check whether we'll have
1017 * to deallocate the hard way.
1018 */
1019
1020 if ((copy_object->shadow != object) ||
1021 (copy_object->ref_count == 1)) {
1022 vm_object_unlock(copy_object)((void)(&(copy_object)->Lock));
1023 vm_object_deallocate(copy_object);
1024 vm_object_lock(object);
1025 continue;
1026 }
1027
1028 /*
1029 * Pick back up the old object's
1030 * lock. [It is safe to do so,
1031 * since it must be deeper in the
1032 * object tree.]
1033 */
1034
1035 vm_object_lock(object);
1036 }
1037
1038 /*
1039 * Because we're pushing a page upward
1040 * in the object tree, we must restart
1041 * any faults that are waiting here.
1042 * [Note that this is an expansion of
1043 * PAGE_WAKEUP that uses the THREAD_RESTART
1044 * wait result]. Can't turn off the page's
1045 * busy bit because we're not done with it.
1046 */
1047
1048 if (m->wanted) {
1049 m->wanted = FALSE((boolean_t) 0);
1050 thread_wakeup_with_result((event_t) m,thread_wakeup_prim(((event_t) m), ((boolean_t) 0), (3))
1051 THREAD_RESTART)thread_wakeup_prim(((event_t) m), ((boolean_t) 0), (3));
1052 }
1053 }
1054
1055 /*
1056 * The reference count on copy_object must be
1057 * at least 2: one for our extra reference,
1058 * and at least one from the outside world
1059 * (we checked that when we last locked
1060 * copy_object).
1061 */
1062 copy_object->ref_count--;
1063 assert(copy_object->ref_count > 0)({ if (!(copy_object->ref_count > 0)) Assert("copy_object->ref_count > 0"
, "../vm/vm_fault.c", 1063); })
;
1064 vm_object_unlock(copy_object)((void)(&(copy_object)->Lock));
1065
1066 break;
1067 }
1068
1069 *result_page = m;
1070 *top_page = first_m;
1071
1072 /*
1073 * If the page can be written, assume that it will be.
1074 * [Earlier, we restrict the permission to allow write
1075 * access only if the fault so required, so we don't
1076 * mark read-only data as dirty.]
1077 */
1078
1079 if (vm_fault_dirty_handling && (*protection & VM_PROT_WRITE((vm_prot_t) 0x02)))
1080 m->dirty = TRUE((boolean_t) 1);
1081
1082 return(VM_FAULT_SUCCESS0);
1083
1084 block_and_backoff:
1085 vm_fault_cleanup(object, first_m);
1086
1087 if (continuation != (void (*)()) 0) {
1088 vm_fault_state_t *state =
1089 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1090
1091 /*
1092 * Save variables in case we must restart.
1093 */
1094
1095 state->vmfp_backoff = TRUE((boolean_t) 1);
1096 state->vmf_prot = *protection;
1097
1098 counter(c_vm_fault_page_block_backoff_user++);
1099 thread_block(continuation);
1100 } else
1101 {
1102 counter(c_vm_fault_page_block_backoff_kernel++);
1103 thread_block((void (*)()) 0);
1104 }
1105 after_block_and_backoff:
1106 if (current_thread()(active_threads[(0)])->wait_result == THREAD_AWAKENED0)
1107 return VM_FAULT_RETRY1;
1108 else
1109 return VM_FAULT_INTERRUPTED2;
1110
1111#undef RELEASE_PAGE
1112}
1113
1114/*
1115 * Routine: vm_fault
1116 * Purpose:
1117 * Handle page faults, including pseudo-faults
1118 * used to change the wiring status of pages.
1119 * Returns:
1120 * If an explicit (expression) continuation is supplied,
1121 * then we call the continuation instead of returning.
1122 * Implementation:
1123 * Explicit continuations make this a little icky,
1124 * because it hasn't been rewritten to embrace CPS.
1125 * Instead, we have resume arguments for vm_fault and
1126 * vm_fault_page, to let continue the fault computation.
1127 *
1128 * vm_fault and vm_fault_page save mucho state
1129 * in the moral equivalent of a closure. The state
1130 * structure is allocated when first entering vm_fault
1131 * and deallocated when leaving vm_fault.
1132 */
1133
1134void
1135vm_fault_continue(void)
1136{
1137 vm_fault_state_t *state =
1138 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1139
1140 (void) vm_fault(state->vmf_map,
1141 state->vmf_vaddr,
1142 state->vmf_fault_type,
1143 state->vmf_change_wiring,
1144 TRUE((boolean_t) 1), state->vmf_continuation);
1145 /*NOTREACHED*/
1146}
1147
1148kern_return_t vm_fault(
1149 vm_map_t map,
1150 vm_offset_t vaddr,
1151 vm_prot_t fault_type,
1152 boolean_t change_wiring,
1153 boolean_t resume,
1154 void (*continuation)())
1155{
1156 vm_map_version_t version; /* Map version for verificiation */
1157 boolean_t wired; /* Should mapping be wired down? */
1158 vm_object_t object; /* Top-level object */
1159 vm_offset_t offset; /* Top-level offset */
1160 vm_prot_t prot; /* Protection for mapping */
1161 vm_object_t old_copy_object; /* Saved copy object */
1162 vm_page_t result_page; /* Result of vm_fault_page */
1163 vm_page_t top_page; /* Placeholder page */
1164 kern_return_t kr;
1165
1166 vm_page_t m; /* Fast access to result_page */
1167
1168 if (resume) {
1169 vm_fault_state_t *state =
1170 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1171
1172 /*
1173 * Retrieve cached variables and
1174 * continue vm_fault_page.
1175 */
1176
1177 object = state->vmf_object;
1178 if (object == VM_OBJECT_NULL((vm_object_t) 0))
1179 goto RetryFault;
1180 version = state->vmf_version;
1181 wired = state->vmf_wired;
1182 offset = state->vmf_offset;
1183 prot = state->vmf_prot;
1184
1185 kr = vm_fault_page(object, offset, fault_type,
1186 (change_wiring && !wired), !change_wiring,
1187 &prot, &result_page, &top_page,
1188 TRUE((boolean_t) 1), vm_fault_continue);
1189 goto after_vm_fault_page;
1190 }
1191
1192 if (continuation != (void (*)()) 0) {
1193 /*
1194 * We will probably need to save state.
1195 */
1196
1197 char * state;
1198
1199 /*
1200 * if this assignment stmt is written as
1201 * 'active_threads[cpu_number()] = kmem_cache_alloc()',
1202 * cpu_number may be evaluated before kmem_cache_alloc;
1203 * if kmem_cache_alloc blocks, cpu_number will be wrong
1204 */
1205
1206 state = (char *) kmem_cache_alloc(&vm_fault_state_cache);
1207 current_thread()(active_threads[(0)])->ith_othersaved.other = state;
1208
1209 }
1210
1211 RetryFault: ;
1212
1213 /*
1214 * Find the backing store object and offset into
1215 * it to begin the search.
1216 */
1217
1218 if ((kr = vm_map_lookup(&map, vaddr, fault_type, &version,
1219 &object, &offset,
1220 &prot, &wired)) != KERN_SUCCESS0) {
1221 goto done;
1222 }
1223
1224 /*
1225 * If the page is wired, we must fault for the current protection
1226 * value, to avoid further faults.
1227 */
1228
1229 if (wired)
1230 fault_type = prot;
1231
1232 /*
1233 * Make a reference to this object to
1234 * prevent its disposal while we are messing with
1235 * it. Once we have the reference, the map is free
1236 * to be diddled. Since objects reference their
1237 * shadows (and copies), they will stay around as well.
1238 */
1239
1240 assert(object->ref_count > 0)({ if (!(object->ref_count > 0)) Assert("object->ref_count > 0"
, "../vm/vm_fault.c", 1240); })
;
1241 object->ref_count++;
1242 vm_object_paging_begin(object)((object)->paging_in_progress++);
1243
1244 if (continuation != (void (*)()) 0) {
1245 vm_fault_state_t *state =
1246 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1247
1248 /*
1249 * Save variables, in case vm_fault_page discards
1250 * our kernel stack and we have to restart.
1251 */
1252
1253 state->vmf_map = map;
1254 state->vmf_vaddr = vaddr;
1255 state->vmf_fault_type = fault_type;
1256 state->vmf_change_wiring = change_wiring;
1257 state->vmf_continuation = continuation;
1258
1259 state->vmf_version = version;
1260 state->vmf_wired = wired;
1261 state->vmf_object = object;
1262 state->vmf_offset = offset;
1263 state->vmf_prot = prot;
1264
1265 kr = vm_fault_page(object, offset, fault_type,
1266 (change_wiring && !wired), !change_wiring,
1267 &prot, &result_page, &top_page,
1268 FALSE((boolean_t) 0), vm_fault_continue);
1269 } else
1270 {
1271 kr = vm_fault_page(object, offset, fault_type,
1272 (change_wiring && !wired), !change_wiring,
1273 &prot, &result_page, &top_page,
1274 FALSE((boolean_t) 0), (void (*)()) 0);
1275 }
1276 after_vm_fault_page:
1277
1278 /*
1279 * If we didn't succeed, lose the object reference immediately.
1280 */
1281
1282 if (kr != VM_FAULT_SUCCESS0)
1283 vm_object_deallocate(object);
1284
1285 /*
1286 * See why we failed, and take corrective action.
1287 */
1288
1289 switch (kr) {
1290 case VM_FAULT_SUCCESS0:
1291 break;
1292 case VM_FAULT_RETRY1:
1293 goto RetryFault;
1294 case VM_FAULT_INTERRUPTED2:
1295 kr = KERN_SUCCESS0;
1296 goto done;
1297 case VM_FAULT_MEMORY_SHORTAGE3:
1298 if (continuation != (void (*)()) 0) {
1299 vm_fault_state_t *state =
1300 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1301
1302 /*
1303 * Save variables in case VM_PAGE_WAIT
1304 * discards our kernel stack.
1305 */
1306
1307 state->vmf_map = map;
1308 state->vmf_vaddr = vaddr;
1309 state->vmf_fault_type = fault_type;
1310 state->vmf_change_wiring = change_wiring;
1311 state->vmf_continuation = continuation;
1312 state->vmf_object = VM_OBJECT_NULL((vm_object_t) 0);
1313
1314 VM_PAGE_WAIT(vm_fault_continue)vm_page_wait(vm_fault_continue);
1315 } else
1316 VM_PAGE_WAIT((void (*)()) 0)vm_page_wait((void (*)()) 0);
1317 goto RetryFault;
1318 case VM_FAULT_FICTITIOUS_SHORTAGE4:
1319 vm_page_more_fictitious();
1320 goto RetryFault;
1321 case VM_FAULT_MEMORY_ERROR5:
1322 kr = KERN_MEMORY_ERROR10;
1323 goto done;
1324 }
1325
1326 m = result_page;
1327
1328 assert((change_wiring && !wired) ?({ if (!((change_wiring && !wired) ? (top_page == ((vm_page_t
) 0)) : ((top_page == ((vm_page_t) 0)) == (m->object == object
)))) Assert("(change_wiring && !wired) ? (top_page == VM_PAGE_NULL) : ((top_page == VM_PAGE_NULL) == (m->object == object))"
, "../vm/vm_fault.c", 1330); })
1329 (top_page == VM_PAGE_NULL) :({ if (!((change_wiring && !wired) ? (top_page == ((vm_page_t
) 0)) : ((top_page == ((vm_page_t) 0)) == (m->object == object
)))) Assert("(change_wiring && !wired) ? (top_page == VM_PAGE_NULL) : ((top_page == VM_PAGE_NULL) == (m->object == object))"
, "../vm/vm_fault.c", 1330); })
1330 ((top_page == VM_PAGE_NULL) == (m->object == object)))({ if (!((change_wiring && !wired) ? (top_page == ((vm_page_t
) 0)) : ((top_page == ((vm_page_t) 0)) == (m->object == object
)))) Assert("(change_wiring && !wired) ? (top_page == VM_PAGE_NULL) : ((top_page == VM_PAGE_NULL) == (m->object == object))"
, "../vm/vm_fault.c", 1330); })
;
1331
1332 /*
1333 * How to clean up the result of vm_fault_page. This
1334 * happens whether the mapping is entered or not.
1335 */
1336
1337#define UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); }
\
1338 MACRO_BEGIN({ \
1339 vm_fault_cleanup(m->object, top_page); \
1340 vm_object_deallocate(object); \
1341 MACRO_END})
1342
1343 /*
1344 * What to do with the resulting page from vm_fault_page
1345 * if it doesn't get entered into the physical map:
1346 */
1347
1348#define RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ((void
)(&vm_page_queue_lock)); }
\
1349 MACRO_BEGIN({ \
1350 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } })
; \
1351 vm_page_lock_queues(); \
1352 if (!m->active && !m->inactive) \
1353 vm_page_activate(m); \
1354 vm_page_unlock_queues()((void)(&vm_page_queue_lock)); \
1355 MACRO_END})
1356
1357 /*
1358 * We must verify that the maps have not changed
1359 * since our last lookup.
1360 */
1361
1362 old_copy_object = m->object->copy;
1363
1364 vm_object_unlock(m->object)((void)(&(m->object)->Lock));
1365 while (!vm_map_verify(map, &version)) {
1366 vm_object_t retry_object;
1367 vm_offset_t retry_offset;
1368 vm_prot_t retry_prot;
1369
1370 /*
1371 * To avoid trying to write_lock the map while another
1372 * thread has it read_locked (in vm_map_pageable), we
1373 * do not try for write permission. If the page is
1374 * still writable, we will get write permission. If it
1375 * is not, or has been marked needs_copy, we enter the
1376 * mapping without write permission, and will merely
1377 * take another fault.
1378 */
1379 kr = vm_map_lookup(&map, vaddr,
1380 fault_type & ~VM_PROT_WRITE((vm_prot_t) 0x02), &version,
1381 &retry_object, &retry_offset, &retry_prot,
1382 &wired);
1383
1384 if (kr != KERN_SUCCESS0) {
1385 vm_object_lock(m->object);
1386 RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ((void
)(&vm_page_queue_lock)); }
;
1387 UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); }
;
1388 goto done;
1389 }
1390
1391 vm_object_unlock(retry_object)((void)(&(retry_object)->Lock));
1392 vm_object_lock(m->object);
1393
1394 if ((retry_object != object) ||
1395 (retry_offset != offset)) {
1396 RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ((void
)(&vm_page_queue_lock)); }
;
1397 UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); }
;
1398 goto RetryFault;
1399 }
1400
1401 /*
1402 * Check whether the protection has changed or the object
1403 * has been copied while we left the map unlocked.
1404 */
1405 prot &= retry_prot;
1406 vm_object_unlock(m->object)((void)(&(m->object)->Lock));
1407 }
1408 vm_object_lock(m->object);
1409
1410 /*
1411 * If the copy object changed while the top-level object
1412 * was unlocked, then we must take away write permission.
1413 */
1414
1415 if (m->object->copy != old_copy_object)
1416 prot &= ~VM_PROT_WRITE((vm_prot_t) 0x02);
1417
1418 /*
1419 * If we want to wire down this page, but no longer have
1420 * adequate permissions, we must start all over.
1421 */
1422
1423 if (wired && (prot != fault_type)) {
1424 vm_map_verify_done(map, &version)(lock_done(&(map)->lock));
1425 RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ((void
)(&vm_page_queue_lock)); }
;
1426 UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); }
;
1427 goto RetryFault;
1428 }
1429
1430 /*
1431 * It's critically important that a wired-down page be faulted
1432 * only once in each map for which it is wired.
1433 */
1434
1435 vm_object_unlock(m->object)((void)(&(m->object)->Lock));
1436
1437 /*
1438 * Put this page into the physical map.
1439 * We had to do the unlock above because pmap_enter
1440 * may cause other faults. The page may be on
1441 * the pageout queues. If the pageout daemon comes
1442 * across the page, it will remove it from the queues.
1443 */
1444
1445 PMAP_ENTER(map->pmap, vaddr, m, prot, wired)({ pmap_enter( (map->pmap), (vaddr), (m)->phys_addr, (prot
) & ~(m)->page_lock, (wired) ); })
;
1446
1447 /*
1448 * If the page is not wired down and isn't already
1449 * on a pageout queue, then put it where the
1450 * pageout daemon can find it.
1451 */
1452 vm_object_lock(m->object);
1453 vm_page_lock_queues();
1454 if (change_wiring) {
1455 if (wired)
1456 vm_page_wire(m);
1457 else
1458 vm_page_unwire(m);
1459 } else if (software_reference_bits) {
1460 if (!m->active && !m->inactive)
1461 vm_page_activate(m);
1462 m->reference = TRUE((boolean_t) 1);
1463 } else {
1464 vm_page_activate(m);
1465 }
1466 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
1467
1468 /*
1469 * Unlock everything, and return
1470 */
1471
1472 vm_map_verify_done(map, &version)(lock_done(&(map)->lock));
1473 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } })
;
1474 kr = KERN_SUCCESS0;
1475 UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); }
;
1476
1477#undef UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); }
1478#undef RELEASE_PAGE
1479
1480 done:
1481 if (continuation != (void (*)()) 0) {
1482 vm_fault_state_t *state =
1483 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1484
1485 kmem_cache_free(&vm_fault_state_cache, (vm_offset_t) state);
1486 (*continuation)(kr);
1487 /*NOTREACHED*/
1488 }
1489
1490 return(kr);
1491}
1492
1493/*
1494 * vm_fault_wire:
1495 *
1496 * Wire down a range of virtual addresses in a map.
1497 */
1498void vm_fault_wire(
1499 vm_map_t map,
1500 vm_map_entry_t entry)
1501{
1502
1503 vm_offset_t va;
1504 pmap_t pmap;
1505 vm_offset_t end_addr = entry->vme_endlinks.end;
1506
1507 pmap = vm_map_pmap(map)((map)->pmap);
1508
1509 /*
1510 * Inform the physical mapping system that the
1511 * range of addresses may not fault, so that
1512 * page tables and such can be locked down as well.
1513 */
1514
1515 pmap_pageable(pmap, entry->vme_startlinks.start, end_addr, FALSE((boolean_t) 0));
1516
1517 /*
1518 * We simulate a fault to get the page and enter it
1519 * in the physical map.
1520 */
1521
1522 for (va = entry->vme_startlinks.start; va < end_addr; va += PAGE_SIZE(1 << 12)) {
1523 if (vm_fault_wire_fast(map, va, entry) != KERN_SUCCESS0)
1524 (void) vm_fault(map, va, VM_PROT_NONE((vm_prot_t) 0x00), TRUE((boolean_t) 1),
1525 FALSE((boolean_t) 0), (void (*)()) 0);
1526 }
1527}
1528
1529/*
1530 * vm_fault_unwire:
1531 *
1532 * Unwire a range of virtual addresses in a map.
1533 */
1534void vm_fault_unwire(
1535 vm_map_t map,
1536 vm_map_entry_t entry)
1537{
1538 vm_offset_t va;
1539 pmap_t pmap;
1540 vm_offset_t end_addr = entry->vme_endlinks.end;
1541 vm_object_t object;
1542
1543 pmap = vm_map_pmap(map)((map)->pmap);
1544
1545 object = (entry->is_sub_map)
1546 ? VM_OBJECT_NULL((vm_object_t) 0) : entry->object.vm_object;
1547
1548 /*
1549 * Since the pages are wired down, we must be able to
1550 * get their mappings from the physical map system.
1551 */
1552
1553 for (va = entry->vme_startlinks.start; va < end_addr; va += PAGE_SIZE(1 << 12)) {
1554 pmap_change_wiring(pmap, va, FALSE((boolean_t) 0));
1555
1556 if (object == VM_OBJECT_NULL((vm_object_t) 0)) {
1557 vm_map_lock_set_recursive(map)lock_set_recursive(&(map)->lock);
1558 (void) vm_fault(map, va, VM_PROT_NONE((vm_prot_t) 0x00), TRUE((boolean_t) 1),
1559 FALSE((boolean_t) 0), (void (*)()) 0);
1560 vm_map_lock_clear_recursive(map)lock_clear_recursive(&(map)->lock);
1561 } else {
1562 vm_prot_t prot;
1563 vm_page_t result_page;
1564 vm_page_t top_page;
1565 vm_fault_return_t result;
1566
1567 do {
1568 prot = VM_PROT_NONE((vm_prot_t) 0x00);
1569
1570 vm_object_lock(object);
1571 vm_object_paging_begin(object)((object)->paging_in_progress++);
1572 result = vm_fault_page(object,
1573 entry->offset +
1574 (va - entry->vme_startlinks.start),
1575 VM_PROT_NONE((vm_prot_t) 0x00), TRUE((boolean_t) 1),
1576 FALSE((boolean_t) 0), &prot,
1577 &result_page,
1578 &top_page,
1579 FALSE((boolean_t) 0), (void (*)()) 0);
1580 } while (result == VM_FAULT_RETRY1);
1581
1582 if (result != VM_FAULT_SUCCESS0)
1583 panic("vm_fault_unwire: failure");
1584
1585 vm_page_lock_queues();
1586 vm_page_unwire(result_page);
1587 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
1588 PAGE_WAKEUP_DONE(result_page)({ (result_page)->busy = ((boolean_t) 0); if ((result_page
)->wanted) { (result_page)->wanted = ((boolean_t) 0); thread_wakeup_prim
((((event_t) result_page)), ((boolean_t) 0), 0); } })
;
1589
1590 vm_fault_cleanup(result_page->object, top_page);
1591 }
1592 }
1593
1594 /*
1595 * Inform the physical mapping system that the range
1596 * of addresses may fault, so that page tables and
1597 * such may be unwired themselves.
1598 */
1599
1600 pmap_pageable(pmap, entry->vme_startlinks.start, end_addr, TRUE((boolean_t) 1));
1601}
1602
1603/*
1604 * vm_fault_wire_fast:
1605 *
1606 * Handle common case of a wire down page fault at the given address.
1607 * If successful, the page is inserted into the associated physical map.
1608 * The map entry is passed in to avoid the overhead of a map lookup.
1609 *
1610 * NOTE: the given address should be truncated to the
1611 * proper page address.
1612 *
1613 * KERN_SUCCESS is returned if the page fault is handled; otherwise,
1614 * a standard error specifying why the fault is fatal is returned.
1615 *
1616 * The map in question must be referenced, and remains so.
1617 * Caller has a read lock on the map.
1618 *
1619 * This is a stripped version of vm_fault() for wiring pages. Anything
1620 * other than the common case will return KERN_FAILURE, and the caller
1621 * is expected to call vm_fault().
1622 */
1623kern_return_t vm_fault_wire_fast(
1624 vm_map_t map,
1625 vm_offset_t va,
1626 vm_map_entry_t entry)
1627{
1628 vm_object_t object;
1629 vm_offset_t offset;
1630 vm_page_t m;
1631 vm_prot_t prot;
1632
1633 vm_stat.faults++; /* needs lock XXX */
1634 current_task()((active_threads[(0)])->task)->faults++;
1635/*
1636 * Recovery actions
1637 */
1638
1639#undef RELEASE_PAGE
1640#define RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ((void
)(&vm_page_queue_lock)); }
{ \
1641 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } })
; \
1642 vm_page_lock_queues(); \
1643 vm_page_unwire(m); \
1644 vm_page_unlock_queues()((void)(&vm_page_queue_lock)); \
1645}
1646
1647
1648#undef UNLOCK_THINGS{ object->paging_in_progress--; ((void)(&(object)->
Lock)); }
1649#define UNLOCK_THINGS{ object->paging_in_progress--; ((void)(&(object)->
Lock)); }
{ \
1650 object->paging_in_progress--; \
1651 vm_object_unlock(object)((void)(&(object)->Lock)); \
1652}
1653
1654#undef UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); }
1655#define UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); }
{ \
1656 UNLOCK_THINGS{ object->paging_in_progress--; ((void)(&(object)->
Lock)); }
; \
1657 vm_object_deallocate(object); \
1658}
1659/*
1660 * Give up and have caller do things the hard way.
1661 */
1662
1663#define GIVE_UP{ { { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); }; return(5); }
{ \
1664 UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); }
; \
1665 return(KERN_FAILURE5); \
1666}
1667
1668
1669 /*
1670 * If this entry is not directly to a vm_object, bail out.
1671 */
1672 if (entry->is_sub_map)
1673 return(KERN_FAILURE5);
1674
1675 /*
1676 * Find the backing store object and offset into it.
1677 */
1678
1679 object = entry->object.vm_object;
1680 offset = (va - entry->vme_startlinks.start) + entry->offset;
1681 prot = entry->protection;
1682
1683 /*
1684 * Make a reference to this object to prevent its
1685 * disposal while we are messing with it.
1686 */
1687
1688 vm_object_lock(object);
1689 assert(object->ref_count > 0)({ if (!(object->ref_count > 0)) Assert("object->ref_count > 0"
, "../vm/vm_fault.c", 1689); })
;
1690 object->ref_count++;
1691 object->paging_in_progress++;
1692
1693 /*
1694 * INVARIANTS (through entire routine):
1695 *
1696 * 1) At all times, we must either have the object
1697 * lock or a busy page in some object to prevent
1698 * some other thread from trying to bring in
1699 * the same page.
1700 *
1701 * 2) Once we have a busy page, we must remove it from
1702 * the pageout queues, so that the pageout daemon
1703 * will not grab it away.
1704 *
1705 */
1706
1707 /*
1708 * Look for page in top-level object. If it's not there or
1709 * there's something going on, give up.
1710 */
1711 m = vm_page_lookup(object, offset);
1712 if ((m == VM_PAGE_NULL((vm_page_t) 0)) || (m->error) ||
1713 (m->busy) || (m->absent) || (prot & m->page_lock)) {
1714 GIVE_UP{ { { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); }; return(5); }
;
1715 }
1716
1717 /*
1718 * Wire the page down now. All bail outs beyond this
1719 * point must unwire the page.
1720 */
1721
1722 vm_page_lock_queues();
1723 vm_page_wire(m);
1724 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
1725
1726 /*
1727 * Mark page busy for other threads.
1728 */
1729 assert(!m->busy)({ if (!(!m->busy)) Assert("!m->busy", "../vm/vm_fault.c"
, 1729); })
;
1730 m->busy = TRUE((boolean_t) 1);
1731 assert(!m->absent)({ if (!(!m->absent)) Assert("!m->absent", "../vm/vm_fault.c"
, 1731); })
;
1732
1733 /*
1734 * Give up if the page is being written and there's a copy object
1735 */
1736 if ((object->copy != VM_OBJECT_NULL((vm_object_t) 0)) && (prot & VM_PROT_WRITE((vm_prot_t) 0x02))) {
1737 RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ((void
)(&vm_page_queue_lock)); }
;
1738 GIVE_UP{ { { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); }; return(5); }
;
1739 }
1740
1741 /*
1742 * Put this page into the physical map.
1743 * We have to unlock the object because pmap_enter
1744 * may cause other faults.
1745 */
1746 vm_object_unlock(object)((void)(&(object)->Lock));
1747
1748 PMAP_ENTER(map->pmap, va, m, prot, TRUE)({ pmap_enter( (map->pmap), (va), (m)->phys_addr, (prot
) & ~(m)->page_lock, (((boolean_t) 1)) ); })
;
1749
1750 /*
1751 * Must relock object so that paging_in_progress can be cleared.
1752 */
1753 vm_object_lock(object);
1754
1755 /*
1756 * Unlock everything, and return
1757 */
1758
1759 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } })
;
1760 UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); }
;
1761
1762 return(KERN_SUCCESS0);
1763
1764}
1765
1766/*
1767 * Routine: vm_fault_copy_cleanup
1768 * Purpose:
1769 * Release a page used by vm_fault_copy.
1770 */
1771
1772void vm_fault_copy_cleanup(
1773 vm_page_t page,
1774 vm_page_t top_page)
1775{
1776 vm_object_t object = page->object;
1777
1778 vm_object_lock(object);
1779 PAGE_WAKEUP_DONE(page)({ (page)->busy = ((boolean_t) 0); if ((page)->wanted) {
(page)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) page)), ((boolean_t) 0), 0); } })
;
1780 vm_page_lock_queues();
1781 if (!page->active && !page->inactive)
1782 vm_page_activate(page);
1783 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
1784 vm_fault_cleanup(object, top_page);
1785}
1786
1787/*
1788 * Routine: vm_fault_copy
1789 *
1790 * Purpose:
1791 * Copy pages from one virtual memory object to another --
1792 * neither the source nor destination pages need be resident.
1793 *
1794 * Before actually copying a page, the version associated with
1795 * the destination address map wil be verified.
1796 *
1797 * In/out conditions:
1798 * The caller must hold a reference, but not a lock, to
1799 * each of the source and destination objects and to the
1800 * destination map.
1801 *
1802 * Results:
1803 * Returns KERN_SUCCESS if no errors were encountered in
1804 * reading or writing the data. Returns KERN_INTERRUPTED if
1805 * the operation was interrupted (only possible if the
1806 * "interruptible" argument is asserted). Other return values
1807 * indicate a permanent error in copying the data.
1808 *
1809 * The actual amount of data copied will be returned in the
1810 * "copy_size" argument. In the event that the destination map
1811 * verification failed, this amount may be less than the amount
1812 * requested.
1813 */
1814kern_return_t vm_fault_copy(
1815 vm_object_t src_object,
1816 vm_offset_t src_offset,
1817 vm_size_t *src_size, /* INOUT */
1818 vm_object_t dst_object,
1819 vm_offset_t dst_offset,
1820 vm_map_t dst_map,
1821 vm_map_version_t *dst_version,
1822 boolean_t interruptible)
1823{
1824 vm_page_t result_page;
1825 vm_prot_t prot;
1826
1827 vm_page_t src_page;
1828 vm_page_t src_top_page;
1829
1830 vm_page_t dst_page;
1831 vm_page_t dst_top_page;
1832
1833 vm_size_t amount_done;
1834 vm_object_t old_copy_object;
1835
1836#define RETURN(x) \
1837 MACRO_BEGIN({ \
1838 *src_size = amount_done; \
1839 MACRO_RETURNif (((boolean_t) 1)) return(x); \
1840 MACRO_END})
1841
1842 amount_done = 0;
1843 do { /* while (amount_done != *src_size) */
1844
1845 RetrySourceFault: ;
1846
1847 if (src_object == VM_OBJECT_NULL((vm_object_t) 0)) {
1848 /*
1849 * No source object. We will just
1850 * zero-fill the page in dst_object.
1851 */
1852
1853 src_page = VM_PAGE_NULL((vm_page_t) 0);
1854 } else {
1855 prot = VM_PROT_READ((vm_prot_t) 0x01);
1856
1857 vm_object_lock(src_object);
1858 vm_object_paging_begin(src_object)((src_object)->paging_in_progress++);
1859
1860 switch (vm_fault_page(src_object, src_offset,
1861 VM_PROT_READ((vm_prot_t) 0x01), FALSE((boolean_t) 0), interruptible,
1862 &prot, &result_page, &src_top_page,
1863 FALSE((boolean_t) 0), (void (*)()) 0)) {
1864
1865 case VM_FAULT_SUCCESS0:
1866 break;
1867 case VM_FAULT_RETRY1:
1868 goto RetrySourceFault;
1869 case VM_FAULT_INTERRUPTED2:
1870 RETURN(MACH_SEND_INTERRUPTED0x10000007);
1871 case VM_FAULT_MEMORY_SHORTAGE3:
1872 VM_PAGE_WAIT((void (*)()) 0)vm_page_wait((void (*)()) 0);
1873 goto RetrySourceFault;
1874 case VM_FAULT_FICTITIOUS_SHORTAGE4:
1875 vm_page_more_fictitious();
1876 goto RetrySourceFault;
1877 case VM_FAULT_MEMORY_ERROR5:
1878 return(KERN_MEMORY_ERROR10);
1879 }
1880
1881 src_page = result_page;
1882
1883 assert((src_top_page == VM_PAGE_NULL) ==({ if (!((src_top_page == ((vm_page_t) 0)) == (src_page->object
== src_object))) Assert("(src_top_page == VM_PAGE_NULL) == (src_page->object == src_object)"
, "../vm/vm_fault.c", 1884); })
1884 (src_page->object == src_object))({ if (!((src_top_page == ((vm_page_t) 0)) == (src_page->object
== src_object))) Assert("(src_top_page == VM_PAGE_NULL) == (src_page->object == src_object)"
, "../vm/vm_fault.c", 1884); })
;
1885
1886 assert ((prot & VM_PROT_READ) != VM_PROT_NONE)({ if (!((prot & ((vm_prot_t) 0x01)) != ((vm_prot_t) 0x00
))) Assert("(prot & VM_PROT_READ) != VM_PROT_NONE", "../vm/vm_fault.c"
, 1886); })
;
1887
1888 vm_object_unlock(src_page->object)((void)(&(src_page->object)->Lock));
1889 }
1890
1891 RetryDestinationFault: ;
1892
1893 prot = VM_PROT_WRITE((vm_prot_t) 0x02);
1894
1895 vm_object_lock(dst_object);
1896 vm_object_paging_begin(dst_object)((dst_object)->paging_in_progress++);
1897
1898 switch (vm_fault_page(dst_object, dst_offset, VM_PROT_WRITE((vm_prot_t) 0x02),
1899 FALSE((boolean_t) 0), FALSE((boolean_t) 0) /* interruptible */,
1900 &prot, &result_page, &dst_top_page,
1901 FALSE((boolean_t) 0), (void (*)()) 0)) {
1902
1903 case VM_FAULT_SUCCESS0:
1904 break;
1905 case VM_FAULT_RETRY1:
1906 goto RetryDestinationFault;
1907 case VM_FAULT_INTERRUPTED2:
1908 if (src_page != VM_PAGE_NULL((vm_page_t) 0))
1909 vm_fault_copy_cleanup(src_page,
1910 src_top_page);
1911 RETURN(MACH_SEND_INTERRUPTED0x10000007);
1912 case VM_FAULT_MEMORY_SHORTAGE3:
1913 VM_PAGE_WAIT((void (*)()) 0)vm_page_wait((void (*)()) 0);
1914 goto RetryDestinationFault;
1915 case VM_FAULT_FICTITIOUS_SHORTAGE4:
1916 vm_page_more_fictitious();
1917 goto RetryDestinationFault;
1918 case VM_FAULT_MEMORY_ERROR5:
1919 if (src_page != VM_PAGE_NULL((vm_page_t) 0))
1920 vm_fault_copy_cleanup(src_page,
1921 src_top_page);
1922 return(KERN_MEMORY_ERROR10);
1923 }
1924 assert ((prot & VM_PROT_WRITE) != VM_PROT_NONE)({ if (!((prot & ((vm_prot_t) 0x02)) != ((vm_prot_t) 0x00
))) Assert("(prot & VM_PROT_WRITE) != VM_PROT_NONE", "../vm/vm_fault.c"
, 1924); })
;
1925
1926 dst_page = result_page;
1927
1928 old_copy_object = dst_page->object->copy;
1929
1930 vm_object_unlock(dst_page->object)((void)(&(dst_page->object)->Lock));
1931
1932 if (!vm_map_verify(dst_map, dst_version)) {
1933
1934 BailOut: ;
1935
1936 if (src_page != VM_PAGE_NULL((vm_page_t) 0))
1937 vm_fault_copy_cleanup(src_page, src_top_page);
1938 vm_fault_copy_cleanup(dst_page, dst_top_page);
1939 break;
1940 }
1941
1942
1943 vm_object_lock(dst_page->object);
1944 if (dst_page->object->copy != old_copy_object) {
1945 vm_object_unlock(dst_page->object)((void)(&(dst_page->object)->Lock));
1946 vm_map_verify_done(dst_map, dst_version)(lock_done(&(dst_map)->lock));
1947 goto BailOut;
1948 }
1949 vm_object_unlock(dst_page->object)((void)(&(dst_page->object)->Lock));
1950
1951 /*
1952 * Copy the page, and note that it is dirty
1953 * immediately.
1954 */
1955
1956 if (src_page == VM_PAGE_NULL((vm_page_t) 0))
1957 vm_page_zero_fill(dst_page);
1958 else
1959 vm_page_copy(src_page, dst_page);
1960 dst_page->dirty = TRUE((boolean_t) 1);
1961
1962 /*
1963 * Unlock everything, and return
1964 */
1965
1966 vm_map_verify_done(dst_map, dst_version)(lock_done(&(dst_map)->lock));
1967
1968 if (src_page != VM_PAGE_NULL((vm_page_t) 0))
1969 vm_fault_copy_cleanup(src_page, src_top_page);
1970 vm_fault_copy_cleanup(dst_page, dst_top_page);
1971
1972 amount_done += PAGE_SIZE(1 << 12);
1973 src_offset += PAGE_SIZE(1 << 12);
1974 dst_offset += PAGE_SIZE(1 << 12);
1975
1976 } while (amount_done != *src_size);
1977
1978 RETURN(KERN_SUCCESS0);
1979#undef RETURN
1980
1981 /*NOTREACHED*/
1982}
1983
1984
1985
1986
1987
1988#ifdef notdef
1989
1990/*
1991 * Routine: vm_fault_page_overwrite
1992 *
1993 * Description:
1994 * A form of vm_fault_page that assumes that the
1995 * resulting page will be overwritten in its entirety,
1996 * making it unnecessary to obtain the correct *contents*
1997 * of the page.
1998 *
1999 * Implementation:
2000 * XXX Untested. Also unused. Eventually, this technology
2001 * could be used in vm_fault_copy() to advantage.
2002 */
2003vm_fault_return_t vm_fault_page_overwrite(
2004 vm_object_t dst_object,
2005 vm_offset_t dst_offset,
2006 vm_page_t *result_page) /* OUT */
2007{
2008 vm_page_t dst_page;
2009
2010#define interruptible FALSE((boolean_t) 0) /* XXX */
2011
2012 while (TRUE((boolean_t) 1)) {
2013 /*
2014 * Look for a page at this offset
2015 */
2016
2017 while ((dst_page = vm_page_lookup(dst_object, dst_offset))
2018 == VM_PAGE_NULL((vm_page_t) 0)) {
2019 /*
2020 * No page, no problem... just allocate one.
2021 */
2022
2023 dst_page = vm_page_alloc(dst_object, dst_offset);
2024 if (dst_page == VM_PAGE_NULL((vm_page_t) 0)) {
2025 vm_object_unlock(dst_object)((void)(&(dst_object)->Lock));
2026 VM_PAGE_WAIT((void (*)()) 0)vm_page_wait((void (*)()) 0);
2027 vm_object_lock(dst_object);
2028 continue;
2029 }
2030
2031 /*
2032 * Pretend that the memory manager
2033 * write-protected the page.
2034 *
2035 * Note that we will be asking for write
2036 * permission without asking for the data
2037 * first.
2038 */
2039
2040 dst_page->overwriting = TRUE((boolean_t) 1);
2041 dst_page->page_lock = VM_PROT_WRITE((vm_prot_t) 0x02);
2042 dst_page->absent = TRUE((boolean_t) 1);
2043 dst_object->absent_count++;
2044
2045 break;
2046
2047 /*
2048 * When we bail out, we might have to throw
2049 * away the page created here.
2050 */
2051
2052#define DISCARD_PAGE \
2053 MACRO_BEGIN({ \
2054 vm_object_lock(dst_object); \
2055 dst_page = vm_page_lookup(dst_object, dst_offset); \
2056 if ((dst_page != VM_PAGE_NULL((vm_page_t) 0)) && dst_page->overwriting) \
2057 VM_PAGE_FREE(dst_page)({ ; vm_page_free(dst_page); ((void)(&vm_page_queue_lock)
); })
; \
2058 vm_object_unlock(dst_object)((void)(&(dst_object)->Lock)); \
2059 MACRO_END})
2060 }
2061
2062 /*
2063 * If the page is write-protected...
2064 */
2065
2066 if (dst_page->page_lock & VM_PROT_WRITE((vm_prot_t) 0x02)) {
2067 /*
2068 * ... and an unlock request hasn't been sent
2069 */
2070
2071 if ( ! (dst_page->unlock_request & VM_PROT_WRITE((vm_prot_t) 0x02))) {
2072 vm_prot_t u;
2073 kern_return_t rc;
2074
2075 /*
2076 * ... then send one now.
2077 */
2078
2079 if (!dst_object->pager_ready) {
2080 vm_object_assert_wait(dst_object,({ (dst_object)->all_wanted |= 1 << (1); assert_wait
((event_t)(((vm_offset_t) dst_object) + (1)), (interruptible)
); })
2081 VM_OBJECT_EVENT_PAGER_READY,({ (dst_object)->all_wanted |= 1 << (1); assert_wait
((event_t)(((vm_offset_t) dst_object) + (1)), (interruptible)
); })
2082 interruptible)({ (dst_object)->all_wanted |= 1 << (1); assert_wait
((event_t)(((vm_offset_t) dst_object) + (1)), (interruptible)
); })
;
2083 vm_object_unlock(dst_object)((void)(&(dst_object)->Lock));
2084 thread_block((void (*)()) 0);
2085 if (current_thread()(active_threads[(0)])->wait_result !=
2086 THREAD_AWAKENED0) {
2087 DISCARD_PAGE;
2088 return(VM_FAULT_INTERRUPTED2);
2089 }
2090 continue;
2091 }
2092
2093 u = dst_page->unlock_request |= VM_PROT_WRITE((vm_prot_t) 0x02);
2094 vm_object_unlock(dst_object)((void)(&(dst_object)->Lock));
2095
2096 if ((rc = memory_object_data_unlock(
2097 dst_object->pager,
2098 dst_object->pager_request,
2099 dst_offset + dst_object->paging_offset,
2100 PAGE_SIZE(1 << 12),
2101 u)) != KERN_SUCCESS0) {
2102 printf("vm_object_overwrite: memory_object_data_unlock failed\n");
2103 DISCARD_PAGE;
2104 return((rc == MACH_SEND_INTERRUPTED0x10000007) ?
2105 VM_FAULT_INTERRUPTED2 :
2106 VM_FAULT_MEMORY_ERROR5);
2107 }
2108 vm_object_lock(dst_object);
2109 continue;
2110 }
2111
2112 /* ... fall through to wait below */
2113 } else {
2114 /*
2115 * If the page isn't being used for other
2116 * purposes, then we're done.
2117 */
2118 if ( ! (dst_page->busy || dst_page->absent || dst_page->error) )
2119 break;
2120 }
2121
2122 PAGE_ASSERT_WAIT(dst_page, interruptible)({ (dst_page)->wanted = ((boolean_t) 1); assert_wait((event_t
) (dst_page), (interruptible)); })
;
2123 vm_object_unlock(dst_object)((void)(&(dst_object)->Lock));
2124 thread_block((void (*)()) 0);
2125 if (current_thread()(active_threads[(0)])->wait_result != THREAD_AWAKENED0) {
2126 DISCARD_PAGE;
2127 return(VM_FAULT_INTERRUPTED2);
2128 }
2129 }
2130
2131 *result_page = dst_page;
2132 return(VM_FAULT_SUCCESS0);
2133
2134#undef interruptible
2135#undef DISCARD_PAGE
2136}
2137
2138#endif /* notdef */