Bug Summary

File:obj-scan-build/../vm/vm_fault.c
Location:line 248, column 2
Description:Access to field 'task' results in a dereference of a null pointer

Annotated Source Code

1/*
2 * Mach Operating System
3 * Copyright (c) 1994,1990,1989,1988,1987 Carnegie Mellon University.
4 * Copyright (c) 1993,1994 The University of Utah and
5 * the Computer Systems Laboratory (CSL).
6 * All rights reserved.
7 *
8 * Permission to use, copy, modify and distribute this software and its
9 * documentation is hereby granted, provided that both the copyright
10 * notice and this permission notice appear in all copies of the
11 * software, derivative works or modified versions, and any portions
12 * thereof, and that both notices appear in supporting documentation.
13 *
14 * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15 * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16 * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17 * THIS SOFTWARE.
18 *
19 * Carnegie Mellon requests users of this software to return to
20 *
21 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
22 * School of Computer Science
23 * Carnegie Mellon University
24 * Pittsburgh PA 15213-3890
25 *
26 * any improvements or extensions that they make and grant Carnegie Mellon
27 * the rights to redistribute these changes.
28 */
29/*
30 * File: vm_fault.c
31 * Author: Avadis Tevanian, Jr., Michael Wayne Young
32 *
33 * Page fault handling module.
34 */
35
36#include <kern/printf.h>
37#include <vm/vm_fault.h>
38#include <mach/kern_return.h>
39#include <mach/message.h> /* for error codes */
40#include <kern/counters.h>
41#include <kern/debug.h>
42#include <kern/thread.h>
43#include <kern/sched_prim.h>
44#include <vm/vm_map.h>
45#include <vm/vm_object.h>
46#include <vm/vm_page.h>
47#include <vm/pmap.h>
48#include <mach/vm_statistics.h>
49#include <vm/vm_pageout.h>
50#include <mach/vm_param.h>
51#include <mach/memory_object.h>
52#include <vm/memory_object_user.user.h>
53 /* For memory_object_data_{request,unlock} */
54#include <kern/macro_help.h>
55#include <kern/slab.h>
56
57#if MACH_PCSAMPLE1
58#include <kern/pc_sample.h>
59#endif
60
61
62
63/*
64 * State needed by vm_fault_continue.
65 * This is a little hefty to drop directly
66 * into the thread structure.
67 */
68typedef struct vm_fault_state {
69 struct vm_map *vmf_map;
70 vm_offset_t vmf_vaddr;
71 vm_prot_t vmf_fault_type;
72 boolean_t vmf_change_wiring;
73 void (*vmf_continuation)();
74 vm_map_version_t vmf_version;
75 boolean_t vmf_wired;
76 struct vm_object *vmf_object;
77 vm_offset_t vmf_offset;
78 vm_prot_t vmf_prot;
79
80 boolean_t vmfp_backoff;
81 struct vm_object *vmfp_object;
82 vm_offset_t vmfp_offset;
83 struct vm_page *vmfp_first_m;
84 vm_prot_t vmfp_access;
85} vm_fault_state_t;
86
87struct kmem_cache vm_fault_state_cache;
88
89int vm_object_absent_max = 50;
90
91boolean_t vm_fault_dirty_handling = FALSE((boolean_t) 0);
92boolean_t vm_fault_interruptible = TRUE((boolean_t) 1);
93
94boolean_t software_reference_bits = TRUE((boolean_t) 1);
95
96#if MACH_KDB1
97extern struct db_watchpoint *db_watchpoint_list;
98#endif /* MACH_KDB */
99
100/*
101 * Routine: vm_fault_init
102 * Purpose:
103 * Initialize our private data structures.
104 */
105void vm_fault_init(void)
106{
107 kmem_cache_init(&vm_fault_state_cache, "vm_fault_state",
108 sizeof(vm_fault_state_t), 0, NULL((void *) 0), NULL((void *) 0), NULL((void *) 0), 0);
109}
110
111/*
112 * Routine: vm_fault_cleanup
113 * Purpose:
114 * Clean up the result of vm_fault_page.
115 * Results:
116 * The paging reference for "object" is released.
117 * "object" is unlocked.
118 * If "top_page" is not null, "top_page" is
119 * freed and the paging reference for the object
120 * containing it is released.
121 *
122 * In/out conditions:
123 * "object" must be locked.
124 */
125void
126vm_fault_cleanup(
127 vm_object_t object,
128 vm_page_t top_page)
129{
130 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 130); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
131 vm_object_unlock(object)((void)(&(object)->Lock));
132
133 if (top_page != VM_PAGE_NULL((vm_page_t) 0)) {
134 object = top_page->object;
135 vm_object_lock(object);
136 VM_PAGE_FREE(top_page)({ ; vm_page_free(top_page); ((void)(&vm_page_queue_lock)
); });
137 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 137); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
138 vm_object_unlock(object)((void)(&(object)->Lock));
139 }
140}
141
142
143#if MACH_PCSAMPLE1
144/*
145 * Do PC sampling on current thread, assuming
146 * that it is the thread taking this page fault.
147 *
148 * Must check for THREAD_NULL, since faults
149 * can occur before threads are running.
150 */
151
152#define vm_stat_sample(flavor)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((flavor))) take_pc_sample((_thread_), &
(_thread_)->pc_sample, ((flavor))); task = (_thread_)->
task; if (task->pc_sample.sampletypes & ((flavor))) take_pc_sample
((_thread_), &task->pc_sample, ((flavor))); }); }) \
153 MACRO_BEGIN({ \
154 thread_t _thread_ = current_thread()(active_threads[(0)]); \
155 \
156 if (_thread_ != THREAD_NULL((thread_t) 0)) \
157 take_pc_sample_macro(_thread_, (flavor))({ task_t task; if ((_thread_)->pc_sample.sampletypes &
((flavor))) take_pc_sample((_thread_), &(_thread_)->pc_sample
, ((flavor))); task = (_thread_)->task; if (task->pc_sample
.sampletypes & ((flavor))) take_pc_sample((_thread_), &
task->pc_sample, ((flavor))); }); \
158 MACRO_END})
159
160#else
161#define vm_stat_sample(x)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((x))) take_pc_sample((_thread_), &(_thread_
)->pc_sample, ((x))); task = (_thread_)->task; if (task
->pc_sample.sampletypes & ((x))) take_pc_sample((_thread_
), &task->pc_sample, ((x))); }); })
162#endif /* MACH_PCSAMPLE */
163
164
165
166/*
167 * Routine: vm_fault_page
168 * Purpose:
169 * Find the resident page for the virtual memory
170 * specified by the given virtual memory object
171 * and offset.
172 * Additional arguments:
173 * The required permissions for the page is given
174 * in "fault_type". Desired permissions are included
175 * in "protection".
176 *
177 * If the desired page is known to be resident (for
178 * example, because it was previously wired down), asserting
179 * the "unwiring" parameter will speed the search.
180 *
181 * If the operation can be interrupted (by thread_abort
182 * or thread_terminate), then the "interruptible"
183 * parameter should be asserted.
184 *
185 * Results:
186 * The page containing the proper data is returned
187 * in "result_page".
188 *
189 * In/out conditions:
190 * The source object must be locked and referenced,
191 * and must donate one paging reference. The reference
192 * is not affected. The paging reference and lock are
193 * consumed.
194 *
195 * If the call succeeds, the object in which "result_page"
196 * resides is left locked and holding a paging reference.
197 * If this is not the original object, a busy page in the
198 * original object is returned in "top_page", to prevent other
199 * callers from pursuing this same data, along with a paging
200 * reference for the original object. The "top_page" should
201 * be destroyed when this guarantee is no longer required.
202 * The "result_page" is also left busy. It is not removed
203 * from the pageout queues.
204 */
205vm_fault_return_t vm_fault_page(
206 /* Arguments: */
207 vm_object_t first_object, /* Object to begin search */
208 vm_offset_t first_offset, /* Offset into object */
209 vm_prot_t fault_type, /* What access is requested */
210 boolean_t must_be_resident,/* Must page be resident? */
211 boolean_t interruptible, /* May fault be interrupted? */
212 /* Modifies in place: */
213 vm_prot_t *protection, /* Protection for mapping */
214 /* Returns: */
215 vm_page_t *result_page, /* Page found, if successful */
216 vm_page_t *top_page, /* Page in top object, if
217 * not result_page.
218 */
219 /* More arguments: */
220 boolean_t resume, /* We are restarting. */
221 void (*continuation)()) /* Continuation for blocking. */
222{
223 vm_page_t m;
224 vm_object_t object;
225 vm_offset_t offset;
226 vm_page_t first_m;
227 vm_object_t next_object;
228 vm_object_t copy_object;
229 boolean_t look_for_page;
230 vm_prot_t access_required;
231
232 if (resume) {
1
Assuming 'resume' is 0
→
2
←
Taking false branch
→
233 vm_fault_state_t *state =
234 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
235
236 if (state->vmfp_backoff)
237 goto after_block_and_backoff;
238
239 object = state->vmfp_object;
240 offset = state->vmfp_offset;
241 first_m = state->vmfp_first_m;
242 access_required = state->vmfp_access;
243 goto after_thread_block;
244 }
245
246 vm_stat_sample(SAMPLED_PC_VM_FAULTS_ANY)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((0x100))) take_pc_sample((_thread_), &
(_thread_)->pc_sample, ((0x100))); task = (_thread_)->task
; if (task->pc_sample.sampletypes & ((0x100))) take_pc_sample
((_thread_), &task->pc_sample, ((0x100))); }); });
3
←
Within the expansion of the macro 'vm_stat_sample':
→
a
Assuming '_thread_' is equal to null
247 vm_stat.faults++; /* needs lock XXX */
248 current_task()((active_threads[(0)])->task)->faults++;
4
←
Within the expansion of the macro 'current_task':
a
Access to field 'task' results in a dereference of a null pointer
249
250/*
251 * Recovery actions
252 */
253#define RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ((void
)(&vm_page_queue_lock)); } \
254 MACRO_BEGIN({ \
255 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } }); \
256 vm_page_lock_queues(); \
257 if (!m->active && !m->inactive) \
258 vm_page_activate(m); \
259 vm_page_unlock_queues()((void)(&vm_page_queue_lock)); \
260 MACRO_END})
261
262 if (vm_fault_dirty_handling
263#if MACH_KDB1
264 /*
265 * If there are watchpoints set, then
266 * we don't want to give away write permission
267 * on a read fault. Make the task write fault,
268 * so that the watchpoint code notices the access.
269 */
270 || db_watchpoint_list
271#endif /* MACH_KDB */
272 ) {
273 /*
274 * If we aren't asking for write permission,
275 * then don't give it away. We're using write
276 * faults to set the dirty bit.
277 */
278 if (!(fault_type & VM_PROT_WRITE((vm_prot_t) 0x02)))
279 *protection &= ~VM_PROT_WRITE((vm_prot_t) 0x02);
280 }
281
282 if (!vm_fault_interruptible)
283 interruptible = FALSE((boolean_t) 0);
284
285 /*
286 * INVARIANTS (through entire routine):
287 *
288 * 1) At all times, we must either have the object
289 * lock or a busy page in some object to prevent
290 * some other thread from trying to bring in
291 * the same page.
292 *
293 * Note that we cannot hold any locks during the
294 * pager access or when waiting for memory, so
295 * we use a busy page then.
296 *
297 * Note also that we aren't as concerned about more than
298 * one thread attempting to memory_object_data_unlock
299 * the same page at once, so we don't hold the page
300 * as busy then, but do record the highest unlock
301 * value so far. [Unlock requests may also be delivered
302 * out of order.]
303 *
304 * 2) To prevent another thread from racing us down the
305 * shadow chain and entering a new page in the top
306 * object before we do, we must keep a busy page in
307 * the top object while following the shadow chain.
308 *
309 * 3) We must increment paging_in_progress on any object
310 * for which we have a busy page, to prevent
311 * vm_object_collapse from removing the busy page
312 * without our noticing.
313 *
314 * 4) We leave busy pages on the pageout queues.
315 * If the pageout daemon comes across a busy page,
316 * it will remove the page from the pageout queues.
317 */
318
319 /*
320 * Search for the page at object/offset.
321 */
322
323 object = first_object;
324 offset = first_offset;
325 first_m = VM_PAGE_NULL((vm_page_t) 0);
326 access_required = fault_type;
327
328 /*
329 * See whether this page is resident
330 */
331
332 while (TRUE((boolean_t) 1)) {
333 m = vm_page_lookup(object, offset);
334 if (m != VM_PAGE_NULL((vm_page_t) 0)) {
335 /*
336 * If the page is being brought in,
337 * wait for it and then retry.
338 *
339 * A possible optimization: if the page
340 * is known to be resident, we can ignore
341 * pages that are absent (regardless of
342 * whether they're busy).
343 */
344
345 if (m->busy) {
346 kern_return_t wait_result;
347
348 PAGE_ASSERT_WAIT(m, interruptible)({ (m)->wanted = ((boolean_t) 1); assert_wait((event_t) (m
), (interruptible)); });
349 vm_object_unlock(object)((void)(&(object)->Lock));
350 if (continuation != (void (*)()) 0) {
351 vm_fault_state_t *state =
352 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
353
354 /*
355 * Save variables in case
356 * thread_block discards
357 * our kernel stack.
358 */
359
360 state->vmfp_backoff = FALSE((boolean_t) 0);
361 state->vmfp_object = object;
362 state->vmfp_offset = offset;
363 state->vmfp_first_m = first_m;
364 state->vmfp_access =
365 access_required;
366 state->vmf_prot = *protection;
367
368 counter(c_vm_fault_page_block_busy_user++);
369 thread_block(continuation);
370 } else
371 {
372 counter(c_vm_fault_page_block_busy_kernel++);
373 thread_block((void (*)()) 0);
374 }
375 after_thread_block:
376 wait_result = current_thread()(active_threads[(0)])->wait_result;
377 vm_object_lock(object);
378 if (wait_result != THREAD_AWAKENED0) {
379 vm_fault_cleanup(object, first_m);
380 if (wait_result == THREAD_RESTART3)
381 return(VM_FAULT_RETRY1);
382 else
383 return(VM_FAULT_INTERRUPTED2);
384 }
385 continue;
386 }
387
388 /*
389 * If the page is in error, give up now.
390 */
391
392 if (m->error) {
393 VM_PAGE_FREE(m)({ ; vm_page_free(m); ((void)(&vm_page_queue_lock)); });
394 vm_fault_cleanup(object, first_m);
395 return(VM_FAULT_MEMORY_ERROR5);
396 }
397
398 /*
399 * If the page isn't busy, but is absent,
400 * then it was deemed "unavailable".
401 */
402
403 if (m->absent) {
404 /*
405 * Remove the non-existent page (unless it's
406 * in the top object) and move on down to the
407 * next object (if there is one).
408 */
409
410 offset += object->shadow_offset;
411 access_required = VM_PROT_READ((vm_prot_t) 0x01);
412 next_object = object->shadow;
413 if (next_object == VM_OBJECT_NULL((vm_object_t) 0)) {
414 vm_page_t real_m;
415
416 assert(!must_be_resident)({ if (!(!must_be_resident)) Assert("!must_be_resident", "../vm/vm_fault.c"
, 416); });
417
418 /*
419 * Absent page at bottom of shadow
420 * chain; zero fill the page we left
421 * busy in the first object, and flush
422 * the absent page. But first we
423 * need to allocate a real page.
424 */
425
426 real_m = vm_page_grab(!object->internal);
427 if (real_m == VM_PAGE_NULL((vm_page_t) 0)) {
428 vm_fault_cleanup(object, first_m);
429 return(VM_FAULT_MEMORY_SHORTAGE3);
430 }
431
432 if (object != first_object) {
433 VM_PAGE_FREE(m)({ ; vm_page_free(m); ((void)(&vm_page_queue_lock)); });
434 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 434); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
435 vm_object_unlock(object)((void)(&(object)->Lock));
436 object = first_object;
437 offset = first_offset;
438 m = first_m;
439 first_m = VM_PAGE_NULL((vm_page_t) 0);
440 vm_object_lock(object);
441 }
442
443 VM_PAGE_FREE(m)({ ; vm_page_free(m); ((void)(&vm_page_queue_lock)); });
444 assert(real_m->busy)({ if (!(real_m->busy)) Assert("real_m->busy", "../vm/vm_fault.c"
, 444); });
445 vm_page_lock_queues();
446 vm_page_insert(real_m, object, offset);
447 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
448 m = real_m;
449
450 /*
451 * Drop the lock while zero filling
452 * page. Then break because this
453 * is the page we wanted. Checking
454 * the page lock is a waste of time;
455 * this page was either absent or
456 * newly allocated -- in both cases
457 * it can't be page locked by a pager.
458 */
459 vm_object_unlock(object)((void)(&(object)->Lock));
460
461 vm_page_zero_fill(m);
462
463 vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((0x10))) take_pc_sample((_thread_), &
(_thread_)->pc_sample, ((0x10))); task = (_thread_)->task
; if (task->pc_sample.sampletypes & ((0x10))) take_pc_sample
((_thread_), &task->pc_sample, ((0x10))); }); });
464
465 vm_stat.zero_fill_count++;
466 current_task()((active_threads[(0)])->task)->zero_fills++;
467 vm_object_lock(object);
468 pmap_clear_modify(m->phys_addr);
469 break;
470 } else {
471 if (must_be_resident) {
472 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 472); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
473 } else if (object != first_object) {
474 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 474); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
475 VM_PAGE_FREE(m)({ ; vm_page_free(m); ((void)(&vm_page_queue_lock)); });
476 } else {
477 first_m = m;
478 m->absent = FALSE((boolean_t) 0);
479 vm_object_absent_release(object)({ (object)->absent_count--; ({ if (((object))->all_wanted
& (1 << (3))) thread_wakeup_prim(((event_t)(((vm_offset_t
) (object)) + (3))), ((boolean_t) 0), 0); ((object))->all_wanted
&= ~(1 << (3)); }); });
480 m->busy = TRUE((boolean_t) 1);
481
482 vm_page_lock_queues();
483 VM_PAGE_QUEUES_REMOVE(m)({ if (m->active) { { queue_entry_t next, prev; next = (m)
->pageq.next; prev = (m)->pageq.prev; if ((&vm_page_queue_active
) == next) (&vm_page_queue_active)->prev = prev; else (
(vm_page_t)next)->pageq.prev = prev; if ((&vm_page_queue_active
) == prev) (&vm_page_queue_active)->next = next; else (
(vm_page_t)prev)->pageq.next = next; }; m->active = ((boolean_t
) 0); vm_page_active_count--; } if (m->inactive) { { queue_entry_t
next, prev; next = (m)->pageq.next; prev = (m)->pageq.
prev; if ((&vm_page_queue_inactive) == next) (&vm_page_queue_inactive
)->prev = prev; else ((vm_page_t)next)->pageq.prev = prev
; if ((&vm_page_queue_inactive) == prev) (&vm_page_queue_inactive
)->next = next; else ((vm_page_t)prev)->pageq.next = next
; }; m->inactive = ((boolean_t) 0); vm_page_inactive_count
--; } });
484 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
485 }
486 vm_object_lock(next_object);
487 vm_object_unlock(object)((void)(&(object)->Lock));
488 object = next_object;
489 vm_object_paging_begin(object)((object)->paging_in_progress++);
490 continue;
491 }
492 }
493
494 /*
495 * If the desired access to this page has
496 * been locked out, request that it be unlocked.
497 */
498
499 if (access_required & m->page_lock) {
500 if ((access_required & m->unlock_request) != access_required) {
501 vm_prot_t new_unlock_request;
502 kern_return_t rc;
503
504 if (!object->pager_ready) {
505 vm_object_assert_wait(object,({ (object)->all_wanted |= 1 << (1); assert_wait((event_t
)(((vm_offset_t) object) + (1)), (interruptible)); })
506 VM_OBJECT_EVENT_PAGER_READY,({ (object)->all_wanted |= 1 << (1); assert_wait((event_t
)(((vm_offset_t) object) + (1)), (interruptible)); })
507 interruptible)({ (object)->all_wanted |= 1 << (1); assert_wait((event_t
)(((vm_offset_t) object) + (1)), (interruptible)); });
508 goto block_and_backoff;
509 }
510
511 new_unlock_request = m->unlock_request =
512 (access_required | m->unlock_request);
513 vm_object_unlock(object)((void)(&(object)->Lock));
514 if ((rc = memory_object_data_unlock(
515 object->pager,
516 object->pager_request,
517 offset + object->paging_offset,
518 PAGE_SIZE(1 << 12),
519 new_unlock_request))
520 != KERN_SUCCESS0) {
521 printf("vm_fault: memory_object_data_unlock failed\n");
522 vm_object_lock(object);
523 vm_fault_cleanup(object, first_m);
524 return((rc == MACH_SEND_INTERRUPTED0x10000007) ?
525 VM_FAULT_INTERRUPTED2 :
526 VM_FAULT_MEMORY_ERROR5);
527 }
528 vm_object_lock(object);
529 continue;
530 }
531
532 PAGE_ASSERT_WAIT(m, interruptible)({ (m)->wanted = ((boolean_t) 1); assert_wait((event_t) (m
), (interruptible)); });
533 goto block_and_backoff;
534 }
535
536 /*
537 * We mark the page busy and leave it on
538 * the pageout queues. If the pageout
539 * deamon comes across it, then it will
540 * remove the page.
541 */
542
543 if (!software_reference_bits) {
544 vm_page_lock_queues();
545 if (m->inactive) {
546 vm_stat_sample(SAMPLED_PC_VM_REACTIVATION_FAULTS)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((0x20))) take_pc_sample((_thread_), &
(_thread_)->pc_sample, ((0x20))); task = (_thread_)->task
; if (task->pc_sample.sampletypes & ((0x20))) take_pc_sample
((_thread_), &task->pc_sample, ((0x20))); }); });
547 vm_stat.reactivations++;
548 current_task()((active_threads[(0)])->task)->reactivations++;
549 }
550
551 VM_PAGE_QUEUES_REMOVE(m)({ if (m->active) { { queue_entry_t next, prev; next = (m)
->pageq.next; prev = (m)->pageq.prev; if ((&vm_page_queue_active
) == next) (&vm_page_queue_active)->prev = prev; else (
(vm_page_t)next)->pageq.prev = prev; if ((&vm_page_queue_active
) == prev) (&vm_page_queue_active)->next = next; else (
(vm_page_t)prev)->pageq.next = next; }; m->active = ((boolean_t
) 0); vm_page_active_count--; } if (m->inactive) { { queue_entry_t
next, prev; next = (m)->pageq.next; prev = (m)->pageq.
prev; if ((&vm_page_queue_inactive) == next) (&vm_page_queue_inactive
)->prev = prev; else ((vm_page_t)next)->pageq.prev = prev
; if ((&vm_page_queue_inactive) == prev) (&vm_page_queue_inactive
)->next = next; else ((vm_page_t)prev)->pageq.next = next
; }; m->inactive = ((boolean_t) 0); vm_page_inactive_count
--; } });
552 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
553 }
554
555 assert(!m->busy)({ if (!(!m->busy)) Assert("!m->busy", "../vm/vm_fault.c"
, 555); });
556 m->busy = TRUE((boolean_t) 1);
557 assert(!m->absent)({ if (!(!m->absent)) Assert("!m->absent", "../vm/vm_fault.c"
, 557); });
558 break;
559 }
560
561 look_for_page =
562 (object->pager_created)
563#if MACH_PAGEMAP1
564 && (vm_external_state_get(object->existence_info, offset + object->paging_offset)(((object->existence_info) != ((vm_external_t) 0)) ? _vm_external_state_get
(object->existence_info, offset + object->paging_offset
) : 2) !=
565 VM_EXTERNAL_STATE_ABSENT3)
566#endif /* MACH_PAGEMAP */
567 ;
568
569 if ((look_for_page || (object == first_object))
570 && !must_be_resident) {
571 /*
572 * Allocate a new page for this object/offset
573 * pair.
574 */
575
576 m = vm_page_grab_fictitious();
577 if (m == VM_PAGE_NULL((vm_page_t) 0)) {
578 vm_fault_cleanup(object, first_m);
579 return(VM_FAULT_FICTITIOUS_SHORTAGE4);
580 }
581
582 vm_page_lock_queues();
583 vm_page_insert(m, object, offset);
584 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
585 }
586
587 if (look_for_page && !must_be_resident) {
588 kern_return_t rc;
589
590 /*
591 * If the memory manager is not ready, we
592 * cannot make requests.
593 */
594 if (!object->pager_ready) {
595 vm_object_assert_wait(object,({ (object)->all_wanted |= 1 << (1); assert_wait((event_t
)(((vm_offset_t) object) + (1)), (interruptible)); })
596 VM_OBJECT_EVENT_PAGER_READY,({ (object)->all_wanted |= 1 << (1); assert_wait((event_t
)(((vm_offset_t) object) + (1)), (interruptible)); })
597 interruptible)({ (object)->all_wanted |= 1 << (1); assert_wait((event_t
)(((vm_offset_t) object) + (1)), (interruptible)); });
598 VM_PAGE_FREE(m)({ ; vm_page_free(m); ((void)(&vm_page_queue_lock)); });
599 goto block_and_backoff;
600 }
601
602 if (object->internal) {
603 /*
604 * Requests to the default pager
605 * must reserve a real page in advance,
606 * because the pager's data-provided
607 * won't block for pages.
608 */
609
610 if (m->fictitious && !vm_page_convert(m, FALSE((boolean_t) 0))) {
611 VM_PAGE_FREE(m)({ ; vm_page_free(m); ((void)(&vm_page_queue_lock)); });
612 vm_fault_cleanup(object, first_m);
613 return(VM_FAULT_MEMORY_SHORTAGE3);
614 }
615 } else if (object->absent_count >
616 vm_object_absent_max) {
617 /*
618 * If there are too many outstanding page
619 * requests pending on this object, we
620 * wait for them to be resolved now.
621 */
622
623 vm_object_absent_assert_wait(object, interruptible)({ ({ ((object))->all_wanted |= 1 << (3); assert_wait
((event_t)(((vm_offset_t) (object)) + (3)), ((interruptible))
); }); });
624 VM_PAGE_FREE(m)({ ; vm_page_free(m); ((void)(&vm_page_queue_lock)); });
625 goto block_and_backoff;
626 }
627
628 /*
629 * Indicate that the page is waiting for data
630 * from the memory manager.
631 */
632
633 m->absent = TRUE((boolean_t) 1);
634 object->absent_count++;
635
636 /*
637 * We have a busy page, so we can
638 * release the object lock.
639 */
640 vm_object_unlock(object)((void)(&(object)->Lock));
641
642 /*
643 * Call the memory manager to retrieve the data.
644 */
645
646 vm_stat.pageins++;
647 vm_stat_sample(SAMPLED_PC_VM_PAGEIN_FAULTS)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((0x40))) take_pc_sample((_thread_), &
(_thread_)->pc_sample, ((0x40))); task = (_thread_)->task
; if (task->pc_sample.sampletypes & ((0x40))) take_pc_sample
((_thread_), &task->pc_sample, ((0x40))); }); });
648 current_task()((active_threads[(0)])->task)->pageins++;
649
650 if ((rc = memory_object_data_request(object->pager,
651 object->pager_request,
652 m->offset + object->paging_offset,
653 PAGE_SIZE(1 << 12), access_required)) != KERN_SUCCESS0) {
654 if (rc != MACH_SEND_INTERRUPTED0x10000007)
655 printf("%s(0x%p, 0x%p, 0x%lx, 0x%x, 0x%x) failed, %x\n",
656 "memory_object_data_request",
657 object->pager,
658 object->pager_request,
659 m->offset + object->paging_offset,
660 PAGE_SIZE(1 << 12), access_required, rc);
661 /*
662 * Don't want to leave a busy page around,
663 * but the data request may have blocked,
664 * so check if it's still there and busy.
665 */
666 vm_object_lock(object);
667 if (m == vm_page_lookup(object,offset) &&
668 m->absent && m->busy)
669 VM_PAGE_FREE(m)({ ; vm_page_free(m); ((void)(&vm_page_queue_lock)); });
670 vm_fault_cleanup(object, first_m);
671 return((rc == MACH_SEND_INTERRUPTED0x10000007) ?
672 VM_FAULT_INTERRUPTED2 :
673 VM_FAULT_MEMORY_ERROR5);
674 }
675
676 /*
677 * Retry with same object/offset, since new data may
678 * be in a different page (i.e., m is meaningless at
679 * this point).
680 */
681 vm_object_lock(object);
682 continue;
683 }
684
685 /*
686 * For the XP system, the only case in which we get here is if
687 * object has no pager (or unwiring). If the pager doesn't
688 * have the page this is handled in the m->absent case above
689 * (and if you change things here you should look above).
690 */
691 if (object == first_object)
692 first_m = m;
693 else
694 {
695 assert(m == VM_PAGE_NULL)({ if (!(m == ((vm_page_t) 0))) Assert("m == VM_PAGE_NULL", "../vm/vm_fault.c"
, 695); });
696 }
697
698 /*
699 * Move on to the next object. Lock the next
700 * object before unlocking the current one.
701 */
702 access_required = VM_PROT_READ((vm_prot_t) 0x01);
703
704 offset += object->shadow_offset;
705 next_object = object->shadow;
706 if (next_object == VM_OBJECT_NULL((vm_object_t) 0)) {
707 assert(!must_be_resident)({ if (!(!must_be_resident)) Assert("!must_be_resident", "../vm/vm_fault.c"
, 707); });
708
709 /*
710 * If there's no object left, fill the page
711 * in the top object with zeros. But first we
712 * need to allocate a real page.
713 */
714
715 if (object != first_object) {
716 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 716); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
717 vm_object_unlock(object)((void)(&(object)->Lock));
718
719 object = first_object;
720 offset = first_offset;
721 vm_object_lock(object);
722 }
723
724 m = first_m;
725 assert(m->object == object)({ if (!(m->object == object)) Assert("m->object == object"
, "../vm/vm_fault.c", 725); });
726 first_m = VM_PAGE_NULL((vm_page_t) 0);
727
728 if (m->fictitious && !vm_page_convert(m, !object->internal)) {
729 VM_PAGE_FREE(m)({ ; vm_page_free(m); ((void)(&vm_page_queue_lock)); });
730 vm_fault_cleanup(object, VM_PAGE_NULL((vm_page_t) 0));
731 return(VM_FAULT_MEMORY_SHORTAGE3);
732 }
733
734 vm_object_unlock(object)((void)(&(object)->Lock));
735 vm_page_zero_fill(m);
736 vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((0x10))) take_pc_sample((_thread_), &
(_thread_)->pc_sample, ((0x10))); task = (_thread_)->task
; if (task->pc_sample.sampletypes & ((0x10))) take_pc_sample
((_thread_), &task->pc_sample, ((0x10))); }); });
737 vm_stat.zero_fill_count++;
738 current_task()((active_threads[(0)])->task)->zero_fills++;
739 vm_object_lock(object);
740 pmap_clear_modify(m->phys_addr);
741 break;
742 }
743 else {
744 vm_object_lock(next_object);
745 if ((object != first_object) || must_be_resident)
746 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 746); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
747 vm_object_unlock(object)((void)(&(object)->Lock));
748 object = next_object;
749 vm_object_paging_begin(object)((object)->paging_in_progress++);
750 }
751 }
752
753 /*
754 * PAGE HAS BEEN FOUND.
755 *
756 * This page (m) is:
757 * busy, so that we can play with it;
758 * not absent, so that nobody else will fill it;
759 * possibly eligible for pageout;
760 *
761 * The top-level page (first_m) is:
762 * VM_PAGE_NULL if the page was found in the
763 * top-level object;
764 * busy, not absent, and ineligible for pageout.
765 *
766 * The current object (object) is locked. A paging
767 * reference is held for the current and top-level
768 * objects.
769 */
770
771#if EXTRA_ASSERTIONS
772 assert(m->busy && !m->absent)({ if (!(m->busy && !m->absent)) Assert("m->busy && !m->absent"
, "../vm/vm_fault.c", 772); });
773 assert((first_m == VM_PAGE_NULL) ||({ if (!((first_m == ((vm_page_t) 0)) || (first_m->busy &&
!first_m->absent && !first_m->active &&
!first_m->inactive))) Assert("(first_m == VM_PAGE_NULL) || (first_m->busy && !first_m->absent && !first_m->active && !first_m->inactive)"
, "../vm/vm_fault.c", 775); })
774 (first_m->busy && !first_m->absent &&({ if (!((first_m == ((vm_page_t) 0)) || (first_m->busy &&
!first_m->absent && !first_m->active &&
!first_m->inactive))) Assert("(first_m == VM_PAGE_NULL) || (first_m->busy && !first_m->absent && !first_m->active && !first_m->inactive)"
, "../vm/vm_fault.c", 775); })
775 !first_m->active && !first_m->inactive))({ if (!((first_m == ((vm_page_t) 0)) || (first_m->busy &&
!first_m->absent && !first_m->active &&
!first_m->inactive))) Assert("(first_m == VM_PAGE_NULL) || (first_m->busy && !first_m->absent && !first_m->active && !first_m->inactive)"
, "../vm/vm_fault.c", 775); });
776#endif /* EXTRA_ASSERTIONS */
777
778 /*
779 * If the page is being written, but isn't
780 * already owned by the top-level object,
781 * we have to copy it into a new page owned
782 * by the top-level object.
783 */
784
785 if (object != first_object) {
786 /*
787 * We only really need to copy if we
788 * want to write it.
789 */
790
791 if (fault_type & VM_PROT_WRITE((vm_prot_t) 0x02)) {
792 vm_page_t copy_m;
793
794 assert(!must_be_resident)({ if (!(!must_be_resident)) Assert("!must_be_resident", "../vm/vm_fault.c"
, 794); });
795
796 /*
797 * If we try to collapse first_object at this
798 * point, we may deadlock when we try to get
799 * the lock on an intermediate object (since we
800 * have the bottom object locked). We can't
801 * unlock the bottom object, because the page
802 * we found may move (by collapse) if we do.
803 *
804 * Instead, we first copy the page. Then, when
805 * we have no more use for the bottom object,
806 * we unlock it and try to collapse.
807 *
808 * Note that we copy the page even if we didn't
809 * need to... that's the breaks.
810 */
811
812 /*
813 * Allocate a page for the copy
814 */
815 copy_m = vm_page_grab(!first_object->internal);
816 if (copy_m == VM_PAGE_NULL((vm_page_t) 0)) {
817 RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ((void
)(&vm_page_queue_lock)); };
818 vm_fault_cleanup(object, first_m);
819 return(VM_FAULT_MEMORY_SHORTAGE3);
820 }
821
822 vm_object_unlock(object)((void)(&(object)->Lock));
823 vm_page_copy(m, copy_m);
824 vm_object_lock(object);
825
826 /*
827 * If another map is truly sharing this
828 * page with us, we have to flush all
829 * uses of the original page, since we
830 * can't distinguish those which want the
831 * original from those which need the
832 * new copy.
833 *
834 * XXXO If we know that only one map has
835 * access to this page, then we could
836 * avoid the pmap_page_protect() call.
837 */
838
839 vm_page_lock_queues();
840 vm_page_deactivate(m);
841 pmap_page_protect(m->phys_addr, VM_PROT_NONE((vm_prot_t) 0x00));
842 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
843
844 /*
845 * We no longer need the old page or object.
846 */
847
848 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } });
849 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 849); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
850 vm_object_unlock(object)((void)(&(object)->Lock));
851
852 vm_stat.cow_faults++;
853 vm_stat_sample(SAMPLED_PC_VM_COW_FAULTS)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((0x80))) take_pc_sample((_thread_), &
(_thread_)->pc_sample, ((0x80))); task = (_thread_)->task
; if (task->pc_sample.sampletypes & ((0x80))) take_pc_sample
((_thread_), &task->pc_sample, ((0x80))); }); });
854 current_task()((active_threads[(0)])->task)->cow_faults++;
855 object = first_object;
856 offset = first_offset;
857
858 vm_object_lock(object);
859 VM_PAGE_FREE(first_m)({ ; vm_page_free(first_m); ((void)(&vm_page_queue_lock))
; });
860 first_m = VM_PAGE_NULL((vm_page_t) 0);
861 assert(copy_m->busy)({ if (!(copy_m->busy)) Assert("copy_m->busy", "../vm/vm_fault.c"
, 861); });
862 vm_page_lock_queues();
863 vm_page_insert(copy_m, object, offset);
864 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
865 m = copy_m;
866
867 /*
868 * Now that we've gotten the copy out of the
869 * way, let's try to collapse the top object.
870 * But we have to play ugly games with
871 * paging_in_progress to do that...
872 */
873
874 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 874); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
875 vm_object_collapse(object);
876 vm_object_paging_begin(object)((object)->paging_in_progress++);
877 }
878 else {
879 *protection &= (~VM_PROT_WRITE((vm_prot_t) 0x02));
880 }
881 }
882
883 /*
884 * Now check whether the page needs to be pushed into the
885 * copy object. The use of asymmetric copy on write for
886 * shared temporary objects means that we may do two copies to
887 * satisfy the fault; one above to get the page from a
888 * shadowed object, and one here to push it into the copy.
889 */
890
891 while ((copy_object = first_object->copy) != VM_OBJECT_NULL((vm_object_t) 0)) {
892 vm_offset_t copy_offset;
893 vm_page_t copy_m;
894
895 /*
896 * If the page is being written, but hasn't been
897 * copied to the copy-object, we have to copy it there.
898 */
899
900 if ((fault_type & VM_PROT_WRITE((vm_prot_t) 0x02)) == 0) {
901 *protection &= ~VM_PROT_WRITE((vm_prot_t) 0x02);
902 break;
903 }
904
905 /*
906 * If the page was guaranteed to be resident,
907 * we must have already performed the copy.
908 */
909
910 if (must_be_resident)
911 break;
912
913 /*
914 * Try to get the lock on the copy_object.
915 */
916 if (!vm_object_lock_try(copy_object)(((boolean_t) 1))) {
917 vm_object_unlock(object)((void)(&(object)->Lock));
918
919 simple_lock_pause(); /* wait a bit */
920
921 vm_object_lock(object);
922 continue;
923 }
924
925 /*
926 * Make another reference to the copy-object,
927 * to keep it from disappearing during the
928 * copy.
929 */
930 assert(copy_object->ref_count > 0)({ if (!(copy_object->ref_count > 0)) Assert("copy_object->ref_count > 0"
, "../vm/vm_fault.c", 930); });
931 copy_object->ref_count++;
932
933 /*
934 * Does the page exist in the copy?
935 */
936 copy_offset = first_offset - copy_object->shadow_offset;
937 copy_m = vm_page_lookup(copy_object, copy_offset);
938 if (copy_m != VM_PAGE_NULL((vm_page_t) 0)) {
939 if (copy_m->busy) {
940 /*
941 * If the page is being brought
942 * in, wait for it and then retry.
943 */
944 PAGE_ASSERT_WAIT(copy_m, interruptible)({ (copy_m)->wanted = ((boolean_t) 1); assert_wait((event_t
) (copy_m), (interruptible)); });
945 RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ((void
)(&vm_page_queue_lock)); };
946 copy_object->ref_count--;
947 assert(copy_object->ref_count > 0)({ if (!(copy_object->ref_count > 0)) Assert("copy_object->ref_count > 0"
, "../vm/vm_fault.c", 947); });
948 vm_object_unlock(copy_object)((void)(&(copy_object)->Lock));
949 goto block_and_backoff;
950 }
951 }
952 else {
953 /*
954 * Allocate a page for the copy
955 */
956 copy_m = vm_page_alloc(copy_object, copy_offset);
957 if (copy_m == VM_PAGE_NULL((vm_page_t) 0)) {
958 RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ((void
)(&vm_page_queue_lock)); };
959 copy_object->ref_count--;
960 assert(copy_object->ref_count > 0)({ if (!(copy_object->ref_count > 0)) Assert("copy_object->ref_count > 0"
, "../vm/vm_fault.c", 960); });
961 vm_object_unlock(copy_object)((void)(&(copy_object)->Lock));
962 vm_fault_cleanup(object, first_m);
963 return(VM_FAULT_MEMORY_SHORTAGE3);
964 }
965
966 /*
967 * Must copy page into copy-object.
968 */
969
970 vm_page_copy(m, copy_m);
971
972 /*
973 * If the old page was in use by any users
974 * of the copy-object, it must be removed
975 * from all pmaps. (We can't know which
976 * pmaps use it.)
977 */
978
979 vm_page_lock_queues();
980 pmap_page_protect(m->phys_addr, VM_PROT_NONE((vm_prot_t) 0x00));
981 copy_m->dirty = TRUE((boolean_t) 1);
982 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
983
984 /*
985 * If there's a pager, then immediately
986 * page out this page, using the "initialize"
987 * option. Else, we use the copy.
988 */
989
990 if (!copy_object->pager_created) {
991 vm_page_lock_queues();
992 vm_page_activate(copy_m);
993 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
994 PAGE_WAKEUP_DONE(copy_m)({ (copy_m)->busy = ((boolean_t) 0); if ((copy_m)->wanted
) { (copy_m)->wanted = ((boolean_t) 0); thread_wakeup_prim
((((event_t) copy_m)), ((boolean_t) 0), 0); } });
995 } else {
996 /*
997 * The page is already ready for pageout:
998 * not on pageout queues and busy.
999 * Unlock everything except the
1000 * copy_object itself.
1001 */
1002
1003 vm_object_unlock(object)((void)(&(object)->Lock));
1004
1005 /*
1006 * Write the page to the copy-object,
1007 * flushing it from the kernel.
1008 */
1009
1010 vm_pageout_page(copy_m, TRUE((boolean_t) 1), TRUE((boolean_t) 1));
1011
1012 /*
1013 * Since the pageout may have
1014 * temporarily dropped the
1015 * copy_object's lock, we
1016 * check whether we'll have
1017 * to deallocate the hard way.
1018 */
1019
1020 if ((copy_object->shadow != object) ||
1021 (copy_object->ref_count == 1)) {
1022 vm_object_unlock(copy_object)((void)(&(copy_object)->Lock));
1023 vm_object_deallocate(copy_object);
1024 vm_object_lock(object);
1025 continue;
1026 }
1027
1028 /*
1029 * Pick back up the old object's
1030 * lock. [It is safe to do so,
1031 * since it must be deeper in the
1032 * object tree.]
1033 */
1034
1035 vm_object_lock(object);
1036 }
1037
1038 /*
1039 * Because we're pushing a page upward
1040 * in the object tree, we must restart
1041 * any faults that are waiting here.
1042 * [Note that this is an expansion of
1043 * PAGE_WAKEUP that uses the THREAD_RESTART
1044 * wait result]. Can't turn off the page's
1045 * busy bit because we're not done with it.
1046 */
1047
1048 if (m->wanted) {
1049 m->wanted = FALSE((boolean_t) 0);
1050 thread_wakeup_with_result((event_t) m,thread_wakeup_prim(((event_t) m), ((boolean_t) 0), (3))
1051 THREAD_RESTART)thread_wakeup_prim(((event_t) m), ((boolean_t) 0), (3));
1052 }
1053 }
1054
1055 /*
1056 * The reference count on copy_object must be
1057 * at least 2: one for our extra reference,
1058 * and at least one from the outside world
1059 * (we checked that when we last locked
1060 * copy_object).
1061 */
1062 copy_object->ref_count--;
1063 assert(copy_object->ref_count > 0)({ if (!(copy_object->ref_count > 0)) Assert("copy_object->ref_count > 0"
, "../vm/vm_fault.c", 1063); });
1064 vm_object_unlock(copy_object)((void)(&(copy_object)->Lock));
1065
1066 break;
1067 }
1068
1069 *result_page = m;
1070 *top_page = first_m;
1071
1072 /*
1073 * If the page can be written, assume that it will be.
1074 * [Earlier, we restrict the permission to allow write
1075 * access only if the fault so required, so we don't
1076 * mark read-only data as dirty.]
1077 */
1078
1079 if (vm_fault_dirty_handling && (*protection & VM_PROT_WRITE((vm_prot_t) 0x02)))
1080 m->dirty = TRUE((boolean_t) 1);
1081
1082 return(VM_FAULT_SUCCESS0);
1083
1084 block_and_backoff:
1085 vm_fault_cleanup(object, first_m);
1086
1087 if (continuation != (void (*)()) 0) {
1088 vm_fault_state_t *state =
1089 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1090
1091 /*
1092 * Save variables in case we must restart.
1093 */
1094
1095 state->vmfp_backoff = TRUE((boolean_t) 1);
1096 state->vmf_prot = *protection;
1097
1098 counter(c_vm_fault_page_block_backoff_user++);
1099 thread_block(continuation);
1100 } else
1101 {
1102 counter(c_vm_fault_page_block_backoff_kernel++);
1103 thread_block((void (*)()) 0);
1104 }
1105 after_block_and_backoff:
1106 if (current_thread()(active_threads[(0)])->wait_result == THREAD_AWAKENED0)
1107 return VM_FAULT_RETRY1;
1108 else
1109 return VM_FAULT_INTERRUPTED2;
1110
1111#undef RELEASE_PAGE
1112}
1113
1114/*
1115 * Routine: vm_fault
1116 * Purpose:
1117 * Handle page faults, including pseudo-faults
1118 * used to change the wiring status of pages.
1119 * Returns:
1120 * If an explicit (expression) continuation is supplied,
1121 * then we call the continuation instead of returning.
1122 * Implementation:
1123 * Explicit continuations make this a little icky,
1124 * because it hasn't been rewritten to embrace CPS.
1125 * Instead, we have resume arguments for vm_fault and
1126 * vm_fault_page, to let continue the fault computation.
1127 *
1128 * vm_fault and vm_fault_page save mucho state
1129 * in the moral equivalent of a closure. The state
1130 * structure is allocated when first entering vm_fault
1131 * and deallocated when leaving vm_fault.
1132 */
1133
1134void
1135vm_fault_continue(void)
1136{
1137 vm_fault_state_t *state =
1138 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1139
1140 (void) vm_fault(state->vmf_map,
1141 state->vmf_vaddr,
1142 state->vmf_fault_type,
1143 state->vmf_change_wiring,
1144 TRUE((boolean_t) 1), state->vmf_continuation);
1145 /*NOTREACHED*/
1146}
1147
1148kern_return_t vm_fault(
1149 vm_map_t map,
1150 vm_offset_t vaddr,
1151 vm_prot_t fault_type,
1152 boolean_t change_wiring,
1153 boolean_t resume,
1154 void (*continuation)())
1155{
1156 vm_map_version_t version; /* Map version for verificiation */
1157 boolean_t wired; /* Should mapping be wired down? */
1158 vm_object_t object; /* Top-level object */
1159 vm_offset_t offset; /* Top-level offset */
1160 vm_prot_t prot; /* Protection for mapping */
1161 vm_object_t old_copy_object; /* Saved copy object */
1162 vm_page_t result_page; /* Result of vm_fault_page */
1163 vm_page_t top_page; /* Placeholder page */
1164 kern_return_t kr;
1165
1166 vm_page_t m; /* Fast access to result_page */
1167
1168 if (resume) {
1169 vm_fault_state_t *state =
1170 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1171
1172 /*
1173 * Retrieve cached variables and
1174 * continue vm_fault_page.
1175 */
1176
1177 object = state->vmf_object;
1178 if (object == VM_OBJECT_NULL((vm_object_t) 0))
1179 goto RetryFault;
1180 version = state->vmf_version;
1181 wired = state->vmf_wired;
1182 offset = state->vmf_offset;
1183 prot = state->vmf_prot;
1184
1185 kr = vm_fault_page(object, offset, fault_type,
1186 (change_wiring && !wired), !change_wiring,
1187 &prot, &result_page, &top_page,
1188 TRUE((boolean_t) 1), vm_fault_continue);
1189 goto after_vm_fault_page;
1190 }
1191
1192 if (continuation != (void (*)()) 0) {
1193 /*
1194 * We will probably need to save state.
1195 */
1196
1197 char * state;
1198
1199 /*
1200 * if this assignment stmt is written as
1201 * 'active_threads[cpu_number()] = kmem_cache_alloc()',
1202 * cpu_number may be evaluated before kmem_cache_alloc;
1203 * if kmem_cache_alloc blocks, cpu_number will be wrong
1204 */
1205
1206 state = (char *) kmem_cache_alloc(&vm_fault_state_cache);
1207 current_thread()(active_threads[(0)])->ith_othersaved.other = state;
1208
1209 }
1210
1211 RetryFault: ;
1212
1213 /*
1214 * Find the backing store object and offset into
1215 * it to begin the search.
1216 */
1217
1218 if ((kr = vm_map_lookup(&map, vaddr, fault_type, &version,
1219 &object, &offset,
1220 &prot, &wired)) != KERN_SUCCESS0) {
1221 goto done;
1222 }
1223
1224 /*
1225 * If the page is wired, we must fault for the current protection
1226 * value, to avoid further faults.
1227 */
1228
1229 if (wired)
1230 fault_type = prot;
1231
1232 /*
1233 * Make a reference to this object to
1234 * prevent its disposal while we are messing with
1235 * it. Once we have the reference, the map is free
1236 * to be diddled. Since objects reference their
1237 * shadows (and copies), they will stay around as well.
1238 */
1239
1240 assert(object->ref_count > 0)({ if (!(object->ref_count > 0)) Assert("object->ref_count > 0"
, "../vm/vm_fault.c", 1240); });
1241 object->ref_count++;
1242 vm_object_paging_begin(object)((object)->paging_in_progress++);
1243
1244 if (continuation != (void (*)()) 0) {
1245 vm_fault_state_t *state =
1246 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1247
1248 /*
1249 * Save variables, in case vm_fault_page discards
1250 * our kernel stack and we have to restart.
1251 */
1252
1253 state->vmf_map = map;
1254 state->vmf_vaddr = vaddr;
1255 state->vmf_fault_type = fault_type;
1256 state->vmf_change_wiring = change_wiring;
1257 state->vmf_continuation = continuation;
1258
1259 state->vmf_version = version;
1260 state->vmf_wired = wired;
1261 state->vmf_object = object;
1262 state->vmf_offset = offset;
1263 state->vmf_prot = prot;
1264
1265 kr = vm_fault_page(object, offset, fault_type,
1266 (change_wiring && !wired), !change_wiring,
1267 &prot, &result_page, &top_page,
1268 FALSE((boolean_t) 0), vm_fault_continue);
1269 } else
1270 {
1271 kr = vm_fault_page(object, offset, fault_type,
1272 (change_wiring && !wired), !change_wiring,
1273 &prot, &result_page, &top_page,
1274 FALSE((boolean_t) 0), (void (*)()) 0);
1275 }
1276 after_vm_fault_page:
1277
1278 /*
1279 * If we didn't succeed, lose the object reference immediately.
1280 */
1281
1282 if (kr != VM_FAULT_SUCCESS0)
1283 vm_object_deallocate(object);
1284
1285 /*
1286 * See why we failed, and take corrective action.
1287 */
1288
1289 switch (kr) {
1290 case VM_FAULT_SUCCESS0:
1291 break;
1292 case VM_FAULT_RETRY1:
1293 goto RetryFault;
1294 case VM_FAULT_INTERRUPTED2:
1295 kr = KERN_SUCCESS0;
1296 goto done;
1297 case VM_FAULT_MEMORY_SHORTAGE3:
1298 if (continuation != (void (*)()) 0) {
1299 vm_fault_state_t *state =
1300 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1301
1302 /*
1303 * Save variables in case VM_PAGE_WAIT
1304 * discards our kernel stack.
1305 */
1306
1307 state->vmf_map = map;
1308 state->vmf_vaddr = vaddr;
1309 state->vmf_fault_type = fault_type;
1310 state->vmf_change_wiring = change_wiring;
1311 state->vmf_continuation = continuation;
1312 state->vmf_object = VM_OBJECT_NULL((vm_object_t) 0);
1313
1314 VM_PAGE_WAIT(vm_fault_continue)vm_page_wait(vm_fault_continue);
1315 } else
1316 VM_PAGE_WAIT((void (*)()) 0)vm_page_wait((void (*)()) 0);
1317 goto RetryFault;
1318 case VM_FAULT_FICTITIOUS_SHORTAGE4:
1319 vm_page_more_fictitious();
1320 goto RetryFault;
1321 case VM_FAULT_MEMORY_ERROR5:
1322 kr = KERN_MEMORY_ERROR10;
1323 goto done;
1324 }
1325
1326 m = result_page;
1327
1328 assert((change_wiring && !wired) ?({ if (!((change_wiring && !wired) ? (top_page == ((vm_page_t
) 0)) : ((top_page == ((vm_page_t) 0)) == (m->object == object
)))) Assert("(change_wiring && !wired) ? (top_page == VM_PAGE_NULL) : ((top_page == VM_PAGE_NULL) == (m->object == object))"
, "../vm/vm_fault.c", 1330); })
1329 (top_page == VM_PAGE_NULL) :({ if (!((change_wiring && !wired) ? (top_page == ((vm_page_t
) 0)) : ((top_page == ((vm_page_t) 0)) == (m->object == object
)))) Assert("(change_wiring && !wired) ? (top_page == VM_PAGE_NULL) : ((top_page == VM_PAGE_NULL) == (m->object == object))"
, "../vm/vm_fault.c", 1330); })
1330 ((top_page == VM_PAGE_NULL) == (m->object == object)))({ if (!((change_wiring && !wired) ? (top_page == ((vm_page_t
) 0)) : ((top_page == ((vm_page_t) 0)) == (m->object == object
)))) Assert("(change_wiring && !wired) ? (top_page == VM_PAGE_NULL) : ((top_page == VM_PAGE_NULL) == (m->object == object))"
, "../vm/vm_fault.c", 1330); });
1331
1332 /*
1333 * How to clean up the result of vm_fault_page. This
1334 * happens whether the mapping is entered or not.
1335 */
1336
1337#define UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); } \
1338 MACRO_BEGIN({ \
1339 vm_fault_cleanup(m->object, top_page); \
1340 vm_object_deallocate(object); \
1341 MACRO_END})
1342
1343 /*
1344 * What to do with the resulting page from vm_fault_page
1345 * if it doesn't get entered into the physical map:
1346 */
1347
1348#define RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ((void
)(&vm_page_queue_lock)); } \
1349 MACRO_BEGIN({ \
1350 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } }); \
1351 vm_page_lock_queues(); \
1352 if (!m->active && !m->inactive) \
1353 vm_page_activate(m); \
1354 vm_page_unlock_queues()((void)(&vm_page_queue_lock)); \
1355 MACRO_END})
1356
1357 /*
1358 * We must verify that the maps have not changed
1359 * since our last lookup.
1360 */
1361
1362 old_copy_object = m->object->copy;
1363
1364 vm_object_unlock(m->object)((void)(&(m->object)->Lock));
1365 while (!vm_map_verify(map, &version)) {
1366 vm_object_t retry_object;
1367 vm_offset_t retry_offset;
1368 vm_prot_t retry_prot;
1369
1370 /*
1371 * To avoid trying to write_lock the map while another
1372 * thread has it read_locked (in vm_map_pageable), we
1373 * do not try for write permission. If the page is
1374 * still writable, we will get write permission. If it
1375 * is not, or has been marked needs_copy, we enter the
1376 * mapping without write permission, and will merely
1377 * take another fault.
1378 */
1379 kr = vm_map_lookup(&map, vaddr,
1380 fault_type & ~VM_PROT_WRITE((vm_prot_t) 0x02), &version,
1381 &retry_object, &retry_offset, &retry_prot,
1382 &wired);
1383
1384 if (kr != KERN_SUCCESS0) {
1385 vm_object_lock(m->object);
1386 RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ((void
)(&vm_page_queue_lock)); };
1387 UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); };
1388 goto done;
1389 }
1390
1391 vm_object_unlock(retry_object)((void)(&(retry_object)->Lock));
1392 vm_object_lock(m->object);
1393
1394 if ((retry_object != object) ||
1395 (retry_offset != offset)) {
1396 RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ((void
)(&vm_page_queue_lock)); };
1397 UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); };
1398 goto RetryFault;
1399 }
1400
1401 /*
1402 * Check whether the protection has changed or the object
1403 * has been copied while we left the map unlocked.
1404 */
1405 prot &= retry_prot;
1406 vm_object_unlock(m->object)((void)(&(m->object)->Lock));
1407 }
1408 vm_object_lock(m->object);
1409
1410 /*
1411 * If the copy object changed while the top-level object
1412 * was unlocked, then we must take away write permission.
1413 */
1414
1415 if (m->object->copy != old_copy_object)
1416 prot &= ~VM_PROT_WRITE((vm_prot_t) 0x02);
1417
1418 /*
1419 * If we want to wire down this page, but no longer have
1420 * adequate permissions, we must start all over.
1421 */
1422
1423 if (wired && (prot != fault_type)) {
1424 vm_map_verify_done(map, &version)(lock_done(&(map)->lock));
1425 RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ((void
)(&vm_page_queue_lock)); };
1426 UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); };
1427 goto RetryFault;
1428 }
1429
1430 /*
1431 * It's critically important that a wired-down page be faulted
1432 * only once in each map for which it is wired.
1433 */
1434
1435 vm_object_unlock(m->object)((void)(&(m->object)->Lock));
1436
1437 /*
1438 * Put this page into the physical map.
1439 * We had to do the unlock above because pmap_enter
1440 * may cause other faults. The page may be on
1441 * the pageout queues. If the pageout daemon comes
1442 * across the page, it will remove it from the queues.
1443 */
1444
1445 PMAP_ENTER(map->pmap, vaddr, m, prot, wired)({ pmap_enter( (map->pmap), (vaddr), (m)->phys_addr, (prot
) & ~(m)->page_lock, (wired) ); });
1446
1447 /*
1448 * If the page is not wired down and isn't already
1449 * on a pageout queue, then put it where the
1450 * pageout daemon can find it.
1451 */
1452 vm_object_lock(m->object);
1453 vm_page_lock_queues();
1454 if (change_wiring) {
1455 if (wired)
1456 vm_page_wire(m);
1457 else
1458 vm_page_unwire(m);
1459 } else if (software_reference_bits) {
1460 if (!m->active && !m->inactive)
1461 vm_page_activate(m);
1462 m->reference = TRUE((boolean_t) 1);
1463 } else {
1464 vm_page_activate(m);
1465 }
1466 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
1467
1468 /*
1469 * Unlock everything, and return
1470 */
1471
1472 vm_map_verify_done(map, &version)(lock_done(&(map)->lock));
1473 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } });
1474 kr = KERN_SUCCESS0;
1475 UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); };
1476
1477#undef UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); }
1478#undef RELEASE_PAGE
1479
1480 done:
1481 if (continuation != (void (*)()) 0) {
1482 vm_fault_state_t *state =
1483 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1484
1485 kmem_cache_free(&vm_fault_state_cache, (vm_offset_t) state);
1486 (*continuation)(kr);
1487 /*NOTREACHED*/
1488 }
1489
1490 return(kr);
1491}
1492
1493/*
1494 * vm_fault_wire:
1495 *
1496 * Wire down a range of virtual addresses in a map.
1497 */
1498void vm_fault_wire(
1499 vm_map_t map,
1500 vm_map_entry_t entry)
1501{
1502
1503 vm_offset_t va;
1504 pmap_t pmap;
1505 vm_offset_t end_addr = entry->vme_endlinks.end;
1506
1507 pmap = vm_map_pmap(map)((map)->pmap);
1508
1509 /*
1510 * Inform the physical mapping system that the
1511 * range of addresses may not fault, so that
1512 * page tables and such can be locked down as well.
1513 */
1514
1515 pmap_pageable(pmap, entry->vme_startlinks.start, end_addr, FALSE((boolean_t) 0));
1516
1517 /*
1518 * We simulate a fault to get the page and enter it
1519 * in the physical map.
1520 */
1521
1522 for (va = entry->vme_startlinks.start; va < end_addr; va += PAGE_SIZE(1 << 12)) {
1523 if (vm_fault_wire_fast(map, va, entry) != KERN_SUCCESS0)
1524 (void) vm_fault(map, va, VM_PROT_NONE((vm_prot_t) 0x00), TRUE((boolean_t) 1),
1525 FALSE((boolean_t) 0), (void (*)()) 0);
1526 }
1527}
1528
1529/*
1530 * vm_fault_unwire:
1531 *
1532 * Unwire a range of virtual addresses in a map.
1533 */
1534void vm_fault_unwire(
1535 vm_map_t map,
1536 vm_map_entry_t entry)
1537{
1538 vm_offset_t va;
1539 pmap_t pmap;
1540 vm_offset_t end_addr = entry->vme_endlinks.end;
1541 vm_object_t object;
1542
1543 pmap = vm_map_pmap(map)((map)->pmap);
1544
1545 object = (entry->is_sub_map)
1546 ? VM_OBJECT_NULL((vm_object_t) 0) : entry->object.vm_object;
1547
1548 /*
1549 * Since the pages are wired down, we must be able to
1550 * get their mappings from the physical map system.
1551 */
1552
1553 for (va = entry->vme_startlinks.start; va < end_addr; va += PAGE_SIZE(1 << 12)) {
1554 pmap_change_wiring(pmap, va, FALSE((boolean_t) 0));
1555
1556 if (object == VM_OBJECT_NULL((vm_object_t) 0)) {
1557 vm_map_lock_set_recursive(map)lock_set_recursive(&(map)->lock);
1558 (void) vm_fault(map, va, VM_PROT_NONE((vm_prot_t) 0x00), TRUE((boolean_t) 1),
1559 FALSE((boolean_t) 0), (void (*)()) 0);
1560 vm_map_lock_clear_recursive(map)lock_clear_recursive(&(map)->lock);
1561 } else {
1562 vm_prot_t prot;
1563 vm_page_t result_page;
1564 vm_page_t top_page;
1565 vm_fault_return_t result;
1566
1567 do {
1568 prot = VM_PROT_NONE((vm_prot_t) 0x00);
1569
1570 vm_object_lock(object);
1571 vm_object_paging_begin(object)((object)->paging_in_progress++);
1572 result = vm_fault_page(object,
1573 entry->offset +
1574 (va - entry->vme_startlinks.start),
1575 VM_PROT_NONE((vm_prot_t) 0x00), TRUE((boolean_t) 1),
1576 FALSE((boolean_t) 0), &prot,
1577 &result_page,
1578 &top_page,
1579 FALSE((boolean_t) 0), (void (*)()) 0);
1580 } while (result == VM_FAULT_RETRY1);
1581
1582 if (result != VM_FAULT_SUCCESS0)
1583 panic("vm_fault_unwire: failure");
1584
1585 vm_page_lock_queues();
1586 vm_page_unwire(result_page);
1587 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
1588 PAGE_WAKEUP_DONE(result_page)({ (result_page)->busy = ((boolean_t) 0); if ((result_page
)->wanted) { (result_page)->wanted = ((boolean_t) 0); thread_wakeup_prim
((((event_t) result_page)), ((boolean_t) 0), 0); } });
1589
1590 vm_fault_cleanup(result_page->object, top_page);
1591 }
1592 }
1593
1594 /*
1595 * Inform the physical mapping system that the range
1596 * of addresses may fault, so that page tables and
1597 * such may be unwired themselves.
1598 */
1599
1600 pmap_pageable(pmap, entry->vme_startlinks.start, end_addr, TRUE((boolean_t) 1));
1601}
1602
1603/*
1604 * vm_fault_wire_fast:
1605 *
1606 * Handle common case of a wire down page fault at the given address.
1607 * If successful, the page is inserted into the associated physical map.
1608 * The map entry is passed in to avoid the overhead of a map lookup.
1609 *
1610 * NOTE: the given address should be truncated to the
1611 * proper page address.
1612 *
1613 * KERN_SUCCESS is returned if the page fault is handled; otherwise,
1614 * a standard error specifying why the fault is fatal is returned.
1615 *
1616 * The map in question must be referenced, and remains so.
1617 * Caller has a read lock on the map.
1618 *
1619 * This is a stripped version of vm_fault() for wiring pages. Anything
1620 * other than the common case will return KERN_FAILURE, and the caller
1621 * is expected to call vm_fault().
1622 */
1623kern_return_t vm_fault_wire_fast(
1624 vm_map_t map,
1625 vm_offset_t va,
1626 vm_map_entry_t entry)
1627{
1628 vm_object_t object;
1629 vm_offset_t offset;
1630 vm_page_t m;
1631 vm_prot_t prot;
1632
1633 vm_stat.faults++; /* needs lock XXX */
1634 current_task()((active_threads[(0)])->task)->faults++;
1635/*
1636 * Recovery actions
1637 */
1638
1639#undef RELEASE_PAGE
1640#define RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ((void
)(&vm_page_queue_lock)); } { \
1641 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } }); \
1642 vm_page_lock_queues(); \
1643 vm_page_unwire(m); \
1644 vm_page_unlock_queues()((void)(&vm_page_queue_lock)); \
1645}
1646
1647
1648#undef UNLOCK_THINGS{ object->paging_in_progress--; ((void)(&(object)->
Lock)); }
1649#define UNLOCK_THINGS{ object->paging_in_progress--; ((void)(&(object)->
Lock)); } { \
1650 object->paging_in_progress--; \
1651 vm_object_unlock(object)((void)(&(object)->Lock)); \
1652}
1653
1654#undef UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); }
1655#define UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); } { \
1656 UNLOCK_THINGS{ object->paging_in_progress--; ((void)(&(object)->
Lock)); }; \
1657 vm_object_deallocate(object); \
1658}
1659/*
1660 * Give up and have caller do things the hard way.
1661 */
1662
1663#define GIVE_UP{ { { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); }; return(5); } { \
1664 UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); }; \
1665 return(KERN_FAILURE5); \
1666}
1667
1668
1669 /*
1670 * If this entry is not directly to a vm_object, bail out.
1671 */
1672 if (entry->is_sub_map)
1673 return(KERN_FAILURE5);
1674
1675 /*
1676 * Find the backing store object and offset into it.
1677 */
1678
1679 object = entry->object.vm_object;
1680 offset = (va - entry->vme_startlinks.start) + entry->offset;
1681 prot = entry->protection;
1682
1683 /*
1684 * Make a reference to this object to prevent its
1685 * disposal while we are messing with it.
1686 */
1687
1688 vm_object_lock(object);
1689 assert(object->ref_count > 0)({ if (!(object->ref_count > 0)) Assert("object->ref_count > 0"
, "../vm/vm_fault.c", 1689); });
1690 object->ref_count++;
1691 object->paging_in_progress++;
1692
1693 /*
1694 * INVARIANTS (through entire routine):
1695 *
1696 * 1) At all times, we must either have the object
1697 * lock or a busy page in some object to prevent
1698 * some other thread from trying to bring in
1699 * the same page.
1700 *
1701 * 2) Once we have a busy page, we must remove it from
1702 * the pageout queues, so that the pageout daemon
1703 * will not grab it away.
1704 *
1705 */
1706
1707 /*
1708 * Look for page in top-level object. If it's not there or
1709 * there's something going on, give up.
1710 */
1711 m = vm_page_lookup(object, offset);
1712 if ((m == VM_PAGE_NULL((vm_page_t) 0)) || (m->error) ||
1713 (m->busy) || (m->absent) || (prot & m->page_lock)) {
1714 GIVE_UP{ { { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); }; return(5); };
1715 }
1716
1717 /*
1718 * Wire the page down now. All bail outs beyond this
1719 * point must unwire the page.
1720 */
1721
1722 vm_page_lock_queues();
1723 vm_page_wire(m);
1724 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
1725
1726 /*
1727 * Mark page busy for other threads.
1728 */
1729 assert(!m->busy)({ if (!(!m->busy)) Assert("!m->busy", "../vm/vm_fault.c"
, 1729); });
1730 m->busy = TRUE((boolean_t) 1);
1731 assert(!m->absent)({ if (!(!m->absent)) Assert("!m->absent", "../vm/vm_fault.c"
, 1731); });
1732
1733 /*
1734 * Give up if the page is being written and there's a copy object
1735 */
1736 if ((object->copy != VM_OBJECT_NULL((vm_object_t) 0)) && (prot & VM_PROT_WRITE((vm_prot_t) 0x02))) {
1737 RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ((void
)(&vm_page_queue_lock)); };
1738 GIVE_UP{ { { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); }; return(5); };
1739 }
1740
1741 /*
1742 * Put this page into the physical map.
1743 * We have to unlock the object because pmap_enter
1744 * may cause other faults.
1745 */
1746 vm_object_unlock(object)((void)(&(object)->Lock));
1747
1748 PMAP_ENTER(map->pmap, va, m, prot, TRUE)({ pmap_enter( (map->pmap), (va), (m)->phys_addr, (prot
) & ~(m)->page_lock, (((boolean_t) 1)) ); });
1749
1750 /*
1751 * Must relock object so that paging_in_progress can be cleared.
1752 */
1753 vm_object_lock(object);
1754
1755 /*
1756 * Unlock everything, and return
1757 */
1758
1759 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } });
1760 UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ((void)(&(object)->
Lock)); }; vm_object_deallocate(object); };
1761
1762 return(KERN_SUCCESS0);
1763
1764}
1765
1766/*
1767 * Routine: vm_fault_copy_cleanup
1768 * Purpose:
1769 * Release a page used by vm_fault_copy.
1770 */
1771
1772void vm_fault_copy_cleanup(
1773 vm_page_t page,
1774 vm_page_t top_page)
1775{
1776 vm_object_t object = page->object;
1777
1778 vm_object_lock(object);
1779 PAGE_WAKEUP_DONE(page)({ (page)->busy = ((boolean_t) 0); if ((page)->wanted) {
(page)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) page)), ((boolean_t) 0), 0); } });
1780 vm_page_lock_queues();
1781 if (!page->active && !page->inactive)
1782 vm_page_activate(page);
1783 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
1784 vm_fault_cleanup(object, top_page);
1785}
1786
1787/*
1788 * Routine: vm_fault_copy
1789 *
1790 * Purpose:
1791 * Copy pages from one virtual memory object to another --
1792 * neither the source nor destination pages need be resident.
1793 *
1794 * Before actually copying a page, the version associated with
1795 * the destination address map wil be verified.
1796 *
1797 * In/out conditions:
1798 * The caller must hold a reference, but not a lock, to
1799 * each of the source and destination objects and to the
1800 * destination map.
1801 *
1802 * Results:
1803 * Returns KERN_SUCCESS if no errors were encountered in
1804 * reading or writing the data. Returns KERN_INTERRUPTED if
1805 * the operation was interrupted (only possible if the
1806 * "interruptible" argument is asserted). Other return values
1807 * indicate a permanent error in copying the data.
1808 *
1809 * The actual amount of data copied will be returned in the
1810 * "copy_size" argument. In the event that the destination map
1811 * verification failed, this amount may be less than the amount
1812 * requested.
1813 */
1814kern_return_t vm_fault_copy(
1815 vm_object_t src_object,
1816 vm_offset_t src_offset,
1817 vm_size_t *src_size, /* INOUT */
1818 vm_object_t dst_object,
1819 vm_offset_t dst_offset,
1820 vm_map_t dst_map,
1821 vm_map_version_t *dst_version,
1822 boolean_t interruptible)
1823{
1824 vm_page_t result_page;
1825 vm_prot_t prot;
1826
1827 vm_page_t src_page;
1828 vm_page_t src_top_page;
1829
1830 vm_page_t dst_page;
1831 vm_page_t dst_top_page;
1832
1833 vm_size_t amount_done;
1834 vm_object_t old_copy_object;
1835
1836#define RETURN(x) \
1837 MACRO_BEGIN({ \
1838 *src_size = amount_done; \
1839 MACRO_RETURNif (((boolean_t) 1)) return(x); \
1840 MACRO_END})
1841
1842 amount_done = 0;
1843 do { /* while (amount_done != *src_size) */
1844
1845 RetrySourceFault: ;
1846
1847 if (src_object == VM_OBJECT_NULL((vm_object_t) 0)) {
1848 /*
1849 * No source object. We will just
1850 * zero-fill the page in dst_object.
1851 */
1852
1853 src_page = VM_PAGE_NULL((vm_page_t) 0);
1854 } else {
1855 prot = VM_PROT_READ((vm_prot_t) 0x01);
1856
1857 vm_object_lock(src_object);
1858 vm_object_paging_begin(src_object)((src_object)->paging_in_progress++);
1859
1860 switch (vm_fault_page(src_object, src_offset,
1861 VM_PROT_READ((vm_prot_t) 0x01), FALSE((boolean_t) 0), interruptible,
1862 &prot, &result_page, &src_top_page,
1863 FALSE((boolean_t) 0), (void (*)()) 0)) {
1864
1865 case VM_FAULT_SUCCESS0:
1866 break;
1867 case VM_FAULT_RETRY1:
1868 goto RetrySourceFault;
1869 case VM_FAULT_INTERRUPTED2:
1870 RETURN(MACH_SEND_INTERRUPTED0x10000007);
1871 case VM_FAULT_MEMORY_SHORTAGE3:
1872 VM_PAGE_WAIT((void (*)()) 0)vm_page_wait((void (*)()) 0);
1873 goto RetrySourceFault;
1874 case VM_FAULT_FICTITIOUS_SHORTAGE4:
1875 vm_page_more_fictitious();
1876 goto RetrySourceFault;
1877 case VM_FAULT_MEMORY_ERROR5:
1878 return(KERN_MEMORY_ERROR10);
1879 }
1880
1881 src_page = result_page;
1882
1883 assert((src_top_page == VM_PAGE_NULL) ==({ if (!((src_top_page == ((vm_page_t) 0)) == (src_page->object
== src_object))) Assert("(src_top_page == VM_PAGE_NULL) == (src_page->object == src_object)"
, "../vm/vm_fault.c", 1884); })
1884 (src_page->object == src_object))({ if (!((src_top_page == ((vm_page_t) 0)) == (src_page->object
== src_object))) Assert("(src_top_page == VM_PAGE_NULL) == (src_page->object == src_object)"
, "../vm/vm_fault.c", 1884); });
1885
1886 assert ((prot & VM_PROT_READ) != VM_PROT_NONE)({ if (!((prot & ((vm_prot_t) 0x01)) != ((vm_prot_t) 0x00
))) Assert("(prot & VM_PROT_READ) != VM_PROT_NONE", "../vm/vm_fault.c"
, 1886); });
1887
1888 vm_object_unlock(src_page->object)((void)(&(src_page->object)->Lock));
1889 }
1890
1891 RetryDestinationFault: ;
1892
1893 prot = VM_PROT_WRITE((vm_prot_t) 0x02);
1894
1895 vm_object_lock(dst_object);
1896 vm_object_paging_begin(dst_object)((dst_object)->paging_in_progress++);
1897
1898 switch (vm_fault_page(dst_object, dst_offset, VM_PROT_WRITE((vm_prot_t) 0x02),
1899 FALSE((boolean_t) 0), FALSE((boolean_t) 0) /* interruptible */,
1900 &prot, &result_page, &dst_top_page,
1901 FALSE((boolean_t) 0), (void (*)()) 0)) {
1902
1903 case VM_FAULT_SUCCESS0:
1904 break;
1905 case VM_FAULT_RETRY1:
1906 goto RetryDestinationFault;
1907 case VM_FAULT_INTERRUPTED2:
1908 if (src_page != VM_PAGE_NULL((vm_page_t) 0))
1909 vm_fault_copy_cleanup(src_page,
1910 src_top_page);
1911 RETURN(MACH_SEND_INTERRUPTED0x10000007);
1912 case VM_FAULT_MEMORY_SHORTAGE3:
1913 VM_PAGE_WAIT((void (*)()) 0)vm_page_wait((void (*)()) 0);
1914 goto RetryDestinationFault;
1915 case VM_FAULT_FICTITIOUS_SHORTAGE4:
1916 vm_page_more_fictitious();
1917 goto RetryDestinationFault;
1918 case VM_FAULT_MEMORY_ERROR5:
1919 if (src_page != VM_PAGE_NULL((vm_page_t) 0))
1920 vm_fault_copy_cleanup(src_page,
1921 src_top_page);
1922 return(KERN_MEMORY_ERROR10);
1923 }
1924 assert ((prot & VM_PROT_WRITE) != VM_PROT_NONE)({ if (!((prot & ((vm_prot_t) 0x02)) != ((vm_prot_t) 0x00
))) Assert("(prot & VM_PROT_WRITE) != VM_PROT_NONE", "../vm/vm_fault.c"
, 1924); });
1925
1926 dst_page = result_page;
1927
1928 old_copy_object = dst_page->object->copy;
1929
1930 vm_object_unlock(dst_page->object)((void)(&(dst_page->object)->Lock));
1931
1932 if (!vm_map_verify(dst_map, dst_version)) {
1933
1934 BailOut: ;
1935
1936 if (src_page != VM_PAGE_NULL((vm_page_t) 0))
1937 vm_fault_copy_cleanup(src_page, src_top_page);
1938 vm_fault_copy_cleanup(dst_page, dst_top_page);
1939 break;
1940 }
1941
1942
1943 vm_object_lock(dst_page->object);
1944 if (dst_page->object->copy != old_copy_object) {
1945 vm_object_unlock(dst_page->object)((void)(&(dst_page->object)->Lock));
1946 vm_map_verify_done(dst_map, dst_version)(lock_done(&(dst_map)->lock));
1947 goto BailOut;
1948 }
1949 vm_object_unlock(dst_page->object)((void)(&(dst_page->object)->Lock));
1950
1951 /*
1952 * Copy the page, and note that it is dirty
1953 * immediately.
1954 */
1955
1956 if (src_page == VM_PAGE_NULL((vm_page_t) 0))
1957 vm_page_zero_fill(dst_page);
1958 else
1959 vm_page_copy(src_page, dst_page);
1960 dst_page->dirty = TRUE((boolean_t) 1);
1961
1962 /*
1963 * Unlock everything, and return
1964 */
1965
1966 vm_map_verify_done(dst_map, dst_version)(lock_done(&(dst_map)->lock));
1967
1968 if (src_page != VM_PAGE_NULL((vm_page_t) 0))
1969 vm_fault_copy_cleanup(src_page, src_top_page);
1970 vm_fault_copy_cleanup(dst_page, dst_top_page);
1971
1972 amount_done += PAGE_SIZE(1 << 12);
1973 src_offset += PAGE_SIZE(1 << 12);
1974 dst_offset += PAGE_SIZE(1 << 12);
1975
1976 } while (amount_done != *src_size);
1977
1978 RETURN(KERN_SUCCESS0);
1979#undef RETURN
1980
1981 /*NOTREACHED*/
1982}
1983
1984
1985
1986
1987
1988#ifdef notdef
1989
1990/*
1991 * Routine: vm_fault_page_overwrite
1992 *
1993 * Description:
1994 * A form of vm_fault_page that assumes that the
1995 * resulting page will be overwritten in its entirety,
1996 * making it unnecessary to obtain the correct *contents*
1997 * of the page.
1998 *
1999 * Implementation:
2000 * XXX Untested. Also unused. Eventually, this technology
2001 * could be used in vm_fault_copy() to advantage.
2002 */
2003vm_fault_return_t vm_fault_page_overwrite(
2004 vm_object_t dst_object,
2005 vm_offset_t dst_offset,
2006 vm_page_t *result_page) /* OUT */
2007{
2008 vm_page_t dst_page;
2009
2010#define interruptible FALSE((boolean_t) 0) /* XXX */
2011
2012 while (TRUE((boolean_t) 1)) {
2013 /*
2014 * Look for a page at this offset
2015 */
2016
2017 while ((dst_page = vm_page_lookup(dst_object, dst_offset))
2018 == VM_PAGE_NULL((vm_page_t) 0)) {
2019 /*
2020 * No page, no problem... just allocate one.
2021 */
2022
2023 dst_page = vm_page_alloc(dst_object, dst_offset);
2024 if (dst_page == VM_PAGE_NULL((vm_page_t) 0)) {
2025 vm_object_unlock(dst_object)((void)(&(dst_object)->Lock));
2026 VM_PAGE_WAIT((void (*)()) 0)vm_page_wait((void (*)()) 0);
2027 vm_object_lock(dst_object);
2028 continue;
2029 }
2030
2031 /*
2032 * Pretend that the memory manager
2033 * write-protected the page.
2034 *
2035 * Note that we will be asking for write
2036 * permission without asking for the data
2037 * first.
2038 */
2039
2040 dst_page->overwriting = TRUE((boolean_t) 1);
2041 dst_page->page_lock = VM_PROT_WRITE((vm_prot_t) 0x02);
2042 dst_page->absent = TRUE((boolean_t) 1);
2043 dst_object->absent_count++;
2044
2045 break;
2046
2047 /*
2048 * When we bail out, we might have to throw
2049 * away the page created here.
2050 */
2051
2052#define DISCARD_PAGE \
2053 MACRO_BEGIN({ \
2054 vm_object_lock(dst_object); \
2055 dst_page = vm_page_lookup(dst_object, dst_offset); \
2056 if ((dst_page != VM_PAGE_NULL((vm_page_t) 0)) && dst_page->overwriting) \
2057 VM_PAGE_FREE(dst_page)({ ; vm_page_free(dst_page); ((void)(&vm_page_queue_lock)
); }); \
2058 vm_object_unlock(dst_object)((void)(&(dst_object)->Lock)); \
2059 MACRO_END})
2060 }
2061
2062 /*
2063 * If the page is write-protected...
2064 */
2065
2066 if (dst_page->page_lock & VM_PROT_WRITE((vm_prot_t) 0x02)) {
2067 /*
2068 * ... and an unlock request hasn't been sent
2069 */
2070
2071 if ( ! (dst_page->unlock_request & VM_PROT_WRITE((vm_prot_t) 0x02))) {
2072 vm_prot_t u;
2073 kern_return_t rc;
2074
2075 /*
2076 * ... then send one now.
2077 */
2078
2079 if (!dst_object->pager_ready) {
2080 vm_object_assert_wait(dst_object,({ (dst_object)->all_wanted |= 1 << (1); assert_wait
((event_t)(((vm_offset_t) dst_object) + (1)), (interruptible)
); })
2081 VM_OBJECT_EVENT_PAGER_READY,({ (dst_object)->all_wanted |= 1 << (1); assert_wait
((event_t)(((vm_offset_t) dst_object) + (1)), (interruptible)
); })
2082 interruptible)({ (dst_object)->all_wanted |= 1 << (1); assert_wait
((event_t)(((vm_offset_t) dst_object) + (1)), (interruptible)
); });
2083 vm_object_unlock(dst_object)((void)(&(dst_object)->Lock));
2084 thread_block((void (*)()) 0);
2085 if (current_thread()(active_threads[(0)])->wait_result !=
2086 THREAD_AWAKENED0) {
2087 DISCARD_PAGE;
2088 return(VM_FAULT_INTERRUPTED2);
2089 }
2090 continue;
2091 }
2092
2093 u = dst_page->unlock_request |= VM_PROT_WRITE((vm_prot_t) 0x02);
2094 vm_object_unlock(dst_object)((void)(&(dst_object)->Lock));
2095
2096 if ((rc = memory_object_data_unlock(
2097 dst_object->pager,
2098 dst_object->pager_request,
2099 dst_offset + dst_object->paging_offset,
2100 PAGE_SIZE(1 << 12),
2101 u)) != KERN_SUCCESS0) {
2102 printf("vm_object_overwrite: memory_object_data_unlock failed\n");
2103 DISCARD_PAGE;
2104 return((rc == MACH_SEND_INTERRUPTED0x10000007) ?
2105 VM_FAULT_INTERRUPTED2 :
2106 VM_FAULT_MEMORY_ERROR5);
2107 }
2108 vm_object_lock(dst_object);
2109 continue;
2110 }
2111
2112 /* ... fall through to wait below */
2113 } else {
2114 /*
2115 * If the page isn't being used for other
2116 * purposes, then we're done.
2117 */
2118 if ( ! (dst_page->busy || dst_page->absent || dst_page->error) )
2119 break;
2120 }
2121
2122 PAGE_ASSERT_WAIT(dst_page, interruptible)({ (dst_page)->wanted = ((boolean_t) 1); assert_wait((event_t
) (dst_page), (interruptible)); });
2123 vm_object_unlock(dst_object)((void)(&(dst_object)->Lock));
2124 thread_block((void (*)()) 0);
2125 if (current_thread()(active_threads[(0)])->wait_result != THREAD_AWAKENED0) {
2126 DISCARD_PAGE;
2127 return(VM_FAULT_INTERRUPTED2);
2128 }
2129 }
2130
2131 *result_page = dst_page;
2132 return(VM_FAULT_SUCCESS0);
2133
2134#undef interruptible
2135#undef DISCARD_PAGE
2136}
2137
2138#endif /* notdef */