Bug Summary

File:obj/../vm/vm_fault.c
Location:line 557, column 6
Description:Access to field 'task' results in a dereference of a null pointer

Annotated Source Code

1/*
2 * Mach Operating System
3 * Copyright (c) 1994,1990,1989,1988,1987 Carnegie Mellon University.
4 * Copyright (c) 1993,1994 The University of Utah and
5 * the Computer Systems Laboratory (CSL).
6 * All rights reserved.
7 *
8 * Permission to use, copy, modify and distribute this software and its
9 * documentation is hereby granted, provided that both the copyright
10 * notice and this permission notice appear in all copies of the
11 * software, derivative works or modified versions, and any portions
12 * thereof, and that both notices appear in supporting documentation.
13 *
14 * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15 * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16 * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17 * THIS SOFTWARE.
18 *
19 * Carnegie Mellon requests users of this software to return to
20 *
21 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
22 * School of Computer Science
23 * Carnegie Mellon University
24 * Pittsburgh PA 15213-3890
25 *
26 * any improvements or extensions that they make and grant Carnegie Mellon
27 * the rights to redistribute these changes.
28 */
29/*
30 * File: vm_fault.c
31 * Author: Avadis Tevanian, Jr., Michael Wayne Young
32 *
33 * Page fault handling module.
34 */
35
36#include <kern/printf.h>
37#include <vm/vm_fault.h>
38#include <mach/kern_return.h>
39#include <mach/message.h> /* for error codes */
40#include <kern/counters.h>
41#include <kern/debug.h>
42#include <kern/thread.h>
43#include <kern/sched_prim.h>
44#include <vm/vm_map.h>
45#include <vm/vm_object.h>
46#include <vm/vm_page.h>
47#include <vm/pmap.h>
48#include <mach/vm_statistics.h>
49#include <vm/vm_pageout.h>
50#include <mach/vm_param.h>
51#include <mach/memory_object.h>
52#include <vm/memory_object_user.user.h>
53 /* For memory_object_data_{request,unlock} */
54#include <kern/macro_help.h>
55#include <kern/slab.h>
56
57#if MACH_PCSAMPLE1
58#include <kern/pc_sample.h>
59#endif
60
61
62
63/*
64 * State needed by vm_fault_continue.
65 * This is a little hefty to drop directly
66 * into the thread structure.
67 */
68typedef struct vm_fault_state {
69 struct vm_map *vmf_map;
70 vm_offset_t vmf_vaddr;
71 vm_prot_t vmf_fault_type;
72 boolean_t vmf_change_wiring;
73 void (*vmf_continuation)();
74 vm_map_version_t vmf_version;
75 boolean_t vmf_wired;
76 struct vm_object *vmf_object;
77 vm_offset_t vmf_offset;
78 vm_prot_t vmf_prot;
79
80 boolean_t vmfp_backoff;
81 struct vm_object *vmfp_object;
82 vm_offset_t vmfp_offset;
83 struct vm_page *vmfp_first_m;
84 vm_prot_t vmfp_access;
85} vm_fault_state_t;
86
87struct kmem_cache vm_fault_state_cache;
88
89int vm_object_absent_max = 50;
90
91int vm_fault_debug = 0;
92
93boolean_t vm_fault_dirty_handling = FALSE((boolean_t) 0);
94boolean_t vm_fault_interruptible = TRUE((boolean_t) 1);
95
96boolean_t software_reference_bits = TRUE((boolean_t) 1);
97
98#if MACH_KDB0
99extern struct db_watchpoint *db_watchpoint_list;
100#endif /* MACH_KDB */
101
102/*
103 * Routine: vm_fault_init
104 * Purpose:
105 * Initialize our private data structures.
106 */
107void vm_fault_init(void)
108{
109 kmem_cache_init(&vm_fault_state_cache, "vm_fault_state",
110 sizeof(vm_fault_state_t), 0, NULL((void *) 0), NULL((void *) 0), NULL((void *) 0), 0);
111}
112
113/*
114 * Routine: vm_fault_cleanup
115 * Purpose:
116 * Clean up the result of vm_fault_page.
117 * Results:
118 * The paging reference for "object" is released.
119 * "object" is unlocked.
120 * If "top_page" is not null, "top_page" is
121 * freed and the paging reference for the object
122 * containing it is released.
123 *
124 * In/out conditions:
125 * "object" must be locked.
126 */
127void
128vm_fault_cleanup(object, top_page)
129 register vm_object_t object;
130 register vm_page_t top_page;
131{
132 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 132); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
133 vm_object_unlock(object);
134
135 if (top_page != VM_PAGE_NULL((vm_page_t) 0)) {
136 object = top_page->object;
137 vm_object_lock(object);
138 VM_PAGE_FREE(top_page)({ ; vm_page_free(top_page); ; });
139 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 139); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
140 vm_object_unlock(object);
141 }
142}
143
144
145#if MACH_PCSAMPLE1
146/*
147 * Do PC sampling on current thread, assuming
148 * that it is the thread taking this page fault.
149 *
150 * Must check for THREAD_NULL, since faults
151 * can occur before threads are running.
152 */
153
154#define vm_stat_sample(flavor)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((flavor))) take_pc_sample((_thread_), &
(_thread_)->pc_sample, ((flavor))); task = (_thread_)->
task; if (task->pc_sample.sampletypes & ((flavor))) take_pc_sample
((_thread_), &task->pc_sample, ((flavor))); }); }) \
155 MACRO_BEGIN({ \
156 thread_t _thread_ = current_thread()(active_threads[(0)]); \
157 \
158 if (_thread_ != THREAD_NULL((thread_t) 0)) \
159 take_pc_sample_macro(_thread_, (flavor))({ task_t task; if ((_thread_)->pc_sample.sampletypes &
((flavor))) take_pc_sample((_thread_), &(_thread_)->pc_sample
, ((flavor))); task = (_thread_)->task; if (task->pc_sample
.sampletypes & ((flavor))) take_pc_sample((_thread_), &
task->pc_sample, ((flavor))); }); \
160 MACRO_END})
161
162#else
163#define vm_stat_sample(x)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((x))) take_pc_sample((_thread_), &(_thread_
)->pc_sample, ((x))); task = (_thread_)->task; if (task
->pc_sample.sampletypes & ((x))) take_pc_sample((_thread_
), &task->pc_sample, ((x))); }); })
164#endif /* MACH_PCSAMPLE */
165
166
167
168/*
169 * Routine: vm_fault_page
170 * Purpose:
171 * Find the resident page for the virtual memory
172 * specified by the given virtual memory object
173 * and offset.
174 * Additional arguments:
175 * The required permissions for the page is given
176 * in "fault_type". Desired permissions are included
177 * in "protection".
178 *
179 * If the desired page is known to be resident (for
180 * example, because it was previously wired down), asserting
181 * the "unwiring" parameter will speed the search.
182 *
183 * If the operation can be interrupted (by thread_abort
184 * or thread_terminate), then the "interruptible"
185 * parameter should be asserted.
186 *
187 * Results:
188 * The page containing the proper data is returned
189 * in "result_page".
190 *
191 * In/out conditions:
192 * The source object must be locked and referenced,
193 * and must donate one paging reference. The reference
194 * is not affected. The paging reference and lock are
195 * consumed.
196 *
197 * If the call succeeds, the object in which "result_page"
198 * resides is left locked and holding a paging reference.
199 * If this is not the original object, a busy page in the
200 * original object is returned in "top_page", to prevent other
201 * callers from pursuing this same data, along with a paging
202 * reference for the original object. The "top_page" should
203 * be destroyed when this guarantee is no longer required.
204 * The "result_page" is also left busy. It is not removed
205 * from the pageout queues.
206 */
207vm_fault_return_t vm_fault_page(first_object, first_offset,
208 fault_type, must_be_resident, interruptible,
209 protection,
210 result_page, top_page,
211 resume, continuation)
212 /* Arguments: */
213 vm_object_t first_object; /* Object to begin search */
214 vm_offset_t first_offset; /* Offset into object */
215 vm_prot_t fault_type; /* What access is requested */
216 boolean_t must_be_resident;/* Must page be resident? */
217 boolean_t interruptible; /* May fault be interrupted? */
218 /* Modifies in place: */
219 vm_prot_t *protection; /* Protection for mapping */
220 /* Returns: */
221 vm_page_t *result_page; /* Page found, if successful */
222 vm_page_t *top_page; /* Page in top object, if
223 * not result_page.
224 */
225 /* More arguments: */
226 boolean_t resume; /* We are restarting. */
227 void (*continuation)(); /* Continuation for blocking. */
228{
229 register
230 vm_page_t m;
231 register
232 vm_object_t object;
233 register
234 vm_offset_t offset;
235 vm_page_t first_m;
236 vm_object_t next_object;
237 vm_object_t copy_object;
238 boolean_t look_for_page;
239 vm_prot_t access_required;
240
241 if (resume) {
1
Assuming 'resume' is 0
→
2
←
Taking false branch
→
242 register vm_fault_state_t *state =
243 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
244
245 if (state->vmfp_backoff)
246 goto after_block_and_backoff;
247
248 object = state->vmfp_object;
249 offset = state->vmfp_offset;
250 first_m = state->vmfp_first_m;
251 access_required = state->vmfp_access;
252 goto after_thread_block;
253 }
254
255 vm_stat_sample(SAMPLED_PC_VM_FAULTS_ANY)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((0x100))) take_pc_sample((_thread_), &
(_thread_)->pc_sample, ((0x100))); task = (_thread_)->task
; if (task->pc_sample.sampletypes & ((0x100))) take_pc_sample
((_thread_), &task->pc_sample, ((0x100))); }); });
256 vm_stat.faults++; /* needs lock XXX */
257 current_task()((active_threads[(0)])->task)->faults++;
258
259/*
260 * Recovery actions
261 */
262#define RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ; } \
263 MACRO_BEGIN({ \
264 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } }); \
265 vm_page_lock_queues(); \
266 if (!m->active && !m->inactive) \
267 vm_page_activate(m); \
268 vm_page_unlock_queues(); \
269 MACRO_END})
270
271 if (vm_fault_dirty_handling
3
←
Assuming 'vm_fault_dirty_handling' is 0
→
4
←
Taking false branch
→
272#if MACH_KDB0
273 /*
274 * If there are watchpoints set, then
275 * we don't want to give away write permission
276 * on a read fault. Make the task write fault,
277 * so that the watchpoint code notices the access.
278 */
279 || db_watchpoint_list
280#endif /* MACH_KDB */
281 ) {
282 /*
283 * If we aren't asking for write permission,
284 * then don't give it away. We're using write
285 * faults to set the dirty bit.
286 */
287 if (!(fault_type & VM_PROT_WRITE((vm_prot_t) 0x02)))
288 *protection &= ~VM_PROT_WRITE((vm_prot_t) 0x02);
289 }
290
291 if (!vm_fault_interruptible)
5
←
Assuming 'vm_fault_interruptible' is not equal to 0
→
6
←
Taking false branch
→
292 interruptible = FALSE((boolean_t) 0);
293
294 /*
295 * INVARIANTS (through entire routine):
296 *
297 * 1) At all times, we must either have the object
298 * lock or a busy page in some object to prevent
299 * some other thread from trying to bring in
300 * the same page.
301 *
302 * Note that we cannot hold any locks during the
303 * pager access or when waiting for memory, so
304 * we use a busy page then.
305 *
306 * Note also that we aren't as concerned about more than
307 * one thread attempting to memory_object_data_unlock
308 * the same page at once, so we don't hold the page
309 * as busy then, but do record the highest unlock
310 * value so far. [Unlock requests may also be delivered
311 * out of order.]
312 *
313 * 2) To prevent another thread from racing us down the
314 * shadow chain and entering a new page in the top
315 * object before we do, we must keep a busy page in
316 * the top object while following the shadow chain.
317 *
318 * 3) We must increment paging_in_progress on any object
319 * for which we have a busy page, to prevent
320 * vm_object_collapse from removing the busy page
321 * without our noticing.
322 *
323 * 4) We leave busy pages on the pageout queues.
324 * If the pageout daemon comes across a busy page,
325 * it will remove the page from the pageout queues.
326 */
327
328 /*
329 * Search for the page at object/offset.
330 */
331
332 object = first_object;
333 offset = first_offset;
334 first_m = VM_PAGE_NULL((vm_page_t) 0);
335 access_required = fault_type;
336
337 /*
338 * See whether this page is resident
339 */
340
341 while (TRUE((boolean_t) 1)) {
7
←
Loop condition is true. Entering loop body
→
16
←
Loop condition is true. Entering loop body
→
23
←
Loop condition is true. Entering loop body
→
29
←
Loop condition is true. Entering loop body
→
342 m = vm_page_lookup(object, offset);
343 if (m != VM_PAGE_NULL((vm_page_t) 0)) {
8
←
Assuming 'm' is equal to null
→
9
←
Taking false branch
→
17
←
Assuming 'm' is equal to null
→
18
←
Taking false branch
→
24
←
Assuming 'm' is equal to null
→
25
←
Taking false branch
→
30
←
Assuming 'm' is not equal to null
→
31
←
Taking true branch
→
344 /*
345 * If the page is being brought in,
346 * wait for it and then retry.
347 *
348 * A possible optimization: if the page
349 * is known to be resident, we can ignore
350 * pages that are absent (regardless of
351 * whether they're busy).
352 */
353
354 if (m->busy) {
32
←
Taking false branch
→
355 kern_return_t wait_result;
356
357 PAGE_ASSERT_WAIT(m, interruptible)({ (m)->wanted = ((boolean_t) 1); assert_wait((event_t) (m
), (interruptible)); });
358 vm_object_unlock(object);
359 if (continuation != (void (*)()) 0) {
360 register vm_fault_state_t *state =
361 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
362
363 /*
364 * Save variables in case
365 * thread_block discards
366 * our kernel stack.
367 */
368
369 state->vmfp_backoff = FALSE((boolean_t) 0);
370 state->vmfp_object = object;
371 state->vmfp_offset = offset;
372 state->vmfp_first_m = first_m;
373 state->vmfp_access =
374 access_required;
375 state->vmf_prot = *protection;
376
377 counter(c_vm_fault_page_block_busy_user++);
378 thread_block(continuation);
379 } else
380 {
381 counter(c_vm_fault_page_block_busy_kernel++);
382 thread_block((void (*)()) 0);
383 }
384 after_thread_block:
385 wait_result = current_thread()(active_threads[(0)])->wait_result;
386 vm_object_lock(object);
387 if (wait_result != THREAD_AWAKENED0) {
388 vm_fault_cleanup(object, first_m);
389 if (wait_result == THREAD_RESTART3)
390 return(VM_FAULT_RETRY1);
391 else
392 return(VM_FAULT_INTERRUPTED2);
393 }
394 continue;
395 }
396
397 /*
398 * If the page is in error, give up now.
399 */
400
401 if (m->error) {
33
←
Taking false branch
→
402 VM_PAGE_FREE(m)({ ; vm_page_free(m); ; });
403 vm_fault_cleanup(object, first_m);
404 return(VM_FAULT_MEMORY_ERROR5);
405 }
406
407 /*
408 * If the page isn't busy, but is absent,
409 * then it was deemed "unavailable".
410 */
411
412 if (m->absent) {
34
←
Taking false branch
→
413 /*
414 * Remove the non-existent page (unless it's
415 * in the top object) and move on down to the
416 * next object (if there is one).
417 */
418
419 offset += object->shadow_offset;
420 access_required = VM_PROT_READ((vm_prot_t) 0x01);
421 next_object = object->shadow;
422 if (next_object == VM_OBJECT_NULL((vm_object_t) 0)) {
423 vm_page_t real_m;
424
425 assert(!must_be_resident)({ if (!(!must_be_resident)) Assert("!must_be_resident", "../vm/vm_fault.c"
, 425); });
426
427 /*
428 * Absent page at bottom of shadow
429 * chain; zero fill the page we left
430 * busy in the first object, and flush
431 * the absent page. But first we
432 * need to allocate a real page.
433 */
434
435 real_m = vm_page_grab(!object->internal);
436 if (real_m == VM_PAGE_NULL((vm_page_t) 0)) {
437 vm_fault_cleanup(object, first_m);
438 return(VM_FAULT_MEMORY_SHORTAGE3);
439 }
440
441 if (object != first_object) {
442 VM_PAGE_FREE(m)({ ; vm_page_free(m); ; });
443 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 443); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
444 vm_object_unlock(object);
445 object = first_object;
446 offset = first_offset;
447 m = first_m;
448 first_m = VM_PAGE_NULL((vm_page_t) 0);
449 vm_object_lock(object);
450 }
451
452 VM_PAGE_FREE(m)({ ; vm_page_free(m); ; });
453 assert(real_m->busy)({ if (!(real_m->busy)) Assert("real_m->busy", "../vm/vm_fault.c"
, 453); });
454 vm_page_lock_queues();
455 vm_page_insert(real_m, object, offset);
456 vm_page_unlock_queues();
457 m = real_m;
458
459 /*
460 * Drop the lock while zero filling
461 * page. Then break because this
462 * is the page we wanted. Checking
463 * the page lock is a waste of time;
464 * this page was either absent or
465 * newly allocated -- in both cases
466 * it can't be page locked by a pager.
467 */
468 vm_object_unlock(object);
469
470 vm_page_zero_fill(m);
471
472 vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((0x10))) take_pc_sample((_thread_), &
(_thread_)->pc_sample, ((0x10))); task = (_thread_)->task
; if (task->pc_sample.sampletypes & ((0x10))) take_pc_sample
((_thread_), &task->pc_sample, ((0x10))); }); });
473
474 vm_stat.zero_fill_count++;
475 current_task()((active_threads[(0)])->task)->zero_fills++;
476 vm_object_lock(object);
477 pmap_clear_modify(m->phys_addr);
478 break;
479 } else {
480 if (must_be_resident) {
481 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 481); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
482 } else if (object != first_object) {
483 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 483); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
484 VM_PAGE_FREE(m)({ ; vm_page_free(m); ; });
485 } else {
486 first_m = m;
487 m->absent = FALSE((boolean_t) 0);
488 vm_object_absent_release(object)({ (object)->absent_count--; ({ if (((object))->all_wanted
& (1 << (3))) thread_wakeup_prim(((event_t)(((vm_offset_t
) (object)) + (3))), ((boolean_t) 0), 0); ((object))->all_wanted
&= ~(1 << (3)); }); });
489 m->busy = TRUE((boolean_t) 1);
490
491 vm_page_lock_queues();
492 VM_PAGE_QUEUES_REMOVE(m)({ if (m->active) { { register queue_entry_t next, prev; next
= (m)->pageq.next; prev = (m)->pageq.prev; if ((&vm_page_queue_active
) == next) (&vm_page_queue_active)->prev = prev; else (
(vm_page_t)next)->pageq.prev = prev; if ((&vm_page_queue_active
) == prev) (&vm_page_queue_active)->next = next; else (
(vm_page_t)prev)->pageq.next = next; }; m->active = ((boolean_t
) 0); vm_page_active_count--; } if (m->inactive) { { register
queue_entry_t next, prev; next = (m)->pageq.next; prev = (
m)->pageq.prev; if ((&vm_page_queue_inactive) == next)
(&vm_page_queue_inactive)->prev = prev; else ((vm_page_t
)next)->pageq.prev = prev; if ((&vm_page_queue_inactive
) == prev) (&vm_page_queue_inactive)->next = next; else
((vm_page_t)prev)->pageq.next = next; }; m->inactive =
((boolean_t) 0); vm_page_inactive_count--; } });
493 vm_page_unlock_queues();
494 }
495 vm_object_lock(next_object);
496 vm_object_unlock(object);
497 object = next_object;
498 vm_object_paging_begin(object)((object)->paging_in_progress++);
499 continue;
500 }
501 }
502
503 /*
504 * If the desired access to this page has
505 * been locked out, request that it be unlocked.
506 */
507
508 if (access_required & m->page_lock) {
35
←
Taking false branch
→
509 if ((access_required & m->unlock_request) != access_required) {
510 vm_prot_t new_unlock_request;
511 kern_return_t rc;
512
513 if (!object->pager_ready) {
514 vm_object_assert_wait(object,({ (object)->all_wanted |= 1 << (1); assert_wait((event_t
)(((vm_offset_t) object) + (1)), (interruptible)); })
515 VM_OBJECT_EVENT_PAGER_READY,({ (object)->all_wanted |= 1 << (1); assert_wait((event_t
)(((vm_offset_t) object) + (1)), (interruptible)); })
516 interruptible)({ (object)->all_wanted |= 1 << (1); assert_wait((event_t
)(((vm_offset_t) object) + (1)), (interruptible)); });
517 goto block_and_backoff;
518 }
519
520 new_unlock_request = m->unlock_request =
521 (access_required | m->unlock_request);
522 vm_object_unlock(object);
523 if ((rc = memory_object_data_unlock(
524 object->pager,
525 object->pager_request,
526 offset + object->paging_offset,
527 PAGE_SIZE(1 << 12),
528 new_unlock_request))
529 != KERN_SUCCESS0) {
530 printf("vm_fault: memory_object_data_unlock failed\n");
531 vm_object_lock(object);
532 vm_fault_cleanup(object, first_m);
533 return((rc == MACH_SEND_INTERRUPTED0x10000007) ?
534 VM_FAULT_INTERRUPTED2 :
535 VM_FAULT_MEMORY_ERROR5);
536 }
537 vm_object_lock(object);
538 continue;
539 }
540
541 PAGE_ASSERT_WAIT(m, interruptible)({ (m)->wanted = ((boolean_t) 1); assert_wait((event_t) (m
), (interruptible)); });
542 goto block_and_backoff;
543 }
544
545 /*
546 * We mark the page busy and leave it on
547 * the pageout queues. If the pageout
548 * deamon comes across it, then it will
549 * remove the page.
550 */
551
552 if (!software_reference_bits) {
36
←
Assuming 'software_reference_bits' is 0
→
37
←
Taking true branch
→
553 vm_page_lock_queues();
554 if (m->inactive) {
38
←
Taking true branch
→
555 vm_stat_sample(SAMPLED_PC_VM_REACTIVATION_FAULTS)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((0x20))) take_pc_sample((_thread_), &
(_thread_)->pc_sample, ((0x20))); task = (_thread_)->task
; if (task->pc_sample.sampletypes & ((0x20))) take_pc_sample
((_thread_), &task->pc_sample, ((0x20))); }); });
39
←
Within the expansion of the macro 'vm_stat_sample':
→
a
Assuming '_thread_' is equal to null
556 vm_stat.reactivations++;
557 current_task()((active_threads[(0)])->task)->reactivations++;
40
←
Within the expansion of the macro 'current_task':
a
Access to field 'task' results in a dereference of a null pointer
558 }
559
560 VM_PAGE_QUEUES_REMOVE(m)({ if (m->active) { { register queue_entry_t next, prev; next
= (m)->pageq.next; prev = (m)->pageq.prev; if ((&vm_page_queue_active
) == next) (&vm_page_queue_active)->prev = prev; else (
(vm_page_t)next)->pageq.prev = prev; if ((&vm_page_queue_active
) == prev) (&vm_page_queue_active)->next = next; else (
(vm_page_t)prev)->pageq.next = next; }; m->active = ((boolean_t
) 0); vm_page_active_count--; } if (m->inactive) { { register
queue_entry_t next, prev; next = (m)->pageq.next; prev = (
m)->pageq.prev; if ((&vm_page_queue_inactive) == next)
(&vm_page_queue_inactive)->prev = prev; else ((vm_page_t
)next)->pageq.prev = prev; if ((&vm_page_queue_inactive
) == prev) (&vm_page_queue_inactive)->next = next; else
((vm_page_t)prev)->pageq.next = next; }; m->inactive =
((boolean_t) 0); vm_page_inactive_count--; } });
561 vm_page_unlock_queues();
562 }
563
564 assert(!m->busy)({ if (!(!m->busy)) Assert("!m->busy", "../vm/vm_fault.c"
, 564); });
565 m->busy = TRUE((boolean_t) 1);
566 assert(!m->absent)({ if (!(!m->absent)) Assert("!m->absent", "../vm/vm_fault.c"
, 566); });
567 break;
568 }
569
570 look_for_page =
571 (object->pager_created)
572#if MACH_PAGEMAP1
573 && (vm_external_state_get(object->existence_info, offset + object->paging_offset)(((object->existence_info) != ((vm_external_t) 0)) ? _vm_external_state_get
(object->existence_info, offset + object->paging_offset
) : 2) !=
574 VM_EXTERNAL_STATE_ABSENT3)
575#endif /* MACH_PAGEMAP */
576 ;
577
578 if ((look_for_page || (object == first_object))
11
←
Taking false branch
→
579 && !must_be_resident) {
10
←
Assuming 'must_be_resident' is not equal to 0
→
580 /*
581 * Allocate a new page for this object/offset
582 * pair.
583 */
584
585 m = vm_page_grab_fictitious();
586 if (m == VM_PAGE_NULL((vm_page_t) 0)) {
587 vm_fault_cleanup(object, first_m);
588 return(VM_FAULT_FICTITIOUS_SHORTAGE4);
589 }
590
591 vm_page_lock_queues();
592 vm_page_insert(m, object, offset);
593 vm_page_unlock_queues();
594 }
595
596 if (look_for_page && !must_be_resident) {
597 kern_return_t rc;
598
599 /*
600 * If the memory manager is not ready, we
601 * cannot make requests.
602 */
603 if (!object->pager_ready) {
604 vm_object_assert_wait(object,({ (object)->all_wanted |= 1 << (1); assert_wait((event_t
)(((vm_offset_t) object) + (1)), (interruptible)); })
605 VM_OBJECT_EVENT_PAGER_READY,({ (object)->all_wanted |= 1 << (1); assert_wait((event_t
)(((vm_offset_t) object) + (1)), (interruptible)); })
606 interruptible)({ (object)->all_wanted |= 1 << (1); assert_wait((event_t
)(((vm_offset_t) object) + (1)), (interruptible)); });
607 VM_PAGE_FREE(m)({ ; vm_page_free(m); ; });
608 goto block_and_backoff;
609 }
610
611 if (object->internal) {
612 /*
613 * Requests to the default pager
614 * must reserve a real page in advance,
615 * because the pager's data-provided
616 * won't block for pages.
617 */
618
619 if (m->fictitious && !vm_page_convert(m, FALSE((boolean_t) 0))) {
620 VM_PAGE_FREE(m)({ ; vm_page_free(m); ; });
621 vm_fault_cleanup(object, first_m);
622 return(VM_FAULT_MEMORY_SHORTAGE3);
623 }
624 } else if (object->absent_count >
625 vm_object_absent_max) {
626 /*
627 * If there are too many outstanding page
628 * requests pending on this object, we
629 * wait for them to be resolved now.
630 */
631
632 vm_object_absent_assert_wait(object, interruptible)({ ({ ((object))->all_wanted |= 1 << (3); assert_wait
((event_t)(((vm_offset_t) (object)) + (3)), ((interruptible))
); }); });
633 VM_PAGE_FREE(m)({ ; vm_page_free(m); ; });
634 goto block_and_backoff;
635 }
636
637 /*
638 * Indicate that the page is waiting for data
639 * from the memory manager.
640 */
641
642 m->absent = TRUE((boolean_t) 1);
643 object->absent_count++;
644
645 /*
646 * We have a busy page, so we can
647 * release the object lock.
648 */
649 vm_object_unlock(object);
650
651 /*
652 * Call the memory manager to retrieve the data.
653 */
654
655 vm_stat.pageins++;
656 vm_stat_sample(SAMPLED_PC_VM_PAGEIN_FAULTS)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((0x40))) take_pc_sample((_thread_), &
(_thread_)->pc_sample, ((0x40))); task = (_thread_)->task
; if (task->pc_sample.sampletypes & ((0x40))) take_pc_sample
((_thread_), &task->pc_sample, ((0x40))); }); });
657 current_task()((active_threads[(0)])->task)->pageins++;
658
659 if ((rc = memory_object_data_request(object->pager,
660 object->pager_request,
661 m->offset + object->paging_offset,
662 PAGE_SIZE(1 << 12), access_required)) != KERN_SUCCESS0) {
663 if (rc != MACH_SEND_INTERRUPTED0x10000007)
664 printf("%s(0x%p, 0x%p, 0x%lx, 0x%x, 0x%x) failed, %x\n",
665 "memory_object_data_request",
666 object->pager,
667 object->pager_request,
668 m->offset + object->paging_offset,
669 PAGE_SIZE(1 << 12), access_required, rc);
670 /*
671 * Don't want to leave a busy page around,
672 * but the data request may have blocked,
673 * so check if it's still there and busy.
674 */
675 vm_object_lock(object);
676 if (m == vm_page_lookup(object,offset) &&
677 m->absent && m->busy)
678 VM_PAGE_FREE(m)({ ; vm_page_free(m); ; });
679 vm_fault_cleanup(object, first_m);
680 return((rc == MACH_SEND_INTERRUPTED0x10000007) ?
681 VM_FAULT_INTERRUPTED2 :
682 VM_FAULT_MEMORY_ERROR5);
683 }
684
685 /*
686 * Retry with same object/offset, since new data may
687 * be in a different page (i.e., m is meaningless at
688 * this point).
689 */
690 vm_object_lock(object);
691 continue;
692 }
693
694 /*
695 * For the XP system, the only case in which we get here is if
696 * object has no pager (or unwiring). If the pager doesn't
697 * have the page this is handled in the m->absent case above
698 * (and if you change things here you should look above).
699 */
700 if (object == first_object)
12
←
Taking true branch
→
19
←
Taking false branch
→
26
←
Taking false branch
→
701 first_m = m;
702 else
703 {
704 assert(m == VM_PAGE_NULL)({ if (!(m == ((vm_page_t) 0))) Assert("m == VM_PAGE_NULL", "../vm/vm_fault.c"
, 704); });
705 }
706
707 /*
708 * Move on to the next object. Lock the next
709 * object before unlocking the current one.
710 */
711 access_required = VM_PROT_READ((vm_prot_t) 0x01);
712
713 offset += object->shadow_offset;
714 next_object = object->shadow;
715 if (next_object == VM_OBJECT_NULL((vm_object_t) 0)) {
13
←
Assuming 'next_object' is not equal to null
→
14
←
Taking false branch
→
20
←
Assuming 'next_object' is not equal to null
→
21
←
Taking false branch
→
27
←
Assuming 'next_object' is not equal to null
→
28
←
Taking false branch
→
716 assert(!must_be_resident)({ if (!(!must_be_resident)) Assert("!must_be_resident", "../vm/vm_fault.c"
, 716); });
717
718 /*
719 * If there's no object left, fill the page
720 * in the top object with zeros. But first we
721 * need to allocate a real page.
722 */
723
724 if (object != first_object) {
725 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 725); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
726 vm_object_unlock(object);
727
728 object = first_object;
729 offset = first_offset;
730 vm_object_lock(object);
731 }
732
733 m = first_m;
734 assert(m->object == object)({ if (!(m->object == object)) Assert("m->object == object"
, "../vm/vm_fault.c", 734); });
735 first_m = VM_PAGE_NULL((vm_page_t) 0);
736
737 if (m->fictitious && !vm_page_convert(m, !object->internal)) {
738 VM_PAGE_FREE(m)({ ; vm_page_free(m); ; });
739 vm_fault_cleanup(object, VM_PAGE_NULL((vm_page_t) 0));
740 return(VM_FAULT_MEMORY_SHORTAGE3);
741 }
742
743 vm_object_unlock(object);
744 vm_page_zero_fill(m);
745 vm_stat_sample(SAMPLED_PC_VM_ZFILL_FAULTS)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((0x10))) take_pc_sample((_thread_), &
(_thread_)->pc_sample, ((0x10))); task = (_thread_)->task
; if (task->pc_sample.sampletypes & ((0x10))) take_pc_sample
((_thread_), &task->pc_sample, ((0x10))); }); });
746 vm_stat.zero_fill_count++;
747 current_task()((active_threads[(0)])->task)->zero_fills++;
748 vm_object_lock(object);
749 pmap_clear_modify(m->phys_addr);
750 break;
751 }
752 else {
753 vm_object_lock(next_object);
754 if ((object != first_object) || must_be_resident)
15
←
Taking true branch
→
22
←
Taking true branch
→
755 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 755); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
756 vm_object_unlock(object);
757 object = next_object;
758 vm_object_paging_begin(object)((object)->paging_in_progress++);
759 }
760 }
761
762 /*
763 * PAGE HAS BEEN FOUND.
764 *
765 * This page (m) is:
766 * busy, so that we can play with it;
767 * not absent, so that nobody else will fill it;
768 * possibly eligible for pageout;
769 *
770 * The top-level page (first_m) is:
771 * VM_PAGE_NULL if the page was found in the
772 * top-level object;
773 * busy, not absent, and ineligible for pageout.
774 *
775 * The current object (object) is locked. A paging
776 * reference is held for the current and top-level
777 * objects.
778 */
779
780#if EXTRA_ASSERTIONS
781 assert(m->busy && !m->absent)({ if (!(m->busy && !m->absent)) Assert("m->busy && !m->absent"
, "../vm/vm_fault.c", 781); });
782 assert((first_m == VM_PAGE_NULL) ||({ if (!((first_m == ((vm_page_t) 0)) || (first_m->busy &&
!first_m->absent && !first_m->active &&
!first_m->inactive))) Assert("(first_m == VM_PAGE_NULL) || (first_m->busy && !first_m->absent && !first_m->active && !first_m->inactive)"
, "../vm/vm_fault.c", 784); })
783 (first_m->busy && !first_m->absent &&({ if (!((first_m == ((vm_page_t) 0)) || (first_m->busy &&
!first_m->absent && !first_m->active &&
!first_m->inactive))) Assert("(first_m == VM_PAGE_NULL) || (first_m->busy && !first_m->absent && !first_m->active && !first_m->inactive)"
, "../vm/vm_fault.c", 784); })
784 !first_m->active && !first_m->inactive))({ if (!((first_m == ((vm_page_t) 0)) || (first_m->busy &&
!first_m->absent && !first_m->active &&
!first_m->inactive))) Assert("(first_m == VM_PAGE_NULL) || (first_m->busy && !first_m->absent && !first_m->active && !first_m->inactive)"
, "../vm/vm_fault.c", 784); });
785#endif /* EXTRA_ASSERTIONS */
786
787 /*
788 * If the page is being written, but isn't
789 * already owned by the top-level object,
790 * we have to copy it into a new page owned
791 * by the top-level object.
792 */
793
794 if (object != first_object) {
795 /*
796 * We only really need to copy if we
797 * want to write it.
798 */
799
800 if (fault_type & VM_PROT_WRITE((vm_prot_t) 0x02)) {
801 vm_page_t copy_m;
802
803 assert(!must_be_resident)({ if (!(!must_be_resident)) Assert("!must_be_resident", "../vm/vm_fault.c"
, 803); });
804
805 /*
806 * If we try to collapse first_object at this
807 * point, we may deadlock when we try to get
808 * the lock on an intermediate object (since we
809 * have the bottom object locked). We can't
810 * unlock the bottom object, because the page
811 * we found may move (by collapse) if we do.
812 *
813 * Instead, we first copy the page. Then, when
814 * we have no more use for the bottom object,
815 * we unlock it and try to collapse.
816 *
817 * Note that we copy the page even if we didn't
818 * need to... that's the breaks.
819 */
820
821 /*
822 * Allocate a page for the copy
823 */
824 copy_m = vm_page_grab(!first_object->internal);
825 if (copy_m == VM_PAGE_NULL((vm_page_t) 0)) {
826 RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ; };
827 vm_fault_cleanup(object, first_m);
828 return(VM_FAULT_MEMORY_SHORTAGE3);
829 }
830
831 vm_object_unlock(object);
832 vm_page_copy(m, copy_m);
833 vm_object_lock(object);
834
835 /*
836 * If another map is truly sharing this
837 * page with us, we have to flush all
838 * uses of the original page, since we
839 * can't distinguish those which want the
840 * original from those which need the
841 * new copy.
842 *
843 * XXXO If we know that only one map has
844 * access to this page, then we could
845 * avoid the pmap_page_protect() call.
846 */
847
848 vm_page_lock_queues();
849 vm_page_deactivate(m);
850 pmap_page_protect(m->phys_addr, VM_PROT_NONE((vm_prot_t) 0x00));
851 vm_page_unlock_queues();
852
853 /*
854 * We no longer need the old page or object.
855 */
856
857 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } });
858 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 858); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
859 vm_object_unlock(object);
860
861 vm_stat.cow_faults++;
862 vm_stat_sample(SAMPLED_PC_VM_COW_FAULTS)({ thread_t _thread_ = (active_threads[(0)]); if (_thread_ !=
((thread_t) 0)) ({ task_t task; if ((_thread_)->pc_sample
.sampletypes & ((0x80))) take_pc_sample((_thread_), &
(_thread_)->pc_sample, ((0x80))); task = (_thread_)->task
; if (task->pc_sample.sampletypes & ((0x80))) take_pc_sample
((_thread_), &task->pc_sample, ((0x80))); }); });
863 current_task()((active_threads[(0)])->task)->cow_faults++;
864 object = first_object;
865 offset = first_offset;
866
867 vm_object_lock(object);
868 VM_PAGE_FREE(first_m)({ ; vm_page_free(first_m); ; });
869 first_m = VM_PAGE_NULL((vm_page_t) 0);
870 assert(copy_m->busy)({ if (!(copy_m->busy)) Assert("copy_m->busy", "../vm/vm_fault.c"
, 870); });
871 vm_page_lock_queues();
872 vm_page_insert(copy_m, object, offset);
873 vm_page_unlock_queues();
874 m = copy_m;
875
876 /*
877 * Now that we've gotten the copy out of the
878 * way, let's try to collapse the top object.
879 * But we have to play ugly games with
880 * paging_in_progress to do that...
881 */
882
883 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_fault.c", 883); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
884 vm_object_collapse(object);
885 vm_object_paging_begin(object)((object)->paging_in_progress++);
886 }
887 else {
888 *protection &= (~VM_PROT_WRITE((vm_prot_t) 0x02));
889 }
890 }
891
892 /*
893 * Now check whether the page needs to be pushed into the
894 * copy object. The use of asymmetric copy on write for
895 * shared temporary objects means that we may do two copies to
896 * satisfy the fault; one above to get the page from a
897 * shadowed object, and one here to push it into the copy.
898 */
899
900 while ((copy_object = first_object->copy) != VM_OBJECT_NULL((vm_object_t) 0)) {
901 vm_offset_t copy_offset;
902 vm_page_t copy_m;
903
904 /*
905 * If the page is being written, but hasn't been
906 * copied to the copy-object, we have to copy it there.
907 */
908
909 if ((fault_type & VM_PROT_WRITE((vm_prot_t) 0x02)) == 0) {
910 *protection &= ~VM_PROT_WRITE((vm_prot_t) 0x02);
911 break;
912 }
913
914 /*
915 * If the page was guaranteed to be resident,
916 * we must have already performed the copy.
917 */
918
919 if (must_be_resident)
920 break;
921
922 /*
923 * Try to get the lock on the copy_object.
924 */
925 if (!vm_object_lock_try(copy_object)(((boolean_t) 1))) {
926 vm_object_unlock(object);
927
928 simple_lock_pause(); /* wait a bit */
929
930 vm_object_lock(object);
931 continue;
932 }
933
934 /*
935 * Make another reference to the copy-object,
936 * to keep it from disappearing during the
937 * copy.
938 */
939 assert(copy_object->ref_count > 0)({ if (!(copy_object->ref_count > 0)) Assert("copy_object->ref_count > 0"
, "../vm/vm_fault.c", 939); });
940 copy_object->ref_count++;
941
942 /*
943 * Does the page exist in the copy?
944 */
945 copy_offset = first_offset - copy_object->shadow_offset;
946 copy_m = vm_page_lookup(copy_object, copy_offset);
947 if (copy_m != VM_PAGE_NULL((vm_page_t) 0)) {
948 if (copy_m->busy) {
949 /*
950 * If the page is being brought
951 * in, wait for it and then retry.
952 */
953 PAGE_ASSERT_WAIT(copy_m, interruptible)({ (copy_m)->wanted = ((boolean_t) 1); assert_wait((event_t
) (copy_m), (interruptible)); });
954 RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ; };
955 copy_object->ref_count--;
956 assert(copy_object->ref_count > 0)({ if (!(copy_object->ref_count > 0)) Assert("copy_object->ref_count > 0"
, "../vm/vm_fault.c", 956); });
957 vm_object_unlock(copy_object);
958 goto block_and_backoff;
959 }
960 }
961 else {
962 /*
963 * Allocate a page for the copy
964 */
965 copy_m = vm_page_alloc(copy_object, copy_offset);
966 if (copy_m == VM_PAGE_NULL((vm_page_t) 0)) {
967 RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ; };
968 copy_object->ref_count--;
969 assert(copy_object->ref_count > 0)({ if (!(copy_object->ref_count > 0)) Assert("copy_object->ref_count > 0"
, "../vm/vm_fault.c", 969); });
970 vm_object_unlock(copy_object);
971 vm_fault_cleanup(object, first_m);
972 return(VM_FAULT_MEMORY_SHORTAGE3);
973 }
974
975 /*
976 * Must copy page into copy-object.
977 */
978
979 vm_page_copy(m, copy_m);
980
981 /*
982 * If the old page was in use by any users
983 * of the copy-object, it must be removed
984 * from all pmaps. (We can't know which
985 * pmaps use it.)
986 */
987
988 vm_page_lock_queues();
989 pmap_page_protect(m->phys_addr, VM_PROT_NONE((vm_prot_t) 0x00));
990 copy_m->dirty = TRUE((boolean_t) 1);
991 vm_page_unlock_queues();
992
993 /*
994 * If there's a pager, then immediately
995 * page out this page, using the "initialize"
996 * option. Else, we use the copy.
997 */
998
999 if (!copy_object->pager_created) {
1000 vm_page_lock_queues();
1001 vm_page_activate(copy_m);
1002 vm_page_unlock_queues();
1003 PAGE_WAKEUP_DONE(copy_m)({ (copy_m)->busy = ((boolean_t) 0); if ((copy_m)->wanted
) { (copy_m)->wanted = ((boolean_t) 0); thread_wakeup_prim
((((event_t) copy_m)), ((boolean_t) 0), 0); } });
1004 } else {
1005 /*
1006 * The page is already ready for pageout:
1007 * not on pageout queues and busy.
1008 * Unlock everything except the
1009 * copy_object itself.
1010 */
1011
1012 vm_object_unlock(object);
1013
1014 /*
1015 * Write the page to the copy-object,
1016 * flushing it from the kernel.
1017 */
1018
1019 vm_pageout_page(copy_m, TRUE((boolean_t) 1), TRUE((boolean_t) 1));
1020
1021 /*
1022 * Since the pageout may have
1023 * temporarily dropped the
1024 * copy_object's lock, we
1025 * check whether we'll have
1026 * to deallocate the hard way.
1027 */
1028
1029 if ((copy_object->shadow != object) ||
1030 (copy_object->ref_count == 1)) {
1031 vm_object_unlock(copy_object);
1032 vm_object_deallocate(copy_object);
1033 vm_object_lock(object);
1034 continue;
1035 }
1036
1037 /*
1038 * Pick back up the old object's
1039 * lock. [It is safe to do so,
1040 * since it must be deeper in the
1041 * object tree.]
1042 */
1043
1044 vm_object_lock(object);
1045 }
1046
1047 /*
1048 * Because we're pushing a page upward
1049 * in the object tree, we must restart
1050 * any faults that are waiting here.
1051 * [Note that this is an expansion of
1052 * PAGE_WAKEUP that uses the THREAD_RESTART
1053 * wait result]. Can't turn off the page's
1054 * busy bit because we're not done with it.
1055 */
1056
1057 if (m->wanted) {
1058 m->wanted = FALSE((boolean_t) 0);
1059 thread_wakeup_with_result((event_t) m,thread_wakeup_prim(((event_t) m), ((boolean_t) 0), (3))
1060 THREAD_RESTART)thread_wakeup_prim(((event_t) m), ((boolean_t) 0), (3));
1061 }
1062 }
1063
1064 /*
1065 * The reference count on copy_object must be
1066 * at least 2: one for our extra reference,
1067 * and at least one from the outside world
1068 * (we checked that when we last locked
1069 * copy_object).
1070 */
1071 copy_object->ref_count--;
1072 assert(copy_object->ref_count > 0)({ if (!(copy_object->ref_count > 0)) Assert("copy_object->ref_count > 0"
, "../vm/vm_fault.c", 1072); });
1073 vm_object_unlock(copy_object);
1074
1075 break;
1076 }
1077
1078 *result_page = m;
1079 *top_page = first_m;
1080
1081 /*
1082 * If the page can be written, assume that it will be.
1083 * [Earlier, we restrict the permission to allow write
1084 * access only if the fault so required, so we don't
1085 * mark read-only data as dirty.]
1086 */
1087
1088 if (vm_fault_dirty_handling && (*protection & VM_PROT_WRITE((vm_prot_t) 0x02)))
1089 m->dirty = TRUE((boolean_t) 1);
1090
1091 return(VM_FAULT_SUCCESS0);
1092
1093 block_and_backoff:
1094 vm_fault_cleanup(object, first_m);
1095
1096 if (continuation != (void (*)()) 0) {
1097 register vm_fault_state_t *state =
1098 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1099
1100 /*
1101 * Save variables in case we must restart.
1102 */
1103
1104 state->vmfp_backoff = TRUE((boolean_t) 1);
1105 state->vmf_prot = *protection;
1106
1107 counter(c_vm_fault_page_block_backoff_user++);
1108 thread_block(continuation);
1109 } else
1110 {
1111 counter(c_vm_fault_page_block_backoff_kernel++);
1112 thread_block((void (*)()) 0);
1113 }
1114 after_block_and_backoff:
1115 if (current_thread()(active_threads[(0)])->wait_result == THREAD_AWAKENED0)
1116 return VM_FAULT_RETRY1;
1117 else
1118 return VM_FAULT_INTERRUPTED2;
1119
1120#undef RELEASE_PAGE
1121}
1122
1123/*
1124 * Routine: vm_fault
1125 * Purpose:
1126 * Handle page faults, including pseudo-faults
1127 * used to change the wiring status of pages.
1128 * Returns:
1129 * If an explicit (expression) continuation is supplied,
1130 * then we call the continuation instead of returning.
1131 * Implementation:
1132 * Explicit continuations make this a little icky,
1133 * because it hasn't been rewritten to embrace CPS.
1134 * Instead, we have resume arguments for vm_fault and
1135 * vm_fault_page, to let continue the fault computation.
1136 *
1137 * vm_fault and vm_fault_page save mucho state
1138 * in the moral equivalent of a closure. The state
1139 * structure is allocated when first entering vm_fault
1140 * and deallocated when leaving vm_fault.
1141 */
1142
1143void
1144vm_fault_continue()
1145{
1146 register vm_fault_state_t *state =
1147 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1148
1149 (void) vm_fault(state->vmf_map,
1150 state->vmf_vaddr,
1151 state->vmf_fault_type,
1152 state->vmf_change_wiring,
1153 TRUE((boolean_t) 1), state->vmf_continuation);
1154 /*NOTREACHED*/
1155}
1156
1157kern_return_t vm_fault(map, vaddr, fault_type, change_wiring,
1158 resume, continuation)
1159 vm_map_t map;
1160 vm_offset_t vaddr;
1161 vm_prot_t fault_type;
1162 boolean_t change_wiring;
1163 boolean_t resume;
1164 void (*continuation)();
1165{
1166 vm_map_version_t version; /* Map version for verificiation */
1167 boolean_t wired; /* Should mapping be wired down? */
1168 vm_object_t object; /* Top-level object */
1169 vm_offset_t offset; /* Top-level offset */
1170 vm_prot_t prot; /* Protection for mapping */
1171 vm_object_t old_copy_object; /* Saved copy object */
1172 vm_page_t result_page; /* Result of vm_fault_page */
1173 vm_page_t top_page; /* Placeholder page */
1174 kern_return_t kr;
1175
1176 register
1177 vm_page_t m; /* Fast access to result_page */
1178
1179 if (resume) {
1180 register vm_fault_state_t *state =
1181 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1182
1183 /*
1184 * Retrieve cached variables and
1185 * continue vm_fault_page.
1186 */
1187
1188 object = state->vmf_object;
1189 if (object == VM_OBJECT_NULL((vm_object_t) 0))
1190 goto RetryFault;
1191 version = state->vmf_version;
1192 wired = state->vmf_wired;
1193 offset = state->vmf_offset;
1194 prot = state->vmf_prot;
1195
1196 kr = vm_fault_page(object, offset, fault_type,
1197 (change_wiring && !wired), !change_wiring,
1198 &prot, &result_page, &top_page,
1199 TRUE((boolean_t) 1), vm_fault_continue);
1200 goto after_vm_fault_page;
1201 }
1202
1203 if (continuation != (void (*)()) 0) {
1204 /*
1205 * We will probably need to save state.
1206 */
1207
1208 char * state;
1209
1210 /*
1211 * if this assignment stmt is written as
1212 * 'active_threads[cpu_number()] = kmem_cache_alloc()',
1213 * cpu_number may be evaluated before kmem_cache_alloc;
1214 * if kmem_cache_alloc blocks, cpu_number will be wrong
1215 */
1216
1217 state = (char *) kmem_cache_alloc(&vm_fault_state_cache);
1218 current_thread()(active_threads[(0)])->ith_othersaved.other = state;
1219
1220 }
1221
1222 RetryFault: ;
1223
1224 /*
1225 * Find the backing store object and offset into
1226 * it to begin the search.
1227 */
1228
1229 if ((kr = vm_map_lookup(&map, vaddr, fault_type, &version,
1230 &object, &offset,
1231 &prot, &wired)) != KERN_SUCCESS0) {
1232 goto done;
1233 }
1234
1235 /*
1236 * If the page is wired, we must fault for the current protection
1237 * value, to avoid further faults.
1238 */
1239
1240 if (wired)
1241 fault_type = prot;
1242
1243 /*
1244 * Make a reference to this object to
1245 * prevent its disposal while we are messing with
1246 * it. Once we have the reference, the map is free
1247 * to be diddled. Since objects reference their
1248 * shadows (and copies), they will stay around as well.
1249 */
1250
1251 assert(object->ref_count > 0)({ if (!(object->ref_count > 0)) Assert("object->ref_count > 0"
, "../vm/vm_fault.c", 1251); });
1252 object->ref_count++;
1253 vm_object_paging_begin(object)((object)->paging_in_progress++);
1254
1255 if (continuation != (void (*)()) 0) {
1256 register vm_fault_state_t *state =
1257 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1258
1259 /*
1260 * Save variables, in case vm_fault_page discards
1261 * our kernel stack and we have to restart.
1262 */
1263
1264 state->vmf_map = map;
1265 state->vmf_vaddr = vaddr;
1266 state->vmf_fault_type = fault_type;
1267 state->vmf_change_wiring = change_wiring;
1268 state->vmf_continuation = continuation;
1269
1270 state->vmf_version = version;
1271 state->vmf_wired = wired;
1272 state->vmf_object = object;
1273 state->vmf_offset = offset;
1274 state->vmf_prot = prot;
1275
1276 kr = vm_fault_page(object, offset, fault_type,
1277 (change_wiring && !wired), !change_wiring,
1278 &prot, &result_page, &top_page,
1279 FALSE((boolean_t) 0), vm_fault_continue);
1280 } else
1281 {
1282 kr = vm_fault_page(object, offset, fault_type,
1283 (change_wiring && !wired), !change_wiring,
1284 &prot, &result_page, &top_page,
1285 FALSE((boolean_t) 0), (void (*)()) 0);
1286 }
1287 after_vm_fault_page:
1288
1289 /*
1290 * If we didn't succeed, lose the object reference immediately.
1291 */
1292
1293 if (kr != VM_FAULT_SUCCESS0)
1294 vm_object_deallocate(object);
1295
1296 /*
1297 * See why we failed, and take corrective action.
1298 */
1299
1300 switch (kr) {
1301 case VM_FAULT_SUCCESS0:
1302 break;
1303 case VM_FAULT_RETRY1:
1304 goto RetryFault;
1305 case VM_FAULT_INTERRUPTED2:
1306 kr = KERN_SUCCESS0;
1307 goto done;
1308 case VM_FAULT_MEMORY_SHORTAGE3:
1309 if (continuation != (void (*)()) 0) {
1310 register vm_fault_state_t *state =
1311 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1312
1313 /*
1314 * Save variables in case VM_PAGE_WAIT
1315 * discards our kernel stack.
1316 */
1317
1318 state->vmf_map = map;
1319 state->vmf_vaddr = vaddr;
1320 state->vmf_fault_type = fault_type;
1321 state->vmf_change_wiring = change_wiring;
1322 state->vmf_continuation = continuation;
1323 state->vmf_object = VM_OBJECT_NULL((vm_object_t) 0);
1324
1325 VM_PAGE_WAIT(vm_fault_continue)vm_page_wait(vm_fault_continue);
1326 } else
1327 VM_PAGE_WAIT((void (*)()) 0)vm_page_wait((void (*)()) 0);
1328 goto RetryFault;
1329 case VM_FAULT_FICTITIOUS_SHORTAGE4:
1330 vm_page_more_fictitious();
1331 goto RetryFault;
1332 case VM_FAULT_MEMORY_ERROR5:
1333 kr = KERN_MEMORY_ERROR10;
1334 goto done;
1335 }
1336
1337 m = result_page;
1338
1339 assert((change_wiring && !wired) ?({ if (!((change_wiring && !wired) ? (top_page == ((vm_page_t
) 0)) : ((top_page == ((vm_page_t) 0)) == (m->object == object
)))) Assert("(change_wiring && !wired) ? (top_page == VM_PAGE_NULL) : ((top_page == VM_PAGE_NULL) == (m->object == object))"
, "../vm/vm_fault.c", 1341); })
1340 (top_page == VM_PAGE_NULL) :({ if (!((change_wiring && !wired) ? (top_page == ((vm_page_t
) 0)) : ((top_page == ((vm_page_t) 0)) == (m->object == object
)))) Assert("(change_wiring && !wired) ? (top_page == VM_PAGE_NULL) : ((top_page == VM_PAGE_NULL) == (m->object == object))"
, "../vm/vm_fault.c", 1341); })
1341 ((top_page == VM_PAGE_NULL) == (m->object == object)))({ if (!((change_wiring && !wired) ? (top_page == ((vm_page_t
) 0)) : ((top_page == ((vm_page_t) 0)) == (m->object == object
)))) Assert("(change_wiring && !wired) ? (top_page == VM_PAGE_NULL) : ((top_page == VM_PAGE_NULL) == (m->object == object))"
, "../vm/vm_fault.c", 1341); });
1342
1343 /*
1344 * How to clean up the result of vm_fault_page. This
1345 * happens whether the mapping is entered or not.
1346 */
1347
1348#define UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ; }; vm_object_deallocate
(object); } \
1349 MACRO_BEGIN({ \
1350 vm_fault_cleanup(m->object, top_page); \
1351 vm_object_deallocate(object); \
1352 MACRO_END})
1353
1354 /*
1355 * What to do with the resulting page from vm_fault_page
1356 * if it doesn't get entered into the physical map:
1357 */
1358
1359#define RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ; } \
1360 MACRO_BEGIN({ \
1361 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } }); \
1362 vm_page_lock_queues(); \
1363 if (!m->active && !m->inactive) \
1364 vm_page_activate(m); \
1365 vm_page_unlock_queues(); \
1366 MACRO_END})
1367
1368 /*
1369 * We must verify that the maps have not changed
1370 * since our last lookup.
1371 */
1372
1373 old_copy_object = m->object->copy;
1374
1375 vm_object_unlock(m->object);
1376 while (!vm_map_verify(map, &version)) {
1377 vm_object_t retry_object;
1378 vm_offset_t retry_offset;
1379 vm_prot_t retry_prot;
1380
1381 /*
1382 * To avoid trying to write_lock the map while another
1383 * thread has it read_locked (in vm_map_pageable), we
1384 * do not try for write permission. If the page is
1385 * still writable, we will get write permission. If it
1386 * is not, or has been marked needs_copy, we enter the
1387 * mapping without write permission, and will merely
1388 * take another fault.
1389 */
1390 kr = vm_map_lookup(&map, vaddr,
1391 fault_type & ~VM_PROT_WRITE((vm_prot_t) 0x02), &version,
1392 &retry_object, &retry_offset, &retry_prot,
1393 &wired);
1394
1395 if (kr != KERN_SUCCESS0) {
1396 vm_object_lock(m->object);
1397 RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ; };
1398 UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ; }; vm_object_deallocate
(object); };
1399 goto done;
1400 }
1401
1402 vm_object_unlock(retry_object);
1403 vm_object_lock(m->object);
1404
1405 if ((retry_object != object) ||
1406 (retry_offset != offset)) {
1407 RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ; };
1408 UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ; }; vm_object_deallocate
(object); };
1409 goto RetryFault;
1410 }
1411
1412 /*
1413 * Check whether the protection has changed or the object
1414 * has been copied while we left the map unlocked.
1415 */
1416 prot &= retry_prot;
1417 vm_object_unlock(m->object);
1418 }
1419 vm_object_lock(m->object);
1420
1421 /*
1422 * If the copy object changed while the top-level object
1423 * was unlocked, then we must take away write permission.
1424 */
1425
1426 if (m->object->copy != old_copy_object)
1427 prot &= ~VM_PROT_WRITE((vm_prot_t) 0x02);
1428
1429 /*
1430 * If we want to wire down this page, but no longer have
1431 * adequate permissions, we must start all over.
1432 */
1433
1434 if (wired && (prot != fault_type)) {
1435 vm_map_verify_done(map, &version)(lock_done(&(map)->lock));
1436 RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ; };
1437 UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ; }; vm_object_deallocate
(object); };
1438 goto RetryFault;
1439 }
1440
1441 /*
1442 * It's critically important that a wired-down page be faulted
1443 * only once in each map for which it is wired.
1444 */
1445
1446 vm_object_unlock(m->object);
1447
1448 /*
1449 * Put this page into the physical map.
1450 * We had to do the unlock above because pmap_enter
1451 * may cause other faults. The page may be on
1452 * the pageout queues. If the pageout daemon comes
1453 * across the page, it will remove it from the queues.
1454 */
1455
1456 PMAP_ENTER(map->pmap, vaddr, m, prot, wired)({ pmap_enter( (map->pmap), (vaddr), (m)->phys_addr, (prot
) & ~(m)->page_lock, (wired) ); });
1457
1458 /*
1459 * If the page is not wired down and isn't already
1460 * on a pageout queue, then put it where the
1461 * pageout daemon can find it.
1462 */
1463 vm_object_lock(m->object);
1464 vm_page_lock_queues();
1465 if (change_wiring) {
1466 if (wired)
1467 vm_page_wire(m);
1468 else
1469 vm_page_unwire(m);
1470 } else if (software_reference_bits) {
1471 if (!m->active && !m->inactive)
1472 vm_page_activate(m);
1473 m->reference = TRUE((boolean_t) 1);
1474 } else {
1475 vm_page_activate(m);
1476 }
1477 vm_page_unlock_queues();
1478
1479 /*
1480 * Unlock everything, and return
1481 */
1482
1483 vm_map_verify_done(map, &version)(lock_done(&(map)->lock));
1484 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } });
1485 kr = KERN_SUCCESS0;
1486 UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ; }; vm_object_deallocate
(object); };
1487
1488#undef UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ; }; vm_object_deallocate
(object); }
1489#undef RELEASE_PAGE
1490
1491 done:
1492 if (continuation != (void (*)()) 0) {
1493 register vm_fault_state_t *state =
1494 (vm_fault_state_t *) current_thread()(active_threads[(0)])->ith_othersaved.other;
1495
1496 kmem_cache_free(&vm_fault_state_cache, (vm_offset_t) state);
1497 (*continuation)(kr);
1498 /*NOTREACHED*/
1499 }
1500
1501 return(kr);
1502}
1503
1504kern_return_t vm_fault_wire_fast();
1505
1506/*
1507 * vm_fault_wire:
1508 *
1509 * Wire down a range of virtual addresses in a map.
1510 */
1511void vm_fault_wire(map, entry)
1512 vm_map_t map;
1513 vm_map_entry_t entry;
1514{
1515
1516 register vm_offset_t va;
1517 register pmap_t pmap;
1518 register vm_offset_t end_addr = entry->vme_endlinks.end;
1519
1520 pmap = vm_map_pmap(map)((map)->pmap);
1521
1522 /*
1523 * Inform the physical mapping system that the
1524 * range of addresses may not fault, so that
1525 * page tables and such can be locked down as well.
1526 */
1527
1528 pmap_pageable(pmap, entry->vme_startlinks.start, end_addr, FALSE((boolean_t) 0));
1529
1530 /*
1531 * We simulate a fault to get the page and enter it
1532 * in the physical map.
1533 */
1534
1535 for (va = entry->vme_startlinks.start; va < end_addr; va += PAGE_SIZE(1 << 12)) {
1536 if (vm_fault_wire_fast(map, va, entry) != KERN_SUCCESS0)
1537 (void) vm_fault(map, va, VM_PROT_NONE((vm_prot_t) 0x00), TRUE((boolean_t) 1),
1538 FALSE((boolean_t) 0), (void (*)()) 0);
1539 }
1540}
1541
1542/*
1543 * vm_fault_unwire:
1544 *
1545 * Unwire a range of virtual addresses in a map.
1546 */
1547void vm_fault_unwire(map, entry)
1548 vm_map_t map;
1549 vm_map_entry_t entry;
1550{
1551 register vm_offset_t va;
1552 register pmap_t pmap;
1553 register vm_offset_t end_addr = entry->vme_endlinks.end;
1554 vm_object_t object;
1555
1556 pmap = vm_map_pmap(map)((map)->pmap);
1557
1558 object = (entry->is_sub_map)
1559 ? VM_OBJECT_NULL((vm_object_t) 0) : entry->object.vm_object;
1560
1561 /*
1562 * Since the pages are wired down, we must be able to
1563 * get their mappings from the physical map system.
1564 */
1565
1566 for (va = entry->vme_startlinks.start; va < end_addr; va += PAGE_SIZE(1 << 12)) {
1567 pmap_change_wiring(pmap, va, FALSE((boolean_t) 0));
1568
1569 if (object == VM_OBJECT_NULL((vm_object_t) 0)) {
1570 vm_map_lock_set_recursive(map)lock_set_recursive(&(map)->lock);
1571 (void) vm_fault(map, va, VM_PROT_NONE((vm_prot_t) 0x00), TRUE((boolean_t) 1),
1572 FALSE((boolean_t) 0), (void (*)()) 0);
1573 vm_map_lock_clear_recursive(map)lock_clear_recursive(&(map)->lock);
1574 } else {
1575 vm_prot_t prot;
1576 vm_page_t result_page;
1577 vm_page_t top_page;
1578 vm_fault_return_t result;
1579
1580 do {
1581 prot = VM_PROT_NONE((vm_prot_t) 0x00);
1582
1583 vm_object_lock(object);
1584 vm_object_paging_begin(object)((object)->paging_in_progress++);
1585 result = vm_fault_page(object,
1586 entry->offset +
1587 (va - entry->vme_startlinks.start),
1588 VM_PROT_NONE((vm_prot_t) 0x00), TRUE((boolean_t) 1),
1589 FALSE((boolean_t) 0), &prot,
1590 &result_page,
1591 &top_page,
1592 FALSE((boolean_t) 0), (void (*)()) 0);
1593 } while (result == VM_FAULT_RETRY1);
1594
1595 if (result != VM_FAULT_SUCCESS0)
1596 panic("vm_fault_unwire: failure");
1597
1598 vm_page_lock_queues();
1599 vm_page_unwire(result_page);
1600 vm_page_unlock_queues();
1601 PAGE_WAKEUP_DONE(result_page)({ (result_page)->busy = ((boolean_t) 0); if ((result_page
)->wanted) { (result_page)->wanted = ((boolean_t) 0); thread_wakeup_prim
((((event_t) result_page)), ((boolean_t) 0), 0); } });
1602
1603 vm_fault_cleanup(result_page->object, top_page);
1604 }
1605 }
1606
1607 /*
1608 * Inform the physical mapping system that the range
1609 * of addresses may fault, so that page tables and
1610 * such may be unwired themselves.
1611 */
1612
1613 pmap_pageable(pmap, entry->vme_startlinks.start, end_addr, TRUE((boolean_t) 1));
1614}
1615
1616/*
1617 * vm_fault_wire_fast:
1618 *
1619 * Handle common case of a wire down page fault at the given address.
1620 * If successful, the page is inserted into the associated physical map.
1621 * The map entry is passed in to avoid the overhead of a map lookup.
1622 *
1623 * NOTE: the given address should be truncated to the
1624 * proper page address.
1625 *
1626 * KERN_SUCCESS is returned if the page fault is handled; otherwise,
1627 * a standard error specifying why the fault is fatal is returned.
1628 *
1629 * The map in question must be referenced, and remains so.
1630 * Caller has a read lock on the map.
1631 *
1632 * This is a stripped version of vm_fault() for wiring pages. Anything
1633 * other than the common case will return KERN_FAILURE, and the caller
1634 * is expected to call vm_fault().
1635 */
1636kern_return_t vm_fault_wire_fast(map, va, entry)
1637 vm_map_t map;
1638 vm_offset_t va;
1639 vm_map_entry_t entry;
1640{
1641 vm_object_t object;
1642 vm_offset_t offset;
1643 register vm_page_t m;
1644 vm_prot_t prot;
1645
1646 vm_stat.faults++; /* needs lock XXX */
1647 current_task()((active_threads[(0)])->task)->faults++;
1648/*
1649 * Recovery actions
1650 */
1651
1652#undef RELEASE_PAGE
1653#define RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ; } { \
1654 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } }); \
1655 vm_page_lock_queues(); \
1656 vm_page_unwire(m); \
1657 vm_page_unlock_queues(); \
1658}
1659
1660
1661#undef UNLOCK_THINGS{ object->paging_in_progress--; ; }
1662#define UNLOCK_THINGS{ object->paging_in_progress--; ; } { \
1663 object->paging_in_progress--; \
1664 vm_object_unlock(object); \
1665}
1666
1667#undef UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ; }; vm_object_deallocate
(object); }
1668#define UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ; }; vm_object_deallocate
(object); } { \
1669 UNLOCK_THINGS{ object->paging_in_progress--; ; }; \
1670 vm_object_deallocate(object); \
1671}
1672/*
1673 * Give up and have caller do things the hard way.
1674 */
1675
1676#define GIVE_UP{ { { object->paging_in_progress--; ; }; vm_object_deallocate
(object); }; return(5); } { \
1677 UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ; }; vm_object_deallocate
(object); }; \
1678 return(KERN_FAILURE5); \
1679}
1680
1681
1682 /*
1683 * If this entry is not directly to a vm_object, bail out.
1684 */
1685 if (entry->is_sub_map)
1686 return(KERN_FAILURE5);
1687
1688 /*
1689 * Find the backing store object and offset into it.
1690 */
1691
1692 object = entry->object.vm_object;
1693 offset = (va - entry->vme_startlinks.start) + entry->offset;
1694 prot = entry->protection;
1695
1696 /*
1697 * Make a reference to this object to prevent its
1698 * disposal while we are messing with it.
1699 */
1700
1701 vm_object_lock(object);
1702 assert(object->ref_count > 0)({ if (!(object->ref_count > 0)) Assert("object->ref_count > 0"
, "../vm/vm_fault.c", 1702); });
1703 object->ref_count++;
1704 object->paging_in_progress++;
1705
1706 /*
1707 * INVARIANTS (through entire routine):
1708 *
1709 * 1) At all times, we must either have the object
1710 * lock or a busy page in some object to prevent
1711 * some other thread from trying to bring in
1712 * the same page.
1713 *
1714 * 2) Once we have a busy page, we must remove it from
1715 * the pageout queues, so that the pageout daemon
1716 * will not grab it away.
1717 *
1718 */
1719
1720 /*
1721 * Look for page in top-level object. If it's not there or
1722 * there's something going on, give up.
1723 */
1724 m = vm_page_lookup(object, offset);
1725 if ((m == VM_PAGE_NULL((vm_page_t) 0)) || (m->error) ||
1726 (m->busy) || (m->absent) || (prot & m->page_lock)) {
1727 GIVE_UP{ { { object->paging_in_progress--; ; }; vm_object_deallocate
(object); }; return(5); };
1728 }
1729
1730 /*
1731 * Wire the page down now. All bail outs beyond this
1732 * point must unwire the page.
1733 */
1734
1735 vm_page_lock_queues();
1736 vm_page_wire(m);
1737 vm_page_unlock_queues();
1738
1739 /*
1740 * Mark page busy for other threads.
1741 */
1742 assert(!m->busy)({ if (!(!m->busy)) Assert("!m->busy", "../vm/vm_fault.c"
, 1742); });
1743 m->busy = TRUE((boolean_t) 1);
1744 assert(!m->absent)({ if (!(!m->absent)) Assert("!m->absent", "../vm/vm_fault.c"
, 1744); });
1745
1746 /*
1747 * Give up if the page is being written and there's a copy object
1748 */
1749 if ((object->copy != VM_OBJECT_NULL((vm_object_t) 0)) && (prot & VM_PROT_WRITE((vm_prot_t) 0x02))) {
1750 RELEASE_PAGE(m){ ({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m
)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) m)), ((boolean_t) 0), 0); } }); ; vm_page_unwire(m); ; };
1751 GIVE_UP{ { { object->paging_in_progress--; ; }; vm_object_deallocate
(object); }; return(5); };
1752 }
1753
1754 /*
1755 * Put this page into the physical map.
1756 * We have to unlock the object because pmap_enter
1757 * may cause other faults.
1758 */
1759 vm_object_unlock(object);
1760
1761 PMAP_ENTER(map->pmap, va, m, prot, TRUE)({ pmap_enter( (map->pmap), (va), (m)->phys_addr, (prot
) & ~(m)->page_lock, (((boolean_t) 1)) ); });
1762
1763 /*
1764 * Must relock object so that paging_in_progress can be cleared.
1765 */
1766 vm_object_lock(object);
1767
1768 /*
1769 * Unlock everything, and return
1770 */
1771
1772 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } });
1773 UNLOCK_AND_DEALLOCATE{ { object->paging_in_progress--; ; }; vm_object_deallocate
(object); };
1774
1775 return(KERN_SUCCESS0);
1776
1777}
1778
1779/*
1780 * Routine: vm_fault_copy_cleanup
1781 * Purpose:
1782 * Release a page used by vm_fault_copy.
1783 */
1784
1785void vm_fault_copy_cleanup(page, top_page)
1786 vm_page_t page;
1787 vm_page_t top_page;
1788{
1789 vm_object_t object = page->object;
1790
1791 vm_object_lock(object);
1792 PAGE_WAKEUP_DONE(page)({ (page)->busy = ((boolean_t) 0); if ((page)->wanted) {
(page)->wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t
) page)), ((boolean_t) 0), 0); } });
1793 vm_page_lock_queues();
1794 if (!page->active && !page->inactive)
1795 vm_page_activate(page);
1796 vm_page_unlock_queues();
1797 vm_fault_cleanup(object, top_page);
1798}
1799
1800/*
1801 * Routine: vm_fault_copy
1802 *
1803 * Purpose:
1804 * Copy pages from one virtual memory object to another --
1805 * neither the source nor destination pages need be resident.
1806 *
1807 * Before actually copying a page, the version associated with
1808 * the destination address map wil be verified.
1809 *
1810 * In/out conditions:
1811 * The caller must hold a reference, but not a lock, to
1812 * each of the source and destination objects and to the
1813 * destination map.
1814 *
1815 * Results:
1816 * Returns KERN_SUCCESS if no errors were encountered in
1817 * reading or writing the data. Returns KERN_INTERRUPTED if
1818 * the operation was interrupted (only possible if the
1819 * "interruptible" argument is asserted). Other return values
1820 * indicate a permanent error in copying the data.
1821 *
1822 * The actual amount of data copied will be returned in the
1823 * "copy_size" argument. In the event that the destination map
1824 * verification failed, this amount may be less than the amount
1825 * requested.
1826 */
1827kern_return_t vm_fault_copy(
1828 src_object,
1829 src_offset,
1830 src_size,
1831 dst_object,
1832 dst_offset,
1833 dst_map,
1834 dst_version,
1835 interruptible
1836 )
1837 vm_object_t src_object;
1838 vm_offset_t src_offset;
1839 vm_size_t *src_size; /* INOUT */
1840 vm_object_t dst_object;
1841 vm_offset_t dst_offset;
1842 vm_map_t dst_map;
1843 vm_map_version_t *dst_version;
1844 boolean_t interruptible;
1845{
1846 vm_page_t result_page;
1847 vm_prot_t prot;
1848
1849 vm_page_t src_page;
1850 vm_page_t src_top_page;
1851
1852 vm_page_t dst_page;
1853 vm_page_t dst_top_page;
1854
1855 vm_size_t amount_done;
1856 vm_object_t old_copy_object;
1857
1858#define RETURN(x) \
1859 MACRO_BEGIN({ \
1860 *src_size = amount_done; \
1861 MACRO_RETURNif (((boolean_t) 1)) return(x); \
1862 MACRO_END})
1863
1864 amount_done = 0;
1865 do { /* while (amount_done != *src_size) */
1866
1867 RetrySourceFault: ;
1868
1869 if (src_object == VM_OBJECT_NULL((vm_object_t) 0)) {
1870 /*
1871 * No source object. We will just
1872 * zero-fill the page in dst_object.
1873 */
1874
1875 src_page = VM_PAGE_NULL((vm_page_t) 0);
1876 } else {
1877 prot = VM_PROT_READ((vm_prot_t) 0x01);
1878
1879 vm_object_lock(src_object);
1880 vm_object_paging_begin(src_object)((src_object)->paging_in_progress++);
1881
1882 switch (vm_fault_page(src_object, src_offset,
1883 VM_PROT_READ((vm_prot_t) 0x01), FALSE((boolean_t) 0), interruptible,
1884 &prot, &result_page, &src_top_page,
1885 FALSE((boolean_t) 0), (void (*)()) 0)) {
1886
1887 case VM_FAULT_SUCCESS0:
1888 break;
1889 case VM_FAULT_RETRY1:
1890 goto RetrySourceFault;
1891 case VM_FAULT_INTERRUPTED2:
1892 RETURN(MACH_SEND_INTERRUPTED0x10000007);
1893 case VM_FAULT_MEMORY_SHORTAGE3:
1894 VM_PAGE_WAIT((void (*)()) 0)vm_page_wait((void (*)()) 0);
1895 goto RetrySourceFault;
1896 case VM_FAULT_FICTITIOUS_SHORTAGE4:
1897 vm_page_more_fictitious();
1898 goto RetrySourceFault;
1899 case VM_FAULT_MEMORY_ERROR5:
1900 return(KERN_MEMORY_ERROR10);
1901 }
1902
1903 src_page = result_page;
1904
1905 assert((src_top_page == VM_PAGE_NULL) ==({ if (!((src_top_page == ((vm_page_t) 0)) == (src_page->object
== src_object))) Assert("(src_top_page == VM_PAGE_NULL) == (src_page->object == src_object)"
, "../vm/vm_fault.c", 1906); })
1906 (src_page->object == src_object))({ if (!((src_top_page == ((vm_page_t) 0)) == (src_page->object
== src_object))) Assert("(src_top_page == VM_PAGE_NULL) == (src_page->object == src_object)"
, "../vm/vm_fault.c", 1906); });
1907
1908 assert ((prot & VM_PROT_READ) != VM_PROT_NONE)({ if (!((prot & ((vm_prot_t) 0x01)) != ((vm_prot_t) 0x00
))) Assert("(prot & VM_PROT_READ) != VM_PROT_NONE", "../vm/vm_fault.c"
, 1908); });
1909
1910 vm_object_unlock(src_page->object);
1911 }
1912
1913 RetryDestinationFault: ;
1914
1915 prot = VM_PROT_WRITE((vm_prot_t) 0x02);
1916
1917 vm_object_lock(dst_object);
1918 vm_object_paging_begin(dst_object)((dst_object)->paging_in_progress++);
1919
1920 switch (vm_fault_page(dst_object, dst_offset, VM_PROT_WRITE((vm_prot_t) 0x02),
1921 FALSE((boolean_t) 0), FALSE((boolean_t) 0) /* interruptible */,
1922 &prot, &result_page, &dst_top_page,
1923 FALSE((boolean_t) 0), (void (*)()) 0)) {
1924
1925 case VM_FAULT_SUCCESS0:
1926 break;
1927 case VM_FAULT_RETRY1:
1928 goto RetryDestinationFault;
1929 case VM_FAULT_INTERRUPTED2:
1930 if (src_page != VM_PAGE_NULL((vm_page_t) 0))
1931 vm_fault_copy_cleanup(src_page,
1932 src_top_page);
1933 RETURN(MACH_SEND_INTERRUPTED0x10000007);
1934 case VM_FAULT_MEMORY_SHORTAGE3:
1935 VM_PAGE_WAIT((void (*)()) 0)vm_page_wait((void (*)()) 0);
1936 goto RetryDestinationFault;
1937 case VM_FAULT_FICTITIOUS_SHORTAGE4:
1938 vm_page_more_fictitious();
1939 goto RetryDestinationFault;
1940 case VM_FAULT_MEMORY_ERROR5:
1941 if (src_page != VM_PAGE_NULL((vm_page_t) 0))
1942 vm_fault_copy_cleanup(src_page,
1943 src_top_page);
1944 return(KERN_MEMORY_ERROR10);
1945 }
1946 assert ((prot & VM_PROT_WRITE) != VM_PROT_NONE)({ if (!((prot & ((vm_prot_t) 0x02)) != ((vm_prot_t) 0x00
))) Assert("(prot & VM_PROT_WRITE) != VM_PROT_NONE", "../vm/vm_fault.c"
, 1946); });
1947
1948 dst_page = result_page;
1949
1950 old_copy_object = dst_page->object->copy;
1951
1952 vm_object_unlock(dst_page->object);
1953
1954 if (!vm_map_verify(dst_map, dst_version)) {
1955
1956 BailOut: ;
1957
1958 if (src_page != VM_PAGE_NULL((vm_page_t) 0))
1959 vm_fault_copy_cleanup(src_page, src_top_page);
1960 vm_fault_copy_cleanup(dst_page, dst_top_page);
1961 break;
1962 }
1963
1964
1965 vm_object_lock(dst_page->object);
1966 if (dst_page->object->copy != old_copy_object) {
1967 vm_object_unlock(dst_page->object);
1968 vm_map_verify_done(dst_map, dst_version)(lock_done(&(dst_map)->lock));
1969 goto BailOut;
1970 }
1971 vm_object_unlock(dst_page->object);
1972
1973 /*
1974 * Copy the page, and note that it is dirty
1975 * immediately.
1976 */
1977
1978 if (src_page == VM_PAGE_NULL((vm_page_t) 0))
1979 vm_page_zero_fill(dst_page);
1980 else
1981 vm_page_copy(src_page, dst_page);
1982 dst_page->dirty = TRUE((boolean_t) 1);
1983
1984 /*
1985 * Unlock everything, and return
1986 */
1987
1988 vm_map_verify_done(dst_map, dst_version)(lock_done(&(dst_map)->lock));
1989
1990 if (src_page != VM_PAGE_NULL((vm_page_t) 0))
1991 vm_fault_copy_cleanup(src_page, src_top_page);
1992 vm_fault_copy_cleanup(dst_page, dst_top_page);
1993
1994 amount_done += PAGE_SIZE(1 << 12);
1995 src_offset += PAGE_SIZE(1 << 12);
1996 dst_offset += PAGE_SIZE(1 << 12);
1997
1998 } while (amount_done != *src_size);
1999
2000 RETURN(KERN_SUCCESS0);
2001#undef RETURN
2002
2003 /*NOTREACHED*/
2004}
2005
2006
2007
2008
2009
2010#ifdef notdef
2011
2012/*
2013 * Routine: vm_fault_page_overwrite
2014 *
2015 * Description:
2016 * A form of vm_fault_page that assumes that the
2017 * resulting page will be overwritten in its entirety,
2018 * making it unnecessary to obtain the correct *contents*
2019 * of the page.
2020 *
2021 * Implementation:
2022 * XXX Untested. Also unused. Eventually, this technology
2023 * could be used in vm_fault_copy() to advantage.
2024 */
2025vm_fault_return_t vm_fault_page_overwrite(dst_object, dst_offset, result_page)
2026 register
2027 vm_object_t dst_object;
2028 vm_offset_t dst_offset;
2029 vm_page_t *result_page; /* OUT */
2030{
2031 register
2032 vm_page_t dst_page;
2033
2034#define interruptible FALSE((boolean_t) 0) /* XXX */
2035
2036 while (TRUE((boolean_t) 1)) {
2037 /*
2038 * Look for a page at this offset
2039 */
2040
2041 while ((dst_page = vm_page_lookup(dst_object, dst_offset))
2042 == VM_PAGE_NULL((vm_page_t) 0)) {
2043 /*
2044 * No page, no problem... just allocate one.
2045 */
2046
2047 dst_page = vm_page_alloc(dst_object, dst_offset);
2048 if (dst_page == VM_PAGE_NULL((vm_page_t) 0)) {
2049 vm_object_unlock(dst_object);
2050 VM_PAGE_WAIT((void (*)()) 0)vm_page_wait((void (*)()) 0);
2051 vm_object_lock(dst_object);
2052 continue;
2053 }
2054
2055 /*
2056 * Pretend that the memory manager
2057 * write-protected the page.
2058 *
2059 * Note that we will be asking for write
2060 * permission without asking for the data
2061 * first.
2062 */
2063
2064 dst_page->overwriting = TRUE((boolean_t) 1);
2065 dst_page->page_lock = VM_PROT_WRITE((vm_prot_t) 0x02);
2066 dst_page->absent = TRUE((boolean_t) 1);
2067 dst_object->absent_count++;
2068
2069 break;
2070
2071 /*
2072 * When we bail out, we might have to throw
2073 * away the page created here.
2074 */
2075
2076#define DISCARD_PAGE \
2077 MACRO_BEGIN({ \
2078 vm_object_lock(dst_object); \
2079 dst_page = vm_page_lookup(dst_object, dst_offset); \
2080 if ((dst_page != VM_PAGE_NULL((vm_page_t) 0)) && dst_page->overwriting) \
2081 VM_PAGE_FREE(dst_page)({ ; vm_page_free(dst_page); ; }); \
2082 vm_object_unlock(dst_object); \
2083 MACRO_END})
2084 }
2085
2086 /*
2087 * If the page is write-protected...
2088 */
2089
2090 if (dst_page->page_lock & VM_PROT_WRITE((vm_prot_t) 0x02)) {
2091 /*
2092 * ... and an unlock request hasn't been sent
2093 */
2094
2095 if ( ! (dst_page->unlock_request & VM_PROT_WRITE((vm_prot_t) 0x02))) {
2096 vm_prot_t u;
2097 kern_return_t rc;
2098
2099 /*
2100 * ... then send one now.
2101 */
2102
2103 if (!dst_object->pager_ready) {
2104 vm_object_assert_wait(dst_object,({ (dst_object)->all_wanted |= 1 << (1); assert_wait
((event_t)(((vm_offset_t) dst_object) + (1)), (interruptible)
); })
2105 VM_OBJECT_EVENT_PAGER_READY,({ (dst_object)->all_wanted |= 1 << (1); assert_wait
((event_t)(((vm_offset_t) dst_object) + (1)), (interruptible)
); })
2106 interruptible)({ (dst_object)->all_wanted |= 1 << (1); assert_wait
((event_t)(((vm_offset_t) dst_object) + (1)), (interruptible)
); });
2107 vm_object_unlock(dst_object);
2108 thread_block((void (*)()) 0);
2109 if (current_thread()(active_threads[(0)])->wait_result !=
2110 THREAD_AWAKENED0) {
2111 DISCARD_PAGE;
2112 return(VM_FAULT_INTERRUPTED2);
2113 }
2114 continue;
2115 }
2116
2117 u = dst_page->unlock_request |= VM_PROT_WRITE((vm_prot_t) 0x02);
2118 vm_object_unlock(dst_object);
2119
2120 if ((rc = memory_object_data_unlock(
2121 dst_object->pager,
2122 dst_object->pager_request,
2123 dst_offset + dst_object->paging_offset,
2124 PAGE_SIZE(1 << 12),
2125 u)) != KERN_SUCCESS0) {
2126 printf("vm_object_overwrite: memory_object_data_unlock failed\n");
2127 DISCARD_PAGE;
2128 return((rc == MACH_SEND_INTERRUPTED0x10000007) ?
2129 VM_FAULT_INTERRUPTED2 :
2130 VM_FAULT_MEMORY_ERROR5);
2131 }
2132 vm_object_lock(dst_object);
2133 continue;
2134 }
2135
2136 /* ... fall through to wait below */
2137 } else {
2138 /*
2139 * If the page isn't being used for other
2140 * purposes, then we're done.
2141 */
2142 if ( ! (dst_page->busy || dst_page->absent || dst_page->error) )
2143 break;
2144 }
2145
2146 PAGE_ASSERT_WAIT(dst_page, interruptible)({ (dst_page)->wanted = ((boolean_t) 1); assert_wait((event_t
) (dst_page), (interruptible)); });
2147 vm_object_unlock(dst_object);
2148 thread_block((void (*)()) 0);
2149 if (current_thread()(active_threads[(0)])->wait_result != THREAD_AWAKENED0) {
2150 DISCARD_PAGE;
2151 return(VM_FAULT_INTERRUPTED2);
2152 }
2153 }
2154
2155 *result_page = dst_page;
2156 return(VM_FAULT_SUCCESS0);
2157
2158#undef interruptible
2159#undef DISCARD_PAGE
2160}
2161
2162#endif /* notdef */