Bug Summary

File:obj-scan-build/../vm/vm_map.c
Location:line 2973, column 7
Description:Dereference of null pointer

Annotated Source Code

1/*
2 * Mach Operating System
3 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4 * Copyright (c) 1993,1994 The University of Utah and
5 * the Computer Systems Laboratory (CSL).
6 * All rights reserved.
7 *
8 * Permission to use, copy, modify and distribute this software and its
9 * documentation is hereby granted, provided that both the copyright
10 * notice and this permission notice appear in all copies of the
11 * software, derivative works or modified versions, and any portions
12 * thereof, and that both notices appear in supporting documentation.
13 *
14 * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15 * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16 * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17 * THIS SOFTWARE.
18 *
19 * Carnegie Mellon requests users of this software to return to
20 *
21 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
22 * School of Computer Science
23 * Carnegie Mellon University
24 * Pittsburgh PA 15213-3890
25 *
26 * any improvements or extensions that they make and grant Carnegie Mellon
27 * the rights to redistribute these changes.
28 */
29/*
30 * File: vm/vm_map.c
31 * Author: Avadis Tevanian, Jr., Michael Wayne Young
32 * Date: 1985
33 *
34 * Virtual memory mapping module.
35 */
36
37#include <kern/printf.h>
38#include <mach/kern_return.h>
39#include <mach/port.h>
40#include <mach/vm_attributes.h>
41#include <mach/vm_param.h>
42#include <kern/assert.h>
43#include <kern/debug.h>
44#include <kern/kalloc.h>
45#include <kern/rbtree.h>
46#include <kern/slab.h>
47#include <vm/pmap.h>
48#include <vm/vm_fault.h>
49#include <vm/vm_map.h>
50#include <vm/vm_object.h>
51#include <vm/vm_page.h>
52#include <vm/vm_resident.h>
53#include <vm/vm_kern.h>
54#include <ipc/ipc_port.h>
55
56#if MACH_KDB0
57#include <ddb/db_output.h>
58#include <vm/vm_print.h>
59#endif /* MACH_KDB */
60
61
62/* Forward declarations */
63kern_return_t vm_map_delete(
64 vm_map_t map,
65 vm_offset_t start,
66 vm_offset_t end);
67
68kern_return_t vm_map_copyout_page_list(
69 vm_map_t dst_map,
70 vm_offset_t *dst_addr, /* OUT */
71 vm_map_copy_t copy);
72
73void vm_map_copy_page_discard (vm_map_copy_t copy);
74
75/*
76 * Macros to copy a vm_map_entry. We must be careful to correctly
77 * manage the wired page count. vm_map_entry_copy() creates a new
78 * map entry to the same memory - the wired count in the new entry
79 * must be set to zero. vm_map_entry_copy_full() creates a new
80 * entry that is identical to the old entry. This preserves the
81 * wire count; it's used for map splitting and cache changing in
82 * vm_map_copyout.
83 */
84#define vm_map_entry_copy(NEW,OLD)({ *(NEW) = *(OLD); (NEW)->is_shared = ((boolean_t) 0); (NEW
)->needs_wakeup = ((boolean_t) 0); (NEW)->in_transition
= ((boolean_t) 0); (NEW)->wired_count = 0; (NEW)->user_wired_count
= 0; }) \({
85MACRO_BEGIN({ \
86 *(NEW) = *(OLD); \
87 (NEW)->is_shared = FALSE((boolean_t) 0); \
88 (NEW)->needs_wakeup = FALSE((boolean_t) 0); \
89 (NEW)->in_transition = FALSE((boolean_t) 0); \
90 (NEW)->wired_count = 0; \
91 (NEW)->user_wired_count = 0; \})
92MACRO_END})
93
94#define vm_map_entry_copy_full(NEW,OLD)(*(NEW) = *(OLD)) (*(NEW) = *(OLD))
95
96/*
97 * Virtual memory maps provide for the mapping, protection,
98 * and sharing of virtual memory objects. In addition,
99 * this module provides for an efficient virtual copy of
100 * memory from one map to another.
101 *
102 * Synchronization is required prior to most operations.
103 *
104 * Maps consist of an ordered doubly-linked list of simple
105 * entries; a hint and a red-black tree are used to speed up lookups.
106 *
107 * Sharing maps have been deleted from this version of Mach.
108 * All shared objects are now mapped directly into the respective
109 * maps. This requires a change in the copy on write strategy;
110 * the asymmetric (delayed) strategy is used for shared temporary
111 * objects instead of the symmetric (shadow) strategy. This is
112 * selected by the (new) use_shared_copy bit in the object. See
113 * vm_object_copy_temporary in vm_object.c for details. All maps
114 * are now "top level" maps (either task map, kernel map or submap
115 * of the kernel map).
116 *
117 * Since portions of maps are specified by start/end addreses,
118 * which may not align with existing map entries, all
119 * routines merely "clip" entries to these start/end values.
120 * [That is, an entry is split into two, bordering at a
121 * start or end value.] Note that these clippings may not
122 * always be necessary (as the two resulting entries are then
123 * not changed); however, the clipping is done for convenience.
124 * No attempt is currently made to "glue back together" two
125 * abutting entries.
126 *
127 * The symmetric (shadow) copy strategy implements virtual copy
128 * by copying VM object references from one map to
129 * another, and then marking both regions as copy-on-write.
130 * It is important to note that only one writeable reference
131 * to a VM object region exists in any map when this strategy
132 * is used -- this means that shadow object creation can be
133 * delayed until a write operation occurs. The asymmetric (delayed)
134 * strategy allows multiple maps to have writeable references to
135 * the same region of a vm object, and hence cannot delay creating
136 * its copy objects. See vm_object_copy_temporary() in vm_object.c.
137 * Copying of permanent objects is completely different; see
138 * vm_object_copy_strategically() in vm_object.c.
139 */
140
141struct kmem_cache vm_map_cache; /* cache for vm_map structures */
142struct kmem_cache vm_map_entry_cache; /* cache for vm_map_entry structures */
143struct kmem_cache vm_map_kentry_cache; /* cache for kernel entry structures */
144struct kmem_cache vm_map_copy_cache; /* cache for vm_map_copy structures */
145
146boolean_t vm_map_lookup_entry(); /* forward declaration */
147
148/*
149 * Placeholder object for submap operations. This object is dropped
150 * into the range by a call to vm_map_find, and removed when
151 * vm_map_submap creates the submap.
152 */
153
154static struct vm_object vm_submap_object_store;
155vm_object_t vm_submap_object = &vm_submap_object_store;
156
157/*
158 * vm_map_init:
159 *
160 * Initialize the vm_map module. Must be called before
161 * any other vm_map routines.
162 *
163 * Map and entry structures are allocated from caches -- we must
164 * initialize those caches.
165 *
166 * There are three caches of interest:
167 *
168 * vm_map_cache: used to allocate maps.
169 * vm_map_entry_cache: used to allocate map entries.
170 * vm_map_kentry_cache: used to allocate map entries for the kernel.
171 *
172 * Kernel map entries are allocated from a special cache, using a custom
173 * page allocation function to avoid recursion. It would be difficult
174 * (perhaps impossible) for the kernel to allocate more memory to an entry
175 * cache when it became empty since the very act of allocating memory
176 * implies the creation of a new entry.
177 */
178
179vm_offset_t kentry_data;
180vm_size_t kentry_data_size = KENTRY_DATA_SIZE(256*(1 << 12));
181
182static vm_offset_t kentry_pagealloc(vm_size_t size)
183{
184 vm_offset_t result;
185
186 if (size > kentry_data_size)
187 panic("vm_map: kentry memory exhausted");
188
189 result = kentry_data;
190 kentry_data += size;
191 kentry_data_size -= size;
192 return result;
193}
194
195void vm_map_init(void)
196{
197 kmem_cache_init(&vm_map_cache, "vm_map", sizeof(struct vm_map), 0,
198 NULL((void *) 0), NULL((void *) 0), NULL((void *) 0), 0);
199 kmem_cache_init(&vm_map_entry_cache, "vm_map_entry",
200 sizeof(struct vm_map_entry), 0, NULL((void *) 0), NULL((void *) 0), NULL((void *) 0), 0);
201 kmem_cache_init(&vm_map_kentry_cache, "vm_map_kentry",
202 sizeof(struct vm_map_entry), 0, NULL((void *) 0), kentry_pagealloc,
203 NULL((void *) 0), KMEM_CACHE_NOCPUPOOL0x1 | KMEM_CACHE_NOOFFSLAB0x2
204 | KMEM_CACHE_NORECLAIM0x4);
205 kmem_cache_init(&vm_map_copy_cache, "vm_map_copy",
206 sizeof(struct vm_map_copy), 0, NULL((void *) 0), NULL((void *) 0), NULL((void *) 0), 0);
207
208 /*
209 * Submap object is initialized by vm_object_init.
210 */
211}
212
213void vm_map_setup(map, pmap, min, max, pageable)
214 vm_map_t map;
215 pmap_t pmap;
216 vm_offset_t min, max;
217 boolean_t pageable;
218{
219 vm_map_first_entry(map)((map)->hdr.links.next) = vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links);
220 vm_map_last_entry(map)((map)->hdr.links.prev) = vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links);
221 map->hdr.nentries = 0;
222 map->hdr.entries_pageable = pageable;
223 rbtree_init(&map->hdr.tree);
224
225 map->size = 0;
226 map->ref_count = 1;
227 map->pmap = pmap;
228 map->min_offsethdr.links.start = min;
229 map->max_offsethdr.links.end = max;
230 map->wiring_required = FALSE((boolean_t) 0);
231 map->wait_for_space = FALSE((boolean_t) 0);
232 map->first_free = vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links);
233 map->hint = vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links);
234 vm_map_lock_init(map)({ lock_init(&(map)->lock, ((boolean_t) 1)); (map)->
timestamp = 0; });
235 simple_lock_init(&map->ref_lock);
236 simple_lock_init(&map->hint_lock);
237}
238
239/*
240 * vm_map_create:
241 *
242 * Creates and returns a new empty VM map with
243 * the given physical map structure, and having
244 * the given lower and upper address bounds.
245 */
246vm_map_t vm_map_create(pmap, min, max, pageable)
247 pmap_t pmap;
248 vm_offset_t min, max;
249 boolean_t pageable;
250{
251 vm_map_t result;
252
253 result = (vm_map_t) kmem_cache_alloc(&vm_map_cache);
254 if (result == VM_MAP_NULL((vm_map_t) 0))
255 panic("vm_map_create");
256
257 vm_map_setup(result, pmap, min, max, pageable);
258
259 return(result);
260}
261
262/*
263 * vm_map_entry_create: [ internal use only ]
264 *
265 * Allocates a VM map entry for insertion in the
266 * given map (or map copy). No fields are filled.
267 */
268#define vm_map_entry_create(map)_vm_map_entry_create(&(map)->hdr) \
269 _vm_map_entry_create(&(map)->hdr)
270
271#define vm_map_copy_entry_create(copy)_vm_map_entry_create(&(copy)->c_u.hdr) \
272 _vm_map_entry_create(&(copy)->cpy_hdrc_u.hdr)
273
274vm_map_entry_t _vm_map_entry_create(map_header)
275 struct vm_map_header *map_header;
276{
277 kmem_cache_t cache;
278 vm_map_entry_t entry;
279
280 if (map_header->entries_pageable)
281 cache = &vm_map_entry_cache;
282 else
283 cache = &vm_map_kentry_cache;
284
285 entry = (vm_map_entry_t) kmem_cache_alloc(cache);
286 if (entry == VM_MAP_ENTRY_NULL((vm_map_entry_t) 0))
287 panic("vm_map_entry_create");
288
289 return(entry);
290}
291
292/*
293 * vm_map_entry_dispose: [ internal use only ]
294 *
295 * Inverse of vm_map_entry_create.
296 */
297#define vm_map_entry_dispose(map, entry)_vm_map_entry_dispose(&(map)->hdr, (entry)) \
298 _vm_map_entry_dispose(&(map)->hdr, (entry))
299
300#define vm_map_copy_entry_dispose(map, entry)_vm_map_entry_dispose(&(copy)->c_u.hdr, (entry)) \
301 _vm_map_entry_dispose(&(copy)->cpy_hdrc_u.hdr, (entry))
302
303void _vm_map_entry_dispose(map_header, entry)
304 struct vm_map_header *map_header;
305 vm_map_entry_t entry;
306{
307 kmem_cache_t cache;
308
309 if (map_header->entries_pageable)
310 cache = &vm_map_entry_cache;
311 else
312 cache = &vm_map_kentry_cache;
313
314 kmem_cache_free(cache, (vm_offset_t) entry);
315}
316
317/*
318 * Red-black tree lookup/insert comparison functions
319 */
320static inline int vm_map_entry_cmp_lookup(vm_offset_t addr,
321 const struct rbtree_node *node)
322{
323 struct vm_map_entry *entry;
324
325 entry = rbtree_entry(node, struct vm_map_entry, tree_node)((struct vm_map_entry *)((char *)node - __builtin_offsetof (struct
vm_map_entry, tree_node)));
326
327 if (addr < entry->vme_startlinks.start)
328 return -1;
329 else if (addr < entry->vme_endlinks.end)
330 return 0;
331 else
332 return 1;
333}
334
335static inline int vm_map_entry_cmp_insert(const struct rbtree_node *a,
336 const struct rbtree_node *b)
337{
338 struct vm_map_entry *entry;
339
340 entry = rbtree_entry(a, struct vm_map_entry, tree_node)((struct vm_map_entry *)((char *)a - __builtin_offsetof (struct
vm_map_entry, tree_node)));
341 return vm_map_entry_cmp_lookup(entry->vme_startlinks.start, b);
342}
343
344/*
345 * vm_map_entry_{un,}link:
346 *
347 * Insert/remove entries from maps (or map copies).
348 *
349 * The start and end addresses of the entries must be properly set
350 * before using these macros.
351 */
352#define vm_map_entry_link(map, after_where, entry)({ (&(map)->hdr)->nentries++; (entry)->links.prev
= (after_where); (entry)->links.next = (after_where)->
links.next; (entry)->links.prev->links.next = (entry)->
links.next->links.prev = (entry); ({ struct rbtree_node *___cur
, *___prev; int ___diff, ___index; ___prev = ((void *) 0); ___index
= -1; ___cur = (&(&(map)->hdr)->tree)->root
; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(entry)->tree_node, ___cur); ({ if (!(___diff != 0))
Assert("___diff != 0", "../vm/vm_map.c", 352); }); ___prev =
___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(map
)->hdr)->tree, ___prev, ___index, &(entry)->tree_node
); }); }) \
353 _vm_map_entry_link(&(map)->hdr, after_where, entry)({ (&(map)->hdr)->nentries++; (entry)->links.prev
= (after_where); (entry)->links.next = (after_where)->
links.next; (entry)->links.prev->links.next = (entry)->
links.next->links.prev = (entry); ({ struct rbtree_node *___cur
, *___prev; int ___diff, ___index; ___prev = ((void *) 0); ___index
= -1; ___cur = (&(&(map)->hdr)->tree)->root
; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(entry)->tree_node, ___cur); ({ if (!(___diff != 0))
Assert("___diff != 0", "../vm/vm_map.c", 353); }); ___prev =
___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(map
)->hdr)->tree, ___prev, ___index, &(entry)->tree_node
); }); })
354
355#define vm_map_copy_entry_link(copy, after_where, entry)({ (&(copy)->c_u.hdr)->nentries++; (entry)->links
.prev = (after_where); (entry)->links.next = (after_where)
->links.next; (entry)->links.prev->links.next = (entry
)->links.next->links.prev = (entry); ({ struct rbtree_node
*___cur, *___prev; int ___diff, ___index; ___prev = ((void *
) 0); ___index = -1; ___cur = (&(&(copy)->c_u.hdr)
->tree)->root; while (___cur != ((void *) 0)) { ___diff
= vm_map_entry_cmp_insert(&(entry)->tree_node, ___cur
); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c"
, 355); }); ___prev = ___cur; ___index = rbtree_d2i(___diff);
___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(&(copy)->c_u.hdr)->tree, ___prev, ___index, &
(entry)->tree_node); }); }) \
356 _vm_map_entry_link(&(copy)->cpy_hdr, after_where, entry)({ (&(copy)->c_u.hdr)->nentries++; (entry)->links
.prev = (after_where); (entry)->links.next = (after_where)
->links.next; (entry)->links.prev->links.next = (entry
)->links.next->links.prev = (entry); ({ struct rbtree_node
*___cur, *___prev; int ___diff, ___index; ___prev = ((void *
) 0); ___index = -1; ___cur = (&(&(copy)->c_u.hdr)
->tree)->root; while (___cur != ((void *) 0)) { ___diff
= vm_map_entry_cmp_insert(&(entry)->tree_node, ___cur
); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c"
, 356); }); ___prev = ___cur; ___index = rbtree_d2i(___diff);
___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(&(copy)->c_u.hdr)->tree, ___prev, ___index, &
(entry)->tree_node); }); })
357
358#define _vm_map_entry_link(hdr, after_where, entry)({ (hdr)->nentries++; (entry)->links.prev = (after_where
); (entry)->links.next = (after_where)->links.next; (entry
)->links.prev->links.next = (entry)->links.next->
links.prev = (entry); ({ struct rbtree_node *___cur, *___prev
; int ___diff, ___index; ___prev = ((void *) 0); ___index = -
1; ___cur = (&(hdr)->tree)->root; while (___cur != (
(void *) 0)) { ___diff = vm_map_entry_cmp_insert(&(entry)
->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0"
, "../vm/vm_map.c", 358); }); ___prev = ___cur; ___index = rbtree_d2i
(___diff); ___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(hdr)->tree, ___prev, ___index, &(entry)->tree_node
); }); }) \
359 MACRO_BEGIN({ \
360 (hdr)->nentries++; \
361 (entry)->vme_prevlinks.prev = (after_where); \
362 (entry)->vme_nextlinks.next = (after_where)->vme_nextlinks.next; \
363 (entry)->vme_prevlinks.prev->vme_nextlinks.next = \
364 (entry)->vme_nextlinks.next->vme_prevlinks.prev = (entry); \
365 rbtree_insert(&(hdr)->tree, &(entry)->tree_node, \({ struct rbtree_node *___cur, *___prev; int ___diff, ___index
; ___prev = ((void *) 0); ___index = -1; ___cur = (&(hdr)
->tree)->root; while (___cur != ((void *) 0)) { ___diff
= vm_map_entry_cmp_insert(&(entry)->tree_node, ___cur
); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c"
, 366); }); ___prev = ___cur; ___index = rbtree_d2i(___diff);
___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(hdr)->tree, ___prev, ___index, &(entry)->tree_node
); })
366 vm_map_entry_cmp_insert)({ struct rbtree_node *___cur, *___prev; int ___diff, ___index
; ___prev = ((void *) 0); ___index = -1; ___cur = (&(hdr)
->tree)->root; while (___cur != ((void *) 0)) { ___diff
= vm_map_entry_cmp_insert(&(entry)->tree_node, ___cur
); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c"
, 366); }); ___prev = ___cur; ___index = rbtree_d2i(___diff);
___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(hdr)->tree, ___prev, ___index, &(entry)->tree_node
); }); \
367 MACRO_END})
368
369#define vm_map_entry_unlink(map, entry)({ (&(map)->hdr)->nentries--; (entry)->links.next
->links.prev = (entry)->links.prev; (entry)->links.prev
->links.next = (entry)->links.next; rbtree_remove(&
(&(map)->hdr)->tree, &(entry)->tree_node); }
) \
370 _vm_map_entry_unlink(&(map)->hdr, entry)({ (&(map)->hdr)->nentries--; (entry)->links.next
->links.prev = (entry)->links.prev; (entry)->links.prev
->links.next = (entry)->links.next; rbtree_remove(&
(&(map)->hdr)->tree, &(entry)->tree_node); }
)
371
372#define vm_map_copy_entry_unlink(copy, entry)({ (&(copy)->c_u.hdr)->nentries--; (entry)->links
.next->links.prev = (entry)->links.prev; (entry)->links
.prev->links.next = (entry)->links.next; rbtree_remove(
&(&(copy)->c_u.hdr)->tree, &(entry)->tree_node
); }) \
373 _vm_map_entry_unlink(&(copy)->cpy_hdr, entry)({ (&(copy)->c_u.hdr)->nentries--; (entry)->links
.next->links.prev = (entry)->links.prev; (entry)->links
.prev->links.next = (entry)->links.next; rbtree_remove(
&(&(copy)->c_u.hdr)->tree, &(entry)->tree_node
); })
374
375#define _vm_map_entry_unlink(hdr, entry)({ (hdr)->nentries--; (entry)->links.next->links.prev
= (entry)->links.prev; (entry)->links.prev->links.next
= (entry)->links.next; rbtree_remove(&(hdr)->tree,
&(entry)->tree_node); }) \
376 MACRO_BEGIN({ \
377 (hdr)->nentries--; \
378 (entry)->vme_nextlinks.next->vme_prevlinks.prev = (entry)->vme_prevlinks.prev; \
379 (entry)->vme_prevlinks.prev->vme_nextlinks.next = (entry)->vme_nextlinks.next; \
380 rbtree_remove(&(hdr)->tree, &(entry)->tree_node); \
381 MACRO_END})
382
383/*
384 * vm_map_reference:
385 *
386 * Creates another valid reference to the given map.
387 *
388 */
389void vm_map_reference(map)
390 vm_map_t map;
391{
392 if (map == VM_MAP_NULL((vm_map_t) 0))
393 return;
394
395 simple_lock(&map->ref_lock);
396 map->ref_count++;
397 simple_unlock(&map->ref_lock);
398}
399
400/*
401 * vm_map_deallocate:
402 *
403 * Removes a reference from the specified map,
404 * destroying it if no references remain.
405 * The map should not be locked.
406 */
407void vm_map_deallocate(map)
408 vm_map_t map;
409{
410 int c;
411
412 if (map == VM_MAP_NULL((vm_map_t) 0))
413 return;
414
415 simple_lock(&map->ref_lock);
416 c = --map->ref_count;
417 simple_unlock(&map->ref_lock);
418
419 if (c > 0) {
420 return;
421 }
422
423 projected_buffer_collect(map);
424 (void) vm_map_delete(map, map->min_offsethdr.links.start, map->max_offsethdr.links.end);
425
426 pmap_destroy(map->pmap);
427
428 kmem_cache_free(&vm_map_cache, (vm_offset_t) map);
429}
430
431/*
432 * SAVE_HINT:
433 *
434 * Saves the specified entry as the hint for
435 * future lookups. Performs necessary interlocks.
436 */
437#define SAVE_HINT(map,value); (map)->hint = (value); ; \
438 simple_lock(&(map)->hint_lock); \
439 (map)->hint = (value); \
440 simple_unlock(&(map)->hint_lock);
441
442/*
443 * vm_map_lookup_entry: [ internal use only ]
444 *
445 * Finds the map entry containing (or
446 * immediately preceding) the specified address
447 * in the given map; the entry is returned
448 * in the "entry" parameter. The boolean
449 * result indicates whether the address is
450 * actually contained in the map.
451 */
452boolean_t vm_map_lookup_entry(map, address, entry)
453 vm_map_t map;
454 vm_offset_t address;
455 vm_map_entry_t *entry; /* OUT */
456{
457 struct rbtree_node *node;
458 vm_map_entry_t hint;
459
460 /*
461 * First, make a quick check to see if we are already
462 * looking at the entry we want (which is often the case).
463 */
464
465 simple_lock(&map->hint_lock);
466 hint = map->hint;
467 simple_unlock(&map->hint_lock);
468
469 if ((hint != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) && (address >= hint->vme_startlinks.start)) {
470 if (address < hint->vme_endlinks.end) {
471 *entry = hint;
472 return(TRUE((boolean_t) 1));
473 } else {
474 vm_map_entry_t next = hint->vme_nextlinks.next;
475
476 if ((next == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links))
477 || (address < next->vme_startlinks.start)) {
478 *entry = hint;
479 return(FALSE((boolean_t) 0));
480 }
481 }
482 }
483
484 /*
485 * If the hint didn't help, use the red-black tree.
486 */
487
488 node = rbtree_lookup_nearest(&map->hdr.tree, address,({ struct rbtree_node *___cur, *___prev; int ___diff, ___index
; ___prev = ((void *) 0); ___index = -1; ___cur = (&map->
hdr.tree)->root; while (___cur != ((void *) 0)) { ___diff =
vm_map_entry_cmp_lookup(address, ___cur); if (___diff == 0) break
; ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur
->children[___index]; } if (___cur == ((void *) 0)) ___cur
= rbtree_nearest(___prev, ___index, 0); ___cur; })
489 vm_map_entry_cmp_lookup, RBTREE_LEFT)({ struct rbtree_node *___cur, *___prev; int ___diff, ___index
; ___prev = ((void *) 0); ___index = -1; ___cur = (&map->
hdr.tree)->root; while (___cur != ((void *) 0)) { ___diff =
vm_map_entry_cmp_lookup(address, ___cur); if (___diff == 0) break
; ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur
->children[___index]; } if (___cur == ((void *) 0)) ___cur
= rbtree_nearest(___prev, ___index, 0); ___cur; });
490
491 if (node == NULL((void *) 0)) {
492 *entry = vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links);
493 SAVE_HINT(map, *entry); (map)->hint = (*entry); ;;
494 return(FALSE((boolean_t) 0));
495 } else {
496 *entry = rbtree_entry(node, struct vm_map_entry, tree_node)((struct vm_map_entry *)((char *)node - __builtin_offsetof (struct
vm_map_entry, tree_node)));
497 SAVE_HINT(map, *entry); (map)->hint = (*entry); ;;
498 return((address < (*entry)->vme_endlinks.end) ? TRUE((boolean_t) 1) : FALSE((boolean_t) 0));
499 }
500}
501
502/*
503 * Routine: invalid_user_access
504 *
505 * Verifies whether user access is valid.
506 */
507
508boolean_t
509invalid_user_access(map, start, end, prot)
510 vm_map_t map;
511 vm_offset_t start, end;
512 vm_prot_t prot;
513{
514 vm_map_entry_t entry;
515
516 return (map == VM_MAP_NULL((vm_map_t) 0) || map == kernel_map ||
517 !vm_map_lookup_entry(map, start, &entry) ||
518 entry->vme_endlinks.end < end ||
519 (prot & ~(entry->protection)));
520}
521
522
523/*
524 * Routine: vm_map_find_entry
525 * Purpose:
526 * Allocate a range in the specified virtual address map,
527 * returning the entry allocated for that range.
528 * Used by kmem_alloc, etc. Returns wired entries.
529 *
530 * The map must be locked.
531 *
532 * If an entry is allocated, the object/offset fields
533 * are initialized to zero. If an object is supplied,
534 * then an existing entry may be extended.
535 */
536kern_return_t vm_map_find_entry(map, address, size, mask, object, o_entry)
537 vm_map_t map;
538 vm_offset_t *address; /* OUT */
539 vm_size_t size;
540 vm_offset_t mask;
541 vm_object_t object;
542 vm_map_entry_t *o_entry; /* OUT */
543{
544 vm_map_entry_t entry, new_entry;
545 vm_offset_t start;
546 vm_offset_t end;
547
548 /*
549 * Look for the first possible address;
550 * if there's already something at this
551 * address, we have to start after it.
552 */
553
554 if ((entry = map->first_free) == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links))
555 start = map->min_offsethdr.links.start;
556 else
557 start = entry->vme_endlinks.end;
558
559 /*
560 * In any case, the "entry" always precedes
561 * the proposed new region throughout the loop:
562 */
563
564 while (TRUE((boolean_t) 1)) {
565 vm_map_entry_t next;
566
567 /*
568 * Find the end of the proposed new region.
569 * Be sure we didn't go beyond the end, or
570 * wrap around the address.
571 */
572
573 if (((start + mask) & ~mask) < start) {
574 printf_once("no more room for vm_map_find_entry in %p\n", map)({ static int __once = 0; if (!__once) { printf("no more room for vm_map_find_entry in %p\n"
, map); __once = 1; } });
575 return(KERN_NO_SPACE3);
576 }
577 start = ((start + mask) & ~mask);
578 end = start + size;
579
580 if ((end > map->max_offsethdr.links.end) || (end < start)) {
581 printf_once("no more room for vm_map_find_entry in %p\n", map)({ static int __once = 0; if (!__once) { printf("no more room for vm_map_find_entry in %p\n"
, map); __once = 1; } });
582 return(KERN_NO_SPACE3);
583 }
584
585 /*
586 * If there are no more entries, we must win.
587 */
588
589 next = entry->vme_nextlinks.next;
590 if (next == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links))
591 break;
592
593 /*
594 * If there is another entry, it must be
595 * after the end of the potential new region.
596 */
597
598 if (next->vme_startlinks.start >= end)
599 break;
600
601 /*
602 * Didn't fit -- move to the next entry.
603 */
604
605 entry = next;
606 start = entry->vme_endlinks.end;
607 }
608
609 /*
610 * At this point,
611 * "start" and "end" should define the endpoints of the
612 * available new range, and
613 * "entry" should refer to the region before the new
614 * range, and
615 *
616 * the map should be locked.
617 */
618
619 *address = start;
620
621 /*
622 * See whether we can avoid creating a new entry by
623 * extending one of our neighbors. [So far, we only attempt to
624 * extend from below.]
625 */
626
627 if ((object != VM_OBJECT_NULL((vm_object_t) 0)) &&
628 (entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
629 (entry->vme_endlinks.end == start) &&
630 (!entry->is_shared) &&
631 (!entry->is_sub_map) &&
632 (entry->object.vm_object == object) &&
633 (entry->needs_copy == FALSE((boolean_t) 0)) &&
634 (entry->inheritance == VM_INHERIT_DEFAULT((vm_inherit_t) 1)) &&
635 (entry->protection == VM_PROT_DEFAULT(((vm_prot_t) 0x01)|((vm_prot_t) 0x02))) &&
636 (entry->max_protection == VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04))) &&
637 (entry->wired_count == 1) &&
638 (entry->user_wired_count == 0) &&
639 (entry->projected_on == 0)) {
640 /*
641 * Because this is a special case,
642 * we don't need to use vm_object_coalesce.
643 */
644
645 entry->vme_endlinks.end = end;
646 new_entry = entry;
647 } else {
648 new_entry = vm_map_entry_create(map)_vm_map_entry_create(&(map)->hdr);
649
650 new_entry->vme_startlinks.start = start;
651 new_entry->vme_endlinks.end = end;
652
653 new_entry->is_shared = FALSE((boolean_t) 0);
654 new_entry->is_sub_map = FALSE((boolean_t) 0);
655 new_entry->object.vm_object = VM_OBJECT_NULL((vm_object_t) 0);
656 new_entry->offset = (vm_offset_t) 0;
657
658 new_entry->needs_copy = FALSE((boolean_t) 0);
659
660 new_entry->inheritance = VM_INHERIT_DEFAULT((vm_inherit_t) 1);
661 new_entry->protection = VM_PROT_DEFAULT(((vm_prot_t) 0x01)|((vm_prot_t) 0x02));
662 new_entry->max_protection = VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04));
663 new_entry->wired_count = 1;
664 new_entry->user_wired_count = 0;
665
666 new_entry->in_transition = FALSE((boolean_t) 0);
667 new_entry->needs_wakeup = FALSE((boolean_t) 0);
668 new_entry->projected_on = 0;
669
670 /*
671 * Insert the new entry into the list
672 */
673
674 vm_map_entry_link(map, entry, new_entry)({ (&(map)->hdr)->nentries++; (new_entry)->links
.prev = (entry); (new_entry)->links.next = (entry)->links
.next; (new_entry)->links.prev->links.next = (new_entry
)->links.next->links.prev = (new_entry); ({ struct rbtree_node
*___cur, *___prev; int ___diff, ___index; ___prev = ((void *
) 0); ___index = -1; ___cur = (&(&(map)->hdr)->
tree)->root; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(new_entry)->tree_node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 674); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(map
)->hdr)->tree, ___prev, ___index, &(new_entry)->
tree_node); }); });
675 }
676
677 map->size += size;
678
679 /*
680 * Update the free space hint and the lookup hint
681 */
682
683 map->first_free = new_entry;
684 SAVE_HINT(map, new_entry); (map)->hint = (new_entry); ;;
685
686 *o_entry = new_entry;
687 return(KERN_SUCCESS0);
688}
689
690int vm_map_pmap_enter_print = FALSE((boolean_t) 0);
691int vm_map_pmap_enter_enable = FALSE((boolean_t) 0);
692
693/*
694 * Routine: vm_map_pmap_enter
695 *
696 * Description:
697 * Force pages from the specified object to be entered into
698 * the pmap at the specified address if they are present.
699 * As soon as a page not found in the object the scan ends.
700 *
701 * Returns:
702 * Nothing.
703 *
704 * In/out conditions:
705 * The source map should not be locked on entry.
706 */
707void
708vm_map_pmap_enter(map, addr, end_addr, object, offset, protection)
709 vm_map_t map;
710 vm_offset_t addr;
711 vm_offset_t end_addr;
712 vm_object_t object;
713 vm_offset_t offset;
714 vm_prot_t protection;
715{
716 while (addr < end_addr) {
717 vm_page_t m;
718
719 vm_object_lock(object);
720 vm_object_paging_begin(object)((object)->paging_in_progress++);
721
722 m = vm_page_lookup(object, offset);
723 if (m == VM_PAGE_NULL((vm_page_t) 0) || m->absent) {
724 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_map.c", 724); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
725 vm_object_unlock(object);
726 return;
727 }
728
729 if (vm_map_pmap_enter_print) {
730 printf("vm_map_pmap_enter:");
731 printf("map: %p, addr: %lx, object: %p, offset: %lx\n",
732 map, addr, object, offset);
733 }
734
735 m->busy = TRUE((boolean_t) 1);
736 vm_object_unlock(object);
737
738 PMAP_ENTER(map->pmap, addr, m,({ pmap_enter( (map->pmap), (addr), (m)->phys_addr, (protection
) & ~(m)->page_lock, (((boolean_t) 0)) ); })
739 protection, FALSE)({ pmap_enter( (map->pmap), (addr), (m)->phys_addr, (protection
) & ~(m)->page_lock, (((boolean_t) 0)) ); });
740
741 vm_object_lock(object);
742 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } });
743 vm_page_lock_queues();
744 if (!m->active && !m->inactive)
745 vm_page_activate(m);
746 vm_page_unlock_queues();
747 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_map.c", 747); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
748 vm_object_unlock(object);
749
750 offset += PAGE_SIZE(1 << 12);
751 addr += PAGE_SIZE(1 << 12);
752 }
753}
754
755/*
756 * Routine: vm_map_enter
757 *
758 * Description:
759 * Allocate a range in the specified virtual address map.
760 * The resulting range will refer to memory defined by
761 * the given memory object and offset into that object.
762 *
763 * Arguments are as defined in the vm_map call.
764 */
765kern_return_t vm_map_enter(
766 map,
767 address, size, mask, anywhere,
768 object, offset, needs_copy,
769 cur_protection, max_protection, inheritance)
770 vm_map_t map;
771 vm_offset_t *address; /* IN/OUT */
772 vm_size_t size;
773 vm_offset_t mask;
774 boolean_t anywhere;
775 vm_object_t object;
776 vm_offset_t offset;
777 boolean_t needs_copy;
778 vm_prot_t cur_protection;
779 vm_prot_t max_protection;
780 vm_inherit_t inheritance;
781{
782 vm_map_entry_t entry;
783 vm_offset_t start;
784 vm_offset_t end;
785 kern_return_t result = KERN_SUCCESS0;
786
787#define RETURN(value) { result = value; goto BailOut; }
788
789 if (size == 0)
790 return KERN_INVALID_ARGUMENT4;
791
792 StartAgain: ;
793
794 start = *address;
795
796 if (anywhere) {
797 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
798
799 /*
800 * Calculate the first possible address.
801 */
802
803 if (start < map->min_offsethdr.links.start)
804 start = map->min_offsethdr.links.start;
805 if (start > map->max_offsethdr.links.end)
806 RETURN(KERN_NO_SPACE3);
807
808 /*
809 * Look for the first possible address;
810 * if there's already something at this
811 * address, we have to start after it.
812 */
813
814 if (start == map->min_offsethdr.links.start) {
815 if ((entry = map->first_free) != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links))
816 start = entry->vme_endlinks.end;
817 } else {
818 vm_map_entry_t tmp_entry;
819 if (vm_map_lookup_entry(map, start, &tmp_entry))
820 start = tmp_entry->vme_endlinks.end;
821 entry = tmp_entry;
822 }
823
824 /*
825 * In any case, the "entry" always precedes
826 * the proposed new region throughout the
827 * loop:
828 */
829
830 while (TRUE((boolean_t) 1)) {
831 vm_map_entry_t next;
832
833 /*
834 * Find the end of the proposed new region.
835 * Be sure we didn't go beyond the end, or
836 * wrap around the address.
837 */
838
839 if (((start + mask) & ~mask) < start) {
840 printf_once("no more room for vm_map_enter in %p\n", map)({ static int __once = 0; if (!__once) { printf("no more room for vm_map_enter in %p\n"
, map); __once = 1; } });
841 RETURN(KERN_NO_SPACE3);
842 }
843 start = ((start + mask) & ~mask);
844 end = start + size;
845
846 if ((end > map->max_offsethdr.links.end) || (end < start)) {
847 if (map->wait_for_space) {
848 if (size <= (map->max_offsethdr.links.end -
849 map->min_offsethdr.links.start)) {
850 assert_wait((event_t) map, TRUE((boolean_t) 1));
851 vm_map_unlock(map)lock_done(&(map)->lock);
852 thread_block((void (*)()) 0);
853 goto StartAgain;
854 }
855 }
856
857 printf_once("no more room for vm_map_enter in %p\n", map)({ static int __once = 0; if (!__once) { printf("no more room for vm_map_enter in %p\n"
, map); __once = 1; } });
858 RETURN(KERN_NO_SPACE3);
859 }
860
861 /*
862 * If there are no more entries, we must win.
863 */
864
865 next = entry->vme_nextlinks.next;
866 if (next == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links))
867 break;
868
869 /*
870 * If there is another entry, it must be
871 * after the end of the potential new region.
872 */
873
874 if (next->vme_startlinks.start >= end)
875 break;
876
877 /*
878 * Didn't fit -- move to the next entry.
879 */
880
881 entry = next;
882 start = entry->vme_endlinks.end;
883 }
884 *address = start;
885 } else {
886 vm_map_entry_t temp_entry;
887
888 /*
889 * Verify that:
890 * the address doesn't itself violate
891 * the mask requirement.
892 */
893
894 if ((start & mask) != 0)
895 return(KERN_NO_SPACE3);
896
897 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
898
899 /*
900 * ... the address is within bounds
901 */
902
903 end = start + size;
904
905 if ((start < map->min_offsethdr.links.start) ||
906 (end > map->max_offsethdr.links.end) ||
907 (start >= end)) {
908 RETURN(KERN_INVALID_ADDRESS1);
909 }
910
911 /*
912 * ... the starting address isn't allocated
913 */
914
915 if (vm_map_lookup_entry(map, start, &temp_entry))
916 RETURN(KERN_NO_SPACE3);
917
918 entry = temp_entry;
919
920 /*
921 * ... the next region doesn't overlap the
922 * end point.
923 */
924
925 if ((entry->vme_nextlinks.next != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
926 (entry->vme_nextlinks.next->vme_startlinks.start < end))
927 RETURN(KERN_NO_SPACE3);
928 }
929
930 /*
931 * At this point,
932 * "start" and "end" should define the endpoints of the
933 * available new range, and
934 * "entry" should refer to the region before the new
935 * range, and
936 *
937 * the map should be locked.
938 */
939
940 /*
941 * See whether we can avoid creating a new entry (and object) by
942 * extending one of our neighbors. [So far, we only attempt to
943 * extend from below.]
944 */
945
946 if ((object == VM_OBJECT_NULL((vm_object_t) 0)) &&
947 (entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
948 (entry->vme_endlinks.end == start) &&
949 (!entry->is_shared) &&
950 (!entry->is_sub_map) &&
951 (entry->inheritance == inheritance) &&
952 (entry->protection == cur_protection) &&
953 (entry->max_protection == max_protection) &&
954 (entry->wired_count == 0) && /* implies user_wired_count == 0 */
955 (entry->projected_on == 0)) {
956 if (vm_object_coalesce(entry->object.vm_object,
957 VM_OBJECT_NULL((vm_object_t) 0),
958 entry->offset,
959 (vm_offset_t) 0,
960 (vm_size_t)(entry->vme_endlinks.end - entry->vme_startlinks.start),
961 (vm_size_t)(end - entry->vme_endlinks.end))) {
962
963 /*
964 * Coalesced the two objects - can extend
965 * the previous map entry to include the
966 * new range.
967 */
968 map->size += (end - entry->vme_endlinks.end);
969 entry->vme_endlinks.end = end;
970 RETURN(KERN_SUCCESS0);
971 }
972 }
973
974 /*
975 * Create a new entry
976 */
977
978 /**/ {
979 vm_map_entry_t new_entry;
980
981 new_entry = vm_map_entry_create(map)_vm_map_entry_create(&(map)->hdr);
982
983 new_entry->vme_startlinks.start = start;
984 new_entry->vme_endlinks.end = end;
985
986 new_entry->is_shared = FALSE((boolean_t) 0);
987 new_entry->is_sub_map = FALSE((boolean_t) 0);
988 new_entry->object.vm_object = object;
989 new_entry->offset = offset;
990
991 new_entry->needs_copy = needs_copy;
992
993 new_entry->inheritance = inheritance;
994 new_entry->protection = cur_protection;
995 new_entry->max_protection = max_protection;
996 new_entry->wired_count = 0;
997 new_entry->user_wired_count = 0;
998
999 new_entry->in_transition = FALSE((boolean_t) 0);
1000 new_entry->needs_wakeup = FALSE((boolean_t) 0);
1001 new_entry->projected_on = 0;
1002
1003 /*
1004 * Insert the new entry into the list
1005 */
1006
1007 vm_map_entry_link(map, entry, new_entry)({ (&(map)->hdr)->nentries++; (new_entry)->links
.prev = (entry); (new_entry)->links.next = (entry)->links
.next; (new_entry)->links.prev->links.next = (new_entry
)->links.next->links.prev = (new_entry); ({ struct rbtree_node
*___cur, *___prev; int ___diff, ___index; ___prev = ((void *
) 0); ___index = -1; ___cur = (&(&(map)->hdr)->
tree)->root; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(new_entry)->tree_node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 1007); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(map
)->hdr)->tree, ___prev, ___index, &(new_entry)->
tree_node); }); });
1008 map->size += size;
1009
1010 /*
1011 * Update the free space hint and the lookup hint
1012 */
1013
1014 if ((map->first_free == entry) &&
1015 ((entry == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links) ? map->min_offsethdr.links.start : entry->vme_endlinks.end)
1016 >= new_entry->vme_startlinks.start))
1017 map->first_free = new_entry;
1018
1019 SAVE_HINT(map, new_entry); (map)->hint = (new_entry); ;;
1020
1021 vm_map_unlock(map)lock_done(&(map)->lock);
1022
1023 if ((object != VM_OBJECT_NULL((vm_object_t) 0)) &&
1024 (vm_map_pmap_enter_enable) &&
1025 (!anywhere) &&
1026 (!needs_copy) &&
1027 (size < (128*1024))) {
1028 vm_map_pmap_enter(map, start, end,
1029 object, offset, cur_protection);
1030 }
1031
1032 return(result);
1033 /**/ }
1034
1035 BailOut: ;
1036
1037 vm_map_unlock(map)lock_done(&(map)->lock);
1038 return(result);
1039
1040#undef RETURN
1041}
1042
1043/*
1044 * vm_map_clip_start: [ internal use only ]
1045 *
1046 * Asserts that the given entry begins at or after
1047 * the specified address; if necessary,
1048 * it splits the entry into two.
1049 */
1050void _vm_map_clip_start();
1051#define vm_map_clip_start(map, entry, startaddr)({ if ((startaddr) > (entry)->links.start) _vm_map_clip_start
(&(map)->hdr,(entry),(startaddr)); }) \
1052 MACRO_BEGIN({ \
1053 if ((startaddr) > (entry)->vme_startlinks.start) \
1054 _vm_map_clip_start(&(map)->hdr,(entry),(startaddr)); \
1055 MACRO_END})
1056
1057void _vm_map_copy_clip_start();
1058#define vm_map_copy_clip_start(copy, entry, startaddr)({ if ((startaddr) > (entry)->links.start) _vm_map_clip_start
(&(copy)->c_u.hdr,(entry),(startaddr)); }) \
1059 MACRO_BEGIN({ \
1060 if ((startaddr) > (entry)->vme_startlinks.start) \
1061 _vm_map_clip_start(&(copy)->cpy_hdrc_u.hdr,(entry),(startaddr)); \
1062 MACRO_END})
1063
1064/*
1065 * This routine is called only when it is known that
1066 * the entry must be split.
1067 */
1068void _vm_map_clip_start(map_header, entry, start)
1069 struct vm_map_header *map_header;
1070 vm_map_entry_t entry;
1071 vm_offset_t start;
1072{
1073 vm_map_entry_t new_entry;
1074
1075 /*
1076 * Split off the front portion --
1077 * note that we must insert the new
1078 * entry BEFORE this one, so that
1079 * this entry has the specified starting
1080 * address.
1081 */
1082
1083 new_entry = _vm_map_entry_create(map_header);
1084 vm_map_entry_copy_full(new_entry, entry)(*(new_entry) = *(entry));
1085
1086 new_entry->vme_endlinks.end = start;
1087 entry->offset += (start - entry->vme_startlinks.start);
1088 entry->vme_startlinks.start = start;
1089
1090 _vm_map_entry_link(map_header, entry->vme_prev, new_entry)({ (map_header)->nentries++; (new_entry)->links.prev = (
entry->links.prev); (new_entry)->links.next = (entry->
links.prev)->links.next; (new_entry)->links.prev->links
.next = (new_entry)->links.next->links.prev = (new_entry
); ({ struct rbtree_node *___cur, *___prev; int ___diff, ___index
; ___prev = ((void *) 0); ___index = -1; ___cur = (&(map_header
)->tree)->root; while (___cur != ((void *) 0)) { ___diff
= vm_map_entry_cmp_insert(&(new_entry)->tree_node, ___cur
); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c"
, 1090); }); ___prev = ___cur; ___index = rbtree_d2i(___diff)
; ___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(map_header)->tree, ___prev, ___index, &(new_entry
)->tree_node); }); });
1091
1092 if (entry->is_sub_map)
1093 vm_map_reference(new_entry->object.sub_map);
1094 else
1095 vm_object_reference(new_entry->object.vm_object);
1096}
1097
1098/*
1099 * vm_map_clip_end: [ internal use only ]
1100 *
1101 * Asserts that the given entry ends at or before
1102 * the specified address; if necessary,
1103 * it splits the entry into two.
1104 */
1105void _vm_map_clip_end();
1106#define vm_map_clip_end(map, entry, endaddr)({ if ((endaddr) < (entry)->links.end) _vm_map_clip_end
(&(map)->hdr,(entry),(endaddr)); }) \
1107 MACRO_BEGIN({ \
1108 if ((endaddr) < (entry)->vme_endlinks.end) \
1109 _vm_map_clip_end(&(map)->hdr,(entry),(endaddr)); \
1110 MACRO_END})
1111
1112void _vm_map_copy_clip_end();
1113#define vm_map_copy_clip_end(copy, entry, endaddr)({ if ((endaddr) < (entry)->links.end) _vm_map_clip_end
(&(copy)->c_u.hdr,(entry),(endaddr)); }) \
1114 MACRO_BEGIN({ \
1115 if ((endaddr) < (entry)->vme_endlinks.end) \
1116 _vm_map_clip_end(&(copy)->cpy_hdrc_u.hdr,(entry),(endaddr)); \
1117 MACRO_END})
1118
1119/*
1120 * This routine is called only when it is known that
1121 * the entry must be split.
1122 */
1123void _vm_map_clip_end(map_header, entry, end)
1124 struct vm_map_header *map_header;
1125 vm_map_entry_t entry;
1126 vm_offset_t end;
1127{
1128 vm_map_entry_t new_entry;
1129
1130 /*
1131 * Create a new entry and insert it
1132 * AFTER the specified entry
1133 */
1134
1135 new_entry = _vm_map_entry_create(map_header);
1136 vm_map_entry_copy_full(new_entry, entry)(*(new_entry) = *(entry));
1137
1138 new_entry->vme_startlinks.start = entry->vme_endlinks.end = end;
1139 new_entry->offset += (end - entry->vme_startlinks.start);
1140
1141 _vm_map_entry_link(map_header, entry, new_entry)({ (map_header)->nentries++; (new_entry)->links.prev = (
entry); (new_entry)->links.next = (entry)->links.next; (
new_entry)->links.prev->links.next = (new_entry)->links
.next->links.prev = (new_entry); ({ struct rbtree_node *___cur
, *___prev; int ___diff, ___index; ___prev = ((void *) 0); ___index
= -1; ___cur = (&(map_header)->tree)->root; while (
___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert(&
(new_entry)->tree_node, ___cur); ({ if (!(___diff != 0)) Assert
("___diff != 0", "../vm/vm_map.c", 1141); }); ___prev = ___cur
; ___index = rbtree_d2i(___diff); ___cur = ___cur->children
[___index]; } rbtree_insert_rebalance(&(map_header)->tree
, ___prev, ___index, &(new_entry)->tree_node); }); });
1142
1143 if (entry->is_sub_map)
1144 vm_map_reference(new_entry->object.sub_map);
1145 else
1146 vm_object_reference(new_entry->object.vm_object);
1147}
1148
1149/*
1150 * VM_MAP_RANGE_CHECK: [ internal use only ]
1151 *
1152 * Asserts that the starting and ending region
1153 * addresses fall within the valid range of the map.
1154 */
1155#define VM_MAP_RANGE_CHECK(map, start, end){ if (start < ((map)->hdr.links.start)) start = ((map)->
hdr.links.start); if (end > ((map)->hdr.links.end)) end
= ((map)->hdr.links.end); if (start > end) start = end
; } \
1156 { \
1157 if (start < vm_map_min(map)((map)->hdr.links.start)) \
1158 start = vm_map_min(map)((map)->hdr.links.start); \
1159 if (end > vm_map_max(map)((map)->hdr.links.end)) \
1160 end = vm_map_max(map)((map)->hdr.links.end); \
1161 if (start > end) \
1162 start = end; \
1163 }
1164
1165/*
1166 * vm_map_submap: [ kernel use only ]
1167 *
1168 * Mark the given range as handled by a subordinate map.
1169 *
1170 * This range must have been created with vm_map_find using
1171 * the vm_submap_object, and no other operations may have been
1172 * performed on this range prior to calling vm_map_submap.
1173 *
1174 * Only a limited number of operations can be performed
1175 * within this rage after calling vm_map_submap:
1176 * vm_fault
1177 * [Don't try vm_map_copyin!]
1178 *
1179 * To remove a submapping, one must first remove the
1180 * range from the superior map, and then destroy the
1181 * submap (if desired). [Better yet, don't try it.]
1182 */
1183kern_return_t vm_map_submap(map, start, end, submap)
1184 vm_map_t map;
1185 vm_offset_t start;
1186 vm_offset_t end;
1187 vm_map_t submap;
1188{
1189 vm_map_entry_t entry;
1190 kern_return_t result = KERN_INVALID_ARGUMENT4;
1191 vm_object_t object;
1192
1193 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
1194
1195 VM_MAP_RANGE_CHECK(map, start, end){ if (start < ((map)->hdr.links.start)) start = ((map)->
hdr.links.start); if (end > ((map)->hdr.links.end)) end
= ((map)->hdr.links.end); if (start > end) start = end
; };
1196
1197 if (vm_map_lookup_entry(map, start, &entry)) {
1198 vm_map_clip_start(map, entry, start)({ if ((start) > (entry)->links.start) _vm_map_clip_start
(&(map)->hdr,(entry),(start)); });
1199 }
1200 else
1201 entry = entry->vme_nextlinks.next;
1202
1203 vm_map_clip_end(map, entry, end)({ if ((end) < (entry)->links.end) _vm_map_clip_end(&
(map)->hdr,(entry),(end)); });
1204
1205 if ((entry->vme_startlinks.start == start) && (entry->vme_endlinks.end == end) &&
1206 (!entry->is_sub_map) &&
1207 ((object = entry->object.vm_object) == vm_submap_object) &&
1208 (object->resident_page_count == 0) &&
1209 (object->copy == VM_OBJECT_NULL((vm_object_t) 0)) &&
1210 (object->shadow == VM_OBJECT_NULL((vm_object_t) 0)) &&
1211 (!object->pager_created)) {
1212 entry->object.vm_object = VM_OBJECT_NULL((vm_object_t) 0);
1213 vm_object_deallocate(object);
1214 entry->is_sub_map = TRUE((boolean_t) 1);
1215 vm_map_reference(entry->object.sub_map = submap);
1216 result = KERN_SUCCESS0;
1217 }
1218 vm_map_unlock(map)lock_done(&(map)->lock);
1219
1220 return(result);
1221}
1222
1223/*
1224 * vm_map_protect:
1225 *
1226 * Sets the protection of the specified address
1227 * region in the target map. If "set_max" is
1228 * specified, the maximum protection is to be set;
1229 * otherwise, only the current protection is affected.
1230 */
1231kern_return_t vm_map_protect(map, start, end, new_prot, set_max)
1232 vm_map_t map;
1233 vm_offset_t start;
1234 vm_offset_t end;
1235 vm_prot_t new_prot;
1236 boolean_t set_max;
1237{
1238 vm_map_entry_t current;
1239 vm_map_entry_t entry;
1240
1241 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
1242
1243 VM_MAP_RANGE_CHECK(map, start, end){ if (start < ((map)->hdr.links.start)) start = ((map)->
hdr.links.start); if (end > ((map)->hdr.links.end)) end
= ((map)->hdr.links.end); if (start > end) start = end
; };
1244
1245 if (vm_map_lookup_entry(map, start, &entry)) {
1246 vm_map_clip_start(map, entry, start)({ if ((start) > (entry)->links.start) _vm_map_clip_start
(&(map)->hdr,(entry),(start)); });
1247 }
1248 else
1249 entry = entry->vme_nextlinks.next;
1250
1251 /*
1252 * Make a first pass to check for protection
1253 * violations.
1254 */
1255
1256 current = entry;
1257 while ((current != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
1258 (current->vme_startlinks.start < end)) {
1259
1260 if (current->is_sub_map) {
1261 vm_map_unlock(map)lock_done(&(map)->lock);
1262 return(KERN_INVALID_ARGUMENT4);
1263 }
1264 if ((new_prot & (VM_PROT_NOTIFY((vm_prot_t) 0x10) | current->max_protection))
1265 != new_prot) {
1266 vm_map_unlock(map)lock_done(&(map)->lock);
1267 return(KERN_PROTECTION_FAILURE2);
1268 }
1269
1270 current = current->vme_nextlinks.next;
1271 }
1272
1273 /*
1274 * Go back and fix up protections.
1275 * [Note that clipping is not necessary the second time.]
1276 */
1277
1278 current = entry;
1279
1280 while ((current != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
1281 (current->vme_startlinks.start < end)) {
1282
1283 vm_prot_t old_prot;
1284
1285 vm_map_clip_end(map, current, end)({ if ((end) < (current)->links.end) _vm_map_clip_end(&
(map)->hdr,(current),(end)); });
1286
1287 old_prot = current->protection;
1288 if (set_max)
1289 current->protection =
1290 (current->max_protection = new_prot) &
1291 old_prot;
1292 else
1293 current->protection = new_prot;
1294
1295 /*
1296 * Update physical map if necessary.
1297 */
1298
1299 if (current->protection != old_prot) {
1300 pmap_protect(map->pmap, current->vme_startlinks.start,
1301 current->vme_endlinks.end,
1302 current->protection);
1303 }
1304 current = current->vme_nextlinks.next;
1305 }
1306
1307 vm_map_unlock(map)lock_done(&(map)->lock);
1308 return(KERN_SUCCESS0);
1309}
1310
1311/*
1312 * vm_map_inherit:
1313 *
1314 * Sets the inheritance of the specified address
1315 * range in the target map. Inheritance
1316 * affects how the map will be shared with
1317 * child maps at the time of vm_map_fork.
1318 */
1319kern_return_t vm_map_inherit(map, start, end, new_inheritance)
1320 vm_map_t map;
1321 vm_offset_t start;
1322 vm_offset_t end;
1323 vm_inherit_t new_inheritance;
1324{
1325 vm_map_entry_t entry;
1326 vm_map_entry_t temp_entry;
1327
1328 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
1329
1330 VM_MAP_RANGE_CHECK(map, start, end){ if (start < ((map)->hdr.links.start)) start = ((map)->
hdr.links.start); if (end > ((map)->hdr.links.end)) end
= ((map)->hdr.links.end); if (start > end) start = end
; };
1331
1332 if (vm_map_lookup_entry(map, start, &temp_entry)) {
1333 entry = temp_entry;
1334 vm_map_clip_start(map, entry, start)({ if ((start) > (entry)->links.start) _vm_map_clip_start
(&(map)->hdr,(entry),(start)); });
1335 }
1336 else
1337 entry = temp_entry->vme_nextlinks.next;
1338
1339 while ((entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) && (entry->vme_startlinks.start < end)) {
1340 vm_map_clip_end(map, entry, end)({ if ((end) < (entry)->links.end) _vm_map_clip_end(&
(map)->hdr,(entry),(end)); });
1341
1342 entry->inheritance = new_inheritance;
1343
1344 entry = entry->vme_nextlinks.next;
1345 }
1346
1347 vm_map_unlock(map)lock_done(&(map)->lock);
1348 return(KERN_SUCCESS0);
1349}
1350
1351/*
1352 * vm_map_pageable_common:
1353 *
1354 * Sets the pageability of the specified address
1355 * range in the target map. Regions specified
1356 * as not pageable require locked-down physical
1357 * memory and physical page maps. access_type indicates
1358 * types of accesses that must not generate page faults.
1359 * This is checked against protection of memory being locked-down.
1360 * access_type of VM_PROT_NONE makes memory pageable.
1361 *
1362 * The map must not be locked, but a reference
1363 * must remain to the map throughout the call.
1364 *
1365 * Callers should use macros in vm/vm_map.h (i.e. vm_map_pageable,
1366 * or vm_map_pageable_user); don't call vm_map_pageable directly.
1367 */
1368kern_return_t vm_map_pageable_common(map, start, end, access_type, user_wire)
1369 vm_map_t map;
1370 vm_offset_t start;
1371 vm_offset_t end;
1372 vm_prot_t access_type;
1373 boolean_t user_wire;
1374{
1375 vm_map_entry_t entry;
1376 vm_map_entry_t start_entry;
1377
1378 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
1379
1380 VM_MAP_RANGE_CHECK(map, start, end){ if (start < ((map)->hdr.links.start)) start = ((map)->
hdr.links.start); if (end > ((map)->hdr.links.end)) end
= ((map)->hdr.links.end); if (start > end) start = end
; };
1381
1382 if (vm_map_lookup_entry(map, start, &start_entry)) {
1383 entry = start_entry;
1384 /*
1385 * vm_map_clip_start will be done later.
1386 */
1387 }
1388 else {
1389 /*
1390 * Start address is not in map; this is fatal.
1391 */
1392 vm_map_unlock(map)lock_done(&(map)->lock);
1393 return(KERN_FAILURE5);
1394 }
1395
1396 /*
1397 * Actions are rather different for wiring and unwiring,
1398 * so we have two separate cases.
1399 */
1400
1401 if (access_type == VM_PROT_NONE((vm_prot_t) 0x00)) {
1402
1403 vm_map_clip_start(map, entry, start)({ if ((start) > (entry)->links.start) _vm_map_clip_start
(&(map)->hdr,(entry),(start)); });
1404
1405 /*
1406 * Unwiring. First ensure that the range to be
1407 * unwired is really wired down.
1408 */
1409 while ((entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
1410 (entry->vme_startlinks.start < end)) {
1411
1412 if ((entry->wired_count == 0) ||
1413 ((entry->vme_endlinks.end < end) &&
1414 ((entry->vme_nextlinks.next == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) ||
1415 (entry->vme_nextlinks.next->vme_startlinks.start > entry->vme_endlinks.end))) ||
1416 (user_wire && (entry->user_wired_count == 0))) {
1417 vm_map_unlock(map)lock_done(&(map)->lock);
1418 return(KERN_INVALID_ARGUMENT4);
1419 }
1420 entry = entry->vme_nextlinks.next;
1421 }
1422
1423 /*
1424 * Now decrement the wiring count for each region.
1425 * If a region becomes completely unwired,
1426 * unwire its physical pages and mappings.
1427 */
1428 entry = start_entry;
1429 while ((entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
1430 (entry->vme_startlinks.start < end)) {
1431 vm_map_clip_end(map, entry, end)({ if ((end) < (entry)->links.end) _vm_map_clip_end(&
(map)->hdr,(entry),(end)); });
1432
1433 if (user_wire) {
1434 if (--(entry->user_wired_count) == 0)
1435 entry->wired_count--;
1436 }
1437 else {
1438 entry->wired_count--;
1439 }
1440
1441 if (entry->wired_count == 0)
1442 vm_fault_unwire(map, entry);
1443
1444 entry = entry->vme_nextlinks.next;
1445 }
1446 }
1447
1448 else {
1449 /*
1450 * Wiring. We must do this in two passes:
1451 *
1452 * 1. Holding the write lock, we create any shadow
1453 * or zero-fill objects that need to be created.
1454 * Then we clip each map entry to the region to be
1455 * wired and increment its wiring count. We
1456 * create objects before clipping the map entries
1457 * to avoid object proliferation.
1458 *
1459 * 2. We downgrade to a read lock, and call
1460 * vm_fault_wire to fault in the pages for any
1461 * newly wired area (wired_count is 1).
1462 *
1463 * Downgrading to a read lock for vm_fault_wire avoids
1464 * a possible deadlock with another thread that may have
1465 * faulted on one of the pages to be wired (it would mark
1466 * the page busy, blocking us, then in turn block on the
1467 * map lock that we hold). Because of problems in the
1468 * recursive lock package, we cannot upgrade to a write
1469 * lock in vm_map_lookup. Thus, any actions that require
1470 * the write lock must be done beforehand. Because we
1471 * keep the read lock on the map, the copy-on-write
1472 * status of the entries we modify here cannot change.
1473 */
1474
1475 /*
1476 * Pass 1.
1477 */
1478 while ((entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
1479 (entry->vme_startlinks.start < end)) {
1480 vm_map_clip_end(map, entry, end)({ if ((end) < (entry)->links.end) _vm_map_clip_end(&
(map)->hdr,(entry),(end)); });
1481
1482 if (entry->wired_count == 0) {
1483
1484 /*
1485 * Perform actions of vm_map_lookup that need
1486 * the write lock on the map: create a shadow
1487 * object for a copy-on-write region, or an
1488 * object for a zero-fill region.
1489 */
1490 if (entry->needs_copy &&
1491 ((entry->protection & VM_PROT_WRITE((vm_prot_t) 0x02)) != 0)) {
1492
1493 vm_object_shadow(&entry->object.vm_object,
1494 &entry->offset,
1495 (vm_size_t)(entry->vme_endlinks.end
1496 - entry->vme_startlinks.start));
1497 entry->needs_copy = FALSE((boolean_t) 0);
1498 }
1499 if (entry->object.vm_object == VM_OBJECT_NULL((vm_object_t) 0)) {
1500 entry->object.vm_object =
1501 vm_object_allocate(
1502 (vm_size_t)(entry->vme_endlinks.end
1503 - entry->vme_startlinks.start));
1504 entry->offset = (vm_offset_t)0;
1505 }
1506 }
1507 vm_map_clip_start(map, entry, start)({ if ((start) > (entry)->links.start) _vm_map_clip_start
(&(map)->hdr,(entry),(start)); });
1508 vm_map_clip_end(map, entry, end)({ if ((end) < (entry)->links.end) _vm_map_clip_end(&
(map)->hdr,(entry),(end)); });
1509
1510 if (user_wire) {
1511 if ((entry->user_wired_count)++ == 0)
1512 entry->wired_count++;
1513 }
1514 else {
1515 entry->wired_count++;
1516 }
1517
1518 /*
1519 * Check for holes and protection mismatch.
1520 * Holes: Next entry should be contiguous unless
1521 * this is the end of the region.
1522 * Protection: Access requested must be allowed.
1523 */
1524 if (((entry->vme_endlinks.end < end) &&
1525 ((entry->vme_nextlinks.next == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) ||
1526 (entry->vme_nextlinks.next->vme_startlinks.start > entry->vme_endlinks.end))) ||
1527 ((entry->protection & access_type) != access_type)) {
1528 /*
1529 * Found a hole or protection problem.
1530 * Object creation actions
1531 * do not need to be undone, but the
1532 * wired counts need to be restored.
1533 */
1534 while ((entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
1535 (entry->vme_endlinks.end > start)) {
1536 if (user_wire) {
1537 if (--(entry->user_wired_count) == 0)
1538 entry->wired_count--;
1539 }
1540 else {
1541 entry->wired_count--;
1542 }
1543
1544 entry = entry->vme_prevlinks.prev;
1545 }
1546
1547 vm_map_unlock(map)lock_done(&(map)->lock);
1548 return(KERN_FAILURE5);
1549 }
1550 entry = entry->vme_nextlinks.next;
1551 }
1552
1553 /*
1554 * Pass 2.
1555 */
1556
1557 /*
1558 * HACK HACK HACK HACK
1559 *
1560 * If we are wiring in the kernel map or a submap of it,
1561 * unlock the map to avoid deadlocks. We trust that the
1562 * kernel threads are well-behaved, and therefore will
1563 * not do anything destructive to this region of the map
1564 * while we have it unlocked. We cannot trust user threads
1565 * to do the same.
1566 *
1567 * HACK HACK HACK HACK
1568 */
1569 if (vm_map_pmap(map)((map)->pmap) == kernel_pmap) {
1570 vm_map_unlock(map)lock_done(&(map)->lock); /* trust me ... */
1571 }
1572 else {
1573 vm_map_lock_set_recursive(map)lock_set_recursive(&(map)->lock);
1574 vm_map_lock_write_to_read(map)lock_write_to_read(&(map)->lock);
1575 }
1576
1577 entry = start_entry;
1578 while (entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links) &&
1579 entry->vme_startlinks.start < end) {
1580 /*
1581 * Wiring cases:
1582 * Kernel: wired == 1 && user_wired == 0
1583 * User: wired == 1 && user_wired == 1
1584 *
1585 * Don't need to wire if either is > 1. wired = 0 &&
1586 * user_wired == 1 can't happen.
1587 */
1588
1589 /*
1590 * XXX This assumes that the faults always succeed.
1591 */
1592 if ((entry->wired_count == 1) &&
1593 (entry->user_wired_count <= 1)) {
1594 vm_fault_wire(map, entry);
1595 }
1596 entry = entry->vme_nextlinks.next;
1597 }
1598
1599 if (vm_map_pmap(map)((map)->pmap) == kernel_pmap) {
1600 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
1601 }
1602 else {
1603 vm_map_lock_clear_recursive(map)lock_clear_recursive(&(map)->lock);
1604 }
1605 }
1606
1607 vm_map_unlock(map)lock_done(&(map)->lock);
1608
1609 return(KERN_SUCCESS0);
1610}
1611
1612/*
1613 * vm_map_entry_delete: [ internal use only ]
1614 *
1615 * Deallocate the given entry from the target map.
1616 */
1617void vm_map_entry_delete(map, entry)
1618 vm_map_t map;
1619 vm_map_entry_t entry;
1620{
1621 vm_offset_t s, e;
1622 vm_object_t object;
1623 extern vm_object_t kernel_object;
1624
1625 s = entry->vme_startlinks.start;
1626 e = entry->vme_endlinks.end;
1627
1628 /*Check if projected buffer*/
1629 if (map != kernel_map && entry->projected_on != 0) {
1630 /*Check if projected kernel entry is persistent;
1631 may only manipulate directly if it is*/
1632 if (entry->projected_on->projected_on == 0)
1633 entry->wired_count = 0; /*Avoid unwire fault*/
1634 else
1635 return;
1636 }
1637
1638 /*
1639 * Get the object. Null objects cannot have pmap entries.
1640 */
1641
1642 if ((object = entry->object.vm_object) != VM_OBJECT_NULL((vm_object_t) 0)) {
1643
1644 /*
1645 * Unwire before removing addresses from the pmap;
1646 * otherwise, unwiring will put the entries back in
1647 * the pmap.
1648 */
1649
1650 if (entry->wired_count != 0) {
1651 vm_fault_unwire(map, entry);
1652 entry->wired_count = 0;
1653 entry->user_wired_count = 0;
1654 }
1655
1656 /*
1657 * If the object is shared, we must remove
1658 * *all* references to this data, since we can't
1659 * find all of the physical maps which are sharing
1660 * it.
1661 */
1662
1663 if (object == kernel_object) {
1664 vm_object_lock(object);
1665 vm_object_page_remove(object, entry->offset,
1666 entry->offset + (e - s));
1667 vm_object_unlock(object);
1668 } else if (entry->is_shared) {
1669 vm_object_pmap_remove(object,
1670 entry->offset,
1671 entry->offset + (e - s));
1672 }
1673 else {
1674 pmap_remove(map->pmap, s, e);
1675 }
1676 }
1677
1678 /*
1679 * Deallocate the object only after removing all
1680 * pmap entries pointing to its pages.
1681 */
1682
1683 if (entry->is_sub_map)
1684 vm_map_deallocate(entry->object.sub_map);
1685 else
1686 vm_object_deallocate(entry->object.vm_object);
1687
1688 vm_map_entry_unlink(map, entry)({ (&(map)->hdr)->nentries--; (entry)->links.next
->links.prev = (entry)->links.prev; (entry)->links.prev
->links.next = (entry)->links.next; rbtree_remove(&
(&(map)->hdr)->tree, &(entry)->tree_node); }
);
1689 map->size -= e - s;
1690
1691 vm_map_entry_dispose(map, entry)_vm_map_entry_dispose(&(map)->hdr, (entry));
1692}
1693
1694/*
1695 * vm_map_delete: [ internal use only ]
1696 *
1697 * Deallocates the given address range from the target
1698 * map.
1699 */
1700
1701kern_return_t vm_map_delete(map, start, end)
1702 vm_map_t map;
1703 vm_offset_t start;
1704 vm_offset_t end;
1705{
1706 vm_map_entry_t entry;
1707 vm_map_entry_t first_entry;
1708
1709 /*
1710 * Find the start of the region, and clip it
1711 */
1712
1713 if (!vm_map_lookup_entry(map, start, &first_entry))
1714 entry = first_entry->vme_nextlinks.next;
1715 else {
1716 entry = first_entry;
1717 vm_map_clip_start(map, entry, start)({ if ((start) > (entry)->links.start) _vm_map_clip_start
(&(map)->hdr,(entry),(start)); });
1718
1719 /*
1720 * Fix the lookup hint now, rather than each
1721 * time though the loop.
1722 */
1723
1724 SAVE_HINT(map, entry->vme_prev); (map)->hint = (entry->links.prev); ;;
1725 }
1726
1727 /*
1728 * Save the free space hint
1729 */
1730
1731 if (map->first_free->vme_startlinks.start >= start)
1732 map->first_free = entry->vme_prevlinks.prev;
1733
1734 /*
1735 * Step through all entries in this region
1736 */
1737
1738 while ((entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) && (entry->vme_startlinks.start < end)) {
1739 vm_map_entry_t next;
1740
1741 vm_map_clip_end(map, entry, end)({ if ((end) < (entry)->links.end) _vm_map_clip_end(&
(map)->hdr,(entry),(end)); });
1742
1743 /*
1744 * If the entry is in transition, we must wait
1745 * for it to exit that state. It could be clipped
1746 * while we leave the map unlocked.
1747 */
1748 if(entry->in_transition) {
1749 /*
1750 * Say that we are waiting, and wait for entry.
1751 */
1752 entry->needs_wakeup = TRUE((boolean_t) 1);
1753 vm_map_entry_wait(map, FALSE)({ assert_wait((event_t)&(map)->hdr, ((boolean_t) 0));
lock_done(&(map)->lock); thread_block((void (*)()) 0)
; });
1754 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
1755
1756 /*
1757 * The entry could have been clipped or it
1758 * may not exist anymore. look it up again.
1759 */
1760 if(!vm_map_lookup_entry(map, start, &entry)) {
1761 entry = entry->vme_nextlinks.next;
1762 }
1763 continue;
1764 }
1765
1766 next = entry->vme_nextlinks.next;
1767
1768 vm_map_entry_delete(map, entry);
1769 entry = next;
1770 }
1771
1772 if (map->wait_for_space)
1773 thread_wakeup((event_t) map)thread_wakeup_prim(((event_t) map), ((boolean_t) 0), 0);
1774
1775 return(KERN_SUCCESS0);
1776}
1777
1778/*
1779 * vm_map_remove:
1780 *
1781 * Remove the given address range from the target map.
1782 * This is the exported form of vm_map_delete.
1783 */
1784kern_return_t vm_map_remove(map, start, end)
1785 vm_map_t map;
1786 vm_offset_t start;
1787 vm_offset_t end;
1788{
1789 kern_return_t result;
1790
1791 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
1792 VM_MAP_RANGE_CHECK(map, start, end){ if (start < ((map)->hdr.links.start)) start = ((map)->
hdr.links.start); if (end > ((map)->hdr.links.end)) end
= ((map)->hdr.links.end); if (start > end) start = end
; };
1793 result = vm_map_delete(map, start, end);
1794 vm_map_unlock(map)lock_done(&(map)->lock);
1795
1796 return(result);
1797}
1798
1799
1800/*
1801 * vm_map_copy_steal_pages:
1802 *
1803 * Steal all the pages from a vm_map_copy page_list by copying ones
1804 * that have not already been stolen.
1805 */
1806void
1807vm_map_copy_steal_pages(copy)
1808vm_map_copy_t copy;
1809{
1810 vm_page_t m, new_m;
1811 int i;
1812 vm_object_t object;
1813
1814 for (i = 0; i < copy->cpy_npagesc_u.c_p.npages; i++) {
1815
1816 /*
1817 * If the page is not tabled, then it's already stolen.
1818 */
1819 m = copy->cpy_page_listc_u.c_p.page_list[i];
1820 if (!m->tabled)
1821 continue;
1822
1823 /*
1824 * Page was not stolen, get a new
1825 * one and do the copy now.
1826 */
1827 while ((new_m = vm_page_grab(FALSE((boolean_t) 0))) == VM_PAGE_NULL((vm_page_t) 0)) {
1828 VM_PAGE_WAIT((void(*)()) 0)vm_page_wait((void(*)()) 0);
1829 }
1830
1831 vm_page_copy(m, new_m);
1832
1833 object = m->object;
1834 vm_object_lock(object);
1835 vm_page_lock_queues();
1836 if (!m->active && !m->inactive)
1837 vm_page_activate(m);
1838 vm_page_unlock_queues();
1839 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } });
1840 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_map.c", 1840); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
1841 vm_object_unlock(object);
1842
1843 copy->cpy_page_listc_u.c_p.page_list[i] = new_m;
1844 }
1845}
1846
1847/*
1848 * vm_map_copy_page_discard:
1849 *
1850 * Get rid of the pages in a page_list copy. If the pages are
1851 * stolen, they are freed. If the pages are not stolen, they
1852 * are unbusied, and associated state is cleaned up.
1853 */
1854void vm_map_copy_page_discard(copy)
1855vm_map_copy_t copy;
1856{
1857 while (copy->cpy_npagesc_u.c_p.npages > 0) {
1858 vm_page_t m;
1859
1860 if((m = copy->cpy_page_listc_u.c_p.page_list[--(copy->cpy_npagesc_u.c_p.npages)]) !=
1861 VM_PAGE_NULL((vm_page_t) 0)) {
1862
1863 /*
1864 * If it's not in the table, then it's
1865 * a stolen page that goes back
1866 * to the free list. Else it belongs
1867 * to some object, and we hold a
1868 * paging reference on that object.
1869 */
1870 if (!m->tabled) {
1871 VM_PAGE_FREE(m)({ ; vm_page_free(m); ; });
1872 }
1873 else {
1874 vm_object_t object;
1875
1876 object = m->object;
1877
1878 vm_object_lock(object);
1879 vm_page_lock_queues();
1880 if (!m->active && !m->inactive)
1881 vm_page_activate(m);
1882 vm_page_unlock_queues();
1883
1884 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } });
1885 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_map.c", 1885); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
1886 vm_object_unlock(object);
1887 }
1888 }
1889 }
1890}
1891
1892/*
1893 * Routine: vm_map_copy_discard
1894 *
1895 * Description:
1896 * Dispose of a map copy object (returned by
1897 * vm_map_copyin).
1898 */
1899void
1900vm_map_copy_discard(copy)
1901 vm_map_copy_t copy;
1902{
1903free_next_copy:
1904 if (copy == VM_MAP_COPY_NULL((vm_map_copy_t) 0))
1905 return;
1906
1907 switch (copy->type) {
1908 case VM_MAP_COPY_ENTRY_LIST1:
1909 while (vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next) !=
1910 vm_map_copy_to_entry(copy)((struct vm_map_entry *) &(copy)->c_u.hdr.links)) {
1911 vm_map_entry_t entry = vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next);
1912
1913 vm_map_copy_entry_unlink(copy, entry)({ (&(copy)->c_u.hdr)->nentries--; (entry)->links
.next->links.prev = (entry)->links.prev; (entry)->links
.prev->links.next = (entry)->links.next; rbtree_remove(
&(&(copy)->c_u.hdr)->tree, &(entry)->tree_node
); });
1914 vm_object_deallocate(entry->object.vm_object);
1915 vm_map_copy_entry_dispose(copy, entry)_vm_map_entry_dispose(&(copy)->c_u.hdr, (entry));
1916 }
1917 break;
1918 case VM_MAP_COPY_OBJECT2:
1919 vm_object_deallocate(copy->cpy_objectc_u.c_o.object);
1920 break;
1921 case VM_MAP_COPY_PAGE_LIST3:
1922
1923 /*
1924 * To clean this up, we have to unbusy all the pages
1925 * and release the paging references in their objects.
1926 */
1927 if (copy->cpy_npagesc_u.c_p.npages > 0)
1928 vm_map_copy_page_discard(copy);
1929
1930 /*
1931 * If there's a continuation, abort it. The
1932 * abort routine releases any storage.
1933 */
1934 if (vm_map_copy_has_cont(copy)(((copy)->c_u.c_p.cont) != (kern_return_t (*)()) 0)) {
1935
1936 /*
1937 * Special case: recognize
1938 * vm_map_copy_discard_cont and optimize
1939 * here to avoid tail recursion.
1940 */
1941 if (copy->cpy_contc_u.c_p.cont == vm_map_copy_discard_cont) {
1942 vm_map_copy_t new_copy;
1943
1944 new_copy = (vm_map_copy_t) copy->cpy_cont_argsc_u.c_p.cont_args;
1945 kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1946 copy = new_copy;
1947 goto free_next_copy;
1948 }
1949 else {
1950 vm_map_copy_abort_cont(copy)({ vm_map_copy_page_discard(copy); (*((copy)->c_u.c_p.cont
))((copy)->c_u.c_p.cont_args, (vm_map_copy_t *) 0); (copy)
->c_u.c_p.cont = (kern_return_t (*)()) 0; (copy)->c_u.c_p
.cont_args = (char *) 0; });
1951 }
1952 }
1953
1954 break;
1955 }
1956 kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1957}
1958
1959/*
1960 * Routine: vm_map_copy_copy
1961 *
1962 * Description:
1963 * Move the information in a map copy object to
1964 * a new map copy object, leaving the old one
1965 * empty.
1966 *
1967 * This is used by kernel routines that need
1968 * to look at out-of-line data (in copyin form)
1969 * before deciding whether to return SUCCESS.
1970 * If the routine returns FAILURE, the original
1971 * copy object will be deallocated; therefore,
1972 * these routines must make a copy of the copy
1973 * object and leave the original empty so that
1974 * deallocation will not fail.
1975 */
1976vm_map_copy_t
1977vm_map_copy_copy(copy)
1978 vm_map_copy_t copy;
1979{
1980 vm_map_copy_t new_copy;
1981
1982 if (copy == VM_MAP_COPY_NULL((vm_map_copy_t) 0))
1983 return VM_MAP_COPY_NULL((vm_map_copy_t) 0);
1984
1985 /*
1986 * Allocate a new copy object, and copy the information
1987 * from the old one into it.
1988 */
1989
1990 new_copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1991 *new_copy = *copy;
1992
1993 if (copy->type == VM_MAP_COPY_ENTRY_LIST1) {
1994 /*
1995 * The links in the entry chain must be
1996 * changed to point to the new copy object.
1997 */
1998 vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next)->vme_prevlinks.prev
1999 = vm_map_copy_to_entry(new_copy)((struct vm_map_entry *) &(new_copy)->c_u.hdr.links);
2000 vm_map_copy_last_entry(copy)((copy)->c_u.hdr.links.prev)->vme_nextlinks.next
2001 = vm_map_copy_to_entry(new_copy)((struct vm_map_entry *) &(new_copy)->c_u.hdr.links);
2002 }
2003
2004 /*
2005 * Change the old copy object into one that contains
2006 * nothing to be deallocated.
2007 */
2008 copy->type = VM_MAP_COPY_OBJECT2;
2009 copy->cpy_objectc_u.c_o.object = VM_OBJECT_NULL((vm_object_t) 0);
2010
2011 /*
2012 * Return the new object.
2013 */
2014 return new_copy;
2015}
2016
2017/*
2018 * Routine: vm_map_copy_discard_cont
2019 *
2020 * Description:
2021 * A version of vm_map_copy_discard that can be called
2022 * as a continuation from a vm_map_copy page list.
2023 */
2024kern_return_t vm_map_copy_discard_cont(cont_args, copy_result)
2025vm_map_copyin_args_t cont_args;
2026vm_map_copy_t *copy_result; /* OUT */
2027{
2028 vm_map_copy_discard((vm_map_copy_t) cont_args);
2029 if (copy_result != (vm_map_copy_t *)0)
2030 *copy_result = VM_MAP_COPY_NULL((vm_map_copy_t) 0);
2031 return(KERN_SUCCESS0);
2032}
2033
2034/*
2035 * Routine: vm_map_copy_overwrite
2036 *
2037 * Description:
2038 * Copy the memory described by the map copy
2039 * object (copy; returned by vm_map_copyin) onto
2040 * the specified destination region (dst_map, dst_addr).
2041 * The destination must be writeable.
2042 *
2043 * Unlike vm_map_copyout, this routine actually
2044 * writes over previously-mapped memory. If the
2045 * previous mapping was to a permanent (user-supplied)
2046 * memory object, it is preserved.
2047 *
2048 * The attributes (protection and inheritance) of the
2049 * destination region are preserved.
2050 *
2051 * If successful, consumes the copy object.
2052 * Otherwise, the caller is responsible for it.
2053 *
2054 * Implementation notes:
2055 * To overwrite temporary virtual memory, it is
2056 * sufficient to remove the previous mapping and insert
2057 * the new copy. This replacement is done either on
2058 * the whole region (if no permanent virtual memory
2059 * objects are embedded in the destination region) or
2060 * in individual map entries.
2061 *
2062 * To overwrite permanent virtual memory, it is
2063 * necessary to copy each page, as the external
2064 * memory management interface currently does not
2065 * provide any optimizations.
2066 *
2067 * Once a page of permanent memory has been overwritten,
2068 * it is impossible to interrupt this function; otherwise,
2069 * the call would be neither atomic nor location-independent.
2070 * The kernel-state portion of a user thread must be
2071 * interruptible.
2072 *
2073 * It may be expensive to forward all requests that might
2074 * overwrite permanent memory (vm_write, vm_copy) to
2075 * uninterruptible kernel threads. This routine may be
2076 * called by interruptible threads; however, success is
2077 * not guaranteed -- if the request cannot be performed
2078 * atomically and interruptibly, an error indication is
2079 * returned.
2080 */
2081kern_return_t vm_map_copy_overwrite(dst_map, dst_addr, copy, interruptible)
2082 vm_map_t dst_map;
2083 vm_offset_t dst_addr;
2084 vm_map_copy_t copy;
2085 boolean_t interruptible;
2086{
2087 vm_size_t size;
2088 vm_offset_t start;
2089 vm_map_entry_t tmp_entry;
2090 vm_map_entry_t entry;
2091
2092 boolean_t contains_permanent_objects = FALSE((boolean_t) 0);
2093
2094 interruptible = FALSE((boolean_t) 0); /* XXX */
2095
2096 /*
2097 * Check for null copy object.
2098 */
2099
2100 if (copy == VM_MAP_COPY_NULL((vm_map_copy_t) 0))
2101 return(KERN_SUCCESS0);
2102
2103 /*
2104 * Only works for entry lists at the moment. Will
2105 * support page lists LATER.
2106 */
2107
2108 assert(copy->type == VM_MAP_COPY_ENTRY_LIST)({ if (!(copy->type == 1)) Assert("copy->type == VM_MAP_COPY_ENTRY_LIST"
, "../vm/vm_map.c", 2108); });
2109
2110 /*
2111 * Currently this routine only handles page-aligned
2112 * regions. Eventually, it should handle misalignments
2113 * by actually copying pages.
2114 */
2115
2116 if (!page_aligned(copy->offset)((((vm_offset_t) (copy->offset)) & ((1 << 12)-1)
) == 0) ||
2117 !page_aligned(copy->size)((((vm_offset_t) (copy->size)) & ((1 << 12)-1)) ==
0) ||
2118 !page_aligned(dst_addr)((((vm_offset_t) (dst_addr)) & ((1 << 12)-1)) == 0))
2119 return(KERN_INVALID_ARGUMENT4);
2120
2121 size = copy->size;
2122
2123 if (size == 0) {
2124 vm_map_copy_discard(copy);
2125 return(KERN_SUCCESS0);
2126 }
2127
2128 /*
2129 * Verify that the destination is all writeable
2130 * initially.
2131 */
2132start_pass_1:
2133 vm_map_lock(dst_map)({ lock_write(&(dst_map)->lock); (dst_map)->timestamp
++; });
2134 if (!vm_map_lookup_entry(dst_map, dst_addr, &tmp_entry)) {
2135 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2136 return(KERN_INVALID_ADDRESS1);
2137 }
2138 vm_map_clip_start(dst_map, tmp_entry, dst_addr)({ if ((dst_addr) > (tmp_entry)->links.start) _vm_map_clip_start
(&(dst_map)->hdr,(tmp_entry),(dst_addr)); });
2139 for (entry = tmp_entry;;) {
2140 vm_size_t sub_size = (entry->vme_endlinks.end - entry->vme_startlinks.start);
2141 vm_map_entry_t next = entry->vme_nextlinks.next;
2142
2143 if ( ! (entry->protection & VM_PROT_WRITE((vm_prot_t) 0x02))) {
2144 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2145 return(KERN_PROTECTION_FAILURE2);
2146 }
2147
2148 /*
2149 * If the entry is in transition, we must wait
2150 * for it to exit that state. Anything could happen
2151 * when we unlock the map, so start over.
2152 */
2153 if (entry->in_transition) {
2154
2155 /*
2156 * Say that we are waiting, and wait for entry.
2157 */
2158 entry->needs_wakeup = TRUE((boolean_t) 1);
2159 vm_map_entry_wait(dst_map, FALSE)({ assert_wait((event_t)&(dst_map)->hdr, ((boolean_t) 0
)); lock_done(&(dst_map)->lock); thread_block((void (*
)()) 0); });
2160
2161 goto start_pass_1;
2162 }
2163
2164 if (size <= sub_size)
2165 break;
2166
2167 if ((next == vm_map_to_entry(dst_map)((struct vm_map_entry *) &(dst_map)->hdr.links)) ||
2168 (next->vme_startlinks.start != entry->vme_endlinks.end)) {
2169 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2170 return(KERN_INVALID_ADDRESS1);
2171 }
2172
2173
2174 /*
2175 * Check for permanent objects in the destination.
2176 */
2177
2178 if ((entry->object.vm_object != VM_OBJECT_NULL((vm_object_t) 0)) &&
2179 !entry->object.vm_object->temporary)
2180 contains_permanent_objects = TRUE((boolean_t) 1);
2181
2182 size -= sub_size;
2183 entry = next;
2184 }
2185
2186 /*
2187 * If there are permanent objects in the destination, then
2188 * the copy cannot be interrupted.
2189 */
2190
2191 if (interruptible && contains_permanent_objects) {
2192 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2193 return(KERN_FAILURE5); /* XXX */
2194 }
2195
2196 /*
2197 * XXXO If there are no permanent objects in the destination,
2198 * XXXO and the source and destination map entry caches match,
2199 * XXXO and the destination map entry is not shared,
2200 * XXXO then the map entries can be deleted and replaced
2201 * XXXO with those from the copy. The following code is the
2202 * XXXO basic idea of what to do, but there are lots of annoying
2203 * XXXO little details about getting protection and inheritance
2204 * XXXO right. Should add protection, inheritance, and sharing checks
2205 * XXXO to the above pass and make sure that no wiring is involved.
2206 */
2207/*
2208 * if (!contains_permanent_objects &&
2209 * copy->cpy_hdr.entries_pageable == dst_map->hdr.entries_pageable) {
2210 *
2211 * *
2212 * * Run over copy and adjust entries. Steal code
2213 * * from vm_map_copyout() to do this.
2214 * *
2215 *
2216 * tmp_entry = tmp_entry->vme_prev;
2217 * vm_map_delete(dst_map, dst_addr, dst_addr + copy->size);
2218 * vm_map_copy_insert(dst_map, tmp_entry, copy);
2219 *
2220 * vm_map_unlock(dst_map);
2221 * vm_map_copy_discard(copy);
2222 * }
2223 */
2224 /*
2225 *
2226 * Make a second pass, overwriting the data
2227 * At the beginning of each loop iteration,
2228 * the next entry to be overwritten is "tmp_entry"
2229 * (initially, the value returned from the lookup above),
2230 * and the starting address expected in that entry
2231 * is "start".
2232 */
2233
2234 start = dst_addr;
2235
2236 while (vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next) != vm_map_copy_to_entry(copy)((struct vm_map_entry *) &(copy)->c_u.hdr.links)) {
2237 vm_map_entry_t copy_entry = vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next);
2238 vm_size_t copy_size = (copy_entry->vme_endlinks.end - copy_entry->vme_startlinks.start);
2239 vm_object_t object;
2240
2241 entry = tmp_entry;
2242 size = (entry->vme_endlinks.end - entry->vme_startlinks.start);
2243 /*
2244 * Make sure that no holes popped up in the
2245 * address map, and that the protection is
2246 * still valid, in case the map was unlocked
2247 * earlier.
2248 */
2249
2250 if (entry->vme_startlinks.start != start) {
2251 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2252 return(KERN_INVALID_ADDRESS1);
2253 }
2254 assert(entry != vm_map_to_entry(dst_map))({ if (!(entry != ((struct vm_map_entry *) &(dst_map)->
hdr.links))) Assert("entry != vm_map_to_entry(dst_map)", "../vm/vm_map.c"
, 2254); });
2255
2256 /*
2257 * Check protection again
2258 */
2259
2260 if ( ! (entry->protection & VM_PROT_WRITE((vm_prot_t) 0x02))) {
2261 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2262 return(KERN_PROTECTION_FAILURE2);
2263 }
2264
2265 /*
2266 * Adjust to source size first
2267 */
2268
2269 if (copy_size < size) {
2270 vm_map_clip_end(dst_map, entry, entry->vme_start + copy_size)({ if ((entry->links.start + copy_size) < (entry)->links
.end) _vm_map_clip_end(&(dst_map)->hdr,(entry),(entry->
links.start + copy_size)); });
2271 size = copy_size;
2272 }
2273
2274 /*
2275 * Adjust to destination size
2276 */
2277
2278 if (size < copy_size) {
2279 vm_map_copy_clip_end(copy, copy_entry,({ if ((copy_entry->links.start + size) < (copy_entry)->
links.end) _vm_map_clip_end(&(copy)->c_u.hdr,(copy_entry
),(copy_entry->links.start + size)); })
2280 copy_entry->vme_start + size)({ if ((copy_entry->links.start + size) < (copy_entry)->
links.end) _vm_map_clip_end(&(copy)->c_u.hdr,(copy_entry
),(copy_entry->links.start + size)); });
2281 copy_size = size;
2282 }
2283
2284 assert((entry->vme_end - entry->vme_start) == size)({ if (!((entry->links.end - entry->links.start) == size
)) Assert("(entry->vme_end - entry->vme_start) == size"
, "../vm/vm_map.c", 2284); });
2285 assert((tmp_entry->vme_end - tmp_entry->vme_start) == size)({ if (!((tmp_entry->links.end - tmp_entry->links.start
) == size)) Assert("(tmp_entry->vme_end - tmp_entry->vme_start) == size"
, "../vm/vm_map.c", 2285); });
2286 assert((copy_entry->vme_end - copy_entry->vme_start) == size)({ if (!((copy_entry->links.end - copy_entry->links.start
) == size)) Assert("(copy_entry->vme_end - copy_entry->vme_start) == size"
, "../vm/vm_map.c", 2286); });
2287
2288 /*
2289 * If the destination contains temporary unshared memory,
2290 * we can perform the copy by throwing it away and
2291 * installing the source data.
2292 */
2293
2294 object = entry->object.vm_object;
2295 if (!entry->is_shared &&
2296 ((object == VM_OBJECT_NULL((vm_object_t) 0)) || object->temporary)) {
2297 vm_object_t old_object = entry->object.vm_object;
2298 vm_offset_t old_offset = entry->offset;
2299
2300 entry->object = copy_entry->object;
2301 entry->offset = copy_entry->offset;
2302 entry->needs_copy = copy_entry->needs_copy;
2303 entry->wired_count = 0;
2304 entry->user_wired_count = 0;
2305
2306 vm_map_copy_entry_unlink(copy, copy_entry)({ (&(copy)->c_u.hdr)->nentries--; (copy_entry)->
links.next->links.prev = (copy_entry)->links.prev; (copy_entry
)->links.prev->links.next = (copy_entry)->links.next
; rbtree_remove(&(&(copy)->c_u.hdr)->tree, &
(copy_entry)->tree_node); });
2307 vm_map_copy_entry_dispose(copy, copy_entry)_vm_map_entry_dispose(&(copy)->c_u.hdr, (copy_entry));
2308
2309 vm_object_pmap_protect(
2310 old_object,
2311 old_offset,
2312 size,
2313 dst_map->pmap,
2314 tmp_entry->vme_startlinks.start,
2315 VM_PROT_NONE((vm_prot_t) 0x00));
2316
2317 vm_object_deallocate(old_object);
2318
2319 /*
2320 * Set up for the next iteration. The map
2321 * has not been unlocked, so the next
2322 * address should be at the end of this
2323 * entry, and the next map entry should be
2324 * the one following it.
2325 */
2326
2327 start = tmp_entry->vme_endlinks.end;
2328 tmp_entry = tmp_entry->vme_nextlinks.next;
2329 } else {
2330 vm_map_version_t version;
2331 vm_object_t dst_object = entry->object.vm_object;
2332 vm_offset_t dst_offset = entry->offset;
2333 kern_return_t r;
2334
2335 /*
2336 * Take an object reference, and record
2337 * the map version information so that the
2338 * map can be safely unlocked.
2339 */
2340
2341 vm_object_reference(dst_object);
2342
2343 version.main_timestamp = dst_map->timestamp;
2344
2345 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2346
2347 /*
2348 * Copy as much as possible in one pass
2349 */
2350
2351 copy_size = size;
2352 r = vm_fault_copy(
2353 copy_entry->object.vm_object,
2354 copy_entry->offset,
2355 &copy_size,
2356 dst_object,
2357 dst_offset,
2358 dst_map,
2359 &version,
2360 FALSE((boolean_t) 0) /* XXX interruptible */ );
2361
2362 /*
2363 * Release the object reference
2364 */
2365
2366 vm_object_deallocate(dst_object);
2367
2368 /*
2369 * If a hard error occurred, return it now
2370 */
2371
2372 if (r != KERN_SUCCESS0)
2373 return(r);
2374
2375 if (copy_size != 0) {
2376 /*
2377 * Dispose of the copied region
2378 */
2379
2380 vm_map_copy_clip_end(copy, copy_entry,({ if ((copy_entry->links.start + copy_size) < (copy_entry
)->links.end) _vm_map_clip_end(&(copy)->c_u.hdr,(copy_entry
),(copy_entry->links.start + copy_size)); })
2381 copy_entry->vme_start + copy_size)({ if ((copy_entry->links.start + copy_size) < (copy_entry
)->links.end) _vm_map_clip_end(&(copy)->c_u.hdr,(copy_entry
),(copy_entry->links.start + copy_size)); });
2382 vm_map_copy_entry_unlink(copy, copy_entry)({ (&(copy)->c_u.hdr)->nentries--; (copy_entry)->
links.next->links.prev = (copy_entry)->links.prev; (copy_entry
)->links.prev->links.next = (copy_entry)->links.next
; rbtree_remove(&(&(copy)->c_u.hdr)->tree, &
(copy_entry)->tree_node); });
2383 vm_object_deallocate(copy_entry->object.vm_object);
2384 vm_map_copy_entry_dispose(copy, copy_entry)_vm_map_entry_dispose(&(copy)->c_u.hdr, (copy_entry));
2385 }
2386
2387 /*
2388 * Pick up in the destination map where we left off.
2389 *
2390 * Use the version information to avoid a lookup
2391 * in the normal case.
2392 */
2393
2394 start += copy_size;
2395 vm_map_lock(dst_map)({ lock_write(&(dst_map)->lock); (dst_map)->timestamp
++; });
2396 if ((version.main_timestamp + 1) == dst_map->timestamp) {
2397 /* We can safely use saved tmp_entry value */
2398
2399 vm_map_clip_end(dst_map, tmp_entry, start)({ if ((start) < (tmp_entry)->links.end) _vm_map_clip_end
(&(dst_map)->hdr,(tmp_entry),(start)); });
2400 tmp_entry = tmp_entry->vme_nextlinks.next;
2401 } else {
2402 /* Must do lookup of tmp_entry */
2403
2404 if (!vm_map_lookup_entry(dst_map, start, &tmp_entry)) {
2405 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2406 return(KERN_INVALID_ADDRESS1);
2407 }
2408 vm_map_clip_start(dst_map, tmp_entry, start)({ if ((start) > (tmp_entry)->links.start) _vm_map_clip_start
(&(dst_map)->hdr,(tmp_entry),(start)); });
2409 }
2410 }
2411
2412 }
2413 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2414
2415 /*
2416 * Throw away the vm_map_copy object
2417 */
2418 vm_map_copy_discard(copy);
2419
2420 return(KERN_SUCCESS0);
2421}
2422
2423/*
2424 * Macro: vm_map_copy_insert
2425 *
2426 * Description:
2427 * Link a copy chain ("copy") into a map at the
2428 * specified location (after "where").
2429 * Side effects:
2430 * The copy chain is destroyed.
2431 * Warning:
2432 * The arguments are evaluated multiple times.
2433 */
2434#define vm_map_copy_insert(map, where, copy)({ struct rbtree_node *node, *tmp; for (node = rbtree_postwalk_deepest
(&(copy)->c_u.hdr.tree), tmp = rbtree_postwalk_unlink(
node); node != ((void *) 0); node = tmp, tmp = rbtree_postwalk_unlink
(node)) ({ struct rbtree_node *___cur, *___prev; int ___diff,
___index; ___prev = ((void *) 0); ___index = -1; ___cur = (&
(map)->hdr.tree)->root; while (___cur != ((void *) 0)) {
___diff = vm_map_entry_cmp_insert(node, ___cur); ({ if (!(___diff
!= 0)) Assert("___diff != 0", "../vm/vm_map.c", 2434); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(map)->
hdr.tree, ___prev, ___index, node); }); (((where)->links.next
)->links.prev = ((copy)->c_u.hdr.links.prev)) ->links
.next = ((where)->links.next); ((where)->links.next = (
(copy)->c_u.hdr.links.next)) ->links.prev = (where); (map
)->hdr.nentries += (copy)->c_u.hdr.nentries; kmem_cache_free
(&vm_map_copy_cache, (vm_offset_t) copy); }) \
2435 MACRO_BEGIN({ \
2436 struct rbtree_node *node, *tmp; \
2437 rbtree_for_each_remove(&(copy)->cpy_hdr.tree, node, tmp)for (node = rbtree_postwalk_deepest(&(copy)->c_u.hdr.tree
), tmp = rbtree_postwalk_unlink(node); node != ((void *) 0); node
= tmp, tmp = rbtree_postwalk_unlink(node)) \
2438 rbtree_insert(&(map)->hdr.tree, node, \({ struct rbtree_node *___cur, *___prev; int ___diff, ___index
; ___prev = ((void *) 0); ___index = -1; ___cur = (&(map)
->hdr.tree)->root; while (___cur != ((void *) 0)) { ___diff
= vm_map_entry_cmp_insert(node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 2439); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(map)->
hdr.tree, ___prev, ___index, node); })
2439 vm_map_entry_cmp_insert)({ struct rbtree_node *___cur, *___prev; int ___diff, ___index
; ___prev = ((void *) 0); ___index = -1; ___cur = (&(map)
->hdr.tree)->root; while (___cur != ((void *) 0)) { ___diff
= vm_map_entry_cmp_insert(node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 2439); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(map)->
hdr.tree, ___prev, ___index, node); }); \
2440 (((where)->vme_nextlinks.next)->vme_prevlinks.prev = vm_map_copy_last_entry(copy)((copy)->c_u.hdr.links.prev)) \
2441 ->vme_nextlinks.next = ((where)->vme_nextlinks.next); \
2442 ((where)->vme_nextlinks.next = vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next)) \
2443 ->vme_prevlinks.prev = (where); \
2444 (map)->hdr.nentries += (copy)->cpy_hdrc_u.hdr.nentries; \
2445 kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); \
2446 MACRO_END})
2447
2448/*
2449 * Routine: vm_map_copyout
2450 *
2451 * Description:
2452 * Copy out a copy chain ("copy") into newly-allocated
2453 * space in the destination map.
2454 *
2455 * If successful, consumes the copy object.
2456 * Otherwise, the caller is responsible for it.
2457 */
2458kern_return_t vm_map_copyout(dst_map, dst_addr, copy)
2459 vm_map_t dst_map;
2460 vm_offset_t *dst_addr; /* OUT */
2461 vm_map_copy_t copy;
2462{
2463 vm_size_t size;
2464 vm_size_t adjustment;
2465 vm_offset_t start;
2466 vm_offset_t vm_copy_start;
2467 vm_map_entry_t last;
2468 vm_map_entry_t entry;
2469
2470 /*
2471 * Check for null copy object.
2472 */
2473
2474 if (copy == VM_MAP_COPY_NULL((vm_map_copy_t) 0)) {
2475 *dst_addr = 0;
2476 return(KERN_SUCCESS0);
2477 }
2478
2479 /*
2480 * Check for special copy object, created
2481 * by vm_map_copyin_object.
2482 */
2483
2484 if (copy->type == VM_MAP_COPY_OBJECT2) {
2485 vm_object_t object = copy->cpy_objectc_u.c_o.object;
2486 vm_size_t offset = copy->offset;
2487 vm_size_t tmp_size = copy->size;
2488 kern_return_t kr;
2489
2490 *dst_addr = 0;
2491 kr = vm_map_enter(dst_map, dst_addr, tmp_size,
2492 (vm_offset_t) 0, TRUE((boolean_t) 1),
2493 object, offset, FALSE((boolean_t) 0),
2494 VM_PROT_DEFAULT(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)), VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04)),
2495 VM_INHERIT_DEFAULT((vm_inherit_t) 1));
2496 if (kr != KERN_SUCCESS0)
2497 return(kr);
2498 kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
2499 return(KERN_SUCCESS0);
2500 }
2501
2502 if (copy->type == VM_MAP_COPY_PAGE_LIST3)
2503 return(vm_map_copyout_page_list(dst_map, dst_addr, copy));
2504
2505 /*
2506 * Find space for the data
2507 */
2508
2509 vm_copy_start = trunc_page(copy->offset)((vm_offset_t)(((vm_offset_t)(copy->offset)) & ~((1 <<
12)-1)));
2510 size = round_page(copy->offset + copy->size)((vm_offset_t)((((vm_offset_t)(copy->offset + copy->size
)) + ((1 << 12)-1)) & ~((1 << 12)-1))) - vm_copy_start;
2511
2512 StartAgain: ;
2513
2514 vm_map_lock(dst_map)({ lock_write(&(dst_map)->lock); (dst_map)->timestamp
++; });
2515 start = ((last = dst_map->first_free) == vm_map_to_entry(dst_map)((struct vm_map_entry *) &(dst_map)->hdr.links)) ?
2516 vm_map_min(dst_map)((dst_map)->hdr.links.start) : last->vme_endlinks.end;
2517
2518 while (TRUE((boolean_t) 1)) {
2519 vm_map_entry_t next = last->vme_nextlinks.next;
2520 vm_offset_t end = start + size;
2521
2522 if ((end > dst_map->max_offsethdr.links.end) || (end < start)) {
2523 if (dst_map->wait_for_space) {
2524 if (size <= (dst_map->max_offsethdr.links.end - dst_map->min_offsethdr.links.start)) {
2525 assert_wait((event_t) dst_map, TRUE((boolean_t) 1));
2526 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2527 thread_block((void (*)()) 0);
2528 goto StartAgain;
2529 }
2530 }
2531 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2532 printf_once("no more room for vm_map_copyout in %p\n", dst_map)({ static int __once = 0; if (!__once) { printf("no more room for vm_map_copyout in %p\n"
, dst_map); __once = 1; } });
2533 return(KERN_NO_SPACE3);
2534 }
2535
2536 if ((next == vm_map_to_entry(dst_map)((struct vm_map_entry *) &(dst_map)->hdr.links)) ||
2537 (next->vme_startlinks.start >= end))
2538 break;
2539
2540 last = next;
2541 start = last->vme_endlinks.end;
2542 }
2543
2544 /*
2545 * Since we're going to just drop the map
2546 * entries from the copy into the destination
2547 * map, they must come from the same pool.
2548 */
2549
2550 if (copy->cpy_hdrc_u.hdr.entries_pageable != dst_map->hdr.entries_pageable) {
2551 /*
2552 * Mismatches occur when dealing with the default
2553 * pager.
2554 */
2555 kmem_cache_t old_cache;
2556 vm_map_entry_t next, new;
2557
2558 /*
2559 * Find the cache that the copies were allocated from
2560 */
2561 old_cache = (copy->cpy_hdrc_u.hdr.entries_pageable)
2562 ? &vm_map_entry_cache
2563 : &vm_map_kentry_cache;
2564 entry = vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next);
2565
2566 /*
2567 * Reinitialize the copy so that vm_map_copy_entry_link
2568 * will work.
2569 */
2570 copy->cpy_hdrc_u.hdr.nentries = 0;
2571 copy->cpy_hdrc_u.hdr.entries_pageable = dst_map->hdr.entries_pageable;
2572 vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next) =
2573 vm_map_copy_last_entry(copy)((copy)->c_u.hdr.links.prev) =
2574 vm_map_copy_to_entry(copy)((struct vm_map_entry *) &(copy)->c_u.hdr.links);
2575
2576 /*
2577 * Copy each entry.
2578 */
2579 while (entry != vm_map_copy_to_entry(copy)((struct vm_map_entry *) &(copy)->c_u.hdr.links)) {
2580 new = vm_map_copy_entry_create(copy)_vm_map_entry_create(&(copy)->c_u.hdr);
2581 vm_map_entry_copy_full(new, entry)(*(new) = *(entry));
2582 vm_map_copy_entry_link(copy,({ (&(copy)->c_u.hdr)->nentries++; (new)->links.
prev = (((copy)->c_u.hdr.links.prev)); (new)->links.next
= (((copy)->c_u.hdr.links.prev))->links.next; (new)->
links.prev->links.next = (new)->links.next->links.prev
= (new); ({ struct rbtree_node *___cur, *___prev; int ___diff
, ___index; ___prev = ((void *) 0); ___index = -1; ___cur = (
&(&(copy)->c_u.hdr)->tree)->root; while (___cur
!= ((void *) 0)) { ___diff = vm_map_entry_cmp_insert(&(new
)->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0"
, "../vm/vm_map.c", 2584); }); ___prev = ___cur; ___index = rbtree_d2i
(___diff); ___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(&(copy)->c_u.hdr)->tree, ___prev, ___index, &
(new)->tree_node); }); })
2583 vm_map_copy_last_entry(copy),({ (&(copy)->c_u.hdr)->nentries++; (new)->links.
prev = (((copy)->c_u.hdr.links.prev)); (new)->links.next
= (((copy)->c_u.hdr.links.prev))->links.next; (new)->
links.prev->links.next = (new)->links.next->links.prev
= (new); ({ struct rbtree_node *___cur, *___prev; int ___diff
, ___index; ___prev = ((void *) 0); ___index = -1; ___cur = (
&(&(copy)->c_u.hdr)->tree)->root; while (___cur
!= ((void *) 0)) { ___diff = vm_map_entry_cmp_insert(&(new
)->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0"
, "../vm/vm_map.c", 2584); }); ___prev = ___cur; ___index = rbtree_d2i
(___diff); ___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(&(copy)->c_u.hdr)->tree, ___prev, ___index, &
(new)->tree_node); }); })
2584 new)({ (&(copy)->c_u.hdr)->nentries++; (new)->links.
prev = (((copy)->c_u.hdr.links.prev)); (new)->links.next
= (((copy)->c_u.hdr.links.prev))->links.next; (new)->
links.prev->links.next = (new)->links.next->links.prev
= (new); ({ struct rbtree_node *___cur, *___prev; int ___diff
, ___index; ___prev = ((void *) 0); ___index = -1; ___cur = (
&(&(copy)->c_u.hdr)->tree)->root; while (___cur
!= ((void *) 0)) { ___diff = vm_map_entry_cmp_insert(&(new
)->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0"
, "../vm/vm_map.c", 2584); }); ___prev = ___cur; ___index = rbtree_d2i
(___diff); ___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(&(copy)->c_u.hdr)->tree, ___prev, ___index, &
(new)->tree_node); }); });
2585 next = entry->vme_nextlinks.next;
2586 kmem_cache_free(old_cache, (vm_offset_t) entry);
2587 entry = next;
2588 }
2589 }
2590
2591 /*
2592 * Adjust the addresses in the copy chain, and
2593 * reset the region attributes.
2594 */
2595
2596 adjustment = start - vm_copy_start;
2597 for (entry = vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next);
2598 entry != vm_map_copy_to_entry(copy)((struct vm_map_entry *) &(copy)->c_u.hdr.links);
2599 entry = entry->vme_nextlinks.next) {
2600 entry->vme_startlinks.start += adjustment;
2601 entry->vme_endlinks.end += adjustment;
2602
2603 entry->inheritance = VM_INHERIT_DEFAULT((vm_inherit_t) 1);
2604 entry->protection = VM_PROT_DEFAULT(((vm_prot_t) 0x01)|((vm_prot_t) 0x02));
2605 entry->max_protection = VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04));
2606 entry->projected_on = 0;
2607
2608 /*
2609 * If the entry is now wired,
2610 * map the pages into the destination map.
2611 */
2612 if (entry->wired_count != 0) {
2613 vm_offset_t va;
2614 vm_offset_t offset;
2615 vm_object_t object;
2616
2617 object = entry->object.vm_object;
2618 offset = entry->offset;
2619 va = entry->vme_startlinks.start;
2620
2621 pmap_pageable(dst_map->pmap,
2622 entry->vme_startlinks.start,
2623 entry->vme_endlinks.end,
2624 TRUE((boolean_t) 1));
2625
2626 while (va < entry->vme_endlinks.end) {
2627 vm_page_t m;
2628
2629 /*
2630 * Look up the page in the object.
2631 * Assert that the page will be found in the
2632 * top object:
2633 * either
2634 * the object was newly created by
2635 * vm_object_copy_slowly, and has
2636 * copies of all of the pages from
2637 * the source object
2638 * or
2639 * the object was moved from the old
2640 * map entry; because the old map
2641 * entry was wired, all of the pages
2642 * were in the top-level object.
2643 * (XXX not true if we wire pages for
2644 * reading)
2645 */
2646 vm_object_lock(object);
2647 vm_object_paging_begin(object)((object)->paging_in_progress++);
2648
2649 m = vm_page_lookup(object, offset);
2650 if (m == VM_PAGE_NULL((vm_page_t) 0) || m->wire_count == 0 ||
2651 m->absent)
2652 panic("vm_map_copyout: wiring 0x%x", m);
2653
2654 m->busy = TRUE((boolean_t) 1);
2655 vm_object_unlock(object);
2656
2657 PMAP_ENTER(dst_map->pmap, va, m,({ pmap_enter( (dst_map->pmap), (va), (m)->phys_addr, (
entry->protection) & ~(m)->page_lock, (((boolean_t)
1)) ); })
2658 entry->protection, TRUE)({ pmap_enter( (dst_map->pmap), (va), (m)->phys_addr, (
entry->protection) & ~(m)->page_lock, (((boolean_t)
1)) ); });
2659
2660 vm_object_lock(object);
2661 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } });
2662 /* the page is wired, so we don't have to activate */
2663 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_map.c", 2663); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
2664 vm_object_unlock(object);
2665
2666 offset += PAGE_SIZE(1 << 12);
2667 va += PAGE_SIZE(1 << 12);
2668 }
2669 }
2670
2671
2672 }
2673
2674 /*
2675 * Correct the page alignment for the result
2676 */
2677
2678 *dst_addr = start + (copy->offset - vm_copy_start);
2679
2680 /*
2681 * Update the hints and the map size
2682 */
2683
2684 if (dst_map->first_free == last)
2685 dst_map->first_free = vm_map_copy_last_entry(copy)((copy)->c_u.hdr.links.prev);
2686 SAVE_HINT(dst_map, vm_map_copy_last_entry(copy)); (dst_map)->hint = (((copy)->c_u.hdr.links.prev)); ;;
2687
2688 dst_map->size += size;
2689
2690 /*
2691 * Link in the copy
2692 */
2693
2694 vm_map_copy_insert(dst_map, last, copy)({ struct rbtree_node *node, *tmp; for (node = rbtree_postwalk_deepest
(&(copy)->c_u.hdr.tree), tmp = rbtree_postwalk_unlink(
node); node != ((void *) 0); node = tmp, tmp = rbtree_postwalk_unlink
(node)) ({ struct rbtree_node *___cur, *___prev; int ___diff,
___index; ___prev = ((void *) 0); ___index = -1; ___cur = (&
(dst_map)->hdr.tree)->root; while (___cur != ((void *) 0
)) { ___diff = vm_map_entry_cmp_insert(node, ___cur); ({ if (
!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c", 2694
); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur
= ___cur->children[___index]; } rbtree_insert_rebalance(&
(dst_map)->hdr.tree, ___prev, ___index, node); }); (((last
)->links.next)->links.prev = ((copy)->c_u.hdr.links.
prev)) ->links.next = ((last)->links.next); ((last)->
links.next = ((copy)->c_u.hdr.links.next)) ->links.prev
= (last); (dst_map)->hdr.nentries += (copy)->c_u.hdr.nentries
; kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy)
; });
2695
2696 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2697
2698 /*
2699 * XXX If wiring_required, call vm_map_pageable
2700 */
2701
2702 return(KERN_SUCCESS0);
2703}
2704
2705/*
2706 *
2707 * vm_map_copyout_page_list:
2708 *
2709 * Version of vm_map_copyout() for page list vm map copies.
2710 *
2711 */
2712kern_return_t vm_map_copyout_page_list(dst_map, dst_addr, copy)
2713 vm_map_t dst_map;
2714 vm_offset_t *dst_addr; /* OUT */
2715 vm_map_copy_t copy;
2716{
2717 vm_size_t size;
2718 vm_offset_t start;
2719 vm_offset_t end;
2720 vm_offset_t offset;
2721 vm_map_entry_t last;
2722 vm_object_t object;
2723 vm_page_t *page_list, m;
2724 vm_map_entry_t entry;
2725 vm_offset_t old_last_offset;
2726 boolean_t cont_invoked, needs_wakeup = FALSE((boolean_t) 0);
2727 kern_return_t result = KERN_SUCCESS0;
2728 vm_map_copy_t orig_copy;
2729 vm_offset_t dst_offset;
2730 boolean_t must_wire;
2731
2732 /*
2733 * Make sure the pages are stolen, because we are
2734 * going to put them in a new object. Assume that
2735 * all pages are identical to first in this regard.
2736 */
2737
2738 page_list = &copy->cpy_page_listc_u.c_p.page_list[0];
2739 if ((*page_list)->tabled)
1
Taking false branch
→
2740 vm_map_copy_steal_pages(copy);
2741
2742 /*
2743 * Find space for the data
2744 */
2745
2746 size = round_page(copy->offset + copy->size)((vm_offset_t)((((vm_offset_t)(copy->offset + copy->size
)) + ((1 << 12)-1)) & ~((1 << 12)-1))) -
2747 trunc_page(copy->offset)((vm_offset_t)(((vm_offset_t)(copy->offset)) & ~((1 <<
12)-1)));
2748StartAgain:
2749 vm_map_lock(dst_map)({ lock_write(&(dst_map)->lock); (dst_map)->timestamp
++; });
2750 must_wire = dst_map->wiring_required;
2751
2752 last = dst_map->first_free;
2753 if (last == vm_map_to_entry(dst_map)((struct vm_map_entry *) &(dst_map)->hdr.links)) {
2
←
Taking false branch
→
2754 start = vm_map_min(dst_map)((dst_map)->hdr.links.start);
2755 } else {
2756 start = last->vme_endlinks.end;
2757 }
2758
2759 while (TRUE((boolean_t) 1)) {
3
←
Loop condition is true. Entering loop body
→
6
←
Loop condition is true. Entering loop body
→
9
←
Loop condition is true. Entering loop body
→
12
←
Loop condition is true. Entering loop body
→
2760 vm_map_entry_t next = last->vme_nextlinks.next;
2761 end = start + size;
2762
2763 if ((end > dst_map->max_offsethdr.links.end) || (end < start)) {
4
←
Taking false branch
→
7
←
Taking false branch
→
10
←
Taking false branch
→
13
←
Taking false branch
→
2764 if (dst_map->wait_for_space) {
2765 if (size <= (dst_map->max_offsethdr.links.end -
2766 dst_map->min_offsethdr.links.start)) {
2767 assert_wait((event_t) dst_map, TRUE((boolean_t) 1));
2768 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2769 thread_block((void (*)()) 0);
2770 goto StartAgain;
2771 }
2772 }
2773 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2774 printf_once("no more room for vm_map_copyout_page_list in %p\n", dst_map)({ static int __once = 0; if (!__once) { printf("no more room for vm_map_copyout_page_list in %p\n"
, dst_map); __once = 1; } });
2775 return(KERN_NO_SPACE3);
2776 }
2777
2778 if ((next == vm_map_to_entry(dst_map)((struct vm_map_entry *) &(dst_map)->hdr.links)) ||
5
←
Taking false branch
→
8
←
Taking false branch
→
11
←
Taking false branch
→
14
←
Taking true branch
→
2779 (next->vme_startlinks.start >= end)) {
2780 break;
15
←
 Execution continues on line 2796
→
2781 }
2782
2783 last = next;
2784 start = last->vme_endlinks.end;
2785 }
2786
2787 /*
2788 * See whether we can avoid creating a new entry (and object) by
2789 * extending one of our neighbors. [So far, we only attempt to
2790 * extend from below.]
2791 *
2792 * The code path below here is a bit twisted. If any of the
2793 * extension checks fails, we branch to create_object. If
2794 * it all works, we fall out the bottom and goto insert_pages.
2795 */
2796 if (last == vm_map_to_entry(dst_map)((struct vm_map_entry *) &(dst_map)->hdr.links) ||
18
←
Taking false branch
→
2797 last->vme_endlinks.end != start ||
2798 last->is_shared != FALSE((boolean_t) 0) ||
2799 last->is_sub_map != FALSE((boolean_t) 0) ||
2800 last->inheritance != VM_INHERIT_DEFAULT((vm_inherit_t) 1) ||
2801 last->protection != VM_PROT_DEFAULT(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)) ||
2802 last->max_protection != VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04)) ||
2803 (must_wire ? (last->wired_count != 1 ||
16
←
Assuming 'must_wire' is 0
→
17
←
'?' condition is false
→
2804 last->user_wired_count != 1) :
2805 (last->wired_count != 0))) {
2806 goto create_object;
2807 }
2808
2809 /*
2810 * If this entry needs an object, make one.
2811 */
2812 if (last->object.vm_object == VM_OBJECT_NULL((vm_object_t) 0)) {
19
←
Taking false branch
→
2813 object = vm_object_allocate(
2814 (vm_size_t)(last->vme_endlinks.end - last->vme_startlinks.start + size));
2815 last->object.vm_object = object;
2816 last->offset = 0;
2817 vm_object_lock(object);
2818 }
2819 else {
2820 vm_offset_t prev_offset = last->offset;
2821 vm_size_t prev_size = start - last->vme_startlinks.start;
2822 vm_size_t new_size;
2823
2824 /*
2825 * This is basically vm_object_coalesce.
2826 */
2827
2828 object = last->object.vm_object;
2829 vm_object_lock(object);
2830
2831 /*
2832 * Try to collapse the object first
2833 */
2834 vm_object_collapse(object);
2835
2836 /*
2837 * Can't coalesce if pages not mapped to
2838 * last may be in use anyway:
2839 * . more than one reference
2840 * . paged out
2841 * . shadows another object
2842 * . has a copy elsewhere
2843 * . paging references (pages might be in page-list)
2844 */
2845
2846 if ((object->ref_count > 1) ||
20
←
Taking false branch
→
2847 object->pager_created ||
2848 (object->shadow != VM_OBJECT_NULL((vm_object_t) 0)) ||
2849 (object->copy != VM_OBJECT_NULL((vm_object_t) 0)) ||
2850 (object->paging_in_progress != 0)) {
2851 vm_object_unlock(object);
2852 goto create_object;
2853 }
2854
2855 /*
2856 * Extend the object if necessary. Don't have to call
2857 * vm_object_page_remove because the pages aren't mapped,
2858 * and vm_page_replace will free up any old ones it encounters.
2859 */
2860 new_size = prev_offset + prev_size + size;
2861 if (new_size > object->size)
21
←
Taking false branch
→
2862 object->size = new_size;
2863 }
2864
2865 /*
2866 * Coalesced the two objects - can extend
2867 * the previous map entry to include the
2868 * new range.
2869 */
2870 dst_map->size += size;
2871 last->vme_endlinks.end = end;
2872
2873 SAVE_HINT(dst_map, last); (dst_map)->hint = (last); ;;
2874
2875 goto insert_pages;
22
←
Control jumps to line 2938
→
2876
2877create_object:
2878
2879 /*
2880 * Create object
2881 */
2882 object = vm_object_allocate(size);
2883
2884 /*
2885 * Create entry
2886 */
2887
2888 entry = vm_map_entry_create(dst_map)_vm_map_entry_create(&(dst_map)->hdr);
2889
2890 entry->object.vm_object = object;
2891 entry->offset = 0;
2892
2893 entry->is_shared = FALSE((boolean_t) 0);
2894 entry->is_sub_map = FALSE((boolean_t) 0);
2895 entry->needs_copy = FALSE((boolean_t) 0);
2896
2897 if (must_wire) {
2898 entry->wired_count = 1;
2899 entry->user_wired_count = 1;
2900 } else {
2901 entry->wired_count = 0;
2902 entry->user_wired_count = 0;
2903 }
2904
2905 entry->in_transition = TRUE((boolean_t) 1);
2906 entry->needs_wakeup = FALSE((boolean_t) 0);
2907
2908 entry->vme_startlinks.start = start;
2909 entry->vme_endlinks.end = start + size;
2910
2911 entry->inheritance = VM_INHERIT_DEFAULT((vm_inherit_t) 1);
2912 entry->protection = VM_PROT_DEFAULT(((vm_prot_t) 0x01)|((vm_prot_t) 0x02));
2913 entry->max_protection = VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04));
2914 entry->projected_on = 0;
2915
2916 vm_object_lock(object);
2917
2918 /*
2919 * Update the hints and the map size
2920 */
2921 if (dst_map->first_free == last) {
2922 dst_map->first_free = entry;
2923 }
2924 SAVE_HINT(dst_map, entry); (dst_map)->hint = (entry); ;;
2925 dst_map->size += size;
2926
2927 /*
2928 * Link in the entry
2929 */
2930 vm_map_entry_link(dst_map, last, entry)({ (&(dst_map)->hdr)->nentries++; (entry)->links
.prev = (last); (entry)->links.next = (last)->links.next
; (entry)->links.prev->links.next = (entry)->links.next
->links.prev = (entry); ({ struct rbtree_node *___cur, *___prev
; int ___diff, ___index; ___prev = ((void *) 0); ___index = -
1; ___cur = (&(&(dst_map)->hdr)->tree)->root
; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(entry)->tree_node, ___cur); ({ if (!(___diff != 0))
Assert("___diff != 0", "../vm/vm_map.c", 2930); }); ___prev =
___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(dst_map
)->hdr)->tree, ___prev, ___index, &(entry)->tree_node
); }); });
2931 last = entry;
2932
2933 /*
2934 * Transfer pages into new object.
2935 * Scan page list in vm_map_copy.
2936 */
2937insert_pages:
2938 dst_offset = copy->offset & PAGE_MASK((1 << 12)-1);
2939 cont_invoked = FALSE((boolean_t) 0);
2940 orig_copy = copy;
2941 last->in_transition = TRUE((boolean_t) 1);
2942 old_last_offset = last->offset
2943 + (start - last->vme_startlinks.start);
2944
2945 vm_page_lock_queues();
2946
2947 for (offset = 0; offset < size; offset += PAGE_SIZE(1 << 12)) {
23
←
Assuming 'offset' is < 'size'
→
24
←
Loop condition is true. Entering loop body
→
31
←
Assuming 'offset' is < 'size'
→
32
←
Loop condition is true. Entering loop body
→
2948 m = *page_list;
2949 assert(m && !m->tabled)({ if (!(m && !m->tabled)) Assert("m && !m->tabled"
, "../vm/vm_map.c", 2949); });
2950
2951 /*
2952 * Must clear busy bit in page before inserting it.
2953 * Ok to skip wakeup logic because nobody else
2954 * can possibly know about this page.
2955 * The page is dirty in its new object.
2956 */
2957
2958 assert(!m->wanted)({ if (!(!m->wanted)) Assert("!m->wanted", "../vm/vm_map.c"
, 2958); });
2959
2960 m->busy = FALSE((boolean_t) 0);
2961 m->dirty = TRUE((boolean_t) 1);
2962 vm_page_replace(m, object, old_last_offset + offset);
2963 if (must_wire) {
25
←
Taking false branch
→
33
←
Taking false branch
→
2964 vm_page_wire(m);
2965 PMAP_ENTER(dst_map->pmap,({ pmap_enter( (dst_map->pmap), (last->links.start + m->
offset - last->offset), (m)->phys_addr, (last->protection
) & ~(m)->page_lock, (((boolean_t) 1)) ); })
2966 last->vme_start + m->offset - last->offset,({ pmap_enter( (dst_map->pmap), (last->links.start + m->
offset - last->offset), (m)->phys_addr, (last->protection
) & ~(m)->page_lock, (((boolean_t) 1)) ); })
2967 m, last->protection, TRUE)({ pmap_enter( (dst_map->pmap), (last->links.start + m->
offset - last->offset), (m)->phys_addr, (last->protection
) & ~(m)->page_lock, (((boolean_t) 1)) ); });
2968 } else {
2969 vm_page_activate(m);
2970 }
2971
2972 *page_list++ = VM_PAGE_NULL((vm_page_t) 0);
2973 if (--(copy->cpy_npagesc_u.c_p.npages) == 0 &&
26
←
Taking true branch
→
34
←
Dereference of null pointer
2974 vm_map_copy_has_cont(copy)(((copy)->c_u.c_p.cont) != (kern_return_t (*)()) 0)) {
2975 vm_map_copy_t new_copy;
2976
2977 /*
2978 * Ok to unlock map because entry is
2979 * marked in_transition.
2980 */
2981 cont_invoked = TRUE((boolean_t) 1);
2982 vm_page_unlock_queues();
2983 vm_object_unlock(object);
2984 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2985 vm_map_copy_invoke_cont(copy, &new_copy, &result)({ vm_map_copy_page_discard(copy); *&result = (*((copy)->
c_u.c_p.cont))((copy)->c_u.c_p.cont_args, &new_copy); (
copy)->c_u.c_p.cont = (kern_return_t (*)()) 0; });
2986
2987 if (result == KERN_SUCCESS0) {
27
←
Assuming 'result' is equal to 0
→
28
←
Taking true branch
→
2988
2989 /*
2990 * If we got back a copy with real pages,
2991 * steal them now. Either all of the
2992 * pages in the list are tabled or none
2993 * of them are; mixtures are not possible.
2994 *
2995 * Save original copy for consume on
2996 * success logic at end of routine.
2997 */
2998 if (copy != orig_copy)
29
←
Taking false branch
→
2999 vm_map_copy_discard(copy);
3000
3001 if ((copy = new_copy) != VM_MAP_COPY_NULL((vm_map_copy_t) 0)) {
30
←
Taking false branch
→
3002 page_list = &copy->cpy_page_listc_u.c_p.page_list[0];
3003 if ((*page_list)->tabled)
3004 vm_map_copy_steal_pages(copy);
3005 }
3006 }
3007 else {
3008 /*
3009 * Continuation failed.
3010 */
3011 vm_map_lock(dst_map)({ lock_write(&(dst_map)->lock); (dst_map)->timestamp
++; });
3012 goto error;
3013 }
3014
3015 vm_map_lock(dst_map)({ lock_write(&(dst_map)->lock); (dst_map)->timestamp
++; });
3016 vm_object_lock(object);
3017 vm_page_lock_queues();
3018 }
3019 }
3020
3021 vm_page_unlock_queues();
3022 vm_object_unlock(object);
3023
3024 *dst_addr = start + dst_offset;
3025
3026 /*
3027 * Clear the in transition bits. This is easy if we
3028 * didn't have a continuation.
3029 */
3030error:
3031 if (!cont_invoked) {
3032 /*
3033 * We didn't unlock the map, so nobody could
3034 * be waiting.
3035 */
3036 last->in_transition = FALSE((boolean_t) 0);
3037 assert(!last->needs_wakeup)({ if (!(!last->needs_wakeup)) Assert("!last->needs_wakeup"
, "../vm/vm_map.c", 3037); });
3038 needs_wakeup = FALSE((boolean_t) 0);
3039 }
3040 else {
3041 if (!vm_map_lookup_entry(dst_map, start, &entry))
3042 panic("vm_map_copyout_page_list: missing entry");
3043
3044 /*
3045 * Clear transition bit for all constituent entries that
3046 * were in the original entry. Also check for waiters.
3047 */
3048 while((entry != vm_map_to_entry(dst_map)((struct vm_map_entry *) &(dst_map)->hdr.links)) &&
3049 (entry->vme_startlinks.start < end)) {
3050 assert(entry->in_transition)({ if (!(entry->in_transition)) Assert("entry->in_transition"
, "../vm/vm_map.c", 3050); });
3051 entry->in_transition = FALSE((boolean_t) 0);
3052 if(entry->needs_wakeup) {
3053 entry->needs_wakeup = FALSE((boolean_t) 0);
3054 needs_wakeup = TRUE((boolean_t) 1);
3055 }
3056 entry = entry->vme_nextlinks.next;
3057 }
3058 }
3059
3060 if (result != KERN_SUCCESS0)
3061 vm_map_delete(dst_map, start, end);
3062
3063 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
3064
3065 if (needs_wakeup)
3066 vm_map_entry_wakeup(dst_map)thread_wakeup_prim(((event_t)&(dst_map)->hdr), ((boolean_t
) 0), 0);
3067
3068 /*
3069 * Consume on success logic.
3070 */
3071 if (copy != orig_copy) {
3072 kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
3073 }
3074 if (result == KERN_SUCCESS0) {
3075 kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) orig_copy);
3076 }
3077
3078 return(result);
3079}
3080
3081/*
3082 * Routine: vm_map_copyin
3083 *
3084 * Description:
3085 * Copy the specified region (src_addr, len) from the
3086 * source address space (src_map), possibly removing
3087 * the region from the source address space (src_destroy).
3088 *
3089 * Returns:
3090 * A vm_map_copy_t object (copy_result), suitable for
3091 * insertion into another address space (using vm_map_copyout),
3092 * copying over another address space region (using
3093 * vm_map_copy_overwrite). If the copy is unused, it
3094 * should be destroyed (using vm_map_copy_discard).
3095 *
3096 * In/out conditions:
3097 * The source map should not be locked on entry.
3098 */
3099kern_return_t vm_map_copyin(src_map, src_addr, len, src_destroy, copy_result)
3100 vm_map_t src_map;
3101 vm_offset_t src_addr;
3102 vm_size_t len;
3103 boolean_t src_destroy;
3104 vm_map_copy_t *copy_result; /* OUT */
3105{
3106 vm_map_entry_t tmp_entry; /* Result of last map lookup --
3107 * in multi-level lookup, this
3108 * entry contains the actual
3109 * vm_object/offset.
3110 */
3111
3112 vm_offset_t src_start; /* Start of current entry --
3113 * where copy is taking place now
3114 */
3115 vm_offset_t src_end; /* End of entire region to be
3116 * copied */
3117
3118 vm_map_copy_t copy; /* Resulting copy */
3119
3120 /*
3121 * Check for copies of zero bytes.
3122 */
3123
3124 if (len == 0) {
3125 *copy_result = VM_MAP_COPY_NULL((vm_map_copy_t) 0);
3126 return(KERN_SUCCESS0);
3127 }
3128
3129 /*
3130 * Compute start and end of region
3131 */
3132
3133 src_start = trunc_page(src_addr)((vm_offset_t)(((vm_offset_t)(src_addr)) & ~((1 << 12
)-1)));
3134 src_end = round_page(src_addr + len)((vm_offset_t)((((vm_offset_t)(src_addr + len)) + ((1 <<
12)-1)) & ~((1 << 12)-1)));
3135
3136 /*
3137 * Check that the end address doesn't overflow
3138 */
3139
3140 if (src_end <= src_start)
3141 if ((src_end < src_start) || (src_start != 0))
3142 return(KERN_INVALID_ADDRESS1);
3143
3144 /*
3145 * Allocate a header element for the list.
3146 *
3147 * Use the start and end in the header to
3148 * remember the endpoints prior to rounding.
3149 */
3150
3151 copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
3152 vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next) =
3153 vm_map_copy_last_entry(copy)((copy)->c_u.hdr.links.prev) = vm_map_copy_to_entry(copy)((struct vm_map_entry *) &(copy)->c_u.hdr.links);
3154 copy->type = VM_MAP_COPY_ENTRY_LIST1;
3155 copy->cpy_hdrc_u.hdr.nentries = 0;
3156 copy->cpy_hdrc_u.hdr.entries_pageable = TRUE((boolean_t) 1);
3157 rbtree_init(&copy->cpy_hdrc_u.hdr.tree);
3158
3159 copy->offset = src_addr;
3160 copy->size = len;
3161
3162#define RETURN(x) \
3163 MACRO_BEGIN({ \
3164 vm_map_unlock(src_map)lock_done(&(src_map)->lock); \
3165 vm_map_copy_discard(copy); \
3166 MACRO_RETURNif (((boolean_t) 1)) return(x); \
3167 MACRO_END})
3168
3169 /*
3170 * Find the beginning of the region.
3171 */
3172
3173 vm_map_lock(src_map)({ lock_write(&(src_map)->lock); (src_map)->timestamp
++; });
3174
3175 if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry))
3176 RETURN(KERN_INVALID_ADDRESS1);
3177 vm_map_clip_start(src_map, tmp_entry, src_start)({ if ((src_start) > (tmp_entry)->links.start) _vm_map_clip_start
(&(src_map)->hdr,(tmp_entry),(src_start)); });
3178
3179 /*
3180 * Go through entries until we get to the end.
3181 */
3182
3183 while (TRUE((boolean_t) 1)) {
3184 vm_map_entry_t src_entry = tmp_entry; /* Top-level entry */
3185 vm_size_t src_size; /* Size of source
3186 * map entry (in both
3187 * maps)
3188 */
3189
3190 vm_object_t src_object; /* Object to copy */
3191 vm_offset_t src_offset;
3192
3193 boolean_t src_needs_copy; /* Should source map
3194 * be made read-only
3195 * for copy-on-write?
3196 */
3197
3198 vm_map_entry_t new_entry; /* Map entry for copy */
3199 boolean_t new_entry_needs_copy; /* Will new entry be COW? */
3200
3201 boolean_t was_wired; /* Was source wired? */
3202 vm_map_version_t version; /* Version before locks
3203 * dropped to make copy
3204 */
3205
3206 /*
3207 * Verify that the region can be read.
3208 */
3209
3210 if (! (src_entry->protection & VM_PROT_READ((vm_prot_t) 0x01)))
3211 RETURN(KERN_PROTECTION_FAILURE2);
3212
3213 /*
3214 * Clip against the endpoints of the entire region.
3215 */
3216
3217 vm_map_clip_end(src_map, src_entry, src_end)({ if ((src_end) < (src_entry)->links.end) _vm_map_clip_end
(&(src_map)->hdr,(src_entry),(src_end)); });
3218
3219 src_size = src_entry->vme_endlinks.end - src_start;
3220 src_object = src_entry->object.vm_object;
3221 src_offset = src_entry->offset;
3222 was_wired = (src_entry->wired_count != 0);
3223
3224 /*
3225 * Create a new address map entry to
3226 * hold the result. Fill in the fields from
3227 * the appropriate source entries.
3228 */
3229
3230 new_entry = vm_map_copy_entry_create(copy)_vm_map_entry_create(&(copy)->c_u.hdr);
3231 vm_map_entry_copy(new_entry, src_entry)({ *(new_entry) = *(src_entry); (new_entry)->is_shared = (
(boolean_t) 0); (new_entry)->needs_wakeup = ((boolean_t) 0
); (new_entry)->in_transition = ((boolean_t) 0); (new_entry
)->wired_count = 0; (new_entry)->user_wired_count = 0; }
);
3232
3233 /*
3234 * Attempt non-blocking copy-on-write optimizations.
3235 */
3236
3237 if (src_destroy &&
3238 (src_object == VM_OBJECT_NULL((vm_object_t) 0) ||
3239 (src_object->temporary && !src_object->use_shared_copy)))
3240 {
3241 /*
3242 * If we are destroying the source, and the object
3243 * is temporary, and not shared writable,
3244 * we can move the object reference
3245 * from the source to the copy. The copy is
3246 * copy-on-write only if the source is.
3247 * We make another reference to the object, because
3248 * destroying the source entry will deallocate it.
3249 */
3250 vm_object_reference(src_object);
3251
3252 /*
3253 * Copy is always unwired. vm_map_copy_entry
3254 * set its wired count to zero.
3255 */
3256
3257 goto CopySuccessful;
3258 }
3259
3260 if (!was_wired &&
3261 vm_object_copy_temporary(
3262 &new_entry->object.vm_object,
3263 &new_entry->offset,
3264 &src_needs_copy,
3265 &new_entry_needs_copy)) {
3266
3267 new_entry->needs_copy = new_entry_needs_copy;
3268
3269 /*
3270 * Handle copy-on-write obligations
3271 */
3272
3273 if (src_needs_copy && !tmp_entry->needs_copy) {
3274 vm_object_pmap_protect(
3275 src_object,
3276 src_offset,
3277 src_size,
3278 (src_entry->is_shared ? PMAP_NULL((pmap_t) 0)
3279 : src_map->pmap),
3280 src_entry->vme_startlinks.start,
3281 src_entry->protection &
3282 ~VM_PROT_WRITE((vm_prot_t) 0x02));
3283
3284 tmp_entry->needs_copy = TRUE((boolean_t) 1);
3285 }
3286
3287 /*
3288 * The map has never been unlocked, so it's safe to
3289 * move to the next entry rather than doing another
3290 * lookup.
3291 */
3292
3293 goto CopySuccessful;
3294 }
3295
3296 new_entry->needs_copy = FALSE((boolean_t) 0);
3297
3298 /*
3299 * Take an object reference, so that we may
3300 * release the map lock(s).
3301 */
3302
3303 assert(src_object != VM_OBJECT_NULL)({ if (!(src_object != ((vm_object_t) 0))) Assert("src_object != VM_OBJECT_NULL"
, "../vm/vm_map.c", 3303); });
3304 vm_object_reference(src_object);
3305
3306 /*
3307 * Record the timestamp for later verification.
3308 * Unlock the map.
3309 */
3310
3311 version.main_timestamp = src_map->timestamp;
3312 vm_map_unlock(src_map)lock_done(&(src_map)->lock);
3313
3314 /*
3315 * Perform the copy
3316 */
3317
3318 if (was_wired) {
3319 vm_object_lock(src_object);
3320 (void) vm_object_copy_slowly(
3321 src_object,
3322 src_offset,
3323 src_size,
3324 FALSE((boolean_t) 0),
3325 &new_entry->object.vm_object);
3326 new_entry->offset = 0;
3327 new_entry->needs_copy = FALSE((boolean_t) 0);
3328 } else {
3329 kern_return_t result;
3330
3331 result = vm_object_copy_strategically(src_object,
3332 src_offset,
3333 src_size,
3334 &new_entry->object.vm_object,
3335 &new_entry->offset,
3336 &new_entry_needs_copy);
3337
3338 new_entry->needs_copy = new_entry_needs_copy;
3339
3340
3341 if (result != KERN_SUCCESS0) {
3342 vm_map_copy_entry_dispose(copy, new_entry)_vm_map_entry_dispose(&(copy)->c_u.hdr, (new_entry));
3343
3344 vm_map_lock(src_map)({ lock_write(&(src_map)->lock); (src_map)->timestamp
++; });
3345 RETURN(result);
3346 }
3347
3348 }
3349
3350 /*
3351 * Throw away the extra reference
3352 */
3353
3354 vm_object_deallocate(src_object);
3355
3356 /*
3357 * Verify that the map has not substantially
3358 * changed while the copy was being made.
3359 */
3360
3361 vm_map_lock(src_map)({ lock_write(&(src_map)->lock); (src_map)->timestamp
++; }); /* Increments timestamp once! */
3362
3363 if ((version.main_timestamp + 1) == src_map->timestamp)
3364 goto CopySuccessful;
3365
3366 /*
3367 * Simple version comparison failed.
3368 *
3369 * Retry the lookup and verify that the
3370 * same object/offset are still present.
3371 *
3372 * [Note: a memory manager that colludes with
3373 * the calling task can detect that we have
3374 * cheated. While the map was unlocked, the
3375 * mapping could have been changed and restored.]
3376 */
3377
3378 if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry)) {
3379 vm_map_copy_entry_dispose(copy, new_entry)_vm_map_entry_dispose(&(copy)->c_u.hdr, (new_entry));
3380 RETURN(KERN_INVALID_ADDRESS1);
3381 }
3382
3383 src_entry = tmp_entry;
3384 vm_map_clip_start(src_map, src_entry, src_start)({ if ((src_start) > (src_entry)->links.start) _vm_map_clip_start
(&(src_map)->hdr,(src_entry),(src_start)); });
3385
3386 if ((src_entry->protection & VM_PROT_READ((vm_prot_t) 0x01)) == VM_PROT_NONE((vm_prot_t) 0x00))
3387 goto VerificationFailed;
3388
3389 if (src_entry->vme_endlinks.end < new_entry->vme_endlinks.end)
3390 src_size = (new_entry->vme_endlinks.end = src_entry->vme_endlinks.end) - src_start;
3391
3392 if ((src_entry->object.vm_object != src_object) ||
3393 (src_entry->offset != src_offset) ) {
3394
3395 /*
3396 * Verification failed.
3397 *
3398 * Start over with this top-level entry.
3399 */
3400
3401 VerificationFailed: ;
3402
3403 vm_object_deallocate(new_entry->object.vm_object);
3404 vm_map_copy_entry_dispose(copy, new_entry)_vm_map_entry_dispose(&(copy)->c_u.hdr, (new_entry));
3405 tmp_entry = src_entry;
3406 continue;
3407 }
3408
3409 /*
3410 * Verification succeeded.
3411 */
3412
3413 CopySuccessful: ;
3414
3415 /*
3416 * Link in the new copy entry.
3417 */
3418
3419 vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy),({ (&(copy)->c_u.hdr)->nentries++; (new_entry)->
links.prev = (((copy)->c_u.hdr.links.prev)); (new_entry)->
links.next = (((copy)->c_u.hdr.links.prev))->links.next
; (new_entry)->links.prev->links.next = (new_entry)->
links.next->links.prev = (new_entry); ({ struct rbtree_node
*___cur, *___prev; int ___diff, ___index; ___prev = ((void *
) 0); ___index = -1; ___cur = (&(&(copy)->c_u.hdr)
->tree)->root; while (___cur != ((void *) 0)) { ___diff
= vm_map_entry_cmp_insert(&(new_entry)->tree_node, ___cur
); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c"
, 3420); }); ___prev = ___cur; ___index = rbtree_d2i(___diff)
; ___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(&(copy)->c_u.hdr)->tree, ___prev, ___index, &
(new_entry)->tree_node); }); })
3420 new_entry)({ (&(copy)->c_u.hdr)->nentries++; (new_entry)->
links.prev = (((copy)->c_u.hdr.links.prev)); (new_entry)->
links.next = (((copy)->c_u.hdr.links.prev))->links.next
; (new_entry)->links.prev->links.next = (new_entry)->
links.next->links.prev = (new_entry); ({ struct rbtree_node
*___cur, *___prev; int ___diff, ___index; ___prev = ((void *
) 0); ___index = -1; ___cur = (&(&(copy)->c_u.hdr)
->tree)->root; while (___cur != ((void *) 0)) { ___diff
= vm_map_entry_cmp_insert(&(new_entry)->tree_node, ___cur
); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c"
, 3420); }); ___prev = ___cur; ___index = rbtree_d2i(___diff)
; ___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(&(copy)->c_u.hdr)->tree, ___prev, ___index, &
(new_entry)->tree_node); }); });
3421
3422 /*
3423 * Determine whether the entire region
3424 * has been copied.
3425 */
3426 src_start = new_entry->vme_endlinks.end;
3427 if ((src_start >= src_end) && (src_end != 0))
3428 break;
3429
3430 /*
3431 * Verify that there are no gaps in the region
3432 */
3433
3434 tmp_entry = src_entry->vme_nextlinks.next;
3435 if (tmp_entry->vme_startlinks.start != src_start)
3436 RETURN(KERN_INVALID_ADDRESS1);
3437 }
3438
3439 /*
3440 * If the source should be destroyed, do it now, since the
3441 * copy was successful.
3442 */
3443 if (src_destroy)
3444 (void) vm_map_delete(src_map, trunc_page(src_addr)((vm_offset_t)(((vm_offset_t)(src_addr)) & ~((1 << 12
)-1))), src_end);
3445
3446 vm_map_unlock(src_map)lock_done(&(src_map)->lock);
3447
3448 *copy_result = copy;
3449 return(KERN_SUCCESS0);
3450
3451#undef RETURN
3452}
3453
3454/*
3455 * vm_map_copyin_object:
3456 *
3457 * Create a copy object from an object.
3458 * Our caller donates an object reference.
3459 */
3460
3461kern_return_t vm_map_copyin_object(object, offset, size, copy_result)
3462 vm_object_t object;
3463 vm_offset_t offset; /* offset of region in object */
3464 vm_size_t size; /* size of region in object */
3465 vm_map_copy_t *copy_result; /* OUT */
3466{
3467 vm_map_copy_t copy; /* Resulting copy */
3468
3469 /*
3470 * We drop the object into a special copy object
3471 * that contains the object directly. These copy objects
3472 * are distinguished by entries_pageable == FALSE
3473 * and null links.
3474 */
3475
3476 copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
3477 vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next) =
3478 vm_map_copy_last_entry(copy)((copy)->c_u.hdr.links.prev) = VM_MAP_ENTRY_NULL((vm_map_entry_t) 0);
3479 copy->type = VM_MAP_COPY_OBJECT2;
3480 copy->cpy_objectc_u.c_o.object = object;
3481 copy->offset = offset;
3482 copy->size = size;
3483
3484 *copy_result = copy;
3485 return(KERN_SUCCESS0);
3486}
3487
3488/*
3489 * vm_map_copyin_page_list_cont:
3490 *
3491 * Continuation routine for vm_map_copyin_page_list.
3492 *
3493 * If vm_map_copyin_page_list can't fit the entire vm range
3494 * into a single page list object, it creates a continuation.
3495 * When the target of the operation has used the pages in the
3496 * initial page list, it invokes the continuation, which calls
3497 * this routine. If an error happens, the continuation is aborted
3498 * (abort arg to this routine is TRUE). To avoid deadlocks, the
3499 * pages are discarded from the initial page list before invoking
3500 * the continuation.
3501 *
3502 * NOTE: This is not the same sort of continuation used by
3503 * the scheduler.
3504 */
3505
3506kern_return_t vm_map_copyin_page_list_cont(cont_args, copy_result)
3507vm_map_copyin_args_t cont_args;
3508vm_map_copy_t *copy_result; /* OUT */
3509{
3510 kern_return_t result = 0; /* '=0' to quiet gcc warnings */
3511 boolean_t do_abort, src_destroy, src_destroy_only;
3512
3513 /*
3514 * Check for cases that only require memory destruction.
3515 */
3516 do_abort = (copy_result == (vm_map_copy_t *) 0);
3517 src_destroy = (cont_args->destroy_len != (vm_size_t) 0);
3518 src_destroy_only = (cont_args->src_len == (vm_size_t) 0);
3519
3520 if (do_abort || src_destroy_only) {
3521 if (src_destroy)
3522 result = vm_map_remove(cont_args->map,
3523 cont_args->destroy_addr,
3524 cont_args->destroy_addr + cont_args->destroy_len);
3525 if (!do_abort)
3526 *copy_result = VM_MAP_COPY_NULL((vm_map_copy_t) 0);
3527 }
3528 else {
3529 result = vm_map_copyin_page_list(cont_args->map,
3530 cont_args->src_addr, cont_args->src_len, src_destroy,
3531 cont_args->steal_pages, copy_result, TRUE((boolean_t) 1));
3532
3533 if (src_destroy && !cont_args->steal_pages &&
3534 vm_map_copy_has_cont(*copy_result)(((*copy_result)->c_u.c_p.cont) != (kern_return_t (*)()) 0
)) {
3535 vm_map_copyin_args_t new_args;
3536 /*
3537 * Transfer old destroy info.
3538 */
3539 new_args = (vm_map_copyin_args_t)
3540 (*copy_result)->cpy_cont_argsc_u.c_p.cont_args;
3541 new_args->destroy_addr = cont_args->destroy_addr;
3542 new_args->destroy_len = cont_args->destroy_len;
3543 }
3544 }
3545
3546 vm_map_deallocate(cont_args->map);
3547 kfree((vm_offset_t)cont_args, sizeof(vm_map_copyin_args_data_t));
3548
3549 return(result);
3550}
3551
3552/*
3553 * vm_map_copyin_page_list:
3554 *
3555 * This is a variant of vm_map_copyin that copies in a list of pages.
3556 * If steal_pages is TRUE, the pages are only in the returned list.
3557 * If steal_pages is FALSE, the pages are busy and still in their
3558 * objects. A continuation may be returned if not all the pages fit:
3559 * the recipient of this copy_result must be prepared to deal with it.
3560 */
3561
3562kern_return_t vm_map_copyin_page_list(src_map, src_addr, len, src_destroy,
3563 steal_pages, copy_result, is_cont)
3564 vm_map_t src_map;
3565 vm_offset_t src_addr;
3566 vm_size_t len;
3567 boolean_t src_destroy;
3568 boolean_t steal_pages;
3569 vm_map_copy_t *copy_result; /* OUT */
3570 boolean_t is_cont;
3571{
3572 vm_map_entry_t src_entry;
3573 vm_page_t m;
3574 vm_offset_t src_start;
3575 vm_offset_t src_end;
3576 vm_size_t src_size;
3577 vm_object_t src_object;
3578 vm_offset_t src_offset;
3579 vm_offset_t src_last_offset;
3580 vm_map_copy_t copy; /* Resulting copy */
3581 kern_return_t result = KERN_SUCCESS0;
3582 boolean_t need_map_lookup;
3583 vm_map_copyin_args_t cont_args;
3584
3585 /*
3586 * If steal_pages is FALSE, this leaves busy pages in
3587 * the object. A continuation must be used if src_destroy
3588 * is true in this case (!steal_pages && src_destroy).
3589 *
3590 * XXX Still have a more general problem of what happens
3591 * XXX if the same page occurs twice in a list. Deadlock
3592 * XXX can happen if vm_fault_page was called. A
3593 * XXX possible solution is to use a continuation if vm_fault_page
3594 * XXX is called and we cross a map entry boundary.
3595 */
3596
3597 /*
3598 * Check for copies of zero bytes.
3599 */
3600
3601 if (len == 0) {
3602 *copy_result = VM_MAP_COPY_NULL((vm_map_copy_t) 0);
3603 return(KERN_SUCCESS0);
3604 }
3605
3606 /*
3607 * Compute start and end of region
3608 */
3609
3610 src_start = trunc_page(src_addr)((vm_offset_t)(((vm_offset_t)(src_addr)) & ~((1 << 12
)-1)));
3611 src_end = round_page(src_addr + len)((vm_offset_t)((((vm_offset_t)(src_addr + len)) + ((1 <<
12)-1)) & ~((1 << 12)-1)));
3612
3613 /*
3614 * Check that the end address doesn't overflow
3615 */
3616
3617 if (src_end <= src_start && (src_end < src_start || src_start != 0)) {
3618 return KERN_INVALID_ADDRESS1;
3619 }
3620
3621 /*
3622 * Allocate a header element for the page list.
3623 *
3624 * Record original offset and size, as caller may not
3625 * be page-aligned.
3626 */
3627
3628 copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
3629 copy->type = VM_MAP_COPY_PAGE_LIST3;
3630 copy->cpy_npagesc_u.c_p.npages = 0;
3631 copy->offset = src_addr;
3632 copy->size = len;
3633 copy->cpy_contc_u.c_p.cont = ((kern_return_t (*)()) 0);
3634 copy->cpy_cont_argsc_u.c_p.cont_args = (char *) VM_MAP_COPYIN_ARGS_NULL((vm_map_copyin_args_t) 0);
3635
3636 /*
3637 * Find the beginning of the region.
3638 */
3639
3640do_map_lookup:
3641
3642 vm_map_lock(src_map)({ lock_write(&(src_map)->lock); (src_map)->timestamp
++; });
3643
3644 if (!vm_map_lookup_entry(src_map, src_start, &src_entry)) {
3645 result = KERN_INVALID_ADDRESS1;
3646 goto error;
3647 }
3648 need_map_lookup = FALSE((boolean_t) 0);
3649
3650 /*
3651 * Go through entries until we get to the end.
3652 */
3653
3654 while (TRUE((boolean_t) 1)) {
3655
3656 if (! (src_entry->protection & VM_PROT_READ((vm_prot_t) 0x01))) {
3657 result = KERN_PROTECTION_FAILURE2;
3658 goto error;
3659 }
3660
3661 if (src_end > src_entry->vme_endlinks.end)
3662 src_size = src_entry->vme_endlinks.end - src_start;
3663 else
3664 src_size = src_end - src_start;
3665
3666 src_object = src_entry->object.vm_object;
3667 src_offset = src_entry->offset +
3668 (src_start - src_entry->vme_startlinks.start);
3669
3670 /*
3671 * If src_object is NULL, allocate it now;
3672 * we're going to fault on it shortly.
3673 */
3674 if (src_object == VM_OBJECT_NULL((vm_object_t) 0)) {
3675 src_object = vm_object_allocate((vm_size_t)
3676 src_entry->vme_endlinks.end -
3677 src_entry->vme_startlinks.start);
3678 src_entry->object.vm_object = src_object;
3679 }
3680
3681 /*
3682 * Iterate over pages. Fault in ones that aren't present.
3683 */
3684 src_last_offset = src_offset + src_size;
3685 for (; (src_offset < src_last_offset && !need_map_lookup);
3686 src_offset += PAGE_SIZE(1 << 12), src_start += PAGE_SIZE(1 << 12)) {
3687
3688 if (copy->cpy_npagesc_u.c_p.npages == VM_MAP_COPY_PAGE_LIST_MAX64) {
3689make_continuation:
3690 /*
3691 * At this point we have the max number of
3692 * pages busy for this thread that we're
3693 * willing to allow. Stop here and record
3694 * arguments for the remainder. Note:
3695 * this means that this routine isn't atomic,
3696 * but that's the breaks. Note that only
3697 * the first vm_map_copy_t that comes back
3698 * from this routine has the right offset
3699 * and size; those from continuations are
3700 * page rounded, and short by the amount
3701 * already done.
3702 *
3703 * Reset src_end so the src_destroy
3704 * code at the bottom doesn't do
3705 * something stupid.
3706 */
3707
3708 cont_args = (vm_map_copyin_args_t)
3709 kalloc(sizeof(vm_map_copyin_args_data_t));
3710 cont_args->map = src_map;
3711 vm_map_reference(src_map);
3712 cont_args->src_addr = src_start;
3713 cont_args->src_len = len - (src_start - src_addr);
3714 if (src_destroy) {
3715 cont_args->destroy_addr = cont_args->src_addr;
3716 cont_args->destroy_len = cont_args->src_len;
3717 }
3718 else {
3719 cont_args->destroy_addr = (vm_offset_t) 0;
3720 cont_args->destroy_len = (vm_offset_t) 0;
3721 }
3722 cont_args->steal_pages = steal_pages;
3723
3724 copy->cpy_cont_argsc_u.c_p.cont_args = (char *) cont_args;
3725 copy->cpy_contc_u.c_p.cont = vm_map_copyin_page_list_cont;
3726
3727 src_end = src_start;
3728 vm_map_clip_end(src_map, src_entry, src_end)({ if ((src_end) < (src_entry)->links.end) _vm_map_clip_end
(&(src_map)->hdr,(src_entry),(src_end)); });
3729 break;
3730 }
3731
3732 /*
3733 * Try to find the page of data.
3734 */
3735 vm_object_lock(src_object);
3736 vm_object_paging_begin(src_object)((src_object)->paging_in_progress++);
3737 if (((m = vm_page_lookup(src_object, src_offset)) !=
3738 VM_PAGE_NULL((vm_page_t) 0)) && !m->busy && !m->fictitious &&
3739 !m->absent && !m->error) {
3740
3741 /*
3742 * This is the page. Mark it busy
3743 * and keep the paging reference on
3744 * the object whilst we do our thing.
3745 */
3746 m->busy = TRUE((boolean_t) 1);
3747
3748 /*
3749 * Also write-protect the page, so
3750 * that the map`s owner cannot change
3751 * the data. The busy bit will prevent
3752 * faults on the page from succeeding
3753 * until the copy is released; after
3754 * that, the page can be re-entered
3755 * as writable, since we didn`t alter
3756 * the map entry. This scheme is a
3757 * cheap copy-on-write.
3758 *
3759 * Don`t forget the protection and
3760 * the page_lock value!
3761 *
3762 * If the source is being destroyed
3763 * AND not shared writable, we don`t
3764 * have to protect the page, since
3765 * we will destroy the (only)
3766 * writable mapping later.
3767 */
3768 if (!src_destroy ||
3769 src_object->use_shared_copy)
3770 {
3771 pmap_page_protect(m->phys_addr,
3772 src_entry->protection
3773 & ~m->page_lock
3774 & ~VM_PROT_WRITE((vm_prot_t) 0x02));
3775 }
3776
3777 }
3778 else {
3779 vm_prot_t result_prot;
3780 vm_page_t top_page;
3781 kern_return_t kr;
3782
3783 /*
3784 * Have to fault the page in; must
3785 * unlock the map to do so. While
3786 * the map is unlocked, anything
3787 * can happen, we must lookup the
3788 * map entry before continuing.
3789 */
3790 vm_map_unlock(src_map)lock_done(&(src_map)->lock);
3791 need_map_lookup = TRUE((boolean_t) 1);
3792retry:
3793 result_prot = VM_PROT_READ((vm_prot_t) 0x01);
3794
3795 kr = vm_fault_page(src_object, src_offset,
3796 VM_PROT_READ((vm_prot_t) 0x01), FALSE((boolean_t) 0), FALSE((boolean_t) 0),
3797 &result_prot, &m, &top_page,
3798 FALSE((boolean_t) 0), (void (*)()) 0);
3799 /*
3800 * Cope with what happened.
3801 */
3802 switch (kr) {
3803 case VM_FAULT_SUCCESS0:
3804 break;
3805 case VM_FAULT_INTERRUPTED2: /* ??? */
3806 case VM_FAULT_RETRY1:
3807 vm_object_lock(src_object);
3808 vm_object_paging_begin(src_object)((src_object)->paging_in_progress++);
3809 goto retry;
3810 case VM_FAULT_MEMORY_SHORTAGE3:
3811 VM_PAGE_WAIT((void (*)()) 0)vm_page_wait((void (*)()) 0);
3812 vm_object_lock(src_object);
3813 vm_object_paging_begin(src_object)((src_object)->paging_in_progress++);
3814 goto retry;
3815 case VM_FAULT_FICTITIOUS_SHORTAGE4:
3816 vm_page_more_fictitious();
3817 vm_object_lock(src_object);
3818 vm_object_paging_begin(src_object)((src_object)->paging_in_progress++);
3819 goto retry;
3820 case VM_FAULT_MEMORY_ERROR5:
3821 /*
3822 * Something broke. If this
3823 * is a continuation, return
3824 * a partial result if possible,
3825 * else fail the whole thing.
3826 * In the continuation case, the
3827 * next continuation call will
3828 * get this error if it persists.
3829 */
3830 vm_map_lock(src_map)({ lock_write(&(src_map)->lock); (src_map)->timestamp
++; });
3831 if (is_cont &&
3832 copy->cpy_npagesc_u.c_p.npages != 0)
3833 goto make_continuation;
3834
3835 result = KERN_MEMORY_ERROR10;
3836 goto error;
3837 }
3838
3839 if (top_page != VM_PAGE_NULL((vm_page_t) 0)) {
3840 vm_object_lock(src_object);
3841 VM_PAGE_FREE(top_page)({ ; vm_page_free(top_page); ; });
3842 vm_object_paging_end(src_object)({ ({ if (!((src_object)->paging_in_progress != 0)) Assert
("(src_object)->paging_in_progress != 0", "../vm/vm_map.c"
, 3842); }); if (--(src_object)->paging_in_progress == 0) {
({ if ((src_object)->all_wanted & (1 << (2))) thread_wakeup_prim
(((event_t)(((vm_offset_t) src_object) + (2))), ((boolean_t) 0
), 0); (src_object)->all_wanted &= ~(1 << (2)); }
); } });
3843 vm_object_unlock(src_object);
3844 }
3845
3846 /*
3847 * We do not need to write-protect
3848 * the page, since it cannot have
3849 * been in the pmap (and we did not
3850 * enter it above). The busy bit
3851 * will protect the page from being
3852 * entered as writable until it is
3853 * unlocked.
3854 */
3855
3856 }
3857
3858 /*
3859 * The page is busy, its object is locked, and
3860 * we have a paging reference on it. Either
3861 * the map is locked, or need_map_lookup is
3862 * TRUE.
3863 *
3864 * Put the page in the page list.
3865 */
3866 copy->cpy_page_listc_u.c_p.page_list[copy->cpy_npagesc_u.c_p.npages++] = m;
3867 vm_object_unlock(m->object);
3868 }
3869
3870 /*
3871 * DETERMINE whether the entire region
3872 * has been copied.
3873 */
3874 if (src_start >= src_end && src_end != 0) {
3875 if (need_map_lookup)
3876 vm_map_lock(src_map)({ lock_write(&(src_map)->lock); (src_map)->timestamp
++; });
3877 break;
3878 }
3879
3880 /*
3881 * If need_map_lookup is TRUE, have to start over with
3882 * another map lookup. Note that we dropped the map
3883 * lock (to call vm_fault_page) above only in this case.
3884 */
3885 if (need_map_lookup)
3886 goto do_map_lookup;
3887
3888 /*
3889 * Verify that there are no gaps in the region
3890 */
3891
3892 src_start = src_entry->vme_endlinks.end;
3893 src_entry = src_entry->vme_nextlinks.next;
3894 if (src_entry->vme_startlinks.start != src_start) {
3895 result = KERN_INVALID_ADDRESS1;
3896 goto error;
3897 }
3898 }
3899
3900 /*
3901 * If steal_pages is true, make sure all
3902 * pages in the copy are not in any object
3903 * We try to remove them from the original
3904 * object, but we may have to copy them.
3905 *
3906 * At this point every page in the list is busy
3907 * and holds a paging reference to its object.
3908 * When we're done stealing, every page is busy,
3909 * and in no object (m->tabled == FALSE).
3910 */
3911 src_start = trunc_page(src_addr)((vm_offset_t)(((vm_offset_t)(src_addr)) & ~((1 << 12
)-1)));
3912 if (steal_pages) {
3913 int i;
3914 vm_offset_t unwire_end;
3915
3916 unwire_end = src_start;
3917 for (i = 0; i < copy->cpy_npagesc_u.c_p.npages; i++) {
3918
3919 /*
3920 * Remove the page from its object if it
3921 * can be stolen. It can be stolen if:
3922 *
3923 * (1) The source is being destroyed,
3924 * the object is temporary, and
3925 * not shared.
3926 * (2) The page is not precious.
3927 *
3928 * The not shared check consists of two
3929 * parts: (a) there are no objects that
3930 * shadow this object. (b) it is not the
3931 * object in any shared map entries (i.e.,
3932 * use_shared_copy is not set).
3933 *
3934 * The first check (a) means that we can't
3935 * steal pages from objects that are not
3936 * at the top of their shadow chains. This
3937 * should not be a frequent occurrence.
3938 *
3939 * Stealing wired pages requires telling the
3940 * pmap module to let go of them.
3941 *
3942 * NOTE: stealing clean pages from objects
3943 * whose mappings survive requires a call to
3944 * the pmap module. Maybe later.
3945 */
3946 m = copy->cpy_page_listc_u.c_p.page_list[i];
3947 src_object = m->object;
3948 vm_object_lock(src_object);
3949
3950 if (src_destroy &&
3951 src_object->temporary &&
3952 (!src_object->shadowed) &&
3953 (!src_object->use_shared_copy) &&
3954 !m->precious) {
3955 vm_offset_t page_vaddr;
3956
3957 page_vaddr = src_start + (i * PAGE_SIZE(1 << 12));
3958 if (m->wire_count > 0) {
3959
3960 assert(m->wire_count == 1)({ if (!(m->wire_count == 1)) Assert("m->wire_count == 1"
, "../vm/vm_map.c", 3960); });
3961 /*
3962 * In order to steal a wired
3963 * page, we have to unwire it
3964 * first. We do this inline
3965 * here because we have the page.
3966 *
3967 * Step 1: Unwire the map entry.
3968 * Also tell the pmap module
3969 * that this piece of the
3970 * pmap is pageable.
3971 */
3972 vm_object_unlock(src_object);
3973 if (page_vaddr >= unwire_end) {
3974 if (!vm_map_lookup_entry(src_map,
3975 page_vaddr, &src_entry))
3976 panic("vm_map_copyin_page_list: missing wired map entry");
3977
3978 vm_map_clip_start(src_map, src_entry,({ if ((page_vaddr) > (src_entry)->links.start) _vm_map_clip_start
(&(src_map)->hdr,(src_entry),(page_vaddr)); })
3979 page_vaddr)({ if ((page_vaddr) > (src_entry)->links.start) _vm_map_clip_start
(&(src_map)->hdr,(src_entry),(page_vaddr)); });
3980 vm_map_clip_end(src_map, src_entry,({ if ((src_start + src_size) < (src_entry)->links.end)
_vm_map_clip_end(&(src_map)->hdr,(src_entry),(src_start
+ src_size)); })
3981 src_start + src_size)({ if ((src_start + src_size) < (src_entry)->links.end)
_vm_map_clip_end(&(src_map)->hdr,(src_entry),(src_start
+ src_size)); });
3982
3983 assert(src_entry->wired_count > 0)({ if (!(src_entry->wired_count > 0)) Assert("src_entry->wired_count > 0"
, "../vm/vm_map.c", 3983); });
3984 src_entry->wired_count = 0;
3985 src_entry->user_wired_count = 0;
3986 unwire_end = src_entry->vme_endlinks.end;
3987 pmap_pageable(vm_map_pmap(src_map)((src_map)->pmap),
3988 page_vaddr, unwire_end, TRUE((boolean_t) 1));
3989 }
3990
3991 /*
3992 * Step 2: Unwire the page.
3993 * pmap_remove handles this for us.
3994 */
3995 vm_object_lock(src_object);
3996 }
3997
3998 /*
3999 * Don't need to remove the mapping;
4000 * vm_map_delete will handle it.
4001 *
4002 * Steal the page. Setting the wire count
4003 * to zero is vm_page_unwire without
4004 * activating the page.
4005 */
4006 vm_page_lock_queues();
4007 vm_page_remove(m);
4008 if (m->wire_count > 0) {
4009 m->wire_count = 0;
4010 vm_page_wire_count--;
4011 } else {
4012 VM_PAGE_QUEUES_REMOVE(m)({ if (m->active) { { queue_entry_t next, prev; next = (m)
->pageq.next; prev = (m)->pageq.prev; if ((&vm_page_queue_active
) == next) (&vm_page_queue_active)->prev = prev; else (
(vm_page_t)next)->pageq.prev = prev; if ((&vm_page_queue_active
) == prev) (&vm_page_queue_active)->next = next; else (
(vm_page_t)prev)->pageq.next = next; }; m->active = ((boolean_t
) 0); vm_page_active_count--; } if (m->inactive) { { queue_entry_t
next, prev; next = (m)->pageq.next; prev = (m)->pageq.
prev; if ((&vm_page_queue_inactive) == next) (&vm_page_queue_inactive
)->prev = prev; else ((vm_page_t)next)->pageq.prev = prev
; if ((&vm_page_queue_inactive) == prev) (&vm_page_queue_inactive
)->next = next; else ((vm_page_t)prev)->pageq.next = next
; }; m->inactive = ((boolean_t) 0); vm_page_inactive_count
--; } });
4013 }
4014 vm_page_unlock_queues();
4015 }
4016 else {
4017 /*
4018 * Have to copy this page. Have to
4019 * unlock the map while copying,
4020 * hence no further page stealing.
4021 * Hence just copy all the pages.
4022 * Unlock the map while copying;
4023 * This means no further page stealing.
4024 */
4025 vm_object_unlock(src_object);
4026 vm_map_unlock(src_map)lock_done(&(src_map)->lock);
4027
4028 vm_map_copy_steal_pages(copy);
4029
4030 vm_map_lock(src_map)({ lock_write(&(src_map)->lock); (src_map)->timestamp
++; });
4031 break;
4032 }
4033
4034 vm_object_paging_end(src_object)({ ({ if (!((src_object)->paging_in_progress != 0)) Assert
("(src_object)->paging_in_progress != 0", "../vm/vm_map.c"
, 4034); }); if (--(src_object)->paging_in_progress == 0) {
({ if ((src_object)->all_wanted & (1 << (2))) thread_wakeup_prim
(((event_t)(((vm_offset_t) src_object) + (2))), ((boolean_t) 0
), 0); (src_object)->all_wanted &= ~(1 << (2)); }
); } });
4035 vm_object_unlock(src_object);
4036 }
4037
4038 /*
4039 * If the source should be destroyed, do it now, since the
4040 * copy was successful.
4041 */
4042
4043 if (src_destroy) {
4044 (void) vm_map_delete(src_map, src_start, src_end);
4045 }
4046 }
4047 else {
4048 /*
4049 * !steal_pages leaves busy pages in the map.
4050 * This will cause src_destroy to hang. Use
4051 * a continuation to prevent this.
4052 */
4053 if (src_destroy && !vm_map_copy_has_cont(copy)(((copy)->c_u.c_p.cont) != (kern_return_t (*)()) 0)) {
4054 cont_args = (vm_map_copyin_args_t)
4055 kalloc(sizeof(vm_map_copyin_args_data_t));
4056 vm_map_reference(src_map);
4057 cont_args->map = src_map;
4058 cont_args->src_addr = (vm_offset_t) 0;
4059 cont_args->src_len = (vm_size_t) 0;
4060 cont_args->destroy_addr = src_start;
4061 cont_args->destroy_len = src_end - src_start;
4062 cont_args->steal_pages = FALSE((boolean_t) 0);
4063
4064 copy->cpy_cont_argsc_u.c_p.cont_args = (char *) cont_args;
4065 copy->cpy_contc_u.c_p.cont = vm_map_copyin_page_list_cont;
4066 }
4067
4068 }
4069
4070 vm_map_unlock(src_map)lock_done(&(src_map)->lock);
4071
4072 *copy_result = copy;
4073 return(result);
4074
4075error:
4076 vm_map_unlock(src_map)lock_done(&(src_map)->lock);
4077 vm_map_copy_discard(copy);
4078 return(result);
4079}
4080
4081/*
4082 * vm_map_fork:
4083 *
4084 * Create and return a new map based on the old
4085 * map, according to the inheritance values on the
4086 * regions in that map.
4087 *
4088 * The source map must not be locked.
4089 */
4090vm_map_t vm_map_fork(old_map)
4091 vm_map_t old_map;
4092{
4093 vm_map_t new_map;
4094 vm_map_entry_t old_entry;
4095 vm_map_entry_t new_entry;
4096 pmap_t new_pmap = pmap_create((vm_size_t) 0);
4097 vm_size_t new_size = 0;
4098 vm_size_t entry_size;
4099 vm_object_t object;
4100
4101 vm_map_lock(old_map)({ lock_write(&(old_map)->lock); (old_map)->timestamp
++; });
4102
4103 new_map = vm_map_create(new_pmap,
4104 old_map->min_offsethdr.links.start,
4105 old_map->max_offsethdr.links.end,
4106 old_map->hdr.entries_pageable);
4107
4108 for (
4109 old_entry = vm_map_first_entry(old_map)((old_map)->hdr.links.next);
4110 old_entry != vm_map_to_entry(old_map)((struct vm_map_entry *) &(old_map)->hdr.links);
4111 ) {
4112 if (old_entry->is_sub_map)
4113 panic("vm_map_fork: encountered a submap");
4114
4115 entry_size = (old_entry->vme_endlinks.end - old_entry->vme_startlinks.start);
4116
4117 switch (old_entry->inheritance) {
4118 case VM_INHERIT_NONE((vm_inherit_t) 2):
4119 break;
4120
4121 case VM_INHERIT_SHARE((vm_inherit_t) 0):
4122 /*
4123 * New sharing code. New map entry
4124 * references original object. Temporary
4125 * objects use asynchronous copy algorithm for
4126 * future copies. First make sure we have
4127 * the right object. If we need a shadow,
4128 * or someone else already has one, then
4129 * make a new shadow and share it.
4130 */
4131
4132 object = old_entry->object.vm_object;
4133 if (object == VM_OBJECT_NULL((vm_object_t) 0)) {
4134 object = vm_object_allocate(
4135 (vm_size_t)(old_entry->vme_endlinks.end -
4136 old_entry->vme_startlinks.start));
4137 old_entry->offset = 0;
4138 old_entry->object.vm_object = object;
4139 assert(!old_entry->needs_copy)({ if (!(!old_entry->needs_copy)) Assert("!old_entry->needs_copy"
, "../vm/vm_map.c", 4139); });
4140 }
4141 else if (old_entry->needs_copy || object->shadowed ||
4142 (object->temporary && !old_entry->is_shared &&
4143 object->size > (vm_size_t)(old_entry->vme_endlinks.end -
4144 old_entry->vme_startlinks.start))) {
4145
4146 assert(object->temporary)({ if (!(object->temporary)) Assert("object->temporary"
, "../vm/vm_map.c", 4146); });
4147 assert(!(object->shadowed && old_entry->is_shared))({ if (!(!(object->shadowed && old_entry->is_shared
))) Assert("!(object->shadowed && old_entry->is_shared)"
, "../vm/vm_map.c", 4147); });
4148 vm_object_shadow(
4149 &old_entry->object.vm_object,
4150 &old_entry->offset,
4151 (vm_size_t) (old_entry->vme_endlinks.end -
4152 old_entry->vme_startlinks.start));
4153
4154 /*
4155 * If we're making a shadow for other than
4156 * copy on write reasons, then we have
4157 * to remove write permission.
4158 */
4159
4160 if (!old_entry->needs_copy &&
4161 (old_entry->protection & VM_PROT_WRITE((vm_prot_t) 0x02))) {
4162 pmap_protect(vm_map_pmap(old_map)((old_map)->pmap),
4163 old_entry->vme_startlinks.start,
4164 old_entry->vme_endlinks.end,
4165 old_entry->protection &
4166 ~VM_PROT_WRITE((vm_prot_t) 0x02));
4167 }
4168 old_entry->needs_copy = FALSE((boolean_t) 0);
4169 object = old_entry->object.vm_object;
4170 }
4171
4172 /*
4173 * Set use_shared_copy to indicate that
4174 * object must use shared (delayed) copy-on
4175 * write. This is ignored for permanent objects.
4176 * Bump the reference count for the new entry
4177 */
4178
4179 vm_object_lock(object);
4180 object->use_shared_copy = TRUE((boolean_t) 1);
4181 object->ref_count++;
4182 vm_object_unlock(object);
4183
4184 new_entry = vm_map_entry_create(new_map)_vm_map_entry_create(&(new_map)->hdr);
4185
4186 if (old_entry->projected_on != 0) {
4187 /*
4188 * If entry is projected buffer, clone the
4189 * entry exactly.
4190 */
4191
4192 vm_map_entry_copy_full(new_entry, old_entry)(*(new_entry) = *(old_entry));
4193
4194 } else {
4195 /*
4196 * Clone the entry, using object ref from above.
4197 * Mark both entries as shared.
4198 */
4199
4200 vm_map_entry_copy(new_entry, old_entry)({ *(new_entry) = *(old_entry); (new_entry)->is_shared = (
(boolean_t) 0); (new_entry)->needs_wakeup = ((boolean_t) 0
); (new_entry)->in_transition = ((boolean_t) 0); (new_entry
)->wired_count = 0; (new_entry)->user_wired_count = 0; }
);
4201 old_entry->is_shared = TRUE((boolean_t) 1);
4202 new_entry->is_shared = TRUE((boolean_t) 1);
4203 }
4204
4205 /*
4206 * Insert the entry into the new map -- we
4207 * know we're inserting at the end of the new
4208 * map.
4209 */
4210
4211 vm_map_entry_link(({ (&(new_map)->hdr)->nentries++; (new_entry)->links
.prev = (((new_map)->hdr.links.prev)); (new_entry)->links
.next = (((new_map)->hdr.links.prev))->links.next; (new_entry
)->links.prev->links.next = (new_entry)->links.next->
links.prev = (new_entry); ({ struct rbtree_node *___cur, *___prev
; int ___diff, ___index; ___prev = ((void *) 0); ___index = -
1; ___cur = (&(&(new_map)->hdr)->tree)->root
; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(new_entry)->tree_node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 4214); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(new_map
)->hdr)->tree, ___prev, ___index, &(new_entry)->
tree_node); }); })
4212 new_map,({ (&(new_map)->hdr)->nentries++; (new_entry)->links
.prev = (((new_map)->hdr.links.prev)); (new_entry)->links
.next = (((new_map)->hdr.links.prev))->links.next; (new_entry
)->links.prev->links.next = (new_entry)->links.next->
links.prev = (new_entry); ({ struct rbtree_node *___cur, *___prev
; int ___diff, ___index; ___prev = ((void *) 0); ___index = -
1; ___cur = (&(&(new_map)->hdr)->tree)->root
; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(new_entry)->tree_node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 4214); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(new_map
)->hdr)->tree, ___prev, ___index, &(new_entry)->
tree_node); }); })
4213 vm_map_last_entry(new_map),({ (&(new_map)->hdr)->nentries++; (new_entry)->links
.prev = (((new_map)->hdr.links.prev)); (new_entry)->links
.next = (((new_map)->hdr.links.prev))->links.next; (new_entry
)->links.prev->links.next = (new_entry)->links.next->
links.prev = (new_entry); ({ struct rbtree_node *___cur, *___prev
; int ___diff, ___index; ___prev = ((void *) 0); ___index = -
1; ___cur = (&(&(new_map)->hdr)->tree)->root
; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(new_entry)->tree_node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 4214); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(new_map
)->hdr)->tree, ___prev, ___index, &(new_entry)->
tree_node); }); })
4214 new_entry)({ (&(new_map)->hdr)->nentries++; (new_entry)->links
.prev = (((new_map)->hdr.links.prev)); (new_entry)->links
.next = (((new_map)->hdr.links.prev))->links.next; (new_entry
)->links.prev->links.next = (new_entry)->links.next->
links.prev = (new_entry); ({ struct rbtree_node *___cur, *___prev
; int ___diff, ___index; ___prev = ((void *) 0); ___index = -
1; ___cur = (&(&(new_map)->hdr)->tree)->root
; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(new_entry)->tree_node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 4214); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(new_map
)->hdr)->tree, ___prev, ___index, &(new_entry)->
tree_node); }); });
4215
4216 /*
4217 * Update the physical map
4218 */
4219
4220 pmap_copy(new_map->pmap, old_map->pmap,
4221 new_entry->vme_start,
4222 entry_size,
4223 old_entry->vme_start);
4224
4225 new_size += entry_size;
4226 break;
4227
4228 case VM_INHERIT_COPY((vm_inherit_t) 1):
4229 if (old_entry->wired_count == 0) {
4230 boolean_t src_needs_copy;
4231 boolean_t new_entry_needs_copy;
4232
4233 new_entry = vm_map_entry_create(new_map)_vm_map_entry_create(&(new_map)->hdr);
4234 vm_map_entry_copy(new_entry, old_entry)({ *(new_entry) = *(old_entry); (new_entry)->is_shared = (
(boolean_t) 0); (new_entry)->needs_wakeup = ((boolean_t) 0
); (new_entry)->in_transition = ((boolean_t) 0); (new_entry
)->wired_count = 0; (new_entry)->user_wired_count = 0; }
);
4235
4236 if (vm_object_copy_temporary(
4237 &new_entry->object.vm_object,
4238 &new_entry->offset,
4239 &src_needs_copy,
4240 &new_entry_needs_copy)) {
4241
4242 /*
4243 * Handle copy-on-write obligations
4244 */
4245
4246 if (src_needs_copy && !old_entry->needs_copy) {
4247 vm_object_pmap_protect(
4248 old_entry->object.vm_object,
4249 old_entry->offset,
4250 entry_size,
4251 (old_entry->is_shared ?
4252 PMAP_NULL((pmap_t) 0) :
4253 old_map->pmap),
4254 old_entry->vme_startlinks.start,
4255 old_entry->protection &
4256 ~VM_PROT_WRITE((vm_prot_t) 0x02));
4257
4258 old_entry->needs_copy = TRUE((boolean_t) 1);
4259 }
4260
4261 new_entry->needs_copy = new_entry_needs_copy;
4262
4263 /*
4264 * Insert the entry at the end
4265 * of the map.
4266 */
4267
4268 vm_map_entry_link(new_map,({ (&(new_map)->hdr)->nentries++; (new_entry)->links
.prev = (((new_map)->hdr.links.prev)); (new_entry)->links
.next = (((new_map)->hdr.links.prev))->links.next; (new_entry
)->links.prev->links.next = (new_entry)->links.next->
links.prev = (new_entry); ({ struct rbtree_node *___cur, *___prev
; int ___diff, ___index; ___prev = ((void *) 0); ___index = -
1; ___cur = (&(&(new_map)->hdr)->tree)->root
; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(new_entry)->tree_node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 4270); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(new_map
)->hdr)->tree, ___prev, ___index, &(new_entry)->
tree_node); }); })
4269 vm_map_last_entry(new_map),({ (&(new_map)->hdr)->nentries++; (new_entry)->links
.prev = (((new_map)->hdr.links.prev)); (new_entry)->links
.next = (((new_map)->hdr.links.prev))->links.next; (new_entry
)->links.prev->links.next = (new_entry)->links.next->
links.prev = (new_entry); ({ struct rbtree_node *___cur, *___prev
; int ___diff, ___index; ___prev = ((void *) 0); ___index = -
1; ___cur = (&(&(new_map)->hdr)->tree)->root
; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(new_entry)->tree_node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 4270); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(new_map
)->hdr)->tree, ___prev, ___index, &(new_entry)->
tree_node); }); })
4270 new_entry)({ (&(new_map)->hdr)->nentries++; (new_entry)->links
.prev = (((new_map)->hdr.links.prev)); (new_entry)->links
.next = (((new_map)->hdr.links.prev))->links.next; (new_entry
)->links.prev->links.next = (new_entry)->links.next->
links.prev = (new_entry); ({ struct rbtree_node *___cur, *___prev
; int ___diff, ___index; ___prev = ((void *) 0); ___index = -
1; ___cur = (&(&(new_map)->hdr)->tree)->root
; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(new_entry)->tree_node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 4270); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(new_map
)->hdr)->tree, ___prev, ___index, &(new_entry)->
tree_node); }); });
4271
4272
4273 new_size += entry_size;
4274 break;
4275 }
4276
4277 vm_map_entry_dispose(new_map, new_entry)_vm_map_entry_dispose(&(new_map)->hdr, (new_entry));
4278 }
4279
4280 /* INNER BLOCK (copy cannot be optimized) */ {
4281
4282 vm_offset_t start = old_entry->vme_startlinks.start;
4283 vm_map_copy_t copy;
4284 vm_map_entry_t last = vm_map_last_entry(new_map)((new_map)->hdr.links.prev);
4285
4286 vm_map_unlock(old_map)lock_done(&(old_map)->lock);
4287 if (vm_map_copyin(old_map,
4288 start,
4289 entry_size,
4290 FALSE((boolean_t) 0),
4291 &copy)
4292 != KERN_SUCCESS0) {
4293 vm_map_lock(old_map)({ lock_write(&(old_map)->lock); (old_map)->timestamp
++; });
4294 if (!vm_map_lookup_entry(old_map, start, &last))
4295 last = last->vme_nextlinks.next;
4296 old_entry = last;
4297 /*
4298 * For some error returns, want to
4299 * skip to the next element.
4300 */
4301
4302 continue;
4303 }
4304
4305 /*
4306 * Insert the copy into the new map
4307 */
4308
4309 vm_map_copy_insert(new_map, last, copy)({ struct rbtree_node *node, *tmp; for (node = rbtree_postwalk_deepest
(&(copy)->c_u.hdr.tree), tmp = rbtree_postwalk_unlink(
node); node != ((void *) 0); node = tmp, tmp = rbtree_postwalk_unlink
(node)) ({ struct rbtree_node *___cur, *___prev; int ___diff,
___index; ___prev = ((void *) 0); ___index = -1; ___cur = (&
(new_map)->hdr.tree)->root; while (___cur != ((void *) 0
)) { ___diff = vm_map_entry_cmp_insert(node, ___cur); ({ if (
!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c", 4309
); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur
= ___cur->children[___index]; } rbtree_insert_rebalance(&
(new_map)->hdr.tree, ___prev, ___index, node); }); (((last
)->links.next)->links.prev = ((copy)->c_u.hdr.links.
prev)) ->links.next = ((last)->links.next); ((last)->
links.next = ((copy)->c_u.hdr.links.next)) ->links.prev
= (last); (new_map)->hdr.nentries += (copy)->c_u.hdr.nentries
; kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy)
; });
4310 new_size += entry_size;
4311
4312 /*
4313 * Pick up the traversal at the end of
4314 * the copied region.
4315 */
4316
4317 vm_map_lock(old_map)({ lock_write(&(old_map)->lock); (old_map)->timestamp
++; });
4318 start += entry_size;
4319 if (!vm_map_lookup_entry(old_map, start, &last))
4320 last = last->vme_nextlinks.next;
4321 else
4322 vm_map_clip_start(old_map, last, start)({ if ((start) > (last)->links.start) _vm_map_clip_start
(&(old_map)->hdr,(last),(start)); });
4323 old_entry = last;
4324
4325 continue;
4326 /* INNER BLOCK (copy cannot be optimized) */ }
4327 }
4328 old_entry = old_entry->vme_nextlinks.next;
4329 }
4330
4331 new_map->size = new_size;
4332 vm_map_unlock(old_map)lock_done(&(old_map)->lock);
4333
4334 return(new_map);
4335}
4336
4337/*
4338 * vm_map_lookup:
4339 *
4340 * Finds the VM object, offset, and
4341 * protection for a given virtual address in the
4342 * specified map, assuming a page fault of the
4343 * type specified.
4344 *
4345 * Returns the (object, offset, protection) for
4346 * this address, whether it is wired down, and whether
4347 * this map has the only reference to the data in question.
4348 * In order to later verify this lookup, a "version"
4349 * is returned.
4350 *
4351 * The map should not be locked; it will not be
4352 * locked on exit. In order to guarantee the
4353 * existence of the returned object, it is returned
4354 * locked.
4355 *
4356 * If a lookup is requested with "write protection"
4357 * specified, the map may be changed to perform virtual
4358 * copying operations, although the data referenced will
4359 * remain the same.
4360 */
4361kern_return_t vm_map_lookup(var_map, vaddr, fault_type, out_version,
4362 object, offset, out_prot, wired)
4363 vm_map_t *var_map; /* IN/OUT */
4364 vm_offset_t vaddr;
4365 vm_prot_t fault_type;
4366
4367 vm_map_version_t *out_version; /* OUT */
4368 vm_object_t *object; /* OUT */
4369 vm_offset_t *offset; /* OUT */
4370 vm_prot_t *out_prot; /* OUT */
4371 boolean_t *wired; /* OUT */
4372{
4373 vm_map_entry_t entry;
4374 vm_map_t map = *var_map;
4375 vm_prot_t prot;
4376
4377 RetryLookup: ;
4378
4379 /*
4380 * Lookup the faulting address.
4381 */
4382
4383 vm_map_lock_read(map)lock_read(&(map)->lock);
4384
4385#define RETURN(why) \
4386 { \
4387 vm_map_unlock_read(map)lock_done(&(map)->lock); \
4388 return(why); \
4389 }
4390
4391 /*
4392 * If the map has an interesting hint, try it before calling
4393 * full blown lookup routine.
4394 */
4395
4396 simple_lock(&map->hint_lock);
4397 entry = map->hint;
4398 simple_unlock(&map->hint_lock);
4399
4400 if ((entry == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) ||
4401 (vaddr < entry->vme_startlinks.start) || (vaddr >= entry->vme_endlinks.end)) {
4402 vm_map_entry_t tmp_entry;
4403
4404 /*
4405 * Entry was either not a valid hint, or the vaddr
4406 * was not contained in the entry, so do a full lookup.
4407 */
4408 if (!vm_map_lookup_entry(map, vaddr, &tmp_entry))
4409 RETURN(KERN_INVALID_ADDRESS1);
4410
4411 entry = tmp_entry;
4412 }
4413
4414 /*
4415 * Handle submaps.
4416 */
4417
4418 if (entry->is_sub_map) {
4419 vm_map_t old_map = map;
4420
4421 *var_map = map = entry->object.sub_map;
4422 vm_map_unlock_read(old_map)lock_done(&(old_map)->lock);
4423 goto RetryLookup;
4424 }
4425
4426 /*
4427 * Check whether this task is allowed to have
4428 * this page.
4429 */
4430
4431 prot = entry->protection;
4432
4433 if ((fault_type & (prot)) != fault_type) {
4434 if ((prot & VM_PROT_NOTIFY((vm_prot_t) 0x10)) && (fault_type & VM_PROT_WRITE((vm_prot_t) 0x02))) {
4435 RETURN(KERN_WRITE_PROTECTION_FAILURE24);
4436 } else {
4437 RETURN(KERN_PROTECTION_FAILURE2);
4438 }
4439 }
4440
4441 /*
4442 * If this page is not pageable, we have to get
4443 * it for all possible accesses.
4444 */
4445
4446 if ((*wired = (entry->wired_count != 0)))
4447 prot = fault_type = entry->protection;
4448
4449 /*
4450 * If the entry was copy-on-write, we either ...
4451 */
4452
4453 if (entry->needs_copy) {
4454 /*
4455 * If we want to write the page, we may as well
4456 * handle that now since we've got the map locked.
4457 *
4458 * If we don't need to write the page, we just
4459 * demote the permissions allowed.
4460 */
4461
4462 if (fault_type & VM_PROT_WRITE((vm_prot_t) 0x02)) {
4463 /*
4464 * Make a new object, and place it in the
4465 * object chain. Note that no new references
4466 * have appeared -- one just moved from the
4467 * map to the new object.
4468 */
4469
4470 if (vm_map_lock_read_to_write(map)(lock_read_to_write(&(map)->lock) || (((map)->timestamp
++), 0))) {
4471 goto RetryLookup;
4472 }
4473 map->timestamp++;
4474
4475 vm_object_shadow(
4476 &entry->object.vm_object,
4477 &entry->offset,
4478 (vm_size_t) (entry->vme_endlinks.end - entry->vme_startlinks.start));
4479
4480 entry->needs_copy = FALSE((boolean_t) 0);
4481
4482 vm_map_lock_write_to_read(map)lock_write_to_read(&(map)->lock);
4483 }
4484 else {
4485 /*
4486 * We're attempting to read a copy-on-write
4487 * page -- don't allow writes.
4488 */
4489
4490 prot &= (~VM_PROT_WRITE((vm_prot_t) 0x02));
4491 }
4492 }
4493
4494 /*
4495 * Create an object if necessary.
4496 */
4497 if (entry->object.vm_object == VM_OBJECT_NULL((vm_object_t) 0)) {
4498
4499 if (vm_map_lock_read_to_write(map)(lock_read_to_write(&(map)->lock) || (((map)->timestamp
++), 0))) {
4500 goto RetryLookup;
4501 }
4502
4503 entry->object.vm_object = vm_object_allocate(
4504 (vm_size_t)(entry->vme_endlinks.end - entry->vme_startlinks.start));
4505 entry->offset = 0;
4506 vm_map_lock_write_to_read(map)lock_write_to_read(&(map)->lock);
4507 }
4508
4509 /*
4510 * Return the object/offset from this entry. If the entry
4511 * was copy-on-write or empty, it has been fixed up. Also
4512 * return the protection.
4513 */
4514
4515 *offset = (vaddr - entry->vme_startlinks.start) + entry->offset;
4516 *object = entry->object.vm_object;
4517 *out_prot = prot;
4518
4519 /*
4520 * Lock the object to prevent it from disappearing
4521 */
4522
4523 vm_object_lock(*object);
4524
4525 /*
4526 * Save the version number and unlock the map.
4527 */
4528
4529 out_version->main_timestamp = map->timestamp;
4530
4531 RETURN(KERN_SUCCESS0);
4532
4533#undef RETURN
4534}
4535
4536/*
4537 * vm_map_verify:
4538 *
4539 * Verifies that the map in question has not changed
4540 * since the given version. If successful, the map
4541 * will not change until vm_map_verify_done() is called.
4542 */
4543boolean_t vm_map_verify(map, version)
4544 vm_map_t map;
4545 vm_map_version_t *version; /* REF */
4546{
4547 boolean_t result;
4548
4549 vm_map_lock_read(map)lock_read(&(map)->lock);
4550 result = (map->timestamp == version->main_timestamp);
4551
4552 if (!result)
4553 vm_map_unlock_read(map)lock_done(&(map)->lock);
4554
4555 return(result);
4556}
4557
4558/*
4559 * vm_map_verify_done:
4560 *
4561 * Releases locks acquired by a vm_map_verify.
4562 *
4563 * This is now a macro in vm/vm_map.h. It does a
4564 * vm_map_unlock_read on the map.
4565 */
4566
4567/*
4568 * vm_region:
4569 *
4570 * User call to obtain information about a region in
4571 * a task's address map.
4572 */
4573
4574kern_return_t vm_region(map, address, size,
4575 protection, max_protection,
4576 inheritance, is_shared,
4577 object_name, offset_in_object)
4578 vm_map_t map;
4579 vm_offset_t *address; /* IN/OUT */
4580 vm_size_t *size; /* OUT */
4581 vm_prot_t *protection; /* OUT */
4582 vm_prot_t *max_protection; /* OUT */
4583 vm_inherit_t *inheritance; /* OUT */
4584 boolean_t *is_shared; /* OUT */
4585 ipc_port_t *object_name; /* OUT */
4586 vm_offset_t *offset_in_object; /* OUT */
4587{
4588 vm_map_entry_t tmp_entry;
4589 vm_map_entry_t entry;
4590 vm_offset_t tmp_offset;
4591 vm_offset_t start;
4592
4593 if (map == VM_MAP_NULL((vm_map_t) 0))
4594 return(KERN_INVALID_ARGUMENT4);
4595
4596 start = *address;
4597
4598 vm_map_lock_read(map)lock_read(&(map)->lock);
4599 if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
4600 if ((entry = tmp_entry->vme_nextlinks.next) == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) {
4601 vm_map_unlock_read(map)lock_done(&(map)->lock);
4602 return(KERN_NO_SPACE3);
4603 }
4604 } else {
4605 entry = tmp_entry;
4606 }
4607
4608 start = entry->vme_startlinks.start;
4609 *protection = entry->protection;
4610 *max_protection = entry->max_protection;
4611 *inheritance = entry->inheritance;
4612 *address = start;
4613 *size = (entry->vme_endlinks.end - start);
4614
4615 tmp_offset = entry->offset;
4616
4617
4618 if (entry->is_sub_map) {
4619 *is_shared = FALSE((boolean_t) 0);
4620 *object_name = IP_NULL((ipc_port_t) ((ipc_object_t) 0));
4621 *offset_in_object = tmp_offset;
4622 } else {
4623 *is_shared = entry->is_shared;
4624 *object_name = vm_object_name(entry->object.vm_object);
4625 *offset_in_object = tmp_offset;
4626 }
4627
4628 vm_map_unlock_read(map)lock_done(&(map)->lock);
4629
4630 return(KERN_SUCCESS0);
4631}
4632
4633/*
4634 * Routine: vm_map_simplify
4635 *
4636 * Description:
4637 * Attempt to simplify the map representation in
4638 * the vicinity of the given starting address.
4639 * Note:
4640 * This routine is intended primarily to keep the
4641 * kernel maps more compact -- they generally don't
4642 * benefit from the "expand a map entry" technology
4643 * at allocation time because the adjacent entry
4644 * is often wired down.
4645 */
4646void vm_map_simplify(map, start)
4647 vm_map_t map;
4648 vm_offset_t start;
4649{
4650 vm_map_entry_t this_entry;
4651 vm_map_entry_t prev_entry;
4652
4653 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
4654 if (
4655 (vm_map_lookup_entry(map, start, &this_entry)) &&
4656 ((prev_entry = this_entry->vme_prevlinks.prev) != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
4657
4658 (prev_entry->vme_endlinks.end == start) &&
4659
4660 (prev_entry->is_shared == FALSE((boolean_t) 0)) &&
4661 (prev_entry->is_sub_map == FALSE((boolean_t) 0)) &&
4662
4663 (this_entry->is_shared == FALSE((boolean_t) 0)) &&
4664 (this_entry->is_sub_map == FALSE((boolean_t) 0)) &&
4665
4666 (prev_entry->inheritance == this_entry->inheritance) &&
4667 (prev_entry->protection == this_entry->protection) &&
4668 (prev_entry->max_protection == this_entry->max_protection) &&
4669 (prev_entry->wired_count == this_entry->wired_count) &&
4670 (prev_entry->user_wired_count == this_entry->user_wired_count) &&
4671
4672 (prev_entry->needs_copy == this_entry->needs_copy) &&
4673
4674 (prev_entry->object.vm_object == this_entry->object.vm_object) &&
4675 ((prev_entry->offset + (prev_entry->vme_endlinks.end - prev_entry->vme_startlinks.start))
4676 == this_entry->offset) &&
4677 (prev_entry->projected_on == 0) &&
4678 (this_entry->projected_on == 0)
4679 ) {
4680 if (map->first_free == this_entry)
4681 map->first_free = prev_entry;
4682
4683 SAVE_HINT(map, prev_entry); (map)->hint = (prev_entry); ;;
4684 vm_map_entry_unlink(map, this_entry)({ (&(map)->hdr)->nentries--; (this_entry)->links
.next->links.prev = (this_entry)->links.prev; (this_entry
)->links.prev->links.next = (this_entry)->links.next
; rbtree_remove(&(&(map)->hdr)->tree, &(this_entry
)->tree_node); });
4685 prev_entry->vme_endlinks.end = this_entry->vme_endlinks.end;
4686 vm_object_deallocate(this_entry->object.vm_object);
4687 vm_map_entry_dispose(map, this_entry)_vm_map_entry_dispose(&(map)->hdr, (this_entry));
4688 }
4689 vm_map_unlock(map)lock_done(&(map)->lock);
4690}
4691
4692
4693/*
4694 * Routine: vm_map_machine_attribute
4695 * Purpose:
4696 * Provide machine-specific attributes to mappings,
4697 * such as cachability etc. for machines that provide
4698 * them. NUMA architectures and machines with big/strange
4699 * caches will use this.
4700 * Note:
4701 * Responsibilities for locking and checking are handled here,
4702 * everything else in the pmap module. If any non-volatile
4703 * information must be kept, the pmap module should handle
4704 * it itself. [This assumes that attributes do not
4705 * need to be inherited, which seems ok to me]
4706 */
4707kern_return_t vm_map_machine_attribute(map, address, size, attribute, value)
4708 vm_map_t map;
4709 vm_offset_t address;
4710 vm_size_t size;
4711 vm_machine_attribute_t attribute;
4712 vm_machine_attribute_val_t* value; /* IN/OUT */
4713{
4714 kern_return_t ret;
4715
4716 if (address < vm_map_min(map)((map)->hdr.links.start) ||
4717 (address + size) > vm_map_max(map)((map)->hdr.links.end))
4718 return KERN_INVALID_ARGUMENT4;
4719
4720 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
4721
4722 ret = pmap_attribute(map->pmap, address, size, attribute, value)(1);
4723
4724 vm_map_unlock(map)lock_done(&(map)->lock);
4725
4726 return ret;
4727}
4728
4729
4730#if MACH_KDB0
4731
4732#define printf kdbprintf
4733
4734/*
4735 * vm_map_print: [ debug ]
4736 */
4737void vm_map_print(map)
4738 vm_map_t map;
4739{
4740 vm_map_entry_t entry;
4741
4742 iprintf("Task map 0x%X: pmap=0x%X,",
4743 (vm_offset_t) map, (vm_offset_t) (map->pmap));
4744 printf("ref=%d,nentries=%d,", map->ref_count, map->hdr.nentries);
4745 printf("version=%d\n", map->timestamp);
4746 indent += 2;
4747 for (entry = vm_map_first_entry(map)((map)->hdr.links.next);
4748 entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links);
4749 entry = entry->vme_nextlinks.next) {
4750 static char *inheritance_name[3] = { "share", "copy", "none"};
4751
4752 iprintf("map entry 0x%X: ", (vm_offset_t) entry);
4753 printf("start=0x%X, end=0x%X, ",
4754 (vm_offset_t) entry->vme_startlinks.start, (vm_offset_t) entry->vme_endlinks.end);
4755 printf("prot=%X/%X/%s, ",
4756 entry->protection,
4757 entry->max_protection,
4758 inheritance_name[entry->inheritance]);
4759 if (entry->wired_count != 0) {
4760 printf("wired(");
4761 if (entry->user_wired_count != 0)
4762 printf("u");
4763 if (entry->wired_count >
4764 ((entry->user_wired_count == 0) ? 0 : 1))
4765 printf("k");
4766 printf(") ");
4767 }
4768 if (entry->in_transition) {
4769 printf("in transition");
4770 if (entry->needs_wakeup)
4771 printf("(wake request)");
4772 printf(", ");
4773 }
4774 if (entry->is_sub_map) {
4775 printf("submap=0x%X, offset=0x%X\n",
4776 (vm_offset_t) entry->object.sub_map,
4777 (vm_offset_t) entry->offset);
4778 } else {
4779 printf("object=0x%X, offset=0x%X",
4780 (vm_offset_t) entry->object.vm_object,
4781 (vm_offset_t) entry->offset);
4782 if (entry->is_shared)
4783 printf(", shared");
4784 if (entry->needs_copy)
4785 printf(", copy needed");
4786 printf("\n");
4787
4788 if ((entry->vme_prevlinks.prev == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) ||
4789 (entry->vme_prevlinks.prev->object.vm_object != entry->object.vm_object)) {
4790 indent += 2;
4791 vm_object_print(entry->object.vm_object);
4792 indent -= 2;
4793 }
4794 }
4795 }
4796 indent -= 2;
4797}
4798
4799/*
4800 * Routine: vm_map_copy_print
4801 * Purpose:
4802 * Pretty-print a copy object for ddb.
4803 */
4804
4805void vm_map_copy_print(copy)
4806 vm_map_copy_t copy;
4807{
4808 int i, npages;
4809
4810 printf("copy object 0x%x\n", copy);
4811
4812 indent += 2;
4813
4814 iprintf("type=%d", copy->type);
4815 switch (copy->type) {
4816 case VM_MAP_COPY_ENTRY_LIST1:
4817 printf("[entry_list]");
4818 break;
4819
4820 case VM_MAP_COPY_OBJECT2:
4821 printf("[object]");
4822 break;
4823
4824 case VM_MAP_COPY_PAGE_LIST3:
4825 printf("[page_list]");
4826 break;
4827
4828 default:
4829 printf("[bad type]");
4830 break;
4831 }
4832 printf(", offset=0x%x", copy->offset);
4833 printf(", size=0x%x\n", copy->size);
4834
4835 switch (copy->type) {
4836 case VM_MAP_COPY_ENTRY_LIST1:
4837 /* XXX add stuff here */
4838 break;
4839
4840 case VM_MAP_COPY_OBJECT2:
4841 iprintf("object=0x%x\n", copy->cpy_objectc_u.c_o.object);
4842 break;
4843
4844 case VM_MAP_COPY_PAGE_LIST3:
4845 iprintf("npages=%d", copy->cpy_npagesc_u.c_p.npages);
4846 printf(", cont=%x", copy->cpy_contc_u.c_p.cont);
4847 printf(", cont_args=%x\n", copy->cpy_cont_argsc_u.c_p.cont_args);
4848 if (copy->cpy_npagesc_u.c_p.npages < 0) {
4849 npages = 0;
4850 } else if (copy->cpy_npagesc_u.c_p.npages > VM_MAP_COPY_PAGE_LIST_MAX64) {
4851 npages = VM_MAP_COPY_PAGE_LIST_MAX64;
4852 } else {
4853 npages = copy->cpy_npagesc_u.c_p.npages;
4854 }
4855 iprintf("copy->cpy_page_list[0..%d] = {", npages);
4856 for (i = 0; i < npages - 1; i++) {
4857 printf("0x%x, ", copy->cpy_page_listc_u.c_p.page_list[i]);
4858 }
4859 if (npages > 0) {
4860 printf("0x%x", copy->cpy_page_listc_u.c_p.page_list[npages - 1]);
4861 }
4862 printf("}\n");
4863 break;
4864 }
4865
4866 indent -= 2;
4867}
4868#endif /* MACH_KDB */