Bug Summary

File:obj-scan-build/../vm/vm_map.c
Location:line 2946, column 7
Description:Dereference of null pointer

Annotated Source Code

1/*
2 * Mach Operating System
3 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4 * Copyright (c) 1993,1994 The University of Utah and
5 * the Computer Systems Laboratory (CSL).
6 * All rights reserved.
7 *
8 * Permission to use, copy, modify and distribute this software and its
9 * documentation is hereby granted, provided that both the copyright
10 * notice and this permission notice appear in all copies of the
11 * software, derivative works or modified versions, and any portions
12 * thereof, and that both notices appear in supporting documentation.
13 *
14 * CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15 * THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16 * OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17 * THIS SOFTWARE.
18 *
19 * Carnegie Mellon requests users of this software to return to
20 *
21 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
22 * School of Computer Science
23 * Carnegie Mellon University
24 * Pittsburgh PA 15213-3890
25 *
26 * any improvements or extensions that they make and grant Carnegie Mellon
27 * the rights to redistribute these changes.
28 */
29/*
30 * File: vm/vm_map.c
31 * Author: Avadis Tevanian, Jr., Michael Wayne Young
32 * Date: 1985
33 *
34 * Virtual memory mapping module.
35 */
36
37#include <kern/printfdb_printf.h>
38#include <mach/kern_return.h>
39#include <mach/port.h>
40#include <mach/vm_attributes.h>
41#include <mach/vm_param.h>
42#include <kern/assert.h>
43#include <kern/debug.h>
44#include <kern/kalloc.h>
45#include <kern/rbtree.h>
46#include <kern/slab.h>
47#include <vm/pmap.h>
48#include <vm/vm_fault.h>
49#include <vm/vm_map.h>
50#include <vm/vm_object.h>
51#include <vm/vm_page.h>
52#include <vm/vm_resident.h>
53#include <vm/vm_kern.h>
54#include <ipc/ipc_port.h>
55
56#if MACH_KDB1
57#include <ddb/db_output.h>
58#include <vm/vm_print.h>
59#endif /* MACH_KDB */
60
61/*
62 * Macros to copy a vm_map_entry. We must be careful to correctly
63 * manage the wired page count. vm_map_entry_copy() creates a new
64 * map entry to the same memory - the wired count in the new entry
65 * must be set to zero. vm_map_entry_copy_full() creates a new
66 * entry that is identical to the old entry. This preserves the
67 * wire count; it's used for map splitting and cache changing in
68 * vm_map_copyout.
69 */
70#define vm_map_entry_copy(NEW,OLD)({ *(NEW) = *(OLD); (NEW)->is_shared = ((boolean_t) 0); (NEW
)->needs_wakeup = ((boolean_t) 0); (NEW)->in_transition
= ((boolean_t) 0); (NEW)->wired_count = 0; (NEW)->user_wired_count
= 0; }) \({
71MACRO_BEGIN({ \
72 *(NEW) = *(OLD); \
73 (NEW)->is_shared = FALSE((boolean_t) 0); \
74 (NEW)->needs_wakeup = FALSE((boolean_t) 0); \
75 (NEW)->in_transition = FALSE((boolean_t) 0); \
76 (NEW)->wired_count = 0; \
77 (NEW)->user_wired_count = 0; \})
78MACRO_END})
79
80#define vm_map_entry_copy_full(NEW,OLD)(*(NEW) = *(OLD)) (*(NEW) = *(OLD))
81
82/*
83 * Virtual memory maps provide for the mapping, protection,
84 * and sharing of virtual memory objects. In addition,
85 * this module provides for an efficient virtual copy of
86 * memory from one map to another.
87 *
88 * Synchronization is required prior to most operations.
89 *
90 * Maps consist of an ordered doubly-linked list of simple
91 * entries; a hint and a red-black tree are used to speed up lookups.
92 *
93 * Sharing maps have been deleted from this version of Mach.
94 * All shared objects are now mapped directly into the respective
95 * maps. This requires a change in the copy on write strategy;
96 * the asymmetric (delayed) strategy is used for shared temporary
97 * objects instead of the symmetric (shadow) strategy. This is
98 * selected by the (new) use_shared_copy bit in the object. See
99 * vm_object_copy_temporary in vm_object.c for details. All maps
100 * are now "top level" maps (either task map, kernel map or submap
101 * of the kernel map).
102 *
103 * Since portions of maps are specified by start/end addreses,
104 * which may not align with existing map entries, all
105 * routines merely "clip" entries to these start/end values.
106 * [That is, an entry is split into two, bordering at a
107 * start or end value.] Note that these clippings may not
108 * always be necessary (as the two resulting entries are then
109 * not changed); however, the clipping is done for convenience.
110 * No attempt is currently made to "glue back together" two
111 * abutting entries.
112 *
113 * The symmetric (shadow) copy strategy implements virtual copy
114 * by copying VM object references from one map to
115 * another, and then marking both regions as copy-on-write.
116 * It is important to note that only one writeable reference
117 * to a VM object region exists in any map when this strategy
118 * is used -- this means that shadow object creation can be
119 * delayed until a write operation occurs. The asymmetric (delayed)
120 * strategy allows multiple maps to have writeable references to
121 * the same region of a vm object, and hence cannot delay creating
122 * its copy objects. See vm_object_copy_temporary() in vm_object.c.
123 * Copying of permanent objects is completely different; see
124 * vm_object_copy_strategically() in vm_object.c.
125 */
126
127struct kmem_cache vm_map_cache; /* cache for vm_map structures */
128struct kmem_cache vm_map_entry_cache; /* cache for vm_map_entry structures */
129struct kmem_cache vm_map_kentry_cache; /* cache for kernel entry structures */
130struct kmem_cache vm_map_copy_cache; /* cache for vm_map_copy structures */
131
132/*
133 * Placeholder object for submap operations. This object is dropped
134 * into the range by a call to vm_map_find, and removed when
135 * vm_map_submap creates the submap.
136 */
137
138static struct vm_object vm_submap_object_store;
139vm_object_t vm_submap_object = &vm_submap_object_store;
140
141/*
142 * vm_map_init:
143 *
144 * Initialize the vm_map module. Must be called before
145 * any other vm_map routines.
146 *
147 * Map and entry structures are allocated from caches -- we must
148 * initialize those caches.
149 *
150 * There are three caches of interest:
151 *
152 * vm_map_cache: used to allocate maps.
153 * vm_map_entry_cache: used to allocate map entries.
154 * vm_map_kentry_cache: used to allocate map entries for the kernel.
155 *
156 * Kernel map entries are allocated from a special cache, using a custom
157 * page allocation function to avoid recursion. It would be difficult
158 * (perhaps impossible) for the kernel to allocate more memory to an entry
159 * cache when it became empty since the very act of allocating memory
160 * implies the creation of a new entry.
161 */
162
163vm_offset_t kentry_data;
164vm_size_t kentry_data_size = KENTRY_DATA_SIZE(256*(1 << 12));
165
166static vm_offset_t kentry_pagealloc(vm_size_t size)
167{
168 vm_offset_t result;
169
170 if (size > kentry_data_size)
171 panic("vm_map: kentry memory exhausted");
172
173 result = kentry_data;
174 kentry_data += size;
175 kentry_data_size -= size;
176 return result;
177}
178
179void vm_map_init(void)
180{
181 kmem_cache_init(&vm_map_cache, "vm_map", sizeof(struct vm_map), 0,
182 NULL((void *) 0), NULL((void *) 0), NULL((void *) 0), 0);
183 kmem_cache_init(&vm_map_entry_cache, "vm_map_entry",
184 sizeof(struct vm_map_entry), 0, NULL((void *) 0), NULL((void *) 0), NULL((void *) 0), 0);
185 kmem_cache_init(&vm_map_kentry_cache, "vm_map_kentry",
186 sizeof(struct vm_map_entry), 0, NULL((void *) 0), kentry_pagealloc,
187 NULL((void *) 0), KMEM_CACHE_NOCPUPOOL0x1 | KMEM_CACHE_NOOFFSLAB0x2
188 | KMEM_CACHE_NORECLAIM0x4);
189 kmem_cache_init(&vm_map_copy_cache, "vm_map_copy",
190 sizeof(struct vm_map_copy), 0, NULL((void *) 0), NULL((void *) 0), NULL((void *) 0), 0);
191
192 /*
193 * Submap object is initialized by vm_object_init.
194 */
195}
196
197void vm_map_setup(
198 vm_map_t map,
199 pmap_t pmap,
200 vm_offset_t min,
201 vm_offset_t max,
202 boolean_t pageable)
203{
204 vm_map_first_entry(map)((map)->hdr.links.next) = vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links);
205 vm_map_last_entry(map)((map)->hdr.links.prev) = vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links);
206 map->hdr.nentries = 0;
207 map->hdr.entries_pageable = pageable;
208 rbtree_init(&map->hdr.tree);
209
210 map->size = 0;
211 map->ref_count = 1;
212 map->pmap = pmap;
213 map->min_offsethdr.links.start = min;
214 map->max_offsethdr.links.end = max;
215 map->wiring_required = FALSE((boolean_t) 0);
216 map->wait_for_space = FALSE((boolean_t) 0);
217 map->first_free = vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links);
218 map->hint = vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links);
219 vm_map_lock_init(map)({ lock_init(&(map)->lock, ((boolean_t) 1)); (map)->
timestamp = 0; });
220 simple_lock_init(&map->ref_lock);
221 simple_lock_init(&map->hint_lock);
222}
223
224/*
225 * vm_map_create:
226 *
227 * Creates and returns a new empty VM map with
228 * the given physical map structure, and having
229 * the given lower and upper address bounds.
230 */
231vm_map_t vm_map_create(
232 pmap_t pmap,
233 vm_offset_t min,
234 vm_offset_t max,
235 boolean_t pageable)
236{
237 vm_map_t result;
238
239 result = (vm_map_t) kmem_cache_alloc(&vm_map_cache);
240 if (result == VM_MAP_NULL((vm_map_t) 0))
241 panic("vm_map_create");
242
243 vm_map_setup(result, pmap, min, max, pageable);
244
245 return(result);
246}
247
248/*
249 * vm_map_entry_create: [ internal use only ]
250 *
251 * Allocates a VM map entry for insertion in the
252 * given map (or map copy). No fields are filled.
253 */
254#define vm_map_entry_create(map)_vm_map_entry_create(&(map)->hdr) \
255 _vm_map_entry_create(&(map)->hdr)
256
257#define vm_map_copy_entry_create(copy)_vm_map_entry_create(&(copy)->c_u.hdr) \
258 _vm_map_entry_create(&(copy)->cpy_hdrc_u.hdr)
259
260vm_map_entry_t _vm_map_entry_create(map_header)
261 const struct vm_map_header *map_header;
262{
263 kmem_cache_t cache;
264 vm_map_entry_t entry;
265
266 if (map_header->entries_pageable)
267 cache = &vm_map_entry_cache;
268 else
269 cache = &vm_map_kentry_cache;
270
271 entry = (vm_map_entry_t) kmem_cache_alloc(cache);
272 if (entry == VM_MAP_ENTRY_NULL((vm_map_entry_t) 0))
273 panic("vm_map_entry_create");
274
275 return(entry);
276}
277
278/*
279 * vm_map_entry_dispose: [ internal use only ]
280 *
281 * Inverse of vm_map_entry_create.
282 */
283#define vm_map_entry_dispose(map, entry)_vm_map_entry_dispose(&(map)->hdr, (entry)) \
284 _vm_map_entry_dispose(&(map)->hdr, (entry))
285
286#define vm_map_copy_entry_dispose(map, entry)_vm_map_entry_dispose(&(copy)->c_u.hdr, (entry)) \
287 _vm_map_entry_dispose(&(copy)->cpy_hdrc_u.hdr, (entry))
288
289void _vm_map_entry_dispose(map_header, entry)
290 const struct vm_map_header *map_header;
291 vm_map_entry_t entry;
292{
293 kmem_cache_t cache;
294
295 if (map_header->entries_pageable)
296 cache = &vm_map_entry_cache;
297 else
298 cache = &vm_map_kentry_cache;
299
300 kmem_cache_free(cache, (vm_offset_t) entry);
301}
302
303/*
304 * Red-black tree lookup/insert comparison functions
305 */
306static inline int vm_map_entry_cmp_lookup(vm_offset_t addr,
307 const struct rbtree_node *node)
308{
309 struct vm_map_entry *entry;
310
311 entry = rbtree_entry(node, struct vm_map_entry, tree_node)((struct vm_map_entry *)((char *)node - __builtin_offsetof (struct
vm_map_entry, tree_node)));
312
313 if (addr < entry->vme_startlinks.start)
314 return -1;
315 else if (addr < entry->vme_endlinks.end)
316 return 0;
317 else
318 return 1;
319}
320
321static inline int vm_map_entry_cmp_insert(const struct rbtree_node *a,
322 const struct rbtree_node *b)
323{
324 struct vm_map_entry *entry;
325
326 entry = rbtree_entry(a, struct vm_map_entry, tree_node)((struct vm_map_entry *)((char *)a - __builtin_offsetof (struct
vm_map_entry, tree_node)));
327 return vm_map_entry_cmp_lookup(entry->vme_startlinks.start, b);
328}
329
330/*
331 * vm_map_entry_{un,}link:
332 *
333 * Insert/remove entries from maps (or map copies).
334 *
335 * The start and end addresses of the entries must be properly set
336 * before using these macros.
337 */
338#define vm_map_entry_link(map, after_where, entry)({ (&(map)->hdr)->nentries++; (entry)->links.prev
= (after_where); (entry)->links.next = (after_where)->
links.next; (entry)->links.prev->links.next = (entry)->
links.next->links.prev = (entry); ({ struct rbtree_node *___cur
, *___prev; int ___diff, ___index; ___prev = ((void *) 0); ___index
= -1; ___cur = (&(&(map)->hdr)->tree)->root
; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(entry)->tree_node, ___cur); ({ if (!(___diff != 0))
Assert("___diff != 0", "../vm/vm_map.c", 338); }); ___prev =
___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(map
)->hdr)->tree, ___prev, ___index, &(entry)->tree_node
); }); }) \
339 _vm_map_entry_link(&(map)->hdr, after_where, entry)({ (&(map)->hdr)->nentries++; (entry)->links.prev
= (after_where); (entry)->links.next = (after_where)->
links.next; (entry)->links.prev->links.next = (entry)->
links.next->links.prev = (entry); ({ struct rbtree_node *___cur
, *___prev; int ___diff, ___index; ___prev = ((void *) 0); ___index
= -1; ___cur = (&(&(map)->hdr)->tree)->root
; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(entry)->tree_node, ___cur); ({ if (!(___diff != 0))
Assert("___diff != 0", "../vm/vm_map.c", 339); }); ___prev =
___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(map
)->hdr)->tree, ___prev, ___index, &(entry)->tree_node
); }); })
340
341#define vm_map_copy_entry_link(copy, after_where, entry)({ (&(copy)->c_u.hdr)->nentries++; (entry)->links
.prev = (after_where); (entry)->links.next = (after_where)
->links.next; (entry)->links.prev->links.next = (entry
)->links.next->links.prev = (entry); ({ struct rbtree_node
*___cur, *___prev; int ___diff, ___index; ___prev = ((void *
) 0); ___index = -1; ___cur = (&(&(copy)->c_u.hdr)
->tree)->root; while (___cur != ((void *) 0)) { ___diff
= vm_map_entry_cmp_insert(&(entry)->tree_node, ___cur
); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c"
, 341); }); ___prev = ___cur; ___index = rbtree_d2i(___diff);
___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(&(copy)->c_u.hdr)->tree, ___prev, ___index, &
(entry)->tree_node); }); }) \
342 _vm_map_entry_link(&(copy)->cpy_hdr, after_where, entry)({ (&(copy)->c_u.hdr)->nentries++; (entry)->links
.prev = (after_where); (entry)->links.next = (after_where)
->links.next; (entry)->links.prev->links.next = (entry
)->links.next->links.prev = (entry); ({ struct rbtree_node
*___cur, *___prev; int ___diff, ___index; ___prev = ((void *
) 0); ___index = -1; ___cur = (&(&(copy)->c_u.hdr)
->tree)->root; while (___cur != ((void *) 0)) { ___diff
= vm_map_entry_cmp_insert(&(entry)->tree_node, ___cur
); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c"
, 342); }); ___prev = ___cur; ___index = rbtree_d2i(___diff);
___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(&(copy)->c_u.hdr)->tree, ___prev, ___index, &
(entry)->tree_node); }); })
343
344#define _vm_map_entry_link(hdr, after_where, entry)({ (hdr)->nentries++; (entry)->links.prev = (after_where
); (entry)->links.next = (after_where)->links.next; (entry
)->links.prev->links.next = (entry)->links.next->
links.prev = (entry); ({ struct rbtree_node *___cur, *___prev
; int ___diff, ___index; ___prev = ((void *) 0); ___index = -
1; ___cur = (&(hdr)->tree)->root; while (___cur != (
(void *) 0)) { ___diff = vm_map_entry_cmp_insert(&(entry)
->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0"
, "../vm/vm_map.c", 344); }); ___prev = ___cur; ___index = rbtree_d2i
(___diff); ___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(hdr)->tree, ___prev, ___index, &(entry)->tree_node
); }); }) \
345 MACRO_BEGIN({ \
346 (hdr)->nentries++; \
347 (entry)->vme_prevlinks.prev = (after_where); \
348 (entry)->vme_nextlinks.next = (after_where)->vme_nextlinks.next; \
349 (entry)->vme_prevlinks.prev->vme_nextlinks.next = \
350 (entry)->vme_nextlinks.next->vme_prevlinks.prev = (entry); \
351 rbtree_insert(&(hdr)->tree, &(entry)->tree_node, \({ struct rbtree_node *___cur, *___prev; int ___diff, ___index
; ___prev = ((void *) 0); ___index = -1; ___cur = (&(hdr)
->tree)->root; while (___cur != ((void *) 0)) { ___diff
= vm_map_entry_cmp_insert(&(entry)->tree_node, ___cur
); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c"
, 352); }); ___prev = ___cur; ___index = rbtree_d2i(___diff);
___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(hdr)->tree, ___prev, ___index, &(entry)->tree_node
); })
352 vm_map_entry_cmp_insert)({ struct rbtree_node *___cur, *___prev; int ___diff, ___index
; ___prev = ((void *) 0); ___index = -1; ___cur = (&(hdr)
->tree)->root; while (___cur != ((void *) 0)) { ___diff
= vm_map_entry_cmp_insert(&(entry)->tree_node, ___cur
); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c"
, 352); }); ___prev = ___cur; ___index = rbtree_d2i(___diff);
___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(hdr)->tree, ___prev, ___index, &(entry)->tree_node
); }); \
353 MACRO_END})
354
355#define vm_map_entry_unlink(map, entry)({ (&(map)->hdr)->nentries--; (entry)->links.next
->links.prev = (entry)->links.prev; (entry)->links.prev
->links.next = (entry)->links.next; rbtree_remove(&
(&(map)->hdr)->tree, &(entry)->tree_node); }
) \
356 _vm_map_entry_unlink(&(map)->hdr, entry)({ (&(map)->hdr)->nentries--; (entry)->links.next
->links.prev = (entry)->links.prev; (entry)->links.prev
->links.next = (entry)->links.next; rbtree_remove(&
(&(map)->hdr)->tree, &(entry)->tree_node); }
)
357
358#define vm_map_copy_entry_unlink(copy, entry)({ (&(copy)->c_u.hdr)->nentries--; (entry)->links
.next->links.prev = (entry)->links.prev; (entry)->links
.prev->links.next = (entry)->links.next; rbtree_remove(
&(&(copy)->c_u.hdr)->tree, &(entry)->tree_node
); }) \
359 _vm_map_entry_unlink(&(copy)->cpy_hdr, entry)({ (&(copy)->c_u.hdr)->nentries--; (entry)->links
.next->links.prev = (entry)->links.prev; (entry)->links
.prev->links.next = (entry)->links.next; rbtree_remove(
&(&(copy)->c_u.hdr)->tree, &(entry)->tree_node
); })
360
361#define _vm_map_entry_unlink(hdr, entry)({ (hdr)->nentries--; (entry)->links.next->links.prev
= (entry)->links.prev; (entry)->links.prev->links.next
= (entry)->links.next; rbtree_remove(&(hdr)->tree,
&(entry)->tree_node); }) \
362 MACRO_BEGIN({ \
363 (hdr)->nentries--; \
364 (entry)->vme_nextlinks.next->vme_prevlinks.prev = (entry)->vme_prevlinks.prev; \
365 (entry)->vme_prevlinks.prev->vme_nextlinks.next = (entry)->vme_nextlinks.next; \
366 rbtree_remove(&(hdr)->tree, &(entry)->tree_node); \
367 MACRO_END})
368
369/*
370 * vm_map_reference:
371 *
372 * Creates another valid reference to the given map.
373 *
374 */
375void vm_map_reference(vm_map_t map)
376{
377 if (map == VM_MAP_NULL((vm_map_t) 0))
378 return;
379
380 simple_lock(&map->ref_lock);
381 map->ref_count++;
382 simple_unlock(&map->ref_lock)((void)(&map->ref_lock));
383}
384
385/*
386 * vm_map_deallocate:
387 *
388 * Removes a reference from the specified map,
389 * destroying it if no references remain.
390 * The map should not be locked.
391 */
392void vm_map_deallocate(vm_map_t map)
393{
394 int c;
395
396 if (map == VM_MAP_NULL((vm_map_t) 0))
397 return;
398
399 simple_lock(&map->ref_lock);
400 c = --map->ref_count;
401 simple_unlock(&map->ref_lock)((void)(&map->ref_lock));
402
403 if (c > 0) {
404 return;
405 }
406
407 projected_buffer_collect(map);
408 (void) vm_map_delete(map, map->min_offsethdr.links.start, map->max_offsethdr.links.end);
409
410 pmap_destroy(map->pmap);
411
412 kmem_cache_free(&vm_map_cache, (vm_offset_t) map);
413}
414
415/*
416 * SAVE_HINT:
417 *
418 * Saves the specified entry as the hint for
419 * future lookups. Performs necessary interlocks.
420 */
421#define SAVE_HINT(map,value); (map)->hint = (value); ((void)(&(map)->hint_lock)
); \
422 simple_lock(&(map)->hint_lock); \
423 (map)->hint = (value); \
424 simple_unlock(&(map)->hint_lock)((void)(&(map)->hint_lock));
425
426/*
427 * vm_map_lookup_entry: [ internal use only ]
428 *
429 * Finds the map entry containing (or
430 * immediately preceding) the specified address
431 * in the given map; the entry is returned
432 * in the "entry" parameter. The boolean
433 * result indicates whether the address is
434 * actually contained in the map.
435 */
436boolean_t vm_map_lookup_entry(
437 vm_map_t map,
438 vm_offset_t address,
439 vm_map_entry_t *entry) /* OUT */
440{
441 struct rbtree_node *node;
442 vm_map_entry_t hint;
443
444 /*
445 * First, make a quick check to see if we are already
446 * looking at the entry we want (which is often the case).
447 */
448
449 simple_lock(&map->hint_lock);
450 hint = map->hint;
451 simple_unlock(&map->hint_lock)((void)(&map->hint_lock));
452
453 if ((hint != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) && (address >= hint->vme_startlinks.start)) {
454 if (address < hint->vme_endlinks.end) {
455 *entry = hint;
456 return(TRUE((boolean_t) 1));
457 } else {
458 vm_map_entry_t next = hint->vme_nextlinks.next;
459
460 if ((next == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links))
461 || (address < next->vme_startlinks.start)) {
462 *entry = hint;
463 return(FALSE((boolean_t) 0));
464 }
465 }
466 }
467
468 /*
469 * If the hint didn't help, use the red-black tree.
470 */
471
472 node = rbtree_lookup_nearest(&map->hdr.tree, address,({ struct rbtree_node *___cur, *___prev; int ___diff, ___index
; ___prev = ((void *) 0); ___index = -1; ___cur = (&map->
hdr.tree)->root; while (___cur != ((void *) 0)) { ___diff =
vm_map_entry_cmp_lookup(address, ___cur); if (___diff == 0) break
; ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur
->children[___index]; } if (___cur == ((void *) 0)) ___cur
= rbtree_nearest(___prev, ___index, 0); ___cur; })
473 vm_map_entry_cmp_lookup, RBTREE_LEFT)({ struct rbtree_node *___cur, *___prev; int ___diff, ___index
; ___prev = ((void *) 0); ___index = -1; ___cur = (&map->
hdr.tree)->root; while (___cur != ((void *) 0)) { ___diff =
vm_map_entry_cmp_lookup(address, ___cur); if (___diff == 0) break
; ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur
->children[___index]; } if (___cur == ((void *) 0)) ___cur
= rbtree_nearest(___prev, ___index, 0); ___cur; });
474
475 if (node == NULL((void *) 0)) {
476 *entry = vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links);
477 SAVE_HINT(map, *entry); (map)->hint = (*entry); ((void)(&(map)->hint_lock
));;
478 return(FALSE((boolean_t) 0));
479 } else {
480 *entry = rbtree_entry(node, struct vm_map_entry, tree_node)((struct vm_map_entry *)((char *)node - __builtin_offsetof (struct
vm_map_entry, tree_node)));
481 SAVE_HINT(map, *entry); (map)->hint = (*entry); ((void)(&(map)->hint_lock
));;
482 return((address < (*entry)->vme_endlinks.end) ? TRUE((boolean_t) 1) : FALSE((boolean_t) 0));
483 }
484}
485
486/*
487 * Routine: invalid_user_access
488 *
489 * Verifies whether user access is valid.
490 */
491
492boolean_t
493invalid_user_access(
494 vm_map_t map,
495 vm_offset_t start,
496 vm_offset_t end,
497 vm_prot_t prot)
498{
499 vm_map_entry_t entry;
500
501 return (map == VM_MAP_NULL((vm_map_t) 0) || map == kernel_map ||
502 !vm_map_lookup_entry(map, start, &entry) ||
503 entry->vme_endlinks.end < end ||
504 (prot & ~(entry->protection)));
505}
506
507
508/*
509 * Routine: vm_map_find_entry
510 * Purpose:
511 * Allocate a range in the specified virtual address map,
512 * returning the entry allocated for that range.
513 * Used by kmem_alloc, etc. Returns wired entries.
514 *
515 * The map must be locked.
516 *
517 * If an entry is allocated, the object/offset fields
518 * are initialized to zero. If an object is supplied,
519 * then an existing entry may be extended.
520 */
521kern_return_t vm_map_find_entry(
522 vm_map_t map,
523 vm_offset_t *address, /* OUT */
524 vm_size_t size,
525 vm_offset_t mask,
526 vm_object_t object,
527 vm_map_entry_t *o_entry) /* OUT */
528{
529 vm_map_entry_t entry, new_entry;
530 vm_offset_t start;
531 vm_offset_t end;
532
533 /*
534 * Look for the first possible address;
535 * if there's already something at this
536 * address, we have to start after it.
537 */
538
539 if ((entry = map->first_free) == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links))
540 start = map->min_offsethdr.links.start;
541 else
542 start = entry->vme_endlinks.end;
543
544 /*
545 * In any case, the "entry" always precedes
546 * the proposed new region throughout the loop:
547 */
548
549 while (TRUE((boolean_t) 1)) {
550 vm_map_entry_t next;
551
552 /*
553 * Find the end of the proposed new region.
554 * Be sure we didn't go beyond the end, or
555 * wrap around the address.
556 */
557
558 if (((start + mask) & ~mask) < start) {
559 printf_once("no more room for vm_map_find_entry in %p\n", map)({ static int __once = 0; if (!__once) { db_printf("no more room for vm_map_find_entry in %p\n"
, map); __once = 1; } });
560 return(KERN_NO_SPACE3);
561 }
562 start = ((start + mask) & ~mask);
563 end = start + size;
564
565 if ((end > map->max_offsethdr.links.end) || (end < start)) {
566 printf_once("no more room for vm_map_find_entry in %p\n", map)({ static int __once = 0; if (!__once) { db_printf("no more room for vm_map_find_entry in %p\n"
, map); __once = 1; } });
567 return(KERN_NO_SPACE3);
568 }
569
570 /*
571 * If there are no more entries, we must win.
572 */
573
574 next = entry->vme_nextlinks.next;
575 if (next == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links))
576 break;
577
578 /*
579 * If there is another entry, it must be
580 * after the end of the potential new region.
581 */
582
583 if (next->vme_startlinks.start >= end)
584 break;
585
586 /*
587 * Didn't fit -- move to the next entry.
588 */
589
590 entry = next;
591 start = entry->vme_endlinks.end;
592 }
593
594 /*
595 * At this point,
596 * "start" and "end" should define the endpoints of the
597 * available new range, and
598 * "entry" should refer to the region before the new
599 * range, and
600 *
601 * the map should be locked.
602 */
603
604 *address = start;
605
606 /*
607 * See whether we can avoid creating a new entry by
608 * extending one of our neighbors. [So far, we only attempt to
609 * extend from below.]
610 */
611
612 if ((object != VM_OBJECT_NULL((vm_object_t) 0)) &&
613 (entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
614 (entry->vme_endlinks.end == start) &&
615 (!entry->is_shared) &&
616 (!entry->is_sub_map) &&
617 (entry->object.vm_object == object) &&
618 (entry->needs_copy == FALSE((boolean_t) 0)) &&
619 (entry->inheritance == VM_INHERIT_DEFAULT((vm_inherit_t) 1)) &&
620 (entry->protection == VM_PROT_DEFAULT(((vm_prot_t) 0x01)|((vm_prot_t) 0x02))) &&
621 (entry->max_protection == VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04))) &&
622 (entry->wired_count == 1) &&
623 (entry->user_wired_count == 0) &&
624 (entry->projected_on == 0)) {
625 /*
626 * Because this is a special case,
627 * we don't need to use vm_object_coalesce.
628 */
629
630 entry->vme_endlinks.end = end;
631 new_entry = entry;
632 } else {
633 new_entry = vm_map_entry_create(map)_vm_map_entry_create(&(map)->hdr);
634
635 new_entry->vme_startlinks.start = start;
636 new_entry->vme_endlinks.end = end;
637
638 new_entry->is_shared = FALSE((boolean_t) 0);
639 new_entry->is_sub_map = FALSE((boolean_t) 0);
640 new_entry->object.vm_object = VM_OBJECT_NULL((vm_object_t) 0);
641 new_entry->offset = (vm_offset_t) 0;
642
643 new_entry->needs_copy = FALSE((boolean_t) 0);
644
645 new_entry->inheritance = VM_INHERIT_DEFAULT((vm_inherit_t) 1);
646 new_entry->protection = VM_PROT_DEFAULT(((vm_prot_t) 0x01)|((vm_prot_t) 0x02));
647 new_entry->max_protection = VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04));
648 new_entry->wired_count = 1;
649 new_entry->user_wired_count = 0;
650
651 new_entry->in_transition = FALSE((boolean_t) 0);
652 new_entry->needs_wakeup = FALSE((boolean_t) 0);
653 new_entry->projected_on = 0;
654
655 /*
656 * Insert the new entry into the list
657 */
658
659 vm_map_entry_link(map, entry, new_entry)({ (&(map)->hdr)->nentries++; (new_entry)->links
.prev = (entry); (new_entry)->links.next = (entry)->links
.next; (new_entry)->links.prev->links.next = (new_entry
)->links.next->links.prev = (new_entry); ({ struct rbtree_node
*___cur, *___prev; int ___diff, ___index; ___prev = ((void *
) 0); ___index = -1; ___cur = (&(&(map)->hdr)->
tree)->root; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(new_entry)->tree_node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 659); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(map
)->hdr)->tree, ___prev, ___index, &(new_entry)->
tree_node); }); });
660 }
661
662 map->size += size;
663
664 /*
665 * Update the free space hint and the lookup hint
666 */
667
668 map->first_free = new_entry;
669 SAVE_HINT(map, new_entry); (map)->hint = (new_entry); ((void)(&(map)->hint_lock
));;
670
671 *o_entry = new_entry;
672 return(KERN_SUCCESS0);
673}
674
675boolean_t vm_map_pmap_enter_print = FALSE((boolean_t) 0);
676boolean_t vm_map_pmap_enter_enable = FALSE((boolean_t) 0);
677
678/*
679 * Routine: vm_map_pmap_enter
680 *
681 * Description:
682 * Force pages from the specified object to be entered into
683 * the pmap at the specified address if they are present.
684 * As soon as a page not found in the object the scan ends.
685 *
686 * Returns:
687 * Nothing.
688 *
689 * In/out conditions:
690 * The source map should not be locked on entry.
691 */
692void
693vm_map_pmap_enter(
694 vm_map_t map,
695 vm_offset_t addr,
696 vm_offset_t end_addr,
697 vm_object_t object,
698 vm_offset_t offset,
699 vm_prot_t protection)
700{
701 while (addr < end_addr) {
702 vm_page_t m;
703
704 vm_object_lock(object);
705 vm_object_paging_begin(object)((object)->paging_in_progress++);
706
707 m = vm_page_lookup(object, offset);
708 if (m == VM_PAGE_NULL((vm_page_t) 0) || m->absent) {
709 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_map.c", 709); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
710 vm_object_unlock(object)((void)(&(object)->Lock));
711 return;
712 }
713
714 if (vm_map_pmap_enter_print) {
715 printfdb_printf("vm_map_pmap_enter:");
716 printfdb_printf("map: %p, addr: %lx, object: %p, offset: %lx\n",
717 map, addr, object, offset);
718 }
719
720 m->busy = TRUE((boolean_t) 1);
721 vm_object_unlock(object)((void)(&(object)->Lock));
722
723 PMAP_ENTER(map->pmap, addr, m,({ pmap_enter( (map->pmap), (addr), (m)->phys_addr, (protection
) & ~(m)->page_lock, (((boolean_t) 0)) ); })
724 protection, FALSE)({ pmap_enter( (map->pmap), (addr), (m)->phys_addr, (protection
) & ~(m)->page_lock, (((boolean_t) 0)) ); });
725
726 vm_object_lock(object);
727 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } });
728 vm_page_lock_queues();
729 if (!m->active && !m->inactive)
730 vm_page_activate(m);
731 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
732 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_map.c", 732); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
733 vm_object_unlock(object)((void)(&(object)->Lock));
734
735 offset += PAGE_SIZE(1 << 12);
736 addr += PAGE_SIZE(1 << 12);
737 }
738}
739
740/*
741 * Routine: vm_map_enter
742 *
743 * Description:
744 * Allocate a range in the specified virtual address map.
745 * The resulting range will refer to memory defined by
746 * the given memory object and offset into that object.
747 *
748 * Arguments are as defined in the vm_map call.
749 */
750kern_return_t vm_map_enter(
751 vm_map_t map,
752 vm_offset_t *address, /* IN/OUT */
753 vm_size_t size,
754 vm_offset_t mask,
755 boolean_t anywhere,
756 vm_object_t object,
757 vm_offset_t offset,
758 boolean_t needs_copy,
759 vm_prot_t cur_protection,
760 vm_prot_t max_protection,
761 vm_inherit_t inheritance)
762{
763 vm_map_entry_t entry;
764 vm_offset_t start;
765 vm_offset_t end;
766 kern_return_t result = KERN_SUCCESS0;
767
768#define RETURN(value) { result = value; goto BailOut; }
769
770 if (size == 0)
771 return KERN_INVALID_ARGUMENT4;
772
773 StartAgain: ;
774
775 start = *address;
776
777 if (anywhere) {
778 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
779
780 /*
781 * Calculate the first possible address.
782 */
783
784 if (start < map->min_offsethdr.links.start)
785 start = map->min_offsethdr.links.start;
786 if (start > map->max_offsethdr.links.end)
787 RETURN(KERN_NO_SPACE3);
788
789 /*
790 * Look for the first possible address;
791 * if there's already something at this
792 * address, we have to start after it.
793 */
794
795 if (start == map->min_offsethdr.links.start) {
796 if ((entry = map->first_free) != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links))
797 start = entry->vme_endlinks.end;
798 } else {
799 vm_map_entry_t tmp_entry;
800 if (vm_map_lookup_entry(map, start, &tmp_entry))
801 start = tmp_entry->vme_endlinks.end;
802 entry = tmp_entry;
803 }
804
805 /*
806 * In any case, the "entry" always precedes
807 * the proposed new region throughout the
808 * loop:
809 */
810
811 while (TRUE((boolean_t) 1)) {
812 vm_map_entry_t next;
813
814 /*
815 * Find the end of the proposed new region.
816 * Be sure we didn't go beyond the end, or
817 * wrap around the address.
818 */
819
820 if (((start + mask) & ~mask) < start) {
821 printf_once("no more room for vm_map_enter in %p\n", map)({ static int __once = 0; if (!__once) { db_printf("no more room for vm_map_enter in %p\n"
, map); __once = 1; } });
822 RETURN(KERN_NO_SPACE3);
823 }
824 start = ((start + mask) & ~mask);
825 end = start + size;
826
827 if ((end > map->max_offsethdr.links.end) || (end < start)) {
828 if (map->wait_for_space) {
829 if (size <= (map->max_offsethdr.links.end -
830 map->min_offsethdr.links.start)) {
831 assert_wait((event_t) map, TRUE((boolean_t) 1));
832 vm_map_unlock(map)lock_done(&(map)->lock);
833 thread_block((void (*)()) 0);
834 goto StartAgain;
835 }
836 }
837
838 printf_once("no more room for vm_map_enter in %p\n", map)({ static int __once = 0; if (!__once) { db_printf("no more room for vm_map_enter in %p\n"
, map); __once = 1; } });
839 RETURN(KERN_NO_SPACE3);
840 }
841
842 /*
843 * If there are no more entries, we must win.
844 */
845
846 next = entry->vme_nextlinks.next;
847 if (next == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links))
848 break;
849
850 /*
851 * If there is another entry, it must be
852 * after the end of the potential new region.
853 */
854
855 if (next->vme_startlinks.start >= end)
856 break;
857
858 /*
859 * Didn't fit -- move to the next entry.
860 */
861
862 entry = next;
863 start = entry->vme_endlinks.end;
864 }
865 *address = start;
866 } else {
867 vm_map_entry_t temp_entry;
868
869 /*
870 * Verify that:
871 * the address doesn't itself violate
872 * the mask requirement.
873 */
874
875 if ((start & mask) != 0)
876 return(KERN_NO_SPACE3);
877
878 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
879
880 /*
881 * ... the address is within bounds
882 */
883
884 end = start + size;
885
886 if ((start < map->min_offsethdr.links.start) ||
887 (end > map->max_offsethdr.links.end) ||
888 (start >= end)) {
889 RETURN(KERN_INVALID_ADDRESS1);
890 }
891
892 /*
893 * ... the starting address isn't allocated
894 */
895
896 if (vm_map_lookup_entry(map, start, &temp_entry))
897 RETURN(KERN_NO_SPACE3);
898
899 entry = temp_entry;
900
901 /*
902 * ... the next region doesn't overlap the
903 * end point.
904 */
905
906 if ((entry->vme_nextlinks.next != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
907 (entry->vme_nextlinks.next->vme_startlinks.start < end))
908 RETURN(KERN_NO_SPACE3);
909 }
910
911 /*
912 * At this point,
913 * "start" and "end" should define the endpoints of the
914 * available new range, and
915 * "entry" should refer to the region before the new
916 * range, and
917 *
918 * the map should be locked.
919 */
920
921 /*
922 * See whether we can avoid creating a new entry (and object) by
923 * extending one of our neighbors. [So far, we only attempt to
924 * extend from below.]
925 */
926
927 if ((object == VM_OBJECT_NULL((vm_object_t) 0)) &&
928 (entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
929 (entry->vme_endlinks.end == start) &&
930 (!entry->is_shared) &&
931 (!entry->is_sub_map) &&
932 (entry->inheritance == inheritance) &&
933 (entry->protection == cur_protection) &&
934 (entry->max_protection == max_protection) &&
935 (entry->wired_count == 0) && /* implies user_wired_count == 0 */
936 (entry->projected_on == 0)) {
937 if (vm_object_coalesce(entry->object.vm_object,
938 VM_OBJECT_NULL((vm_object_t) 0),
939 entry->offset,
940 (vm_offset_t) 0,
941 (vm_size_t)(entry->vme_endlinks.end - entry->vme_startlinks.start),
942 (vm_size_t)(end - entry->vme_endlinks.end))) {
943
944 /*
945 * Coalesced the two objects - can extend
946 * the previous map entry to include the
947 * new range.
948 */
949 map->size += (end - entry->vme_endlinks.end);
950 entry->vme_endlinks.end = end;
951 RETURN(KERN_SUCCESS0);
952 }
953 }
954
955 /*
956 * Create a new entry
957 */
958
959 /**/ {
960 vm_map_entry_t new_entry;
961
962 new_entry = vm_map_entry_create(map)_vm_map_entry_create(&(map)->hdr);
963
964 new_entry->vme_startlinks.start = start;
965 new_entry->vme_endlinks.end = end;
966
967 new_entry->is_shared = FALSE((boolean_t) 0);
968 new_entry->is_sub_map = FALSE((boolean_t) 0);
969 new_entry->object.vm_object = object;
970 new_entry->offset = offset;
971
972 new_entry->needs_copy = needs_copy;
973
974 new_entry->inheritance = inheritance;
975 new_entry->protection = cur_protection;
976 new_entry->max_protection = max_protection;
977 new_entry->wired_count = 0;
978 new_entry->user_wired_count = 0;
979
980 new_entry->in_transition = FALSE((boolean_t) 0);
981 new_entry->needs_wakeup = FALSE((boolean_t) 0);
982 new_entry->projected_on = 0;
983
984 /*
985 * Insert the new entry into the list
986 */
987
988 vm_map_entry_link(map, entry, new_entry)({ (&(map)->hdr)->nentries++; (new_entry)->links
.prev = (entry); (new_entry)->links.next = (entry)->links
.next; (new_entry)->links.prev->links.next = (new_entry
)->links.next->links.prev = (new_entry); ({ struct rbtree_node
*___cur, *___prev; int ___diff, ___index; ___prev = ((void *
) 0); ___index = -1; ___cur = (&(&(map)->hdr)->
tree)->root; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(new_entry)->tree_node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 988); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(map
)->hdr)->tree, ___prev, ___index, &(new_entry)->
tree_node); }); });
989 map->size += size;
990
991 /*
992 * Update the free space hint and the lookup hint
993 */
994
995 if ((map->first_free == entry) &&
996 ((entry == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links) ? map->min_offsethdr.links.start : entry->vme_endlinks.end)
997 >= new_entry->vme_startlinks.start))
998 map->first_free = new_entry;
999
1000 SAVE_HINT(map, new_entry); (map)->hint = (new_entry); ((void)(&(map)->hint_lock
));;
1001
1002 vm_map_unlock(map)lock_done(&(map)->lock);
1003
1004 if ((object != VM_OBJECT_NULL((vm_object_t) 0)) &&
1005 (vm_map_pmap_enter_enable) &&
1006 (!anywhere) &&
1007 (!needs_copy) &&
1008 (size < (128*1024))) {
1009 vm_map_pmap_enter(map, start, end,
1010 object, offset, cur_protection);
1011 }
1012
1013 return(result);
1014 /**/ }
1015
1016 BailOut: ;
1017
1018 vm_map_unlock(map)lock_done(&(map)->lock);
1019 return(result);
1020
1021#undef RETURN
1022}
1023
1024/*
1025 * vm_map_clip_start: [ internal use only ]
1026 *
1027 * Asserts that the given entry begins at or after
1028 * the specified address; if necessary,
1029 * it splits the entry into two.
1030 */
1031#define vm_map_clip_start(map, entry, startaddr)({ if ((startaddr) > (entry)->links.start) _vm_map_clip_start
(&(map)->hdr,(entry),(startaddr)); }) \
1032 MACRO_BEGIN({ \
1033 if ((startaddr) > (entry)->vme_startlinks.start) \
1034 _vm_map_clip_start(&(map)->hdr,(entry),(startaddr)); \
1035 MACRO_END})
1036
1037#define vm_map_copy_clip_start(copy, entry, startaddr)({ if ((startaddr) > (entry)->links.start) _vm_map_clip_start
(&(copy)->c_u.hdr,(entry),(startaddr)); }) \
1038 MACRO_BEGIN({ \
1039 if ((startaddr) > (entry)->vme_startlinks.start) \
1040 _vm_map_clip_start(&(copy)->cpy_hdrc_u.hdr,(entry),(startaddr)); \
1041 MACRO_END})
1042
1043/*
1044 * This routine is called only when it is known that
1045 * the entry must be split.
1046 */
1047void _vm_map_clip_start(
1048 struct vm_map_header *map_header,
1049 vm_map_entry_t entry,
1050 vm_offset_t start)
1051{
1052 vm_map_entry_t new_entry;
1053
1054 /*
1055 * Split off the front portion --
1056 * note that we must insert the new
1057 * entry BEFORE this one, so that
1058 * this entry has the specified starting
1059 * address.
1060 */
1061
1062 new_entry = _vm_map_entry_create(map_header);
1063 vm_map_entry_copy_full(new_entry, entry)(*(new_entry) = *(entry));
1064
1065 new_entry->vme_endlinks.end = start;
1066 entry->offset += (start - entry->vme_startlinks.start);
1067 entry->vme_startlinks.start = start;
1068
1069 _vm_map_entry_link(map_header, entry->vme_prev, new_entry)({ (map_header)->nentries++; (new_entry)->links.prev = (
entry->links.prev); (new_entry)->links.next = (entry->
links.prev)->links.next; (new_entry)->links.prev->links
.next = (new_entry)->links.next->links.prev = (new_entry
); ({ struct rbtree_node *___cur, *___prev; int ___diff, ___index
; ___prev = ((void *) 0); ___index = -1; ___cur = (&(map_header
)->tree)->root; while (___cur != ((void *) 0)) { ___diff
= vm_map_entry_cmp_insert(&(new_entry)->tree_node, ___cur
); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c"
, 1069); }); ___prev = ___cur; ___index = rbtree_d2i(___diff)
; ___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(map_header)->tree, ___prev, ___index, &(new_entry
)->tree_node); }); });
1070
1071 if (entry->is_sub_map)
1072 vm_map_reference(new_entry->object.sub_map);
1073 else
1074 vm_object_reference(new_entry->object.vm_object);
1075}
1076
1077/*
1078 * vm_map_clip_end: [ internal use only ]
1079 *
1080 * Asserts that the given entry ends at or before
1081 * the specified address; if necessary,
1082 * it splits the entry into two.
1083 */
1084#define vm_map_clip_end(map, entry, endaddr)({ if ((endaddr) < (entry)->links.end) _vm_map_clip_end
(&(map)->hdr,(entry),(endaddr)); }) \
1085 MACRO_BEGIN({ \
1086 if ((endaddr) < (entry)->vme_endlinks.end) \
1087 _vm_map_clip_end(&(map)->hdr,(entry),(endaddr)); \
1088 MACRO_END})
1089
1090#define vm_map_copy_clip_end(copy, entry, endaddr)({ if ((endaddr) < (entry)->links.end) _vm_map_clip_end
(&(copy)->c_u.hdr,(entry),(endaddr)); }) \
1091 MACRO_BEGIN({ \
1092 if ((endaddr) < (entry)->vme_endlinks.end) \
1093 _vm_map_clip_end(&(copy)->cpy_hdrc_u.hdr,(entry),(endaddr)); \
1094 MACRO_END})
1095
1096/*
1097 * This routine is called only when it is known that
1098 * the entry must be split.
1099 */
1100void _vm_map_clip_end(
1101 struct vm_map_header *map_header,
1102 vm_map_entry_t entry,
1103 vm_offset_t end)
1104{
1105 vm_map_entry_t new_entry;
1106
1107 /*
1108 * Create a new entry and insert it
1109 * AFTER the specified entry
1110 */
1111
1112 new_entry = _vm_map_entry_create(map_header);
1113 vm_map_entry_copy_full(new_entry, entry)(*(new_entry) = *(entry));
1114
1115 new_entry->vme_startlinks.start = entry->vme_endlinks.end = end;
1116 new_entry->offset += (end - entry->vme_startlinks.start);
1117
1118 _vm_map_entry_link(map_header, entry, new_entry)({ (map_header)->nentries++; (new_entry)->links.prev = (
entry); (new_entry)->links.next = (entry)->links.next; (
new_entry)->links.prev->links.next = (new_entry)->links
.next->links.prev = (new_entry); ({ struct rbtree_node *___cur
, *___prev; int ___diff, ___index; ___prev = ((void *) 0); ___index
= -1; ___cur = (&(map_header)->tree)->root; while (
___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert(&
(new_entry)->tree_node, ___cur); ({ if (!(___diff != 0)) Assert
("___diff != 0", "../vm/vm_map.c", 1118); }); ___prev = ___cur
; ___index = rbtree_d2i(___diff); ___cur = ___cur->children
[___index]; } rbtree_insert_rebalance(&(map_header)->tree
, ___prev, ___index, &(new_entry)->tree_node); }); });
1119
1120 if (entry->is_sub_map)
1121 vm_map_reference(new_entry->object.sub_map);
1122 else
1123 vm_object_reference(new_entry->object.vm_object);
1124}
1125
1126/*
1127 * VM_MAP_RANGE_CHECK: [ internal use only ]
1128 *
1129 * Asserts that the starting and ending region
1130 * addresses fall within the valid range of the map.
1131 */
1132#define VM_MAP_RANGE_CHECK(map, start, end){ if (start < ((map)->hdr.links.start)) start = ((map)->
hdr.links.start); if (end > ((map)->hdr.links.end)) end
= ((map)->hdr.links.end); if (start > end) start = end
; } \
1133 { \
1134 if (start < vm_map_min(map)((map)->hdr.links.start)) \
1135 start = vm_map_min(map)((map)->hdr.links.start); \
1136 if (end > vm_map_max(map)((map)->hdr.links.end)) \
1137 end = vm_map_max(map)((map)->hdr.links.end); \
1138 if (start > end) \
1139 start = end; \
1140 }
1141
1142/*
1143 * vm_map_submap: [ kernel use only ]
1144 *
1145 * Mark the given range as handled by a subordinate map.
1146 *
1147 * This range must have been created with vm_map_find using
1148 * the vm_submap_object, and no other operations may have been
1149 * performed on this range prior to calling vm_map_submap.
1150 *
1151 * Only a limited number of operations can be performed
1152 * within this rage after calling vm_map_submap:
1153 * vm_fault
1154 * [Don't try vm_map_copyin!]
1155 *
1156 * To remove a submapping, one must first remove the
1157 * range from the superior map, and then destroy the
1158 * submap (if desired). [Better yet, don't try it.]
1159 */
1160kern_return_t vm_map_submap(
1161 vm_map_t map,
1162 vm_offset_t start,
1163 vm_offset_t end,
1164 vm_map_t submap)
1165{
1166 vm_map_entry_t entry;
1167 kern_return_t result = KERN_INVALID_ARGUMENT4;
1168 vm_object_t object;
1169
1170 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
1171
1172 VM_MAP_RANGE_CHECK(map, start, end){ if (start < ((map)->hdr.links.start)) start = ((map)->
hdr.links.start); if (end > ((map)->hdr.links.end)) end
= ((map)->hdr.links.end); if (start > end) start = end
; };
1173
1174 if (vm_map_lookup_entry(map, start, &entry)) {
1175 vm_map_clip_start(map, entry, start)({ if ((start) > (entry)->links.start) _vm_map_clip_start
(&(map)->hdr,(entry),(start)); });
1176 }
1177 else
1178 entry = entry->vme_nextlinks.next;
1179
1180 vm_map_clip_end(map, entry, end)({ if ((end) < (entry)->links.end) _vm_map_clip_end(&
(map)->hdr,(entry),(end)); });
1181
1182 if ((entry->vme_startlinks.start == start) && (entry->vme_endlinks.end == end) &&
1183 (!entry->is_sub_map) &&
1184 ((object = entry->object.vm_object) == vm_submap_object) &&
1185 (object->resident_page_count == 0) &&
1186 (object->copy == VM_OBJECT_NULL((vm_object_t) 0)) &&
1187 (object->shadow == VM_OBJECT_NULL((vm_object_t) 0)) &&
1188 (!object->pager_created)) {
1189 entry->object.vm_object = VM_OBJECT_NULL((vm_object_t) 0);
1190 vm_object_deallocate(object);
1191 entry->is_sub_map = TRUE((boolean_t) 1);
1192 vm_map_reference(entry->object.sub_map = submap);
1193 result = KERN_SUCCESS0;
1194 }
1195 vm_map_unlock(map)lock_done(&(map)->lock);
1196
1197 return(result);
1198}
1199
1200/*
1201 * vm_map_protect:
1202 *
1203 * Sets the protection of the specified address
1204 * region in the target map. If "set_max" is
1205 * specified, the maximum protection is to be set;
1206 * otherwise, only the current protection is affected.
1207 */
1208kern_return_t vm_map_protect(
1209 vm_map_t map,
1210 vm_offset_t start,
1211 vm_offset_t end,
1212 vm_prot_t new_prot,
1213 boolean_t set_max)
1214{
1215 vm_map_entry_t current;
1216 vm_map_entry_t entry;
1217
1218 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
1219
1220 VM_MAP_RANGE_CHECK(map, start, end){ if (start < ((map)->hdr.links.start)) start = ((map)->
hdr.links.start); if (end > ((map)->hdr.links.end)) end
= ((map)->hdr.links.end); if (start > end) start = end
; };
1221
1222 if (vm_map_lookup_entry(map, start, &entry)) {
1223 vm_map_clip_start(map, entry, start)({ if ((start) > (entry)->links.start) _vm_map_clip_start
(&(map)->hdr,(entry),(start)); });
1224 }
1225 else
1226 entry = entry->vme_nextlinks.next;
1227
1228 /*
1229 * Make a first pass to check for protection
1230 * violations.
1231 */
1232
1233 current = entry;
1234 while ((current != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
1235 (current->vme_startlinks.start < end)) {
1236
1237 if (current->is_sub_map) {
1238 vm_map_unlock(map)lock_done(&(map)->lock);
1239 return(KERN_INVALID_ARGUMENT4);
1240 }
1241 if ((new_prot & (VM_PROT_NOTIFY((vm_prot_t) 0x10) | current->max_protection))
1242 != new_prot) {
1243 vm_map_unlock(map)lock_done(&(map)->lock);
1244 return(KERN_PROTECTION_FAILURE2);
1245 }
1246
1247 current = current->vme_nextlinks.next;
1248 }
1249
1250 /*
1251 * Go back and fix up protections.
1252 * [Note that clipping is not necessary the second time.]
1253 */
1254
1255 current = entry;
1256
1257 while ((current != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
1258 (current->vme_startlinks.start < end)) {
1259
1260 vm_prot_t old_prot;
1261
1262 vm_map_clip_end(map, current, end)({ if ((end) < (current)->links.end) _vm_map_clip_end(&
(map)->hdr,(current),(end)); });
1263
1264 old_prot = current->protection;
1265 if (set_max)
1266 current->protection =
1267 (current->max_protection = new_prot) &
1268 old_prot;
1269 else
1270 current->protection = new_prot;
1271
1272 /*
1273 * Update physical map if necessary.
1274 */
1275
1276 if (current->protection != old_prot) {
1277 pmap_protect(map->pmap, current->vme_startlinks.start,
1278 current->vme_endlinks.end,
1279 current->protection);
1280 }
1281 current = current->vme_nextlinks.next;
1282 }
1283
1284 vm_map_unlock(map)lock_done(&(map)->lock);
1285 return(KERN_SUCCESS0);
1286}
1287
1288/*
1289 * vm_map_inherit:
1290 *
1291 * Sets the inheritance of the specified address
1292 * range in the target map. Inheritance
1293 * affects how the map will be shared with
1294 * child maps at the time of vm_map_fork.
1295 */
1296kern_return_t vm_map_inherit(
1297 vm_map_t map,
1298 vm_offset_t start,
1299 vm_offset_t end,
1300 vm_inherit_t new_inheritance)
1301{
1302 vm_map_entry_t entry;
1303 vm_map_entry_t temp_entry;
1304
1305 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
1306
1307 VM_MAP_RANGE_CHECK(map, start, end){ if (start < ((map)->hdr.links.start)) start = ((map)->
hdr.links.start); if (end > ((map)->hdr.links.end)) end
= ((map)->hdr.links.end); if (start > end) start = end
; };
1308
1309 if (vm_map_lookup_entry(map, start, &temp_entry)) {
1310 entry = temp_entry;
1311 vm_map_clip_start(map, entry, start)({ if ((start) > (entry)->links.start) _vm_map_clip_start
(&(map)->hdr,(entry),(start)); });
1312 }
1313 else
1314 entry = temp_entry->vme_nextlinks.next;
1315
1316 while ((entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) && (entry->vme_startlinks.start < end)) {
1317 vm_map_clip_end(map, entry, end)({ if ((end) < (entry)->links.end) _vm_map_clip_end(&
(map)->hdr,(entry),(end)); });
1318
1319 entry->inheritance = new_inheritance;
1320
1321 entry = entry->vme_nextlinks.next;
1322 }
1323
1324 vm_map_unlock(map)lock_done(&(map)->lock);
1325 return(KERN_SUCCESS0);
1326}
1327
1328/*
1329 * vm_map_pageable_common:
1330 *
1331 * Sets the pageability of the specified address
1332 * range in the target map. Regions specified
1333 * as not pageable require locked-down physical
1334 * memory and physical page maps. access_type indicates
1335 * types of accesses that must not generate page faults.
1336 * This is checked against protection of memory being locked-down.
1337 * access_type of VM_PROT_NONE makes memory pageable.
1338 *
1339 * The map must not be locked, but a reference
1340 * must remain to the map throughout the call.
1341 *
1342 * Callers should use macros in vm/vm_map.h (i.e. vm_map_pageable,
1343 * or vm_map_pageable_user); don't call vm_map_pageable directly.
1344 */
1345kern_return_t vm_map_pageable_common(
1346 vm_map_t map,
1347 vm_offset_t start,
1348 vm_offset_t end,
1349 vm_prot_t access_type,
1350 boolean_t user_wire)
1351{
1352 vm_map_entry_t entry;
1353 vm_map_entry_t start_entry;
1354
1355 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
1356
1357 VM_MAP_RANGE_CHECK(map, start, end){ if (start < ((map)->hdr.links.start)) start = ((map)->
hdr.links.start); if (end > ((map)->hdr.links.end)) end
= ((map)->hdr.links.end); if (start > end) start = end
; };
1358
1359 if (vm_map_lookup_entry(map, start, &start_entry)) {
1360 entry = start_entry;
1361 /*
1362 * vm_map_clip_start will be done later.
1363 */
1364 }
1365 else {
1366 /*
1367 * Start address is not in map; this is fatal.
1368 */
1369 vm_map_unlock(map)lock_done(&(map)->lock);
1370 return(KERN_FAILURE5);
1371 }
1372
1373 /*
1374 * Actions are rather different for wiring and unwiring,
1375 * so we have two separate cases.
1376 */
1377
1378 if (access_type == VM_PROT_NONE((vm_prot_t) 0x00)) {
1379
1380 vm_map_clip_start(map, entry, start)({ if ((start) > (entry)->links.start) _vm_map_clip_start
(&(map)->hdr,(entry),(start)); });
1381
1382 /*
1383 * Unwiring. First ensure that the range to be
1384 * unwired is really wired down.
1385 */
1386 while ((entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
1387 (entry->vme_startlinks.start < end)) {
1388
1389 if ((entry->wired_count == 0) ||
1390 ((entry->vme_endlinks.end < end) &&
1391 ((entry->vme_nextlinks.next == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) ||
1392 (entry->vme_nextlinks.next->vme_startlinks.start > entry->vme_endlinks.end))) ||
1393 (user_wire && (entry->user_wired_count == 0))) {
1394 vm_map_unlock(map)lock_done(&(map)->lock);
1395 return(KERN_INVALID_ARGUMENT4);
1396 }
1397 entry = entry->vme_nextlinks.next;
1398 }
1399
1400 /*
1401 * Now decrement the wiring count for each region.
1402 * If a region becomes completely unwired,
1403 * unwire its physical pages and mappings.
1404 */
1405 entry = start_entry;
1406 while ((entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
1407 (entry->vme_startlinks.start < end)) {
1408 vm_map_clip_end(map, entry, end)({ if ((end) < (entry)->links.end) _vm_map_clip_end(&
(map)->hdr,(entry),(end)); });
1409
1410 if (user_wire) {
1411 if (--(entry->user_wired_count) == 0)
1412 entry->wired_count--;
1413 }
1414 else {
1415 entry->wired_count--;
1416 }
1417
1418 if (entry->wired_count == 0)
1419 vm_fault_unwire(map, entry);
1420
1421 entry = entry->vme_nextlinks.next;
1422 }
1423 }
1424
1425 else {
1426 /*
1427 * Wiring. We must do this in two passes:
1428 *
1429 * 1. Holding the write lock, we create any shadow
1430 * or zero-fill objects that need to be created.
1431 * Then we clip each map entry to the region to be
1432 * wired and increment its wiring count. We
1433 * create objects before clipping the map entries
1434 * to avoid object proliferation.
1435 *
1436 * 2. We downgrade to a read lock, and call
1437 * vm_fault_wire to fault in the pages for any
1438 * newly wired area (wired_count is 1).
1439 *
1440 * Downgrading to a read lock for vm_fault_wire avoids
1441 * a possible deadlock with another thread that may have
1442 * faulted on one of the pages to be wired (it would mark
1443 * the page busy, blocking us, then in turn block on the
1444 * map lock that we hold). Because of problems in the
1445 * recursive lock package, we cannot upgrade to a write
1446 * lock in vm_map_lookup. Thus, any actions that require
1447 * the write lock must be done beforehand. Because we
1448 * keep the read lock on the map, the copy-on-write
1449 * status of the entries we modify here cannot change.
1450 */
1451
1452 /*
1453 * Pass 1.
1454 */
1455 while ((entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
1456 (entry->vme_startlinks.start < end)) {
1457 vm_map_clip_end(map, entry, end)({ if ((end) < (entry)->links.end) _vm_map_clip_end(&
(map)->hdr,(entry),(end)); });
1458
1459 if (entry->wired_count == 0) {
1460
1461 /*
1462 * Perform actions of vm_map_lookup that need
1463 * the write lock on the map: create a shadow
1464 * object for a copy-on-write region, or an
1465 * object for a zero-fill region.
1466 */
1467 if (entry->needs_copy &&
1468 ((entry->protection & VM_PROT_WRITE((vm_prot_t) 0x02)) != 0)) {
1469
1470 vm_object_shadow(&entry->object.vm_object,
1471 &entry->offset,
1472 (vm_size_t)(entry->vme_endlinks.end
1473 - entry->vme_startlinks.start));
1474 entry->needs_copy = FALSE((boolean_t) 0);
1475 }
1476 if (entry->object.vm_object == VM_OBJECT_NULL((vm_object_t) 0)) {
1477 entry->object.vm_object =
1478 vm_object_allocate(
1479 (vm_size_t)(entry->vme_endlinks.end
1480 - entry->vme_startlinks.start));
1481 entry->offset = (vm_offset_t)0;
1482 }
1483 }
1484 vm_map_clip_start(map, entry, start)({ if ((start) > (entry)->links.start) _vm_map_clip_start
(&(map)->hdr,(entry),(start)); });
1485 vm_map_clip_end(map, entry, end)({ if ((end) < (entry)->links.end) _vm_map_clip_end(&
(map)->hdr,(entry),(end)); });
1486
1487 if (user_wire) {
1488 if ((entry->user_wired_count)++ == 0)
1489 entry->wired_count++;
1490 }
1491 else {
1492 entry->wired_count++;
1493 }
1494
1495 /*
1496 * Check for holes and protection mismatch.
1497 * Holes: Next entry should be contiguous unless
1498 * this is the end of the region.
1499 * Protection: Access requested must be allowed.
1500 */
1501 if (((entry->vme_endlinks.end < end) &&
1502 ((entry->vme_nextlinks.next == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) ||
1503 (entry->vme_nextlinks.next->vme_startlinks.start > entry->vme_endlinks.end))) ||
1504 ((entry->protection & access_type) != access_type)) {
1505 /*
1506 * Found a hole or protection problem.
1507 * Object creation actions
1508 * do not need to be undone, but the
1509 * wired counts need to be restored.
1510 */
1511 while ((entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
1512 (entry->vme_endlinks.end > start)) {
1513 if (user_wire) {
1514 if (--(entry->user_wired_count) == 0)
1515 entry->wired_count--;
1516 }
1517 else {
1518 entry->wired_count--;
1519 }
1520
1521 entry = entry->vme_prevlinks.prev;
1522 }
1523
1524 vm_map_unlock(map)lock_done(&(map)->lock);
1525 return(KERN_FAILURE5);
1526 }
1527 entry = entry->vme_nextlinks.next;
1528 }
1529
1530 /*
1531 * Pass 2.
1532 */
1533
1534 /*
1535 * HACK HACK HACK HACK
1536 *
1537 * If we are wiring in the kernel map or a submap of it,
1538 * unlock the map to avoid deadlocks. We trust that the
1539 * kernel threads are well-behaved, and therefore will
1540 * not do anything destructive to this region of the map
1541 * while we have it unlocked. We cannot trust user threads
1542 * to do the same.
1543 *
1544 * HACK HACK HACK HACK
1545 */
1546 if (vm_map_pmap(map)((map)->pmap) == kernel_pmap) {
1547 vm_map_unlock(map)lock_done(&(map)->lock); /* trust me ... */
1548 }
1549 else {
1550 vm_map_lock_set_recursive(map)lock_set_recursive(&(map)->lock);
1551 vm_map_lock_write_to_read(map)lock_write_to_read(&(map)->lock);
1552 }
1553
1554 entry = start_entry;
1555 while (entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links) &&
1556 entry->vme_startlinks.start < end) {
1557 /*
1558 * Wiring cases:
1559 * Kernel: wired == 1 && user_wired == 0
1560 * User: wired == 1 && user_wired == 1
1561 *
1562 * Don't need to wire if either is > 1. wired = 0 &&
1563 * user_wired == 1 can't happen.
1564 */
1565
1566 /*
1567 * XXX This assumes that the faults always succeed.
1568 */
1569 if ((entry->wired_count == 1) &&
1570 (entry->user_wired_count <= 1)) {
1571 vm_fault_wire(map, entry);
1572 }
1573 entry = entry->vme_nextlinks.next;
1574 }
1575
1576 if (vm_map_pmap(map)((map)->pmap) == kernel_pmap) {
1577 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
1578 }
1579 else {
1580 vm_map_lock_clear_recursive(map)lock_clear_recursive(&(map)->lock);
1581 }
1582 }
1583
1584 vm_map_unlock(map)lock_done(&(map)->lock);
1585
1586 return(KERN_SUCCESS0);
1587}
1588
1589/*
1590 * vm_map_entry_delete: [ internal use only ]
1591 *
1592 * Deallocate the given entry from the target map.
1593 */
1594void vm_map_entry_delete(
1595 vm_map_t map,
1596 vm_map_entry_t entry)
1597{
1598 vm_offset_t s, e;
1599 vm_object_t object;
1600 extern vm_object_t kernel_object;
1601
1602 s = entry->vme_startlinks.start;
1603 e = entry->vme_endlinks.end;
1604
1605 /*Check if projected buffer*/
1606 if (map != kernel_map && entry->projected_on != 0) {
1607 /*Check if projected kernel entry is persistent;
1608 may only manipulate directly if it is*/
1609 if (entry->projected_on->projected_on == 0)
1610 entry->wired_count = 0; /*Avoid unwire fault*/
1611 else
1612 return;
1613 }
1614
1615 /*
1616 * Get the object. Null objects cannot have pmap entries.
1617 */
1618
1619 if ((object = entry->object.vm_object) != VM_OBJECT_NULL((vm_object_t) 0)) {
1620
1621 /*
1622 * Unwire before removing addresses from the pmap;
1623 * otherwise, unwiring will put the entries back in
1624 * the pmap.
1625 */
1626
1627 if (entry->wired_count != 0) {
1628 vm_fault_unwire(map, entry);
1629 entry->wired_count = 0;
1630 entry->user_wired_count = 0;
1631 }
1632
1633 /*
1634 * If the object is shared, we must remove
1635 * *all* references to this data, since we can't
1636 * find all of the physical maps which are sharing
1637 * it.
1638 */
1639
1640 if (object == kernel_object) {
1641 vm_object_lock(object);
1642 vm_object_page_remove(object, entry->offset,
1643 entry->offset + (e - s));
1644 vm_object_unlock(object)((void)(&(object)->Lock));
1645 } else if (entry->is_shared) {
1646 vm_object_pmap_remove(object,
1647 entry->offset,
1648 entry->offset + (e - s));
1649 }
1650 else {
1651 pmap_remove(map->pmap, s, e);
1652 }
1653 }
1654
1655 /*
1656 * Deallocate the object only after removing all
1657 * pmap entries pointing to its pages.
1658 */
1659
1660 if (entry->is_sub_map)
1661 vm_map_deallocate(entry->object.sub_map);
1662 else
1663 vm_object_deallocate(entry->object.vm_object);
1664
1665 vm_map_entry_unlink(map, entry)({ (&(map)->hdr)->nentries--; (entry)->links.next
->links.prev = (entry)->links.prev; (entry)->links.prev
->links.next = (entry)->links.next; rbtree_remove(&
(&(map)->hdr)->tree, &(entry)->tree_node); }
);
1666 map->size -= e - s;
1667
1668 vm_map_entry_dispose(map, entry)_vm_map_entry_dispose(&(map)->hdr, (entry));
1669}
1670
1671/*
1672 * vm_map_delete: [ internal use only ]
1673 *
1674 * Deallocates the given address range from the target
1675 * map.
1676 */
1677
1678kern_return_t vm_map_delete(
1679 vm_map_t map,
1680 vm_offset_t start,
1681 vm_offset_t end)
1682{
1683 vm_map_entry_t entry;
1684 vm_map_entry_t first_entry;
1685
1686 /*
1687 * Find the start of the region, and clip it
1688 */
1689
1690 if (!vm_map_lookup_entry(map, start, &first_entry))
1691 entry = first_entry->vme_nextlinks.next;
1692 else {
1693 entry = first_entry;
1694 vm_map_clip_start(map, entry, start)({ if ((start) > (entry)->links.start) _vm_map_clip_start
(&(map)->hdr,(entry),(start)); });
1695
1696 /*
1697 * Fix the lookup hint now, rather than each
1698 * time though the loop.
1699 */
1700
1701 SAVE_HINT(map, entry->vme_prev); (map)->hint = (entry->links.prev); ((void)(&(map)
->hint_lock));;
1702 }
1703
1704 /*
1705 * Save the free space hint
1706 */
1707
1708 if (map->first_free->vme_startlinks.start >= start)
1709 map->first_free = entry->vme_prevlinks.prev;
1710
1711 /*
1712 * Step through all entries in this region
1713 */
1714
1715 while ((entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) && (entry->vme_startlinks.start < end)) {
1716 vm_map_entry_t next;
1717
1718 vm_map_clip_end(map, entry, end)({ if ((end) < (entry)->links.end) _vm_map_clip_end(&
(map)->hdr,(entry),(end)); });
1719
1720 /*
1721 * If the entry is in transition, we must wait
1722 * for it to exit that state. It could be clipped
1723 * while we leave the map unlocked.
1724 */
1725 if(entry->in_transition) {
1726 /*
1727 * Say that we are waiting, and wait for entry.
1728 */
1729 entry->needs_wakeup = TRUE((boolean_t) 1);
1730 vm_map_entry_wait(map, FALSE)({ assert_wait((event_t)&(map)->hdr, ((boolean_t) 0));
lock_done(&(map)->lock); thread_block((void (*)()) 0)
; });
1731 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
1732
1733 /*
1734 * The entry could have been clipped or it
1735 * may not exist anymore. look it up again.
1736 */
1737 if(!vm_map_lookup_entry(map, start, &entry)) {
1738 entry = entry->vme_nextlinks.next;
1739 }
1740 continue;
1741 }
1742
1743 next = entry->vme_nextlinks.next;
1744
1745 vm_map_entry_delete(map, entry);
1746 entry = next;
1747 }
1748
1749 if (map->wait_for_space)
1750 thread_wakeup((event_t) map)thread_wakeup_prim(((event_t) map), ((boolean_t) 0), 0);
1751
1752 return(KERN_SUCCESS0);
1753}
1754
1755/*
1756 * vm_map_remove:
1757 *
1758 * Remove the given address range from the target map.
1759 * This is the exported form of vm_map_delete.
1760 */
1761kern_return_t vm_map_remove(
1762 vm_map_t map,
1763 vm_offset_t start,
1764 vm_offset_t end)
1765{
1766 kern_return_t result;
1767
1768 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
1769 VM_MAP_RANGE_CHECK(map, start, end){ if (start < ((map)->hdr.links.start)) start = ((map)->
hdr.links.start); if (end > ((map)->hdr.links.end)) end
= ((map)->hdr.links.end); if (start > end) start = end
; };
1770 result = vm_map_delete(map, start, end);
1771 vm_map_unlock(map)lock_done(&(map)->lock);
1772
1773 return(result);
1774}
1775
1776
1777/*
1778 * vm_map_copy_steal_pages:
1779 *
1780 * Steal all the pages from a vm_map_copy page_list by copying ones
1781 * that have not already been stolen.
1782 */
1783void
1784vm_map_copy_steal_pages(vm_map_copy_t copy)
1785{
1786 vm_page_t m, new_m;
1787 int i;
1788 vm_object_t object;
1789
1790 for (i = 0; i < copy->cpy_npagesc_u.c_p.npages; i++) {
1791
1792 /*
1793 * If the page is not tabled, then it's already stolen.
1794 */
1795 m = copy->cpy_page_listc_u.c_p.page_list[i];
1796 if (!m->tabled)
1797 continue;
1798
1799 /*
1800 * Page was not stolen, get a new
1801 * one and do the copy now.
1802 */
1803 while ((new_m = vm_page_grab(FALSE((boolean_t) 0))) == VM_PAGE_NULL((vm_page_t) 0)) {
1804 VM_PAGE_WAIT((void(*)()) 0)vm_page_wait((void(*)()) 0);
1805 }
1806
1807 vm_page_copy(m, new_m);
1808
1809 object = m->object;
1810 vm_object_lock(object);
1811 vm_page_lock_queues();
1812 if (!m->active && !m->inactive)
1813 vm_page_activate(m);
1814 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
1815 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } });
1816 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_map.c", 1816); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
1817 vm_object_unlock(object)((void)(&(object)->Lock));
1818
1819 copy->cpy_page_listc_u.c_p.page_list[i] = new_m;
1820 }
1821}
1822
1823/*
1824 * vm_map_copy_page_discard:
1825 *
1826 * Get rid of the pages in a page_list copy. If the pages are
1827 * stolen, they are freed. If the pages are not stolen, they
1828 * are unbusied, and associated state is cleaned up.
1829 */
1830void vm_map_copy_page_discard(vm_map_copy_t copy)
1831{
1832 while (copy->cpy_npagesc_u.c_p.npages > 0) {
1833 vm_page_t m;
1834
1835 if((m = copy->cpy_page_listc_u.c_p.page_list[--(copy->cpy_npagesc_u.c_p.npages)]) !=
1836 VM_PAGE_NULL((vm_page_t) 0)) {
1837
1838 /*
1839 * If it's not in the table, then it's
1840 * a stolen page that goes back
1841 * to the free list. Else it belongs
1842 * to some object, and we hold a
1843 * paging reference on that object.
1844 */
1845 if (!m->tabled) {
1846 VM_PAGE_FREE(m)({ ; vm_page_free(m); ((void)(&vm_page_queue_lock)); });
1847 }
1848 else {
1849 vm_object_t object;
1850
1851 object = m->object;
1852
1853 vm_object_lock(object);
1854 vm_page_lock_queues();
1855 if (!m->active && !m->inactive)
1856 vm_page_activate(m);
1857 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
1858
1859 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } });
1860 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_map.c", 1860); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
1861 vm_object_unlock(object)((void)(&(object)->Lock));
1862 }
1863 }
1864 }
1865}
1866
1867/*
1868 * Routine: vm_map_copy_discard
1869 *
1870 * Description:
1871 * Dispose of a map copy object (returned by
1872 * vm_map_copyin).
1873 */
1874void
1875vm_map_copy_discard(vm_map_copy_t copy)
1876{
1877free_next_copy:
1878 if (copy == VM_MAP_COPY_NULL((vm_map_copy_t) 0))
1879 return;
1880
1881 switch (copy->type) {
1882 case VM_MAP_COPY_ENTRY_LIST1:
1883 while (vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next) !=
1884 vm_map_copy_to_entry(copy)((struct vm_map_entry *) &(copy)->c_u.hdr.links)) {
1885 vm_map_entry_t entry = vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next);
1886
1887 vm_map_copy_entry_unlink(copy, entry)({ (&(copy)->c_u.hdr)->nentries--; (entry)->links
.next->links.prev = (entry)->links.prev; (entry)->links
.prev->links.next = (entry)->links.next; rbtree_remove(
&(&(copy)->c_u.hdr)->tree, &(entry)->tree_node
); });
1888 vm_object_deallocate(entry->object.vm_object);
1889 vm_map_copy_entry_dispose(copy, entry)_vm_map_entry_dispose(&(copy)->c_u.hdr, (entry));
1890 }
1891 break;
1892 case VM_MAP_COPY_OBJECT2:
1893 vm_object_deallocate(copy->cpy_objectc_u.c_o.object);
1894 break;
1895 case VM_MAP_COPY_PAGE_LIST3:
1896
1897 /*
1898 * To clean this up, we have to unbusy all the pages
1899 * and release the paging references in their objects.
1900 */
1901 if (copy->cpy_npagesc_u.c_p.npages > 0)
1902 vm_map_copy_page_discard(copy);
1903
1904 /*
1905 * If there's a continuation, abort it. The
1906 * abort routine releases any storage.
1907 */
1908 if (vm_map_copy_has_cont(copy)(((copy)->c_u.c_p.cont) != (kern_return_t (*)()) 0)) {
1909
1910 /*
1911 * Special case: recognize
1912 * vm_map_copy_discard_cont and optimize
1913 * here to avoid tail recursion.
1914 */
1915 if (copy->cpy_contc_u.c_p.cont == vm_map_copy_discard_cont) {
1916 vm_map_copy_t new_copy;
1917
1918 new_copy = (vm_map_copy_t) copy->cpy_cont_argsc_u.c_p.cont_args;
1919 kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1920 copy = new_copy;
1921 goto free_next_copy;
1922 }
1923 else {
1924 vm_map_copy_abort_cont(copy)({ vm_map_copy_page_discard(copy); (*((copy)->c_u.c_p.cont
))((copy)->c_u.c_p.cont_args, (vm_map_copy_t *) 0); (copy)
->c_u.c_p.cont = (kern_return_t (*)()) 0; (copy)->c_u.c_p
.cont_args = (char *) 0; });
1925 }
1926 }
1927
1928 break;
1929 }
1930 kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1931}
1932
1933/*
1934 * Routine: vm_map_copy_copy
1935 *
1936 * Description:
1937 * Move the information in a map copy object to
1938 * a new map copy object, leaving the old one
1939 * empty.
1940 *
1941 * This is used by kernel routines that need
1942 * to look at out-of-line data (in copyin form)
1943 * before deciding whether to return SUCCESS.
1944 * If the routine returns FAILURE, the original
1945 * copy object will be deallocated; therefore,
1946 * these routines must make a copy of the copy
1947 * object and leave the original empty so that
1948 * deallocation will not fail.
1949 */
1950vm_map_copy_t
1951vm_map_copy_copy(vm_map_copy_t copy)
1952{
1953 vm_map_copy_t new_copy;
1954
1955 if (copy == VM_MAP_COPY_NULL((vm_map_copy_t) 0))
1956 return VM_MAP_COPY_NULL((vm_map_copy_t) 0);
1957
1958 /*
1959 * Allocate a new copy object, and copy the information
1960 * from the old one into it.
1961 */
1962
1963 new_copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1964 *new_copy = *copy;
1965
1966 if (copy->type == VM_MAP_COPY_ENTRY_LIST1) {
1967 /*
1968 * The links in the entry chain must be
1969 * changed to point to the new copy object.
1970 */
1971 vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next)->vme_prevlinks.prev
1972 = vm_map_copy_to_entry(new_copy)((struct vm_map_entry *) &(new_copy)->c_u.hdr.links);
1973 vm_map_copy_last_entry(copy)((copy)->c_u.hdr.links.prev)->vme_nextlinks.next
1974 = vm_map_copy_to_entry(new_copy)((struct vm_map_entry *) &(new_copy)->c_u.hdr.links);
1975 }
1976
1977 /*
1978 * Change the old copy object into one that contains
1979 * nothing to be deallocated.
1980 */
1981 copy->type = VM_MAP_COPY_OBJECT2;
1982 copy->cpy_objectc_u.c_o.object = VM_OBJECT_NULL((vm_object_t) 0);
1983
1984 /*
1985 * Return the new object.
1986 */
1987 return new_copy;
1988}
1989
1990/*
1991 * Routine: vm_map_copy_discard_cont
1992 *
1993 * Description:
1994 * A version of vm_map_copy_discard that can be called
1995 * as a continuation from a vm_map_copy page list.
1996 */
1997kern_return_t vm_map_copy_discard_cont(
1998vm_map_copyin_args_t cont_args,
1999vm_map_copy_t *copy_result) /* OUT */
2000{
2001 vm_map_copy_discard((vm_map_copy_t) cont_args);
2002 if (copy_result != (vm_map_copy_t *)0)
2003 *copy_result = VM_MAP_COPY_NULL((vm_map_copy_t) 0);
2004 return(KERN_SUCCESS0);
2005}
2006
2007/*
2008 * Routine: vm_map_copy_overwrite
2009 *
2010 * Description:
2011 * Copy the memory described by the map copy
2012 * object (copy; returned by vm_map_copyin) onto
2013 * the specified destination region (dst_map, dst_addr).
2014 * The destination must be writeable.
2015 *
2016 * Unlike vm_map_copyout, this routine actually
2017 * writes over previously-mapped memory. If the
2018 * previous mapping was to a permanent (user-supplied)
2019 * memory object, it is preserved.
2020 *
2021 * The attributes (protection and inheritance) of the
2022 * destination region are preserved.
2023 *
2024 * If successful, consumes the copy object.
2025 * Otherwise, the caller is responsible for it.
2026 *
2027 * Implementation notes:
2028 * To overwrite temporary virtual memory, it is
2029 * sufficient to remove the previous mapping and insert
2030 * the new copy. This replacement is done either on
2031 * the whole region (if no permanent virtual memory
2032 * objects are embedded in the destination region) or
2033 * in individual map entries.
2034 *
2035 * To overwrite permanent virtual memory, it is
2036 * necessary to copy each page, as the external
2037 * memory management interface currently does not
2038 * provide any optimizations.
2039 *
2040 * Once a page of permanent memory has been overwritten,
2041 * it is impossible to interrupt this function; otherwise,
2042 * the call would be neither atomic nor location-independent.
2043 * The kernel-state portion of a user thread must be
2044 * interruptible.
2045 *
2046 * It may be expensive to forward all requests that might
2047 * overwrite permanent memory (vm_write, vm_copy) to
2048 * uninterruptible kernel threads. This routine may be
2049 * called by interruptible threads; however, success is
2050 * not guaranteed -- if the request cannot be performed
2051 * atomically and interruptibly, an error indication is
2052 * returned.
2053 */
2054kern_return_t vm_map_copy_overwrite(
2055 vm_map_t dst_map,
2056 vm_offset_t dst_addr,
2057 vm_map_copy_t copy,
2058 boolean_t interruptible)
2059{
2060 vm_size_t size;
2061 vm_offset_t start;
2062 vm_map_entry_t tmp_entry;
2063 vm_map_entry_t entry;
2064
2065 boolean_t contains_permanent_objects = FALSE((boolean_t) 0);
2066
2067 interruptible = FALSE((boolean_t) 0); /* XXX */
2068
2069 /*
2070 * Check for null copy object.
2071 */
2072
2073 if (copy == VM_MAP_COPY_NULL((vm_map_copy_t) 0))
2074 return(KERN_SUCCESS0);
2075
2076 /*
2077 * Only works for entry lists at the moment. Will
2078 * support page lists LATER.
2079 */
2080
2081 assert(copy->type == VM_MAP_COPY_ENTRY_LIST)({ if (!(copy->type == 1)) Assert("copy->type == VM_MAP_COPY_ENTRY_LIST"
, "../vm/vm_map.c", 2081); });
2082
2083 /*
2084 * Currently this routine only handles page-aligned
2085 * regions. Eventually, it should handle misalignments
2086 * by actually copying pages.
2087 */
2088
2089 if (!page_aligned(copy->offset)((((vm_offset_t) (copy->offset)) & ((1 << 12)-1)
) == 0) ||
2090 !page_aligned(copy->size)((((vm_offset_t) (copy->size)) & ((1 << 12)-1)) ==
0) ||
2091 !page_aligned(dst_addr)((((vm_offset_t) (dst_addr)) & ((1 << 12)-1)) == 0))
2092 return(KERN_INVALID_ARGUMENT4);
2093
2094 size = copy->size;
2095
2096 if (size == 0) {
2097 vm_map_copy_discard(copy);
2098 return(KERN_SUCCESS0);
2099 }
2100
2101 /*
2102 * Verify that the destination is all writeable
2103 * initially.
2104 */
2105start_pass_1:
2106 vm_map_lock(dst_map)({ lock_write(&(dst_map)->lock); (dst_map)->timestamp
++; });
2107 if (!vm_map_lookup_entry(dst_map, dst_addr, &tmp_entry)) {
2108 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2109 return(KERN_INVALID_ADDRESS1);
2110 }
2111 vm_map_clip_start(dst_map, tmp_entry, dst_addr)({ if ((dst_addr) > (tmp_entry)->links.start) _vm_map_clip_start
(&(dst_map)->hdr,(tmp_entry),(dst_addr)); });
2112 for (entry = tmp_entry;;) {
2113 vm_size_t sub_size = (entry->vme_endlinks.end - entry->vme_startlinks.start);
2114 vm_map_entry_t next = entry->vme_nextlinks.next;
2115
2116 if ( ! (entry->protection & VM_PROT_WRITE((vm_prot_t) 0x02))) {
2117 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2118 return(KERN_PROTECTION_FAILURE2);
2119 }
2120
2121 /*
2122 * If the entry is in transition, we must wait
2123 * for it to exit that state. Anything could happen
2124 * when we unlock the map, so start over.
2125 */
2126 if (entry->in_transition) {
2127
2128 /*
2129 * Say that we are waiting, and wait for entry.
2130 */
2131 entry->needs_wakeup = TRUE((boolean_t) 1);
2132 vm_map_entry_wait(dst_map, FALSE)({ assert_wait((event_t)&(dst_map)->hdr, ((boolean_t) 0
)); lock_done(&(dst_map)->lock); thread_block((void (*
)()) 0); });
2133
2134 goto start_pass_1;
2135 }
2136
2137 if (size <= sub_size)
2138 break;
2139
2140 if ((next == vm_map_to_entry(dst_map)((struct vm_map_entry *) &(dst_map)->hdr.links)) ||
2141 (next->vme_startlinks.start != entry->vme_endlinks.end)) {
2142 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2143 return(KERN_INVALID_ADDRESS1);
2144 }
2145
2146
2147 /*
2148 * Check for permanent objects in the destination.
2149 */
2150
2151 if ((entry->object.vm_object != VM_OBJECT_NULL((vm_object_t) 0)) &&
2152 !entry->object.vm_object->temporary)
2153 contains_permanent_objects = TRUE((boolean_t) 1);
2154
2155 size -= sub_size;
2156 entry = next;
2157 }
2158
2159 /*
2160 * If there are permanent objects in the destination, then
2161 * the copy cannot be interrupted.
2162 */
2163
2164 if (interruptible && contains_permanent_objects) {
2165 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2166 return(KERN_FAILURE5); /* XXX */
2167 }
2168
2169 /*
2170 * XXXO If there are no permanent objects in the destination,
2171 * XXXO and the source and destination map entry caches match,
2172 * XXXO and the destination map entry is not shared,
2173 * XXXO then the map entries can be deleted and replaced
2174 * XXXO with those from the copy. The following code is the
2175 * XXXO basic idea of what to do, but there are lots of annoying
2176 * XXXO little details about getting protection and inheritance
2177 * XXXO right. Should add protection, inheritance, and sharing checks
2178 * XXXO to the above pass and make sure that no wiring is involved.
2179 */
2180/*
2181 * if (!contains_permanent_objects &&
2182 * copy->cpy_hdr.entries_pageable == dst_map->hdr.entries_pageable) {
2183 *
2184 * *
2185 * * Run over copy and adjust entries. Steal code
2186 * * from vm_map_copyout() to do this.
2187 * *
2188 *
2189 * tmp_entry = tmp_entry->vme_prev;
2190 * vm_map_delete(dst_map, dst_addr, dst_addr + copy->size);
2191 * vm_map_copy_insert(dst_map, tmp_entry, copy);
2192 *
2193 * vm_map_unlock(dst_map);
2194 * vm_map_copy_discard(copy);
2195 * }
2196 */
2197 /*
2198 *
2199 * Make a second pass, overwriting the data
2200 * At the beginning of each loop iteration,
2201 * the next entry to be overwritten is "tmp_entry"
2202 * (initially, the value returned from the lookup above),
2203 * and the starting address expected in that entry
2204 * is "start".
2205 */
2206
2207 start = dst_addr;
2208
2209 while (vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next) != vm_map_copy_to_entry(copy)((struct vm_map_entry *) &(copy)->c_u.hdr.links)) {
2210 vm_map_entry_t copy_entry = vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next);
2211 vm_size_t copy_size = (copy_entry->vme_endlinks.end - copy_entry->vme_startlinks.start);
2212 vm_object_t object;
2213
2214 entry = tmp_entry;
2215 size = (entry->vme_endlinks.end - entry->vme_startlinks.start);
2216 /*
2217 * Make sure that no holes popped up in the
2218 * address map, and that the protection is
2219 * still valid, in case the map was unlocked
2220 * earlier.
2221 */
2222
2223 if (entry->vme_startlinks.start != start) {
2224 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2225 return(KERN_INVALID_ADDRESS1);
2226 }
2227 assert(entry != vm_map_to_entry(dst_map))({ if (!(entry != ((struct vm_map_entry *) &(dst_map)->
hdr.links))) Assert("entry != vm_map_to_entry(dst_map)", "../vm/vm_map.c"
, 2227); });
2228
2229 /*
2230 * Check protection again
2231 */
2232
2233 if ( ! (entry->protection & VM_PROT_WRITE((vm_prot_t) 0x02))) {
2234 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2235 return(KERN_PROTECTION_FAILURE2);
2236 }
2237
2238 /*
2239 * Adjust to source size first
2240 */
2241
2242 if (copy_size < size) {
2243 vm_map_clip_end(dst_map, entry, entry->vme_start + copy_size)({ if ((entry->links.start + copy_size) < (entry)->links
.end) _vm_map_clip_end(&(dst_map)->hdr,(entry),(entry->
links.start + copy_size)); });
2244 size = copy_size;
2245 }
2246
2247 /*
2248 * Adjust to destination size
2249 */
2250
2251 if (size < copy_size) {
2252 vm_map_copy_clip_end(copy, copy_entry,({ if ((copy_entry->links.start + size) < (copy_entry)->
links.end) _vm_map_clip_end(&(copy)->c_u.hdr,(copy_entry
),(copy_entry->links.start + size)); })
2253 copy_entry->vme_start + size)({ if ((copy_entry->links.start + size) < (copy_entry)->
links.end) _vm_map_clip_end(&(copy)->c_u.hdr,(copy_entry
),(copy_entry->links.start + size)); });
2254 copy_size = size;
2255 }
2256
2257 assert((entry->vme_end - entry->vme_start) == size)({ if (!((entry->links.end - entry->links.start) == size
)) Assert("(entry->vme_end - entry->vme_start) == size"
, "../vm/vm_map.c", 2257); });
2258 assert((tmp_entry->vme_end - tmp_entry->vme_start) == size)({ if (!((tmp_entry->links.end - tmp_entry->links.start
) == size)) Assert("(tmp_entry->vme_end - tmp_entry->vme_start) == size"
, "../vm/vm_map.c", 2258); });
2259 assert((copy_entry->vme_end - copy_entry->vme_start) == size)({ if (!((copy_entry->links.end - copy_entry->links.start
) == size)) Assert("(copy_entry->vme_end - copy_entry->vme_start) == size"
, "../vm/vm_map.c", 2259); });
2260
2261 /*
2262 * If the destination contains temporary unshared memory,
2263 * we can perform the copy by throwing it away and
2264 * installing the source data.
2265 */
2266
2267 object = entry->object.vm_object;
2268 if (!entry->is_shared &&
2269 ((object == VM_OBJECT_NULL((vm_object_t) 0)) || object->temporary)) {
2270 vm_object_t old_object = entry->object.vm_object;
2271 vm_offset_t old_offset = entry->offset;
2272
2273 entry->object = copy_entry->object;
2274 entry->offset = copy_entry->offset;
2275 entry->needs_copy = copy_entry->needs_copy;
2276 entry->wired_count = 0;
2277 entry->user_wired_count = 0;
2278
2279 vm_map_copy_entry_unlink(copy, copy_entry)({ (&(copy)->c_u.hdr)->nentries--; (copy_entry)->
links.next->links.prev = (copy_entry)->links.prev; (copy_entry
)->links.prev->links.next = (copy_entry)->links.next
; rbtree_remove(&(&(copy)->c_u.hdr)->tree, &
(copy_entry)->tree_node); });
2280 vm_map_copy_entry_dispose(copy, copy_entry)_vm_map_entry_dispose(&(copy)->c_u.hdr, (copy_entry));
2281
2282 vm_object_pmap_protect(
2283 old_object,
2284 old_offset,
2285 size,
2286 dst_map->pmap,
2287 tmp_entry->vme_startlinks.start,
2288 VM_PROT_NONE((vm_prot_t) 0x00));
2289
2290 vm_object_deallocate(old_object);
2291
2292 /*
2293 * Set up for the next iteration. The map
2294 * has not been unlocked, so the next
2295 * address should be at the end of this
2296 * entry, and the next map entry should be
2297 * the one following it.
2298 */
2299
2300 start = tmp_entry->vme_endlinks.end;
2301 tmp_entry = tmp_entry->vme_nextlinks.next;
2302 } else {
2303 vm_map_version_t version;
2304 vm_object_t dst_object = entry->object.vm_object;
2305 vm_offset_t dst_offset = entry->offset;
2306 kern_return_t r;
2307
2308 /*
2309 * Take an object reference, and record
2310 * the map version information so that the
2311 * map can be safely unlocked.
2312 */
2313
2314 vm_object_reference(dst_object);
2315
2316 version.main_timestamp = dst_map->timestamp;
2317
2318 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2319
2320 /*
2321 * Copy as much as possible in one pass
2322 */
2323
2324 copy_size = size;
2325 r = vm_fault_copy(
2326 copy_entry->object.vm_object,
2327 copy_entry->offset,
2328 &copy_size,
2329 dst_object,
2330 dst_offset,
2331 dst_map,
2332 &version,
2333 FALSE((boolean_t) 0) /* XXX interruptible */ );
2334
2335 /*
2336 * Release the object reference
2337 */
2338
2339 vm_object_deallocate(dst_object);
2340
2341 /*
2342 * If a hard error occurred, return it now
2343 */
2344
2345 if (r != KERN_SUCCESS0)
2346 return(r);
2347
2348 if (copy_size != 0) {
2349 /*
2350 * Dispose of the copied region
2351 */
2352
2353 vm_map_copy_clip_end(copy, copy_entry,({ if ((copy_entry->links.start + copy_size) < (copy_entry
)->links.end) _vm_map_clip_end(&(copy)->c_u.hdr,(copy_entry
),(copy_entry->links.start + copy_size)); })
2354 copy_entry->vme_start + copy_size)({ if ((copy_entry->links.start + copy_size) < (copy_entry
)->links.end) _vm_map_clip_end(&(copy)->c_u.hdr,(copy_entry
),(copy_entry->links.start + copy_size)); });
2355 vm_map_copy_entry_unlink(copy, copy_entry)({ (&(copy)->c_u.hdr)->nentries--; (copy_entry)->
links.next->links.prev = (copy_entry)->links.prev; (copy_entry
)->links.prev->links.next = (copy_entry)->links.next
; rbtree_remove(&(&(copy)->c_u.hdr)->tree, &
(copy_entry)->tree_node); });
2356 vm_object_deallocate(copy_entry->object.vm_object);
2357 vm_map_copy_entry_dispose(copy, copy_entry)_vm_map_entry_dispose(&(copy)->c_u.hdr, (copy_entry));
2358 }
2359
2360 /*
2361 * Pick up in the destination map where we left off.
2362 *
2363 * Use the version information to avoid a lookup
2364 * in the normal case.
2365 */
2366
2367 start += copy_size;
2368 vm_map_lock(dst_map)({ lock_write(&(dst_map)->lock); (dst_map)->timestamp
++; });
2369 if ((version.main_timestamp + 1) == dst_map->timestamp) {
2370 /* We can safely use saved tmp_entry value */
2371
2372 vm_map_clip_end(dst_map, tmp_entry, start)({ if ((start) < (tmp_entry)->links.end) _vm_map_clip_end
(&(dst_map)->hdr,(tmp_entry),(start)); });
2373 tmp_entry = tmp_entry->vme_nextlinks.next;
2374 } else {
2375 /* Must do lookup of tmp_entry */
2376
2377 if (!vm_map_lookup_entry(dst_map, start, &tmp_entry)) {
2378 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2379 return(KERN_INVALID_ADDRESS1);
2380 }
2381 vm_map_clip_start(dst_map, tmp_entry, start)({ if ((start) > (tmp_entry)->links.start) _vm_map_clip_start
(&(dst_map)->hdr,(tmp_entry),(start)); });
2382 }
2383 }
2384
2385 }
2386 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2387
2388 /*
2389 * Throw away the vm_map_copy object
2390 */
2391 vm_map_copy_discard(copy);
2392
2393 return(KERN_SUCCESS0);
2394}
2395
2396/*
2397 * Macro: vm_map_copy_insert
2398 *
2399 * Description:
2400 * Link a copy chain ("copy") into a map at the
2401 * specified location (after "where").
2402 * Side effects:
2403 * The copy chain is destroyed.
2404 * Warning:
2405 * The arguments are evaluated multiple times.
2406 */
2407#define vm_map_copy_insert(map, where, copy)({ struct rbtree_node *node, *tmp; for (node = rbtree_postwalk_deepest
(&(copy)->c_u.hdr.tree), tmp = rbtree_postwalk_unlink(
node); node != ((void *) 0); node = tmp, tmp = rbtree_postwalk_unlink
(node)) ({ struct rbtree_node *___cur, *___prev; int ___diff,
___index; ___prev = ((void *) 0); ___index = -1; ___cur = (&
(map)->hdr.tree)->root; while (___cur != ((void *) 0)) {
___diff = vm_map_entry_cmp_insert(node, ___cur); ({ if (!(___diff
!= 0)) Assert("___diff != 0", "../vm/vm_map.c", 2407); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(map)->
hdr.tree, ___prev, ___index, node); }); (((where)->links.next
)->links.prev = ((copy)->c_u.hdr.links.prev)) ->links
.next = ((where)->links.next); ((where)->links.next = (
(copy)->c_u.hdr.links.next)) ->links.prev = (where); (map
)->hdr.nentries += (copy)->c_u.hdr.nentries; kmem_cache_free
(&vm_map_copy_cache, (vm_offset_t) copy); }) \
2408 MACRO_BEGIN({ \
2409 struct rbtree_node *node, *tmp; \
2410 rbtree_for_each_remove(&(copy)->cpy_hdr.tree, node, tmp)for (node = rbtree_postwalk_deepest(&(copy)->c_u.hdr.tree
), tmp = rbtree_postwalk_unlink(node); node != ((void *) 0); node
= tmp, tmp = rbtree_postwalk_unlink(node)) \
2411 rbtree_insert(&(map)->hdr.tree, node, \({ struct rbtree_node *___cur, *___prev; int ___diff, ___index
; ___prev = ((void *) 0); ___index = -1; ___cur = (&(map)
->hdr.tree)->root; while (___cur != ((void *) 0)) { ___diff
= vm_map_entry_cmp_insert(node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 2412); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(map)->
hdr.tree, ___prev, ___index, node); })
2412 vm_map_entry_cmp_insert)({ struct rbtree_node *___cur, *___prev; int ___diff, ___index
; ___prev = ((void *) 0); ___index = -1; ___cur = (&(map)
->hdr.tree)->root; while (___cur != ((void *) 0)) { ___diff
= vm_map_entry_cmp_insert(node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 2412); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(map)->
hdr.tree, ___prev, ___index, node); }); \
2413 (((where)->vme_nextlinks.next)->vme_prevlinks.prev = vm_map_copy_last_entry(copy)((copy)->c_u.hdr.links.prev)) \
2414 ->vme_nextlinks.next = ((where)->vme_nextlinks.next); \
2415 ((where)->vme_nextlinks.next = vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next)) \
2416 ->vme_prevlinks.prev = (where); \
2417 (map)->hdr.nentries += (copy)->cpy_hdrc_u.hdr.nentries; \
2418 kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); \
2419 MACRO_END})
2420
2421/*
2422 * Routine: vm_map_copyout
2423 *
2424 * Description:
2425 * Copy out a copy chain ("copy") into newly-allocated
2426 * space in the destination map.
2427 *
2428 * If successful, consumes the copy object.
2429 * Otherwise, the caller is responsible for it.
2430 */
2431kern_return_t vm_map_copyout(
2432 vm_map_t dst_map,
2433 vm_offset_t *dst_addr, /* OUT */
2434 vm_map_copy_t copy)
2435{
2436 vm_size_t size;
2437 vm_size_t adjustment;
2438 vm_offset_t start;
2439 vm_offset_t vm_copy_start;
2440 vm_map_entry_t last;
2441 vm_map_entry_t entry;
2442
2443 /*
2444 * Check for null copy object.
2445 */
2446
2447 if (copy == VM_MAP_COPY_NULL((vm_map_copy_t) 0)) {
2448 *dst_addr = 0;
2449 return(KERN_SUCCESS0);
2450 }
2451
2452 /*
2453 * Check for special copy object, created
2454 * by vm_map_copyin_object.
2455 */
2456
2457 if (copy->type == VM_MAP_COPY_OBJECT2) {
2458 vm_object_t object = copy->cpy_objectc_u.c_o.object;
2459 vm_size_t offset = copy->offset;
2460 vm_size_t tmp_size = copy->size;
2461 kern_return_t kr;
2462
2463 *dst_addr = 0;
2464 kr = vm_map_enter(dst_map, dst_addr, tmp_size,
2465 (vm_offset_t) 0, TRUE((boolean_t) 1),
2466 object, offset, FALSE((boolean_t) 0),
2467 VM_PROT_DEFAULT(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)), VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04)),
2468 VM_INHERIT_DEFAULT((vm_inherit_t) 1));
2469 if (kr != KERN_SUCCESS0)
2470 return(kr);
2471 kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
2472 return(KERN_SUCCESS0);
2473 }
2474
2475 if (copy->type == VM_MAP_COPY_PAGE_LIST3)
2476 return(vm_map_copyout_page_list(dst_map, dst_addr, copy));
2477
2478 /*
2479 * Find space for the data
2480 */
2481
2482 vm_copy_start = trunc_page(copy->offset)((vm_offset_t)(((vm_offset_t)(copy->offset)) & ~((1 <<
12)-1)));
2483 size = round_page(copy->offset + copy->size)((vm_offset_t)((((vm_offset_t)(copy->offset + copy->size
)) + ((1 << 12)-1)) & ~((1 << 12)-1))) - vm_copy_start;
2484
2485 StartAgain: ;
2486
2487 vm_map_lock(dst_map)({ lock_write(&(dst_map)->lock); (dst_map)->timestamp
++; });
2488 start = ((last = dst_map->first_free) == vm_map_to_entry(dst_map)((struct vm_map_entry *) &(dst_map)->hdr.links)) ?
2489 vm_map_min(dst_map)((dst_map)->hdr.links.start) : last->vme_endlinks.end;
2490
2491 while (TRUE((boolean_t) 1)) {
2492 vm_map_entry_t next = last->vme_nextlinks.next;
2493 vm_offset_t end = start + size;
2494
2495 if ((end > dst_map->max_offsethdr.links.end) || (end < start)) {
2496 if (dst_map->wait_for_space) {
2497 if (size <= (dst_map->max_offsethdr.links.end - dst_map->min_offsethdr.links.start)) {
2498 assert_wait((event_t) dst_map, TRUE((boolean_t) 1));
2499 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2500 thread_block((void (*)()) 0);
2501 goto StartAgain;
2502 }
2503 }
2504 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2505 printf_once("no more room for vm_map_copyout in %p\n", dst_map)({ static int __once = 0; if (!__once) { db_printf("no more room for vm_map_copyout in %p\n"
, dst_map); __once = 1; } });
2506 return(KERN_NO_SPACE3);
2507 }
2508
2509 if ((next == vm_map_to_entry(dst_map)((struct vm_map_entry *) &(dst_map)->hdr.links)) ||
2510 (next->vme_startlinks.start >= end))
2511 break;
2512
2513 last = next;
2514 start = last->vme_endlinks.end;
2515 }
2516
2517 /*
2518 * Since we're going to just drop the map
2519 * entries from the copy into the destination
2520 * map, they must come from the same pool.
2521 */
2522
2523 if (copy->cpy_hdrc_u.hdr.entries_pageable != dst_map->hdr.entries_pageable) {
2524 /*
2525 * Mismatches occur when dealing with the default
2526 * pager.
2527 */
2528 kmem_cache_t old_cache;
2529 vm_map_entry_t next, new;
2530
2531 /*
2532 * Find the cache that the copies were allocated from
2533 */
2534 old_cache = (copy->cpy_hdrc_u.hdr.entries_pageable)
2535 ? &vm_map_entry_cache
2536 : &vm_map_kentry_cache;
2537 entry = vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next);
2538
2539 /*
2540 * Reinitialize the copy so that vm_map_copy_entry_link
2541 * will work.
2542 */
2543 copy->cpy_hdrc_u.hdr.nentries = 0;
2544 copy->cpy_hdrc_u.hdr.entries_pageable = dst_map->hdr.entries_pageable;
2545 vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next) =
2546 vm_map_copy_last_entry(copy)((copy)->c_u.hdr.links.prev) =
2547 vm_map_copy_to_entry(copy)((struct vm_map_entry *) &(copy)->c_u.hdr.links);
2548
2549 /*
2550 * Copy each entry.
2551 */
2552 while (entry != vm_map_copy_to_entry(copy)((struct vm_map_entry *) &(copy)->c_u.hdr.links)) {
2553 new = vm_map_copy_entry_create(copy)_vm_map_entry_create(&(copy)->c_u.hdr);
2554 vm_map_entry_copy_full(new, entry)(*(new) = *(entry));
2555 vm_map_copy_entry_link(copy,({ (&(copy)->c_u.hdr)->nentries++; (new)->links.
prev = (((copy)->c_u.hdr.links.prev)); (new)->links.next
= (((copy)->c_u.hdr.links.prev))->links.next; (new)->
links.prev->links.next = (new)->links.next->links.prev
= (new); ({ struct rbtree_node *___cur, *___prev; int ___diff
, ___index; ___prev = ((void *) 0); ___index = -1; ___cur = (
&(&(copy)->c_u.hdr)->tree)->root; while (___cur
!= ((void *) 0)) { ___diff = vm_map_entry_cmp_insert(&(new
)->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0"
, "../vm/vm_map.c", 2557); }); ___prev = ___cur; ___index = rbtree_d2i
(___diff); ___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(&(copy)->c_u.hdr)->tree, ___prev, ___index, &
(new)->tree_node); }); })
2556 vm_map_copy_last_entry(copy),({ (&(copy)->c_u.hdr)->nentries++; (new)->links.
prev = (((copy)->c_u.hdr.links.prev)); (new)->links.next
= (((copy)->c_u.hdr.links.prev))->links.next; (new)->
links.prev->links.next = (new)->links.next->links.prev
= (new); ({ struct rbtree_node *___cur, *___prev; int ___diff
, ___index; ___prev = ((void *) 0); ___index = -1; ___cur = (
&(&(copy)->c_u.hdr)->tree)->root; while (___cur
!= ((void *) 0)) { ___diff = vm_map_entry_cmp_insert(&(new
)->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0"
, "../vm/vm_map.c", 2557); }); ___prev = ___cur; ___index = rbtree_d2i
(___diff); ___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(&(copy)->c_u.hdr)->tree, ___prev, ___index, &
(new)->tree_node); }); })
2557 new)({ (&(copy)->c_u.hdr)->nentries++; (new)->links.
prev = (((copy)->c_u.hdr.links.prev)); (new)->links.next
= (((copy)->c_u.hdr.links.prev))->links.next; (new)->
links.prev->links.next = (new)->links.next->links.prev
= (new); ({ struct rbtree_node *___cur, *___prev; int ___diff
, ___index; ___prev = ((void *) 0); ___index = -1; ___cur = (
&(&(copy)->c_u.hdr)->tree)->root; while (___cur
!= ((void *) 0)) { ___diff = vm_map_entry_cmp_insert(&(new
)->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0"
, "../vm/vm_map.c", 2557); }); ___prev = ___cur; ___index = rbtree_d2i
(___diff); ___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(&(copy)->c_u.hdr)->tree, ___prev, ___index, &
(new)->tree_node); }); });
2558 next = entry->vme_nextlinks.next;
2559 kmem_cache_free(old_cache, (vm_offset_t) entry);
2560 entry = next;
2561 }
2562 }
2563
2564 /*
2565 * Adjust the addresses in the copy chain, and
2566 * reset the region attributes.
2567 */
2568
2569 adjustment = start - vm_copy_start;
2570 for (entry = vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next);
2571 entry != vm_map_copy_to_entry(copy)((struct vm_map_entry *) &(copy)->c_u.hdr.links);
2572 entry = entry->vme_nextlinks.next) {
2573 entry->vme_startlinks.start += adjustment;
2574 entry->vme_endlinks.end += adjustment;
2575
2576 entry->inheritance = VM_INHERIT_DEFAULT((vm_inherit_t) 1);
2577 entry->protection = VM_PROT_DEFAULT(((vm_prot_t) 0x01)|((vm_prot_t) 0x02));
2578 entry->max_protection = VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04));
2579 entry->projected_on = 0;
2580
2581 /*
2582 * If the entry is now wired,
2583 * map the pages into the destination map.
2584 */
2585 if (entry->wired_count != 0) {
2586 vm_offset_t va;
2587 vm_offset_t offset;
2588 vm_object_t object;
2589
2590 object = entry->object.vm_object;
2591 offset = entry->offset;
2592 va = entry->vme_startlinks.start;
2593
2594 pmap_pageable(dst_map->pmap,
2595 entry->vme_startlinks.start,
2596 entry->vme_endlinks.end,
2597 TRUE((boolean_t) 1));
2598
2599 while (va < entry->vme_endlinks.end) {
2600 vm_page_t m;
2601
2602 /*
2603 * Look up the page in the object.
2604 * Assert that the page will be found in the
2605 * top object:
2606 * either
2607 * the object was newly created by
2608 * vm_object_copy_slowly, and has
2609 * copies of all of the pages from
2610 * the source object
2611 * or
2612 * the object was moved from the old
2613 * map entry; because the old map
2614 * entry was wired, all of the pages
2615 * were in the top-level object.
2616 * (XXX not true if we wire pages for
2617 * reading)
2618 */
2619 vm_object_lock(object);
2620 vm_object_paging_begin(object)((object)->paging_in_progress++);
2621
2622 m = vm_page_lookup(object, offset);
2623 if (m == VM_PAGE_NULL((vm_page_t) 0) || m->wire_count == 0 ||
2624 m->absent)
2625 panic("vm_map_copyout: wiring 0x%x", m);
2626
2627 m->busy = TRUE((boolean_t) 1);
2628 vm_object_unlock(object)((void)(&(object)->Lock));
2629
2630 PMAP_ENTER(dst_map->pmap, va, m,({ pmap_enter( (dst_map->pmap), (va), (m)->phys_addr, (
entry->protection) & ~(m)->page_lock, (((boolean_t)
1)) ); })
2631 entry->protection, TRUE)({ pmap_enter( (dst_map->pmap), (va), (m)->phys_addr, (
entry->protection) & ~(m)->page_lock, (((boolean_t)
1)) ); });
2632
2633 vm_object_lock(object);
2634 PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)->
wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)),
((boolean_t) 0), 0); } });
2635 /* the page is wired, so we don't have to activate */
2636 vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0"
, "../vm/vm_map.c", 2636); }); if (--(object)->paging_in_progress
== 0) { ({ if ((object)->all_wanted & (1 << (2)
)) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2)
)), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 <<
(2)); }); } });
2637 vm_object_unlock(object)((void)(&(object)->Lock));
2638
2639 offset += PAGE_SIZE(1 << 12);
2640 va += PAGE_SIZE(1 << 12);
2641 }
2642 }
2643
2644
2645 }
2646
2647 /*
2648 * Correct the page alignment for the result
2649 */
2650
2651 *dst_addr = start + (copy->offset - vm_copy_start);
2652
2653 /*
2654 * Update the hints and the map size
2655 */
2656
2657 if (dst_map->first_free == last)
2658 dst_map->first_free = vm_map_copy_last_entry(copy)((copy)->c_u.hdr.links.prev);
2659 SAVE_HINT(dst_map, vm_map_copy_last_entry(copy)); (dst_map)->hint = (((copy)->c_u.hdr.links.prev)); ((void
)(&(dst_map)->hint_lock));;
2660
2661 dst_map->size += size;
2662
2663 /*
2664 * Link in the copy
2665 */
2666
2667 vm_map_copy_insert(dst_map, last, copy)({ struct rbtree_node *node, *tmp; for (node = rbtree_postwalk_deepest
(&(copy)->c_u.hdr.tree), tmp = rbtree_postwalk_unlink(
node); node != ((void *) 0); node = tmp, tmp = rbtree_postwalk_unlink
(node)) ({ struct rbtree_node *___cur, *___prev; int ___diff,
___index; ___prev = ((void *) 0); ___index = -1; ___cur = (&
(dst_map)->hdr.tree)->root; while (___cur != ((void *) 0
)) { ___diff = vm_map_entry_cmp_insert(node, ___cur); ({ if (
!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c", 2667
); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur
= ___cur->children[___index]; } rbtree_insert_rebalance(&
(dst_map)->hdr.tree, ___prev, ___index, node); }); (((last
)->links.next)->links.prev = ((copy)->c_u.hdr.links.
prev)) ->links.next = ((last)->links.next); ((last)->
links.next = ((copy)->c_u.hdr.links.next)) ->links.prev
= (last); (dst_map)->hdr.nentries += (copy)->c_u.hdr.nentries
; kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy)
; });
2668
2669 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2670
2671 /*
2672 * XXX If wiring_required, call vm_map_pageable
2673 */
2674
2675 return(KERN_SUCCESS0);
2676}
2677
2678/*
2679 *
2680 * vm_map_copyout_page_list:
2681 *
2682 * Version of vm_map_copyout() for page list vm map copies.
2683 *
2684 */
2685kern_return_t vm_map_copyout_page_list(
2686 vm_map_t dst_map,
2687 vm_offset_t *dst_addr, /* OUT */
2688 vm_map_copy_t copy)
2689{
2690 vm_size_t size;
2691 vm_offset_t start;
2692 vm_offset_t end;
2693 vm_offset_t offset;
2694 vm_map_entry_t last;
2695 vm_object_t object;
2696 vm_page_t *page_list, m;
2697 vm_map_entry_t entry;
2698 vm_offset_t old_last_offset;
2699 boolean_t cont_invoked, needs_wakeup = FALSE((boolean_t) 0);
2700 kern_return_t result = KERN_SUCCESS0;
2701 vm_map_copy_t orig_copy;
2702 vm_offset_t dst_offset;
2703 boolean_t must_wire;
2704
2705 /*
2706 * Make sure the pages are stolen, because we are
2707 * going to put them in a new object. Assume that
2708 * all pages are identical to first in this regard.
2709 */
2710
2711 page_list = &copy->cpy_page_listc_u.c_p.page_list[0];
2712 if ((*page_list)->tabled)
1
Taking false branch
→
2713 vm_map_copy_steal_pages(copy);
2714
2715 /*
2716 * Find space for the data
2717 */
2718
2719 size = round_page(copy->offset + copy->size)((vm_offset_t)((((vm_offset_t)(copy->offset + copy->size
)) + ((1 << 12)-1)) & ~((1 << 12)-1))) -
2720 trunc_page(copy->offset)((vm_offset_t)(((vm_offset_t)(copy->offset)) & ~((1 <<
12)-1)));
2721StartAgain:
2722 vm_map_lock(dst_map)({ lock_write(&(dst_map)->lock); (dst_map)->timestamp
++; });
2723 must_wire = dst_map->wiring_required;
2724
2725 last = dst_map->first_free;
2726 if (last == vm_map_to_entry(dst_map)((struct vm_map_entry *) &(dst_map)->hdr.links)) {
2
←
Taking false branch
→
2727 start = vm_map_min(dst_map)((dst_map)->hdr.links.start);
2728 } else {
2729 start = last->vme_endlinks.end;
2730 }
2731
2732 while (TRUE((boolean_t) 1)) {
3
←
Loop condition is true. Entering loop body
→
6
←
Loop condition is true. Entering loop body
→
9
←
Loop condition is true. Entering loop body
→
12
←
Loop condition is true. Entering loop body
→
2733 vm_map_entry_t next = last->vme_nextlinks.next;
2734 end = start + size;
2735
2736 if ((end > dst_map->max_offsethdr.links.end) || (end < start)) {
4
←
Taking false branch
→
7
←
Taking false branch
→
10
←
Taking false branch
→
13
←
Taking false branch
→
2737 if (dst_map->wait_for_space) {
2738 if (size <= (dst_map->max_offsethdr.links.end -
2739 dst_map->min_offsethdr.links.start)) {
2740 assert_wait((event_t) dst_map, TRUE((boolean_t) 1));
2741 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2742 thread_block((void (*)()) 0);
2743 goto StartAgain;
2744 }
2745 }
2746 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2747 printf_once("no more room for vm_map_copyout_page_list in %p\n", dst_map)({ static int __once = 0; if (!__once) { db_printf("no more room for vm_map_copyout_page_list in %p\n"
, dst_map); __once = 1; } });
2748 return(KERN_NO_SPACE3);
2749 }
2750
2751 if ((next == vm_map_to_entry(dst_map)((struct vm_map_entry *) &(dst_map)->hdr.links)) ||
5
←
Taking false branch
→
8
←
Taking false branch
→
11
←
Taking false branch
→
14
←
Taking true branch
→
2752 (next->vme_startlinks.start >= end)) {
2753 break;
15
←
 Execution continues on line 2769
→
2754 }
2755
2756 last = next;
2757 start = last->vme_endlinks.end;
2758 }
2759
2760 /*
2761 * See whether we can avoid creating a new entry (and object) by
2762 * extending one of our neighbors. [So far, we only attempt to
2763 * extend from below.]
2764 *
2765 * The code path below here is a bit twisted. If any of the
2766 * extension checks fails, we branch to create_object. If
2767 * it all works, we fall out the bottom and goto insert_pages.
2768 */
2769 if (last == vm_map_to_entry(dst_map)((struct vm_map_entry *) &(dst_map)->hdr.links) ||
18
←
Taking false branch
→
2770 last->vme_endlinks.end != start ||
2771 last->is_shared != FALSE((boolean_t) 0) ||
2772 last->is_sub_map != FALSE((boolean_t) 0) ||
2773 last->inheritance != VM_INHERIT_DEFAULT((vm_inherit_t) 1) ||
2774 last->protection != VM_PROT_DEFAULT(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)) ||
2775 last->max_protection != VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04)) ||
2776 (must_wire ? (last->wired_count != 1 ||
16
←
Assuming 'must_wire' is 0
→
17
←
'?' condition is false
→
2777 last->user_wired_count != 1) :
2778 (last->wired_count != 0))) {
2779 goto create_object;
2780 }
2781
2782 /*
2783 * If this entry needs an object, make one.
2784 */
2785 if (last->object.vm_object == VM_OBJECT_NULL((vm_object_t) 0)) {
19
←
Taking false branch
→
2786 object = vm_object_allocate(
2787 (vm_size_t)(last->vme_endlinks.end - last->vme_startlinks.start + size));
2788 last->object.vm_object = object;
2789 last->offset = 0;
2790 vm_object_lock(object);
2791 }
2792 else {
2793 vm_offset_t prev_offset = last->offset;
2794 vm_size_t prev_size = start - last->vme_startlinks.start;
2795 vm_size_t new_size;
2796
2797 /*
2798 * This is basically vm_object_coalesce.
2799 */
2800
2801 object = last->object.vm_object;
2802 vm_object_lock(object);
2803
2804 /*
2805 * Try to collapse the object first
2806 */
2807 vm_object_collapse(object);
2808
2809 /*
2810 * Can't coalesce if pages not mapped to
2811 * last may be in use anyway:
2812 * . more than one reference
2813 * . paged out
2814 * . shadows another object
2815 * . has a copy elsewhere
2816 * . paging references (pages might be in page-list)
2817 */
2818
2819 if ((object->ref_count > 1) ||
20
←
Taking false branch
→
2820 object->pager_created ||
2821 (object->shadow != VM_OBJECT_NULL((vm_object_t) 0)) ||
2822 (object->copy != VM_OBJECT_NULL((vm_object_t) 0)) ||
2823 (object->paging_in_progress != 0)) {
2824 vm_object_unlock(object)((void)(&(object)->Lock));
2825 goto create_object;
2826 }
2827
2828 /*
2829 * Extend the object if necessary. Don't have to call
2830 * vm_object_page_remove because the pages aren't mapped,
2831 * and vm_page_replace will free up any old ones it encounters.
2832 */
2833 new_size = prev_offset + prev_size + size;
2834 if (new_size > object->size)
21
←
Taking false branch
→
2835 object->size = new_size;
2836 }
2837
2838 /*
2839 * Coalesced the two objects - can extend
2840 * the previous map entry to include the
2841 * new range.
2842 */
2843 dst_map->size += size;
2844 last->vme_endlinks.end = end;
2845
2846 SAVE_HINT(dst_map, last); (dst_map)->hint = (last); ((void)(&(dst_map)->hint_lock
));;
2847
2848 goto insert_pages;
22
←
Control jumps to line 2911
→
2849
2850create_object:
2851
2852 /*
2853 * Create object
2854 */
2855 object = vm_object_allocate(size);
2856
2857 /*
2858 * Create entry
2859 */
2860
2861 entry = vm_map_entry_create(dst_map)_vm_map_entry_create(&(dst_map)->hdr);
2862
2863 entry->object.vm_object = object;
2864 entry->offset = 0;
2865
2866 entry->is_shared = FALSE((boolean_t) 0);
2867 entry->is_sub_map = FALSE((boolean_t) 0);
2868 entry->needs_copy = FALSE((boolean_t) 0);
2869
2870 if (must_wire) {
2871 entry->wired_count = 1;
2872 entry->user_wired_count = 1;
2873 } else {
2874 entry->wired_count = 0;
2875 entry->user_wired_count = 0;
2876 }
2877
2878 entry->in_transition = TRUE((boolean_t) 1);
2879 entry->needs_wakeup = FALSE((boolean_t) 0);
2880
2881 entry->vme_startlinks.start = start;
2882 entry->vme_endlinks.end = start + size;
2883
2884 entry->inheritance = VM_INHERIT_DEFAULT((vm_inherit_t) 1);
2885 entry->protection = VM_PROT_DEFAULT(((vm_prot_t) 0x01)|((vm_prot_t) 0x02));
2886 entry->max_protection = VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04));
2887 entry->projected_on = 0;
2888
2889 vm_object_lock(object);
2890
2891 /*
2892 * Update the hints and the map size
2893 */
2894 if (dst_map->first_free == last) {
2895 dst_map->first_free = entry;
2896 }
2897 SAVE_HINT(dst_map, entry); (dst_map)->hint = (entry); ((void)(&(dst_map)->hint_lock
));;
2898 dst_map->size += size;
2899
2900 /*
2901 * Link in the entry
2902 */
2903 vm_map_entry_link(dst_map, last, entry)({ (&(dst_map)->hdr)->nentries++; (entry)->links
.prev = (last); (entry)->links.next = (last)->links.next
; (entry)->links.prev->links.next = (entry)->links.next
->links.prev = (entry); ({ struct rbtree_node *___cur, *___prev
; int ___diff, ___index; ___prev = ((void *) 0); ___index = -
1; ___cur = (&(&(dst_map)->hdr)->tree)->root
; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(entry)->tree_node, ___cur); ({ if (!(___diff != 0))
Assert("___diff != 0", "../vm/vm_map.c", 2903); }); ___prev =
___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(dst_map
)->hdr)->tree, ___prev, ___index, &(entry)->tree_node
); }); });
2904 last = entry;
2905
2906 /*
2907 * Transfer pages into new object.
2908 * Scan page list in vm_map_copy.
2909 */
2910insert_pages:
2911 dst_offset = copy->offset & PAGE_MASK((1 << 12)-1);
2912 cont_invoked = FALSE((boolean_t) 0);
2913 orig_copy = copy;
2914 last->in_transition = TRUE((boolean_t) 1);
2915 old_last_offset = last->offset
2916 + (start - last->vme_startlinks.start);
2917
2918 vm_page_lock_queues();
2919
2920 for (offset = 0; offset < size; offset += PAGE_SIZE(1 << 12)) {
23
←
Assuming 'offset' is < 'size'
→
24
←
Loop condition is true. Entering loop body
→
31
←
Assuming 'offset' is < 'size'
→
32
←
Loop condition is true. Entering loop body
→
2921 m = *page_list;
2922 assert(m && !m->tabled)({ if (!(m && !m->tabled)) Assert("m && !m->tabled"
, "../vm/vm_map.c", 2922); });
2923
2924 /*
2925 * Must clear busy bit in page before inserting it.
2926 * Ok to skip wakeup logic because nobody else
2927 * can possibly know about this page.
2928 * The page is dirty in its new object.
2929 */
2930
2931 assert(!m->wanted)({ if (!(!m->wanted)) Assert("!m->wanted", "../vm/vm_map.c"
, 2931); });
2932
2933 m->busy = FALSE((boolean_t) 0);
2934 m->dirty = TRUE((boolean_t) 1);
2935 vm_page_replace(m, object, old_last_offset + offset);
2936 if (must_wire) {
25
←
Taking false branch
→
33
←
Taking false branch
→
2937 vm_page_wire(m);
2938 PMAP_ENTER(dst_map->pmap,({ pmap_enter( (dst_map->pmap), (last->links.start + m->
offset - last->offset), (m)->phys_addr, (last->protection
) & ~(m)->page_lock, (((boolean_t) 1)) ); })
2939 last->vme_start + m->offset - last->offset,({ pmap_enter( (dst_map->pmap), (last->links.start + m->
offset - last->offset), (m)->phys_addr, (last->protection
) & ~(m)->page_lock, (((boolean_t) 1)) ); })
2940 m, last->protection, TRUE)({ pmap_enter( (dst_map->pmap), (last->links.start + m->
offset - last->offset), (m)->phys_addr, (last->protection
) & ~(m)->page_lock, (((boolean_t) 1)) ); });
2941 } else {
2942 vm_page_activate(m);
2943 }
2944
2945 *page_list++ = VM_PAGE_NULL((vm_page_t) 0);
2946 if (--(copy->cpy_npagesc_u.c_p.npages) == 0 &&
26
←
Taking true branch
→
34
←
Dereference of null pointer
2947 vm_map_copy_has_cont(copy)(((copy)->c_u.c_p.cont) != (kern_return_t (*)()) 0)) {
2948 vm_map_copy_t new_copy;
2949
2950 /*
2951 * Ok to unlock map because entry is
2952 * marked in_transition.
2953 */
2954 cont_invoked = TRUE((boolean_t) 1);
2955 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
2956 vm_object_unlock(object)((void)(&(object)->Lock));
2957 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2958 vm_map_copy_invoke_cont(copy, &new_copy, &result)({ vm_map_copy_page_discard(copy); *&result = (*((copy)->
c_u.c_p.cont))((copy)->c_u.c_p.cont_args, &new_copy); (
copy)->c_u.c_p.cont = (kern_return_t (*)()) 0; });
2959
2960 if (result == KERN_SUCCESS0) {
27
←
Assuming 'result' is equal to 0
→
28
←
Taking true branch
→
2961
2962 /*
2963 * If we got back a copy with real pages,
2964 * steal them now. Either all of the
2965 * pages in the list are tabled or none
2966 * of them are; mixtures are not possible.
2967 *
2968 * Save original copy for consume on
2969 * success logic at end of routine.
2970 */
2971 if (copy != orig_copy)
29
←
Taking false branch
→
2972 vm_map_copy_discard(copy);
2973
2974 if ((copy = new_copy) != VM_MAP_COPY_NULL((vm_map_copy_t) 0)) {
30
←
Taking false branch
→
2975 page_list = &copy->cpy_page_listc_u.c_p.page_list[0];
2976 if ((*page_list)->tabled)
2977 vm_map_copy_steal_pages(copy);
2978 }
2979 }
2980 else {
2981 /*
2982 * Continuation failed.
2983 */
2984 vm_map_lock(dst_map)({ lock_write(&(dst_map)->lock); (dst_map)->timestamp
++; });
2985 goto error;
2986 }
2987
2988 vm_map_lock(dst_map)({ lock_write(&(dst_map)->lock); (dst_map)->timestamp
++; });
2989 vm_object_lock(object);
2990 vm_page_lock_queues();
2991 }
2992 }
2993
2994 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
2995 vm_object_unlock(object)((void)(&(object)->Lock));
2996
2997 *dst_addr = start + dst_offset;
2998
2999 /*
3000 * Clear the in transition bits. This is easy if we
3001 * didn't have a continuation.
3002 */
3003error:
3004 if (!cont_invoked) {
3005 /*
3006 * We didn't unlock the map, so nobody could
3007 * be waiting.
3008 */
3009 last->in_transition = FALSE((boolean_t) 0);
3010 assert(!last->needs_wakeup)({ if (!(!last->needs_wakeup)) Assert("!last->needs_wakeup"
, "../vm/vm_map.c", 3010); });
3011 needs_wakeup = FALSE((boolean_t) 0);
3012 }
3013 else {
3014 if (!vm_map_lookup_entry(dst_map, start, &entry))
3015 panic("vm_map_copyout_page_list: missing entry");
3016
3017 /*
3018 * Clear transition bit for all constituent entries that
3019 * were in the original entry. Also check for waiters.
3020 */
3021 while((entry != vm_map_to_entry(dst_map)((struct vm_map_entry *) &(dst_map)->hdr.links)) &&
3022 (entry->vme_startlinks.start < end)) {
3023 assert(entry->in_transition)({ if (!(entry->in_transition)) Assert("entry->in_transition"
, "../vm/vm_map.c", 3023); });
3024 entry->in_transition = FALSE((boolean_t) 0);
3025 if(entry->needs_wakeup) {
3026 entry->needs_wakeup = FALSE((boolean_t) 0);
3027 needs_wakeup = TRUE((boolean_t) 1);
3028 }
3029 entry = entry->vme_nextlinks.next;
3030 }
3031 }
3032
3033 if (result != KERN_SUCCESS0)
3034 vm_map_delete(dst_map, start, end);
3035
3036 vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
3037
3038 if (needs_wakeup)
3039 vm_map_entry_wakeup(dst_map)thread_wakeup_prim(((event_t)&(dst_map)->hdr), ((boolean_t
) 0), 0);
3040
3041 /*
3042 * Consume on success logic.
3043 */
3044 if (copy != orig_copy) {
3045 kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
3046 }
3047 if (result == KERN_SUCCESS0) {
3048 kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) orig_copy);
3049 }
3050
3051 return(result);
3052}
3053
3054/*
3055 * Routine: vm_map_copyin
3056 *
3057 * Description:
3058 * Copy the specified region (src_addr, len) from the
3059 * source address space (src_map), possibly removing
3060 * the region from the source address space (src_destroy).
3061 *
3062 * Returns:
3063 * A vm_map_copy_t object (copy_result), suitable for
3064 * insertion into another address space (using vm_map_copyout),
3065 * copying over another address space region (using
3066 * vm_map_copy_overwrite). If the copy is unused, it
3067 * should be destroyed (using vm_map_copy_discard).
3068 *
3069 * In/out conditions:
3070 * The source map should not be locked on entry.
3071 */
3072kern_return_t vm_map_copyin(
3073 vm_map_t src_map,
3074 vm_offset_t src_addr,
3075 vm_size_t len,
3076 boolean_t src_destroy,
3077 vm_map_copy_t *copy_result) /* OUT */
3078{
3079 vm_map_entry_t tmp_entry; /* Result of last map lookup --
3080 * in multi-level lookup, this
3081 * entry contains the actual
3082 * vm_object/offset.
3083 */
3084
3085 vm_offset_t src_start; /* Start of current entry --
3086 * where copy is taking place now
3087 */
3088 vm_offset_t src_end; /* End of entire region to be
3089 * copied */
3090
3091 vm_map_copy_t copy; /* Resulting copy */
3092
3093 /*
3094 * Check for copies of zero bytes.
3095 */
3096
3097 if (len == 0) {
3098 *copy_result = VM_MAP_COPY_NULL((vm_map_copy_t) 0);
3099 return(KERN_SUCCESS0);
3100 }
3101
3102 /*
3103 * Compute start and end of region
3104 */
3105
3106 src_start = trunc_page(src_addr)((vm_offset_t)(((vm_offset_t)(src_addr)) & ~((1 << 12
)-1)));
3107 src_end = round_page(src_addr + len)((vm_offset_t)((((vm_offset_t)(src_addr + len)) + ((1 <<
12)-1)) & ~((1 << 12)-1)));
3108
3109 /*
3110 * Check that the end address doesn't overflow
3111 */
3112
3113 if (src_end <= src_start)
3114 if ((src_end < src_start) || (src_start != 0))
3115 return(KERN_INVALID_ADDRESS1);
3116
3117 /*
3118 * Allocate a header element for the list.
3119 *
3120 * Use the start and end in the header to
3121 * remember the endpoints prior to rounding.
3122 */
3123
3124 copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
3125 vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next) =
3126 vm_map_copy_last_entry(copy)((copy)->c_u.hdr.links.prev) = vm_map_copy_to_entry(copy)((struct vm_map_entry *) &(copy)->c_u.hdr.links);
3127 copy->type = VM_MAP_COPY_ENTRY_LIST1;
3128 copy->cpy_hdrc_u.hdr.nentries = 0;
3129 copy->cpy_hdrc_u.hdr.entries_pageable = TRUE((boolean_t) 1);
3130 rbtree_init(&copy->cpy_hdrc_u.hdr.tree);
3131
3132 copy->offset = src_addr;
3133 copy->size = len;
3134
3135#define RETURN(x) \
3136 MACRO_BEGIN({ \
3137 vm_map_unlock(src_map)lock_done(&(src_map)->lock); \
3138 vm_map_copy_discard(copy); \
3139 MACRO_RETURNif (((boolean_t) 1)) return(x); \
3140 MACRO_END})
3141
3142 /*
3143 * Find the beginning of the region.
3144 */
3145
3146 vm_map_lock(src_map)({ lock_write(&(src_map)->lock); (src_map)->timestamp
++; });
3147
3148 if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry))
3149 RETURN(KERN_INVALID_ADDRESS1);
3150 vm_map_clip_start(src_map, tmp_entry, src_start)({ if ((src_start) > (tmp_entry)->links.start) _vm_map_clip_start
(&(src_map)->hdr,(tmp_entry),(src_start)); });
3151
3152 /*
3153 * Go through entries until we get to the end.
3154 */
3155
3156 while (TRUE((boolean_t) 1)) {
3157 vm_map_entry_t src_entry = tmp_entry; /* Top-level entry */
3158 vm_size_t src_size; /* Size of source
3159 * map entry (in both
3160 * maps)
3161 */
3162
3163 vm_object_t src_object; /* Object to copy */
3164 vm_offset_t src_offset;
3165
3166 boolean_t src_needs_copy; /* Should source map
3167 * be made read-only
3168 * for copy-on-write?
3169 */
3170
3171 vm_map_entry_t new_entry; /* Map entry for copy */
3172 boolean_t new_entry_needs_copy; /* Will new entry be COW? */
3173
3174 boolean_t was_wired; /* Was source wired? */
3175 vm_map_version_t version; /* Version before locks
3176 * dropped to make copy
3177 */
3178
3179 /*
3180 * Verify that the region can be read.
3181 */
3182
3183 if (! (src_entry->protection & VM_PROT_READ((vm_prot_t) 0x01)))
3184 RETURN(KERN_PROTECTION_FAILURE2);
3185
3186 /*
3187 * Clip against the endpoints of the entire region.
3188 */
3189
3190 vm_map_clip_end(src_map, src_entry, src_end)({ if ((src_end) < (src_entry)->links.end) _vm_map_clip_end
(&(src_map)->hdr,(src_entry),(src_end)); });
3191
3192 src_size = src_entry->vme_endlinks.end - src_start;
3193 src_object = src_entry->object.vm_object;
3194 src_offset = src_entry->offset;
3195 was_wired = (src_entry->wired_count != 0);
3196
3197 /*
3198 * Create a new address map entry to
3199 * hold the result. Fill in the fields from
3200 * the appropriate source entries.
3201 */
3202
3203 new_entry = vm_map_copy_entry_create(copy)_vm_map_entry_create(&(copy)->c_u.hdr);
3204 vm_map_entry_copy(new_entry, src_entry)({ *(new_entry) = *(src_entry); (new_entry)->is_shared = (
(boolean_t) 0); (new_entry)->needs_wakeup = ((boolean_t) 0
); (new_entry)->in_transition = ((boolean_t) 0); (new_entry
)->wired_count = 0; (new_entry)->user_wired_count = 0; }
);
3205
3206 /*
3207 * Attempt non-blocking copy-on-write optimizations.
3208 */
3209
3210 if (src_destroy &&
3211 (src_object == VM_OBJECT_NULL((vm_object_t) 0) ||
3212 (src_object->temporary && !src_object->use_shared_copy)))
3213 {
3214 /*
3215 * If we are destroying the source, and the object
3216 * is temporary, and not shared writable,
3217 * we can move the object reference
3218 * from the source to the copy. The copy is
3219 * copy-on-write only if the source is.
3220 * We make another reference to the object, because
3221 * destroying the source entry will deallocate it.
3222 */
3223 vm_object_reference(src_object);
3224
3225 /*
3226 * Copy is always unwired. vm_map_copy_entry
3227 * set its wired count to zero.
3228 */
3229
3230 goto CopySuccessful;
3231 }
3232
3233 if (!was_wired &&
3234 vm_object_copy_temporary(
3235 &new_entry->object.vm_object,
3236 &new_entry->offset,
3237 &src_needs_copy,
3238 &new_entry_needs_copy)) {
3239
3240 new_entry->needs_copy = new_entry_needs_copy;
3241
3242 /*
3243 * Handle copy-on-write obligations
3244 */
3245
3246 if (src_needs_copy && !tmp_entry->needs_copy) {
3247 vm_object_pmap_protect(
3248 src_object,
3249 src_offset,
3250 src_size,
3251 (src_entry->is_shared ? PMAP_NULL((pmap_t) 0)
3252 : src_map->pmap),
3253 src_entry->vme_startlinks.start,
3254 src_entry->protection &
3255 ~VM_PROT_WRITE((vm_prot_t) 0x02));
3256
3257 tmp_entry->needs_copy = TRUE((boolean_t) 1);
3258 }
3259
3260 /*
3261 * The map has never been unlocked, so it's safe to
3262 * move to the next entry rather than doing another
3263 * lookup.
3264 */
3265
3266 goto CopySuccessful;
3267 }
3268
3269 new_entry->needs_copy = FALSE((boolean_t) 0);
3270
3271 /*
3272 * Take an object reference, so that we may
3273 * release the map lock(s).
3274 */
3275
3276 assert(src_object != VM_OBJECT_NULL)({ if (!(src_object != ((vm_object_t) 0))) Assert("src_object != VM_OBJECT_NULL"
, "../vm/vm_map.c", 3276); });
3277 vm_object_reference(src_object);
3278
3279 /*
3280 * Record the timestamp for later verification.
3281 * Unlock the map.
3282 */
3283
3284 version.main_timestamp = src_map->timestamp;
3285 vm_map_unlock(src_map)lock_done(&(src_map)->lock);
3286
3287 /*
3288 * Perform the copy
3289 */
3290
3291 if (was_wired) {
3292 vm_object_lock(src_object);
3293 (void) vm_object_copy_slowly(
3294 src_object,
3295 src_offset,
3296 src_size,
3297 FALSE((boolean_t) 0),
3298 &new_entry->object.vm_object);
3299 new_entry->offset = 0;
3300 new_entry->needs_copy = FALSE((boolean_t) 0);
3301 } else {
3302 kern_return_t result;
3303
3304 result = vm_object_copy_strategically(src_object,
3305 src_offset,
3306 src_size,
3307 &new_entry->object.vm_object,
3308 &new_entry->offset,
3309 &new_entry_needs_copy);
3310
3311 new_entry->needs_copy = new_entry_needs_copy;
3312
3313
3314 if (result != KERN_SUCCESS0) {
3315 vm_map_copy_entry_dispose(copy, new_entry)_vm_map_entry_dispose(&(copy)->c_u.hdr, (new_entry));
3316
3317 vm_map_lock(src_map)({ lock_write(&(src_map)->lock); (src_map)->timestamp
++; });
3318 RETURN(result);
3319 }
3320
3321 }
3322
3323 /*
3324 * Throw away the extra reference
3325 */
3326
3327 vm_object_deallocate(src_object);
3328
3329 /*
3330 * Verify that the map has not substantially
3331 * changed while the copy was being made.
3332 */
3333
3334 vm_map_lock(src_map)({ lock_write(&(src_map)->lock); (src_map)->timestamp
++; }); /* Increments timestamp once! */
3335
3336 if ((version.main_timestamp + 1) == src_map->timestamp)
3337 goto CopySuccessful;
3338
3339 /*
3340 * Simple version comparison failed.
3341 *
3342 * Retry the lookup and verify that the
3343 * same object/offset are still present.
3344 *
3345 * [Note: a memory manager that colludes with
3346 * the calling task can detect that we have
3347 * cheated. While the map was unlocked, the
3348 * mapping could have been changed and restored.]
3349 */
3350
3351 if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry)) {
3352 vm_map_copy_entry_dispose(copy, new_entry)_vm_map_entry_dispose(&(copy)->c_u.hdr, (new_entry));
3353 RETURN(KERN_INVALID_ADDRESS1);
3354 }
3355
3356 src_entry = tmp_entry;
3357 vm_map_clip_start(src_map, src_entry, src_start)({ if ((src_start) > (src_entry)->links.start) _vm_map_clip_start
(&(src_map)->hdr,(src_entry),(src_start)); });
3358
3359 if ((src_entry->protection & VM_PROT_READ((vm_prot_t) 0x01)) == VM_PROT_NONE((vm_prot_t) 0x00))
3360 goto VerificationFailed;
3361
3362 if (src_entry->vme_endlinks.end < new_entry->vme_endlinks.end)
3363 src_size = (new_entry->vme_endlinks.end = src_entry->vme_endlinks.end) - src_start;
3364
3365 if ((src_entry->object.vm_object != src_object) ||
3366 (src_entry->offset != src_offset) ) {
3367
3368 /*
3369 * Verification failed.
3370 *
3371 * Start over with this top-level entry.
3372 */
3373
3374 VerificationFailed: ;
3375
3376 vm_object_deallocate(new_entry->object.vm_object);
3377 vm_map_copy_entry_dispose(copy, new_entry)_vm_map_entry_dispose(&(copy)->c_u.hdr, (new_entry));
3378 tmp_entry = src_entry;
3379 continue;
3380 }
3381
3382 /*
3383 * Verification succeeded.
3384 */
3385
3386 CopySuccessful: ;
3387
3388 /*
3389 * Link in the new copy entry.
3390 */
3391
3392 vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy),({ (&(copy)->c_u.hdr)->nentries++; (new_entry)->
links.prev = (((copy)->c_u.hdr.links.prev)); (new_entry)->
links.next = (((copy)->c_u.hdr.links.prev))->links.next
; (new_entry)->links.prev->links.next = (new_entry)->
links.next->links.prev = (new_entry); ({ struct rbtree_node
*___cur, *___prev; int ___diff, ___index; ___prev = ((void *
) 0); ___index = -1; ___cur = (&(&(copy)->c_u.hdr)
->tree)->root; while (___cur != ((void *) 0)) { ___diff
= vm_map_entry_cmp_insert(&(new_entry)->tree_node, ___cur
); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c"
, 3393); }); ___prev = ___cur; ___index = rbtree_d2i(___diff)
; ___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(&(copy)->c_u.hdr)->tree, ___prev, ___index, &
(new_entry)->tree_node); }); })
3393 new_entry)({ (&(copy)->c_u.hdr)->nentries++; (new_entry)->
links.prev = (((copy)->c_u.hdr.links.prev)); (new_entry)->
links.next = (((copy)->c_u.hdr.links.prev))->links.next
; (new_entry)->links.prev->links.next = (new_entry)->
links.next->links.prev = (new_entry); ({ struct rbtree_node
*___cur, *___prev; int ___diff, ___index; ___prev = ((void *
) 0); ___index = -1; ___cur = (&(&(copy)->c_u.hdr)
->tree)->root; while (___cur != ((void *) 0)) { ___diff
= vm_map_entry_cmp_insert(&(new_entry)->tree_node, ___cur
); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c"
, 3393); }); ___prev = ___cur; ___index = rbtree_d2i(___diff)
; ___cur = ___cur->children[___index]; } rbtree_insert_rebalance
(&(&(copy)->c_u.hdr)->tree, ___prev, ___index, &
(new_entry)->tree_node); }); });
3394
3395 /*
3396 * Determine whether the entire region
3397 * has been copied.
3398 */
3399 src_start = new_entry->vme_endlinks.end;
3400 if ((src_start >= src_end) && (src_end != 0))
3401 break;
3402
3403 /*
3404 * Verify that there are no gaps in the region
3405 */
3406
3407 tmp_entry = src_entry->vme_nextlinks.next;
3408 if (tmp_entry->vme_startlinks.start != src_start)
3409 RETURN(KERN_INVALID_ADDRESS1);
3410 }
3411
3412 /*
3413 * If the source should be destroyed, do it now, since the
3414 * copy was successful.
3415 */
3416 if (src_destroy)
3417 (void) vm_map_delete(src_map, trunc_page(src_addr)((vm_offset_t)(((vm_offset_t)(src_addr)) & ~((1 << 12
)-1))), src_end);
3418
3419 vm_map_unlock(src_map)lock_done(&(src_map)->lock);
3420
3421 *copy_result = copy;
3422 return(KERN_SUCCESS0);
3423
3424#undef RETURN
3425}
3426
3427/*
3428 * vm_map_copyin_object:
3429 *
3430 * Create a copy object from an object.
3431 * Our caller donates an object reference.
3432 */
3433
3434kern_return_t vm_map_copyin_object(
3435 vm_object_t object,
3436 vm_offset_t offset, /* offset of region in object */
3437 vm_size_t size, /* size of region in object */
3438 vm_map_copy_t *copy_result) /* OUT */
3439{
3440 vm_map_copy_t copy; /* Resulting copy */
3441
3442 /*
3443 * We drop the object into a special copy object
3444 * that contains the object directly. These copy objects
3445 * are distinguished by entries_pageable == FALSE
3446 * and null links.
3447 */
3448
3449 copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
3450 vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next) =
3451 vm_map_copy_last_entry(copy)((copy)->c_u.hdr.links.prev) = VM_MAP_ENTRY_NULL((vm_map_entry_t) 0);
3452 copy->type = VM_MAP_COPY_OBJECT2;
3453 copy->cpy_objectc_u.c_o.object = object;
3454 copy->offset = offset;
3455 copy->size = size;
3456
3457 *copy_result = copy;
3458 return(KERN_SUCCESS0);
3459}
3460
3461/*
3462 * vm_map_copyin_page_list_cont:
3463 *
3464 * Continuation routine for vm_map_copyin_page_list.
3465 *
3466 * If vm_map_copyin_page_list can't fit the entire vm range
3467 * into a single page list object, it creates a continuation.
3468 * When the target of the operation has used the pages in the
3469 * initial page list, it invokes the continuation, which calls
3470 * this routine. If an error happens, the continuation is aborted
3471 * (abort arg to this routine is TRUE). To avoid deadlocks, the
3472 * pages are discarded from the initial page list before invoking
3473 * the continuation.
3474 *
3475 * NOTE: This is not the same sort of continuation used by
3476 * the scheduler.
3477 */
3478
3479kern_return_t vm_map_copyin_page_list_cont(
3480 vm_map_copyin_args_t cont_args,
3481 vm_map_copy_t *copy_result) /* OUT */
3482{
3483 kern_return_t result = 0; /* '=0' to quiet gcc warnings */
3484 boolean_t do_abort, src_destroy, src_destroy_only;
3485
3486 /*
3487 * Check for cases that only require memory destruction.
3488 */
3489 do_abort = (copy_result == (vm_map_copy_t *) 0);
3490 src_destroy = (cont_args->destroy_len != (vm_size_t) 0);
3491 src_destroy_only = (cont_args->src_len == (vm_size_t) 0);
3492
3493 if (do_abort || src_destroy_only) {
3494 if (src_destroy)
3495 result = vm_map_remove(cont_args->map,
3496 cont_args->destroy_addr,
3497 cont_args->destroy_addr + cont_args->destroy_len);
3498 if (!do_abort)
3499 *copy_result = VM_MAP_COPY_NULL((vm_map_copy_t) 0);
3500 }
3501 else {
3502 result = vm_map_copyin_page_list(cont_args->map,
3503 cont_args->src_addr, cont_args->src_len, src_destroy,
3504 cont_args->steal_pages, copy_result, TRUE((boolean_t) 1));
3505
3506 if (src_destroy && !cont_args->steal_pages &&
3507 vm_map_copy_has_cont(*copy_result)(((*copy_result)->c_u.c_p.cont) != (kern_return_t (*)()) 0
)) {
3508 vm_map_copyin_args_t new_args;
3509 /*
3510 * Transfer old destroy info.
3511 */
3512 new_args = (vm_map_copyin_args_t)
3513 (*copy_result)->cpy_cont_argsc_u.c_p.cont_args;
3514 new_args->destroy_addr = cont_args->destroy_addr;
3515 new_args->destroy_len = cont_args->destroy_len;
3516 }
3517 }
3518
3519 vm_map_deallocate(cont_args->map);
3520 kfree((vm_offset_t)cont_args, sizeof(vm_map_copyin_args_data_t));
3521
3522 return(result);
3523}
3524
3525/*
3526 * vm_map_copyin_page_list:
3527 *
3528 * This is a variant of vm_map_copyin that copies in a list of pages.
3529 * If steal_pages is TRUE, the pages are only in the returned list.
3530 * If steal_pages is FALSE, the pages are busy and still in their
3531 * objects. A continuation may be returned if not all the pages fit:
3532 * the recipient of this copy_result must be prepared to deal with it.
3533 */
3534
3535kern_return_t vm_map_copyin_page_list(
3536 vm_map_t src_map,
3537 vm_offset_t src_addr,
3538 vm_size_t len,
3539 boolean_t src_destroy,
3540 boolean_t steal_pages,
3541 vm_map_copy_t *copy_result, /* OUT */
3542 boolean_t is_cont)
3543{
3544 vm_map_entry_t src_entry;
3545 vm_page_t m;
3546 vm_offset_t src_start;
3547 vm_offset_t src_end;
3548 vm_size_t src_size;
3549 vm_object_t src_object;
3550 vm_offset_t src_offset;
3551 vm_offset_t src_last_offset;
3552 vm_map_copy_t copy; /* Resulting copy */
3553 kern_return_t result = KERN_SUCCESS0;
3554 boolean_t need_map_lookup;
3555 vm_map_copyin_args_t cont_args;
3556
3557 /*
3558 * If steal_pages is FALSE, this leaves busy pages in
3559 * the object. A continuation must be used if src_destroy
3560 * is true in this case (!steal_pages && src_destroy).
3561 *
3562 * XXX Still have a more general problem of what happens
3563 * XXX if the same page occurs twice in a list. Deadlock
3564 * XXX can happen if vm_fault_page was called. A
3565 * XXX possible solution is to use a continuation if vm_fault_page
3566 * XXX is called and we cross a map entry boundary.
3567 */
3568
3569 /*
3570 * Check for copies of zero bytes.
3571 */
3572
3573 if (len == 0) {
3574 *copy_result = VM_MAP_COPY_NULL((vm_map_copy_t) 0);
3575 return(KERN_SUCCESS0);
3576 }
3577
3578 /*
3579 * Compute start and end of region
3580 */
3581
3582 src_start = trunc_page(src_addr)((vm_offset_t)(((vm_offset_t)(src_addr)) & ~((1 << 12
)-1)));
3583 src_end = round_page(src_addr + len)((vm_offset_t)((((vm_offset_t)(src_addr + len)) + ((1 <<
12)-1)) & ~((1 << 12)-1)));
3584
3585 /*
3586 * Check that the end address doesn't overflow
3587 */
3588
3589 if (src_end <= src_start && (src_end < src_start || src_start != 0)) {
3590 return KERN_INVALID_ADDRESS1;
3591 }
3592
3593 /*
3594 * Allocate a header element for the page list.
3595 *
3596 * Record original offset and size, as caller may not
3597 * be page-aligned.
3598 */
3599
3600 copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
3601 copy->type = VM_MAP_COPY_PAGE_LIST3;
3602 copy->cpy_npagesc_u.c_p.npages = 0;
3603 copy->offset = src_addr;
3604 copy->size = len;
3605 copy->cpy_contc_u.c_p.cont = ((kern_return_t (*)()) 0);
3606 copy->cpy_cont_argsc_u.c_p.cont_args = (char *) VM_MAP_COPYIN_ARGS_NULL((vm_map_copyin_args_t) 0);
3607
3608 /*
3609 * Find the beginning of the region.
3610 */
3611
3612do_map_lookup:
3613
3614 vm_map_lock(src_map)({ lock_write(&(src_map)->lock); (src_map)->timestamp
++; });
3615
3616 if (!vm_map_lookup_entry(src_map, src_start, &src_entry)) {
3617 result = KERN_INVALID_ADDRESS1;
3618 goto error;
3619 }
3620 need_map_lookup = FALSE((boolean_t) 0);
3621
3622 /*
3623 * Go through entries until we get to the end.
3624 */
3625
3626 while (TRUE((boolean_t) 1)) {
3627
3628 if (! (src_entry->protection & VM_PROT_READ((vm_prot_t) 0x01))) {
3629 result = KERN_PROTECTION_FAILURE2;
3630 goto error;
3631 }
3632
3633 if (src_end > src_entry->vme_endlinks.end)
3634 src_size = src_entry->vme_endlinks.end - src_start;
3635 else
3636 src_size = src_end - src_start;
3637
3638 src_object = src_entry->object.vm_object;
3639 src_offset = src_entry->offset +
3640 (src_start - src_entry->vme_startlinks.start);
3641
3642 /*
3643 * If src_object is NULL, allocate it now;
3644 * we're going to fault on it shortly.
3645 */
3646 if (src_object == VM_OBJECT_NULL((vm_object_t) 0)) {
3647 src_object = vm_object_allocate((vm_size_t)
3648 src_entry->vme_endlinks.end -
3649 src_entry->vme_startlinks.start);
3650 src_entry->object.vm_object = src_object;
3651 }
3652
3653 /*
3654 * Iterate over pages. Fault in ones that aren't present.
3655 */
3656 src_last_offset = src_offset + src_size;
3657 for (; (src_offset < src_last_offset && !need_map_lookup);
3658 src_offset += PAGE_SIZE(1 << 12), src_start += PAGE_SIZE(1 << 12)) {
3659
3660 if (copy->cpy_npagesc_u.c_p.npages == VM_MAP_COPY_PAGE_LIST_MAX64) {
3661make_continuation:
3662 /*
3663 * At this point we have the max number of
3664 * pages busy for this thread that we're
3665 * willing to allow. Stop here and record
3666 * arguments for the remainder. Note:
3667 * this means that this routine isn't atomic,
3668 * but that's the breaks. Note that only
3669 * the first vm_map_copy_t that comes back
3670 * from this routine has the right offset
3671 * and size; those from continuations are
3672 * page rounded, and short by the amount
3673 * already done.
3674 *
3675 * Reset src_end so the src_destroy
3676 * code at the bottom doesn't do
3677 * something stupid.
3678 */
3679
3680 cont_args = (vm_map_copyin_args_t)
3681 kalloc(sizeof(vm_map_copyin_args_data_t));
3682 cont_args->map = src_map;
3683 vm_map_reference(src_map);
3684 cont_args->src_addr = src_start;
3685 cont_args->src_len = len - (src_start - src_addr);
3686 if (src_destroy) {
3687 cont_args->destroy_addr = cont_args->src_addr;
3688 cont_args->destroy_len = cont_args->src_len;
3689 }
3690 else {
3691 cont_args->destroy_addr = (vm_offset_t) 0;
3692 cont_args->destroy_len = (vm_offset_t) 0;
3693 }
3694 cont_args->steal_pages = steal_pages;
3695
3696 copy->cpy_cont_argsc_u.c_p.cont_args = (char *) cont_args;
3697 copy->cpy_contc_u.c_p.cont = vm_map_copyin_page_list_cont;
3698
3699 src_end = src_start;
3700 vm_map_clip_end(src_map, src_entry, src_end)({ if ((src_end) < (src_entry)->links.end) _vm_map_clip_end
(&(src_map)->hdr,(src_entry),(src_end)); });
3701 break;
3702 }
3703
3704 /*
3705 * Try to find the page of data.
3706 */
3707 vm_object_lock(src_object);
3708 vm_object_paging_begin(src_object)((src_object)->paging_in_progress++);
3709 if (((m = vm_page_lookup(src_object, src_offset)) !=
3710 VM_PAGE_NULL((vm_page_t) 0)) && !m->busy && !m->fictitious &&
3711 !m->absent && !m->error) {
3712
3713 /*
3714 * This is the page. Mark it busy
3715 * and keep the paging reference on
3716 * the object whilst we do our thing.
3717 */
3718 m->busy = TRUE((boolean_t) 1);
3719
3720 /*
3721 * Also write-protect the page, so
3722 * that the map`s owner cannot change
3723 * the data. The busy bit will prevent
3724 * faults on the page from succeeding
3725 * until the copy is released; after
3726 * that, the page can be re-entered
3727 * as writable, since we didn`t alter
3728 * the map entry. This scheme is a
3729 * cheap copy-on-write.
3730 *
3731 * Don`t forget the protection and
3732 * the page_lock value!
3733 *
3734 * If the source is being destroyed
3735 * AND not shared writable, we don`t
3736 * have to protect the page, since
3737 * we will destroy the (only)
3738 * writable mapping later.
3739 */
3740 if (!src_destroy ||
3741 src_object->use_shared_copy)
3742 {
3743 pmap_page_protect(m->phys_addr,
3744 src_entry->protection
3745 & ~m->page_lock
3746 & ~VM_PROT_WRITE((vm_prot_t) 0x02));
3747 }
3748
3749 }
3750 else {
3751 vm_prot_t result_prot;
3752 vm_page_t top_page;
3753 kern_return_t kr;
3754
3755 /*
3756 * Have to fault the page in; must
3757 * unlock the map to do so. While
3758 * the map is unlocked, anything
3759 * can happen, we must lookup the
3760 * map entry before continuing.
3761 */
3762 vm_map_unlock(src_map)lock_done(&(src_map)->lock);
3763 need_map_lookup = TRUE((boolean_t) 1);
3764retry:
3765 result_prot = VM_PROT_READ((vm_prot_t) 0x01);
3766
3767 kr = vm_fault_page(src_object, src_offset,
3768 VM_PROT_READ((vm_prot_t) 0x01), FALSE((boolean_t) 0), FALSE((boolean_t) 0),
3769 &result_prot, &m, &top_page,
3770 FALSE((boolean_t) 0), (void (*)()) 0);
3771 /*
3772 * Cope with what happened.
3773 */
3774 switch (kr) {
3775 case VM_FAULT_SUCCESS0:
3776 break;
3777 case VM_FAULT_INTERRUPTED2: /* ??? */
3778 case VM_FAULT_RETRY1:
3779 vm_object_lock(src_object);
3780 vm_object_paging_begin(src_object)((src_object)->paging_in_progress++);
3781 goto retry;
3782 case VM_FAULT_MEMORY_SHORTAGE3:
3783 VM_PAGE_WAIT((void (*)()) 0)vm_page_wait((void (*)()) 0);
3784 vm_object_lock(src_object);
3785 vm_object_paging_begin(src_object)((src_object)->paging_in_progress++);
3786 goto retry;
3787 case VM_FAULT_FICTITIOUS_SHORTAGE4:
3788 vm_page_more_fictitious();
3789 vm_object_lock(src_object);
3790 vm_object_paging_begin(src_object)((src_object)->paging_in_progress++);
3791 goto retry;
3792 case VM_FAULT_MEMORY_ERROR5:
3793 /*
3794 * Something broke. If this
3795 * is a continuation, return
3796 * a partial result if possible,
3797 * else fail the whole thing.
3798 * In the continuation case, the
3799 * next continuation call will
3800 * get this error if it persists.
3801 */
3802 vm_map_lock(src_map)({ lock_write(&(src_map)->lock); (src_map)->timestamp
++; });
3803 if (is_cont &&
3804 copy->cpy_npagesc_u.c_p.npages != 0)
3805 goto make_continuation;
3806
3807 result = KERN_MEMORY_ERROR10;
3808 goto error;
3809 }
3810
3811 if (top_page != VM_PAGE_NULL((vm_page_t) 0)) {
3812 vm_object_lock(src_object);
3813 VM_PAGE_FREE(top_page)({ ; vm_page_free(top_page); ((void)(&vm_page_queue_lock)
); });
3814 vm_object_paging_end(src_object)({ ({ if (!((src_object)->paging_in_progress != 0)) Assert
("(src_object)->paging_in_progress != 0", "../vm/vm_map.c"
, 3814); }); if (--(src_object)->paging_in_progress == 0) {
({ if ((src_object)->all_wanted & (1 << (2))) thread_wakeup_prim
(((event_t)(((vm_offset_t) src_object) + (2))), ((boolean_t) 0
), 0); (src_object)->all_wanted &= ~(1 << (2)); }
); } });
3815 vm_object_unlock(src_object)((void)(&(src_object)->Lock));
3816 }
3817
3818 /*
3819 * We do not need to write-protect
3820 * the page, since it cannot have
3821 * been in the pmap (and we did not
3822 * enter it above). The busy bit
3823 * will protect the page from being
3824 * entered as writable until it is
3825 * unlocked.
3826 */
3827
3828 }
3829
3830 /*
3831 * The page is busy, its object is locked, and
3832 * we have a paging reference on it. Either
3833 * the map is locked, or need_map_lookup is
3834 * TRUE.
3835 *
3836 * Put the page in the page list.
3837 */
3838 copy->cpy_page_listc_u.c_p.page_list[copy->cpy_npagesc_u.c_p.npages++] = m;
3839 vm_object_unlock(m->object)((void)(&(m->object)->Lock));
3840 }
3841
3842 /*
3843 * DETERMINE whether the entire region
3844 * has been copied.
3845 */
3846 if (src_start >= src_end && src_end != 0) {
3847 if (need_map_lookup)
3848 vm_map_lock(src_map)({ lock_write(&(src_map)->lock); (src_map)->timestamp
++; });
3849 break;
3850 }
3851
3852 /*
3853 * If need_map_lookup is TRUE, have to start over with
3854 * another map lookup. Note that we dropped the map
3855 * lock (to call vm_fault_page) above only in this case.
3856 */
3857 if (need_map_lookup)
3858 goto do_map_lookup;
3859
3860 /*
3861 * Verify that there are no gaps in the region
3862 */
3863
3864 src_start = src_entry->vme_endlinks.end;
3865 src_entry = src_entry->vme_nextlinks.next;
3866 if (src_entry->vme_startlinks.start != src_start) {
3867 result = KERN_INVALID_ADDRESS1;
3868 goto error;
3869 }
3870 }
3871
3872 /*
3873 * If steal_pages is true, make sure all
3874 * pages in the copy are not in any object
3875 * We try to remove them from the original
3876 * object, but we may have to copy them.
3877 *
3878 * At this point every page in the list is busy
3879 * and holds a paging reference to its object.
3880 * When we're done stealing, every page is busy,
3881 * and in no object (m->tabled == FALSE).
3882 */
3883 src_start = trunc_page(src_addr)((vm_offset_t)(((vm_offset_t)(src_addr)) & ~((1 << 12
)-1)));
3884 if (steal_pages) {
3885 int i;
3886 vm_offset_t unwire_end;
3887
3888 unwire_end = src_start;
3889 for (i = 0; i < copy->cpy_npagesc_u.c_p.npages; i++) {
3890
3891 /*
3892 * Remove the page from its object if it
3893 * can be stolen. It can be stolen if:
3894 *
3895 * (1) The source is being destroyed,
3896 * the object is temporary, and
3897 * not shared.
3898 * (2) The page is not precious.
3899 *
3900 * The not shared check consists of two
3901 * parts: (a) there are no objects that
3902 * shadow this object. (b) it is not the
3903 * object in any shared map entries (i.e.,
3904 * use_shared_copy is not set).
3905 *
3906 * The first check (a) means that we can't
3907 * steal pages from objects that are not
3908 * at the top of their shadow chains. This
3909 * should not be a frequent occurrence.
3910 *
3911 * Stealing wired pages requires telling the
3912 * pmap module to let go of them.
3913 *
3914 * NOTE: stealing clean pages from objects
3915 * whose mappings survive requires a call to
3916 * the pmap module. Maybe later.
3917 */
3918 m = copy->cpy_page_listc_u.c_p.page_list[i];
3919 src_object = m->object;
3920 vm_object_lock(src_object);
3921
3922 if (src_destroy &&
3923 src_object->temporary &&
3924 (!src_object->shadowed) &&
3925 (!src_object->use_shared_copy) &&
3926 !m->precious) {
3927 vm_offset_t page_vaddr;
3928
3929 page_vaddr = src_start + (i * PAGE_SIZE(1 << 12));
3930 if (m->wire_count > 0) {
3931
3932 assert(m->wire_count == 1)({ if (!(m->wire_count == 1)) Assert("m->wire_count == 1"
, "../vm/vm_map.c", 3932); });
3933 /*
3934 * In order to steal a wired
3935 * page, we have to unwire it
3936 * first. We do this inline
3937 * here because we have the page.
3938 *
3939 * Step 1: Unwire the map entry.
3940 * Also tell the pmap module
3941 * that this piece of the
3942 * pmap is pageable.
3943 */
3944 vm_object_unlock(src_object)((void)(&(src_object)->Lock));
3945 if (page_vaddr >= unwire_end) {
3946 if (!vm_map_lookup_entry(src_map,
3947 page_vaddr, &src_entry))
3948 panic("vm_map_copyin_page_list: missing wired map entry");
3949
3950 vm_map_clip_start(src_map, src_entry,({ if ((page_vaddr) > (src_entry)->links.start) _vm_map_clip_start
(&(src_map)->hdr,(src_entry),(page_vaddr)); })
3951 page_vaddr)({ if ((page_vaddr) > (src_entry)->links.start) _vm_map_clip_start
(&(src_map)->hdr,(src_entry),(page_vaddr)); });
3952 vm_map_clip_end(src_map, src_entry,({ if ((src_start + src_size) < (src_entry)->links.end)
_vm_map_clip_end(&(src_map)->hdr,(src_entry),(src_start
+ src_size)); })
3953 src_start + src_size)({ if ((src_start + src_size) < (src_entry)->links.end)
_vm_map_clip_end(&(src_map)->hdr,(src_entry),(src_start
+ src_size)); });
3954
3955 assert(src_entry->wired_count > 0)({ if (!(src_entry->wired_count > 0)) Assert("src_entry->wired_count > 0"
, "../vm/vm_map.c", 3955); });
3956 src_entry->wired_count = 0;
3957 src_entry->user_wired_count = 0;
3958 unwire_end = src_entry->vme_endlinks.end;
3959 pmap_pageable(vm_map_pmap(src_map)((src_map)->pmap),
3960 page_vaddr, unwire_end, TRUE((boolean_t) 1));
3961 }
3962
3963 /*
3964 * Step 2: Unwire the page.
3965 * pmap_remove handles this for us.
3966 */
3967 vm_object_lock(src_object);
3968 }
3969
3970 /*
3971 * Don't need to remove the mapping;
3972 * vm_map_delete will handle it.
3973 *
3974 * Steal the page. Setting the wire count
3975 * to zero is vm_page_unwire without
3976 * activating the page.
3977 */
3978 vm_page_lock_queues();
3979 vm_page_remove(m);
3980 if (m->wire_count > 0) {
3981 m->wire_count = 0;
3982 vm_page_wire_count--;
3983 } else {
3984 VM_PAGE_QUEUES_REMOVE(m)({ if (m->active) { { queue_entry_t next, prev; next = (m)
->pageq.next; prev = (m)->pageq.prev; if ((&vm_page_queue_active
) == next) (&vm_page_queue_active)->prev = prev; else (
(vm_page_t)next)->pageq.prev = prev; if ((&vm_page_queue_active
) == prev) (&vm_page_queue_active)->next = next; else (
(vm_page_t)prev)->pageq.next = next; }; m->active = ((boolean_t
) 0); vm_page_active_count--; } if (m->inactive) { { queue_entry_t
next, prev; next = (m)->pageq.next; prev = (m)->pageq.
prev; if ((&vm_page_queue_inactive) == next) (&vm_page_queue_inactive
)->prev = prev; else ((vm_page_t)next)->pageq.prev = prev
; if ((&vm_page_queue_inactive) == prev) (&vm_page_queue_inactive
)->next = next; else ((vm_page_t)prev)->pageq.next = next
; }; m->inactive = ((boolean_t) 0); vm_page_inactive_count
--; } });
3985 }
3986 vm_page_unlock_queues()((void)(&vm_page_queue_lock));
3987 }
3988 else {
3989 /*
3990 * Have to copy this page. Have to
3991 * unlock the map while copying,
3992 * hence no further page stealing.
3993 * Hence just copy all the pages.
3994 * Unlock the map while copying;
3995 * This means no further page stealing.
3996 */
3997 vm_object_unlock(src_object)((void)(&(src_object)->Lock));
3998 vm_map_unlock(src_map)lock_done(&(src_map)->lock);
3999
4000 vm_map_copy_steal_pages(copy);
4001
4002 vm_map_lock(src_map)({ lock_write(&(src_map)->lock); (src_map)->timestamp
++; });
4003 break;
4004 }
4005
4006 vm_object_paging_end(src_object)({ ({ if (!((src_object)->paging_in_progress != 0)) Assert
("(src_object)->paging_in_progress != 0", "../vm/vm_map.c"
, 4006); }); if (--(src_object)->paging_in_progress == 0) {
({ if ((src_object)->all_wanted & (1 << (2))) thread_wakeup_prim
(((event_t)(((vm_offset_t) src_object) + (2))), ((boolean_t) 0
), 0); (src_object)->all_wanted &= ~(1 << (2)); }
); } });
4007 vm_object_unlock(src_object)((void)(&(src_object)->Lock));
4008 }
4009
4010 /*
4011 * If the source should be destroyed, do it now, since the
4012 * copy was successful.
4013 */
4014
4015 if (src_destroy) {
4016 (void) vm_map_delete(src_map, src_start, src_end);
4017 }
4018 }
4019 else {
4020 /*
4021 * !steal_pages leaves busy pages in the map.
4022 * This will cause src_destroy to hang. Use
4023 * a continuation to prevent this.
4024 */
4025 if (src_destroy && !vm_map_copy_has_cont(copy)(((copy)->c_u.c_p.cont) != (kern_return_t (*)()) 0)) {
4026 cont_args = (vm_map_copyin_args_t)
4027 kalloc(sizeof(vm_map_copyin_args_data_t));
4028 vm_map_reference(src_map);
4029 cont_args->map = src_map;
4030 cont_args->src_addr = (vm_offset_t) 0;
4031 cont_args->src_len = (vm_size_t) 0;
4032 cont_args->destroy_addr = src_start;
4033 cont_args->destroy_len = src_end - src_start;
4034 cont_args->steal_pages = FALSE((boolean_t) 0);
4035
4036 copy->cpy_cont_argsc_u.c_p.cont_args = (char *) cont_args;
4037 copy->cpy_contc_u.c_p.cont = vm_map_copyin_page_list_cont;
4038 }
4039
4040 }
4041
4042 vm_map_unlock(src_map)lock_done(&(src_map)->lock);
4043
4044 *copy_result = copy;
4045 return(result);
4046
4047error:
4048 vm_map_unlock(src_map)lock_done(&(src_map)->lock);
4049 vm_map_copy_discard(copy);
4050 return(result);
4051}
4052
4053/*
4054 * vm_map_fork:
4055 *
4056 * Create and return a new map based on the old
4057 * map, according to the inheritance values on the
4058 * regions in that map.
4059 *
4060 * The source map must not be locked.
4061 */
4062vm_map_t vm_map_fork(vm_map_t old_map)
4063{
4064 vm_map_t new_map;
4065 vm_map_entry_t old_entry;
4066 vm_map_entry_t new_entry;
4067 pmap_t new_pmap = pmap_create((vm_size_t) 0);
4068 vm_size_t new_size = 0;
4069 vm_size_t entry_size;
4070 vm_object_t object;
4071
4072 vm_map_lock(old_map)({ lock_write(&(old_map)->lock); (old_map)->timestamp
++; });
4073
4074 new_map = vm_map_create(new_pmap,
4075 old_map->min_offsethdr.links.start,
4076 old_map->max_offsethdr.links.end,
4077 old_map->hdr.entries_pageable);
4078
4079 for (
4080 old_entry = vm_map_first_entry(old_map)((old_map)->hdr.links.next);
4081 old_entry != vm_map_to_entry(old_map)((struct vm_map_entry *) &(old_map)->hdr.links);
4082 ) {
4083 if (old_entry->is_sub_map)
4084 panic("vm_map_fork: encountered a submap");
4085
4086 entry_size = (old_entry->vme_endlinks.end - old_entry->vme_startlinks.start);
4087
4088 switch (old_entry->inheritance) {
4089 case VM_INHERIT_NONE((vm_inherit_t) 2):
4090 break;
4091
4092 case VM_INHERIT_SHARE((vm_inherit_t) 0):
4093 /*
4094 * New sharing code. New map entry
4095 * references original object. Temporary
4096 * objects use asynchronous copy algorithm for
4097 * future copies. First make sure we have
4098 * the right object. If we need a shadow,
4099 * or someone else already has one, then
4100 * make a new shadow and share it.
4101 */
4102
4103 object = old_entry->object.vm_object;
4104 if (object == VM_OBJECT_NULL((vm_object_t) 0)) {
4105 object = vm_object_allocate(
4106 (vm_size_t)(old_entry->vme_endlinks.end -
4107 old_entry->vme_startlinks.start));
4108 old_entry->offset = 0;
4109 old_entry->object.vm_object = object;
4110 assert(!old_entry->needs_copy)({ if (!(!old_entry->needs_copy)) Assert("!old_entry->needs_copy"
, "../vm/vm_map.c", 4110); });
4111 }
4112 else if (old_entry->needs_copy || object->shadowed ||
4113 (object->temporary && !old_entry->is_shared &&
4114 object->size > (vm_size_t)(old_entry->vme_endlinks.end -
4115 old_entry->vme_startlinks.start))) {
4116
4117 assert(object->temporary)({ if (!(object->temporary)) Assert("object->temporary"
, "../vm/vm_map.c", 4117); });
4118 assert(!(object->shadowed && old_entry->is_shared))({ if (!(!(object->shadowed && old_entry->is_shared
))) Assert("!(object->shadowed && old_entry->is_shared)"
, "../vm/vm_map.c", 4118); });
4119 vm_object_shadow(
4120 &old_entry->object.vm_object,
4121 &old_entry->offset,
4122 (vm_size_t) (old_entry->vme_endlinks.end -
4123 old_entry->vme_startlinks.start));
4124
4125 /*
4126 * If we're making a shadow for other than
4127 * copy on write reasons, then we have
4128 * to remove write permission.
4129 */
4130
4131 if (!old_entry->needs_copy &&
4132 (old_entry->protection & VM_PROT_WRITE((vm_prot_t) 0x02))) {
4133 pmap_protect(vm_map_pmap(old_map)((old_map)->pmap),
4134 old_entry->vme_startlinks.start,
4135 old_entry->vme_endlinks.end,
4136 old_entry->protection &
4137 ~VM_PROT_WRITE((vm_prot_t) 0x02));
4138 }
4139 old_entry->needs_copy = FALSE((boolean_t) 0);
4140 object = old_entry->object.vm_object;
4141 }
4142
4143 /*
4144 * Set use_shared_copy to indicate that
4145 * object must use shared (delayed) copy-on
4146 * write. This is ignored for permanent objects.
4147 * Bump the reference count for the new entry
4148 */
4149
4150 vm_object_lock(object);
4151 object->use_shared_copy = TRUE((boolean_t) 1);
4152 object->ref_count++;
4153 vm_object_unlock(object)((void)(&(object)->Lock));
4154
4155 new_entry = vm_map_entry_create(new_map)_vm_map_entry_create(&(new_map)->hdr);
4156
4157 if (old_entry->projected_on != 0) {
4158 /*
4159 * If entry is projected buffer, clone the
4160 * entry exactly.
4161 */
4162
4163 vm_map_entry_copy_full(new_entry, old_entry)(*(new_entry) = *(old_entry));
4164
4165 } else {
4166 /*
4167 * Clone the entry, using object ref from above.
4168 * Mark both entries as shared.
4169 */
4170
4171 vm_map_entry_copy(new_entry, old_entry)({ *(new_entry) = *(old_entry); (new_entry)->is_shared = (
(boolean_t) 0); (new_entry)->needs_wakeup = ((boolean_t) 0
); (new_entry)->in_transition = ((boolean_t) 0); (new_entry
)->wired_count = 0; (new_entry)->user_wired_count = 0; }
);
4172 old_entry->is_shared = TRUE((boolean_t) 1);
4173 new_entry->is_shared = TRUE((boolean_t) 1);
4174 }
4175
4176 /*
4177 * Insert the entry into the new map -- we
4178 * know we're inserting at the end of the new
4179 * map.
4180 */
4181
4182 vm_map_entry_link(({ (&(new_map)->hdr)->nentries++; (new_entry)->links
.prev = (((new_map)->hdr.links.prev)); (new_entry)->links
.next = (((new_map)->hdr.links.prev))->links.next; (new_entry
)->links.prev->links.next = (new_entry)->links.next->
links.prev = (new_entry); ({ struct rbtree_node *___cur, *___prev
; int ___diff, ___index; ___prev = ((void *) 0); ___index = -
1; ___cur = (&(&(new_map)->hdr)->tree)->root
; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(new_entry)->tree_node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 4185); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(new_map
)->hdr)->tree, ___prev, ___index, &(new_entry)->
tree_node); }); })
4183 new_map,({ (&(new_map)->hdr)->nentries++; (new_entry)->links
.prev = (((new_map)->hdr.links.prev)); (new_entry)->links
.next = (((new_map)->hdr.links.prev))->links.next; (new_entry
)->links.prev->links.next = (new_entry)->links.next->
links.prev = (new_entry); ({ struct rbtree_node *___cur, *___prev
; int ___diff, ___index; ___prev = ((void *) 0); ___index = -
1; ___cur = (&(&(new_map)->hdr)->tree)->root
; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(new_entry)->tree_node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 4185); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(new_map
)->hdr)->tree, ___prev, ___index, &(new_entry)->
tree_node); }); })
4184 vm_map_last_entry(new_map),({ (&(new_map)->hdr)->nentries++; (new_entry)->links
.prev = (((new_map)->hdr.links.prev)); (new_entry)->links
.next = (((new_map)->hdr.links.prev))->links.next; (new_entry
)->links.prev->links.next = (new_entry)->links.next->
links.prev = (new_entry); ({ struct rbtree_node *___cur, *___prev
; int ___diff, ___index; ___prev = ((void *) 0); ___index = -
1; ___cur = (&(&(new_map)->hdr)->tree)->root
; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(new_entry)->tree_node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 4185); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(new_map
)->hdr)->tree, ___prev, ___index, &(new_entry)->
tree_node); }); })
4185 new_entry)({ (&(new_map)->hdr)->nentries++; (new_entry)->links
.prev = (((new_map)->hdr.links.prev)); (new_entry)->links
.next = (((new_map)->hdr.links.prev))->links.next; (new_entry
)->links.prev->links.next = (new_entry)->links.next->
links.prev = (new_entry); ({ struct rbtree_node *___cur, *___prev
; int ___diff, ___index; ___prev = ((void *) 0); ___index = -
1; ___cur = (&(&(new_map)->hdr)->tree)->root
; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(new_entry)->tree_node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 4185); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(new_map
)->hdr)->tree, ___prev, ___index, &(new_entry)->
tree_node); }); });
4186
4187 /*
4188 * Update the physical map
4189 */
4190
4191 pmap_copy(new_map->pmap, old_map->pmap,
4192 new_entry->vme_start,
4193 entry_size,
4194 old_entry->vme_start);
4195
4196 new_size += entry_size;
4197 break;
4198
4199 case VM_INHERIT_COPY((vm_inherit_t) 1):
4200 if (old_entry->wired_count == 0) {
4201 boolean_t src_needs_copy;
4202 boolean_t new_entry_needs_copy;
4203
4204 new_entry = vm_map_entry_create(new_map)_vm_map_entry_create(&(new_map)->hdr);
4205 vm_map_entry_copy(new_entry, old_entry)({ *(new_entry) = *(old_entry); (new_entry)->is_shared = (
(boolean_t) 0); (new_entry)->needs_wakeup = ((boolean_t) 0
); (new_entry)->in_transition = ((boolean_t) 0); (new_entry
)->wired_count = 0; (new_entry)->user_wired_count = 0; }
);
4206
4207 if (vm_object_copy_temporary(
4208 &new_entry->object.vm_object,
4209 &new_entry->offset,
4210 &src_needs_copy,
4211 &new_entry_needs_copy)) {
4212
4213 /*
4214 * Handle copy-on-write obligations
4215 */
4216
4217 if (src_needs_copy && !old_entry->needs_copy) {
4218 vm_object_pmap_protect(
4219 old_entry->object.vm_object,
4220 old_entry->offset,
4221 entry_size,
4222 (old_entry->is_shared ?
4223 PMAP_NULL((pmap_t) 0) :
4224 old_map->pmap),
4225 old_entry->vme_startlinks.start,
4226 old_entry->protection &
4227 ~VM_PROT_WRITE((vm_prot_t) 0x02));
4228
4229 old_entry->needs_copy = TRUE((boolean_t) 1);
4230 }
4231
4232 new_entry->needs_copy = new_entry_needs_copy;
4233
4234 /*
4235 * Insert the entry at the end
4236 * of the map.
4237 */
4238
4239 vm_map_entry_link(new_map,({ (&(new_map)->hdr)->nentries++; (new_entry)->links
.prev = (((new_map)->hdr.links.prev)); (new_entry)->links
.next = (((new_map)->hdr.links.prev))->links.next; (new_entry
)->links.prev->links.next = (new_entry)->links.next->
links.prev = (new_entry); ({ struct rbtree_node *___cur, *___prev
; int ___diff, ___index; ___prev = ((void *) 0); ___index = -
1; ___cur = (&(&(new_map)->hdr)->tree)->root
; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(new_entry)->tree_node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 4241); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(new_map
)->hdr)->tree, ___prev, ___index, &(new_entry)->
tree_node); }); })
4240 vm_map_last_entry(new_map),({ (&(new_map)->hdr)->nentries++; (new_entry)->links
.prev = (((new_map)->hdr.links.prev)); (new_entry)->links
.next = (((new_map)->hdr.links.prev))->links.next; (new_entry
)->links.prev->links.next = (new_entry)->links.next->
links.prev = (new_entry); ({ struct rbtree_node *___cur, *___prev
; int ___diff, ___index; ___prev = ((void *) 0); ___index = -
1; ___cur = (&(&(new_map)->hdr)->tree)->root
; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(new_entry)->tree_node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 4241); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(new_map
)->hdr)->tree, ___prev, ___index, &(new_entry)->
tree_node); }); })
4241 new_entry)({ (&(new_map)->hdr)->nentries++; (new_entry)->links
.prev = (((new_map)->hdr.links.prev)); (new_entry)->links
.next = (((new_map)->hdr.links.prev))->links.next; (new_entry
)->links.prev->links.next = (new_entry)->links.next->
links.prev = (new_entry); ({ struct rbtree_node *___cur, *___prev
; int ___diff, ___index; ___prev = ((void *) 0); ___index = -
1; ___cur = (&(&(new_map)->hdr)->tree)->root
; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert
(&(new_entry)->tree_node, ___cur); ({ if (!(___diff !=
0)) Assert("___diff != 0", "../vm/vm_map.c", 4241); }); ___prev
= ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->
children[___index]; } rbtree_insert_rebalance(&(&(new_map
)->hdr)->tree, ___prev, ___index, &(new_entry)->
tree_node); }); });
4242
4243
4244 new_size += entry_size;
4245 break;
4246 }
4247
4248 vm_map_entry_dispose(new_map, new_entry)_vm_map_entry_dispose(&(new_map)->hdr, (new_entry));
4249 }
4250
4251 /* INNER BLOCK (copy cannot be optimized) */ {
4252
4253 vm_offset_t start = old_entry->vme_startlinks.start;
4254 vm_map_copy_t copy;
4255 vm_map_entry_t last = vm_map_last_entry(new_map)((new_map)->hdr.links.prev);
4256
4257 vm_map_unlock(old_map)lock_done(&(old_map)->lock);
4258 if (vm_map_copyin(old_map,
4259 start,
4260 entry_size,
4261 FALSE((boolean_t) 0),
4262 &copy)
4263 != KERN_SUCCESS0) {
4264 vm_map_lock(old_map)({ lock_write(&(old_map)->lock); (old_map)->timestamp
++; });
4265 if (!vm_map_lookup_entry(old_map, start, &last))
4266 last = last->vme_nextlinks.next;
4267 old_entry = last;
4268 /*
4269 * For some error returns, want to
4270 * skip to the next element.
4271 */
4272
4273 continue;
4274 }
4275
4276 /*
4277 * Insert the copy into the new map
4278 */
4279
4280 vm_map_copy_insert(new_map, last, copy)({ struct rbtree_node *node, *tmp; for (node = rbtree_postwalk_deepest
(&(copy)->c_u.hdr.tree), tmp = rbtree_postwalk_unlink(
node); node != ((void *) 0); node = tmp, tmp = rbtree_postwalk_unlink
(node)) ({ struct rbtree_node *___cur, *___prev; int ___diff,
___index; ___prev = ((void *) 0); ___index = -1; ___cur = (&
(new_map)->hdr.tree)->root; while (___cur != ((void *) 0
)) { ___diff = vm_map_entry_cmp_insert(node, ___cur); ({ if (
!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c", 4280
); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur
= ___cur->children[___index]; } rbtree_insert_rebalance(&
(new_map)->hdr.tree, ___prev, ___index, node); }); (((last
)->links.next)->links.prev = ((copy)->c_u.hdr.links.
prev)) ->links.next = ((last)->links.next); ((last)->
links.next = ((copy)->c_u.hdr.links.next)) ->links.prev
= (last); (new_map)->hdr.nentries += (copy)->c_u.hdr.nentries
; kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy)
; });
4281 new_size += entry_size;
4282
4283 /*
4284 * Pick up the traversal at the end of
4285 * the copied region.
4286 */
4287
4288 vm_map_lock(old_map)({ lock_write(&(old_map)->lock); (old_map)->timestamp
++; });
4289 start += entry_size;
4290 if (!vm_map_lookup_entry(old_map, start, &last))
4291 last = last->vme_nextlinks.next;
4292 else
4293 vm_map_clip_start(old_map, last, start)({ if ((start) > (last)->links.start) _vm_map_clip_start
(&(old_map)->hdr,(last),(start)); });
4294 old_entry = last;
4295
4296 continue;
4297 /* INNER BLOCK (copy cannot be optimized) */ }
4298 }
4299 old_entry = old_entry->vme_nextlinks.next;
4300 }
4301
4302 new_map->size = new_size;
4303 vm_map_unlock(old_map)lock_done(&(old_map)->lock);
4304
4305 return(new_map);
4306}
4307
4308/*
4309 * vm_map_lookup:
4310 *
4311 * Finds the VM object, offset, and
4312 * protection for a given virtual address in the
4313 * specified map, assuming a page fault of the
4314 * type specified.
4315 *
4316 * Returns the (object, offset, protection) for
4317 * this address, whether it is wired down, and whether
4318 * this map has the only reference to the data in question.
4319 * In order to later verify this lookup, a "version"
4320 * is returned.
4321 *
4322 * The map should not be locked; it will not be
4323 * locked on exit. In order to guarantee the
4324 * existence of the returned object, it is returned
4325 * locked.
4326 *
4327 * If a lookup is requested with "write protection"
4328 * specified, the map may be changed to perform virtual
4329 * copying operations, although the data referenced will
4330 * remain the same.
4331 */
4332kern_return_t vm_map_lookup(
4333 vm_map_t *var_map, /* IN/OUT */
4334 vm_offset_t vaddr,
4335 vm_prot_t fault_type,
4336
4337 vm_map_version_t *out_version, /* OUT */
4338 vm_object_t *object, /* OUT */
4339 vm_offset_t *offset, /* OUT */
4340 vm_prot_t *out_prot, /* OUT */
4341 boolean_t *wired) /* OUT */
4342{
4343 vm_map_entry_t entry;
4344 vm_map_t map = *var_map;
4345 vm_prot_t prot;
4346
4347 RetryLookup: ;
4348
4349 /*
4350 * Lookup the faulting address.
4351 */
4352
4353 vm_map_lock_read(map)lock_read(&(map)->lock);
4354
4355#define RETURN(why) \
4356 { \
4357 vm_map_unlock_read(map)lock_done(&(map)->lock); \
4358 return(why); \
4359 }
4360
4361 /*
4362 * If the map has an interesting hint, try it before calling
4363 * full blown lookup routine.
4364 */
4365
4366 simple_lock(&map->hint_lock);
4367 entry = map->hint;
4368 simple_unlock(&map->hint_lock)((void)(&map->hint_lock));
4369
4370 if ((entry == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) ||
4371 (vaddr < entry->vme_startlinks.start) || (vaddr >= entry->vme_endlinks.end)) {
4372 vm_map_entry_t tmp_entry;
4373
4374 /*
4375 * Entry was either not a valid hint, or the vaddr
4376 * was not contained in the entry, so do a full lookup.
4377 */
4378 if (!vm_map_lookup_entry(map, vaddr, &tmp_entry))
4379 RETURN(KERN_INVALID_ADDRESS1);
4380
4381 entry = tmp_entry;
4382 }
4383
4384 /*
4385 * Handle submaps.
4386 */
4387
4388 if (entry->is_sub_map) {
4389 vm_map_t old_map = map;
4390
4391 *var_map = map = entry->object.sub_map;
4392 vm_map_unlock_read(old_map)lock_done(&(old_map)->lock);
4393 goto RetryLookup;
4394 }
4395
4396 /*
4397 * Check whether this task is allowed to have
4398 * this page.
4399 */
4400
4401 prot = entry->protection;
4402
4403 if ((fault_type & (prot)) != fault_type) {
4404 if ((prot & VM_PROT_NOTIFY((vm_prot_t) 0x10)) && (fault_type & VM_PROT_WRITE((vm_prot_t) 0x02))) {
4405 RETURN(KERN_WRITE_PROTECTION_FAILURE24);
4406 } else {
4407 RETURN(KERN_PROTECTION_FAILURE2);
4408 }
4409 }
4410
4411 /*
4412 * If this page is not pageable, we have to get
4413 * it for all possible accesses.
4414 */
4415
4416 if ((*wired = (entry->wired_count != 0)))
4417 prot = fault_type = entry->protection;
4418
4419 /*
4420 * If the entry was copy-on-write, we either ...
4421 */
4422
4423 if (entry->needs_copy) {
4424 /*
4425 * If we want to write the page, we may as well
4426 * handle that now since we've got the map locked.
4427 *
4428 * If we don't need to write the page, we just
4429 * demote the permissions allowed.
4430 */
4431
4432 if (fault_type & VM_PROT_WRITE((vm_prot_t) 0x02)) {
4433 /*
4434 * Make a new object, and place it in the
4435 * object chain. Note that no new references
4436 * have appeared -- one just moved from the
4437 * map to the new object.
4438 */
4439
4440 if (vm_map_lock_read_to_write(map)(lock_read_to_write(&(map)->lock) || (((map)->timestamp
++), 0))) {
4441 goto RetryLookup;
4442 }
4443 map->timestamp++;
4444
4445 vm_object_shadow(
4446 &entry->object.vm_object,
4447 &entry->offset,
4448 (vm_size_t) (entry->vme_endlinks.end - entry->vme_startlinks.start));
4449
4450 entry->needs_copy = FALSE((boolean_t) 0);
4451
4452 vm_map_lock_write_to_read(map)lock_write_to_read(&(map)->lock);
4453 }
4454 else {
4455 /*
4456 * We're attempting to read a copy-on-write
4457 * page -- don't allow writes.
4458 */
4459
4460 prot &= (~VM_PROT_WRITE((vm_prot_t) 0x02));
4461 }
4462 }
4463
4464 /*
4465 * Create an object if necessary.
4466 */
4467 if (entry->object.vm_object == VM_OBJECT_NULL((vm_object_t) 0)) {
4468
4469 if (vm_map_lock_read_to_write(map)(lock_read_to_write(&(map)->lock) || (((map)->timestamp
++), 0))) {
4470 goto RetryLookup;
4471 }
4472
4473 entry->object.vm_object = vm_object_allocate(
4474 (vm_size_t)(entry->vme_endlinks.end - entry->vme_startlinks.start));
4475 entry->offset = 0;
4476 vm_map_lock_write_to_read(map)lock_write_to_read(&(map)->lock);
4477 }
4478
4479 /*
4480 * Return the object/offset from this entry. If the entry
4481 * was copy-on-write or empty, it has been fixed up. Also
4482 * return the protection.
4483 */
4484
4485 *offset = (vaddr - entry->vme_startlinks.start) + entry->offset;
4486 *object = entry->object.vm_object;
4487 *out_prot = prot;
4488
4489 /*
4490 * Lock the object to prevent it from disappearing
4491 */
4492
4493 vm_object_lock(*object);
4494
4495 /*
4496 * Save the version number and unlock the map.
4497 */
4498
4499 out_version->main_timestamp = map->timestamp;
4500
4501 RETURN(KERN_SUCCESS0);
4502
4503#undef RETURN
4504}
4505
4506/*
4507 * vm_map_verify:
4508 *
4509 * Verifies that the map in question has not changed
4510 * since the given version. If successful, the map
4511 * will not change until vm_map_verify_done() is called.
4512 */
4513boolean_t vm_map_verify(
4514 vm_map_t map,
4515 vm_map_version_t *version) /* REF */
4516{
4517 boolean_t result;
4518
4519 vm_map_lock_read(map)lock_read(&(map)->lock);
4520 result = (map->timestamp == version->main_timestamp);
4521
4522 if (!result)
4523 vm_map_unlock_read(map)lock_done(&(map)->lock);
4524
4525 return(result);
4526}
4527
4528/*
4529 * vm_map_verify_done:
4530 *
4531 * Releases locks acquired by a vm_map_verify.
4532 *
4533 * This is now a macro in vm/vm_map.h. It does a
4534 * vm_map_unlock_read on the map.
4535 */
4536
4537/*
4538 * vm_region:
4539 *
4540 * User call to obtain information about a region in
4541 * a task's address map.
4542 */
4543
4544kern_return_t vm_region(
4545 vm_map_t map,
4546 vm_offset_t *address, /* IN/OUT */
4547 vm_size_t *size, /* OUT */
4548 vm_prot_t *protection, /* OUT */
4549 vm_prot_t *max_protection, /* OUT */
4550 vm_inherit_t *inheritance, /* OUT */
4551 boolean_t *is_shared, /* OUT */
4552 ipc_port_t *object_name, /* OUT */
4553 vm_offset_t *offset_in_object) /* OUT */
4554{
4555 vm_map_entry_t tmp_entry;
4556 vm_map_entry_t entry;
4557 vm_offset_t tmp_offset;
4558 vm_offset_t start;
4559
4560 if (map == VM_MAP_NULL((vm_map_t) 0))
4561 return(KERN_INVALID_ARGUMENT4);
4562
4563 start = *address;
4564
4565 vm_map_lock_read(map)lock_read(&(map)->lock);
4566 if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
4567 if ((entry = tmp_entry->vme_nextlinks.next) == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) {
4568 vm_map_unlock_read(map)lock_done(&(map)->lock);
4569 return(KERN_NO_SPACE3);
4570 }
4571 } else {
4572 entry = tmp_entry;
4573 }
4574
4575 start = entry->vme_startlinks.start;
4576 *protection = entry->protection;
4577 *max_protection = entry->max_protection;
4578 *inheritance = entry->inheritance;
4579 *address = start;
4580 *size = (entry->vme_endlinks.end - start);
4581
4582 tmp_offset = entry->offset;
4583
4584
4585 if (entry->is_sub_map) {
4586 *is_shared = FALSE((boolean_t) 0);
4587 *object_name = IP_NULL((ipc_port_t) ((ipc_object_t) 0));
4588 *offset_in_object = tmp_offset;
4589 } else {
4590 *is_shared = entry->is_shared;
4591 *object_name = vm_object_name(entry->object.vm_object);
4592 *offset_in_object = tmp_offset;
4593 }
4594
4595 vm_map_unlock_read(map)lock_done(&(map)->lock);
4596
4597 return(KERN_SUCCESS0);
4598}
4599
4600/*
4601 * Routine: vm_map_simplify
4602 *
4603 * Description:
4604 * Attempt to simplify the map representation in
4605 * the vicinity of the given starting address.
4606 * Note:
4607 * This routine is intended primarily to keep the
4608 * kernel maps more compact -- they generally don't
4609 * benefit from the "expand a map entry" technology
4610 * at allocation time because the adjacent entry
4611 * is often wired down.
4612 */
4613void vm_map_simplify(
4614 vm_map_t map,
4615 vm_offset_t start)
4616{
4617 vm_map_entry_t this_entry;
4618 vm_map_entry_t prev_entry;
4619
4620 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
4621 if (
4622 (vm_map_lookup_entry(map, start, &this_entry)) &&
4623 ((prev_entry = this_entry->vme_prevlinks.prev) != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
4624
4625 (prev_entry->vme_endlinks.end == start) &&
4626
4627 (prev_entry->is_shared == FALSE((boolean_t) 0)) &&
4628 (prev_entry->is_sub_map == FALSE((boolean_t) 0)) &&
4629
4630 (this_entry->is_shared == FALSE((boolean_t) 0)) &&
4631 (this_entry->is_sub_map == FALSE((boolean_t) 0)) &&
4632
4633 (prev_entry->inheritance == this_entry->inheritance) &&
4634 (prev_entry->protection == this_entry->protection) &&
4635 (prev_entry->max_protection == this_entry->max_protection) &&
4636 (prev_entry->wired_count == this_entry->wired_count) &&
4637 (prev_entry->user_wired_count == this_entry->user_wired_count) &&
4638
4639 (prev_entry->needs_copy == this_entry->needs_copy) &&
4640
4641 (prev_entry->object.vm_object == this_entry->object.vm_object) &&
4642 ((prev_entry->offset + (prev_entry->vme_endlinks.end - prev_entry->vme_startlinks.start))
4643 == this_entry->offset) &&
4644 (prev_entry->projected_on == 0) &&
4645 (this_entry->projected_on == 0)
4646 ) {
4647 if (map->first_free == this_entry)
4648 map->first_free = prev_entry;
4649
4650 SAVE_HINT(map, prev_entry); (map)->hint = (prev_entry); ((void)(&(map)->hint_lock
));;
4651 vm_map_entry_unlink(map, this_entry)({ (&(map)->hdr)->nentries--; (this_entry)->links
.next->links.prev = (this_entry)->links.prev; (this_entry
)->links.prev->links.next = (this_entry)->links.next
; rbtree_remove(&(&(map)->hdr)->tree, &(this_entry
)->tree_node); });
4652 prev_entry->vme_endlinks.end = this_entry->vme_endlinks.end;
4653 vm_object_deallocate(this_entry->object.vm_object);
4654 vm_map_entry_dispose(map, this_entry)_vm_map_entry_dispose(&(map)->hdr, (this_entry));
4655 }
4656 vm_map_unlock(map)lock_done(&(map)->lock);
4657}
4658
4659
4660/*
4661 * Routine: vm_map_machine_attribute
4662 * Purpose:
4663 * Provide machine-specific attributes to mappings,
4664 * such as cachability etc. for machines that provide
4665 * them. NUMA architectures and machines with big/strange
4666 * caches will use this.
4667 * Note:
4668 * Responsibilities for locking and checking are handled here,
4669 * everything else in the pmap module. If any non-volatile
4670 * information must be kept, the pmap module should handle
4671 * it itself. [This assumes that attributes do not
4672 * need to be inherited, which seems ok to me]
4673 */
4674kern_return_t vm_map_machine_attribute(
4675 vm_map_t map,
4676 vm_offset_t address,
4677 vm_size_t size,
4678 vm_machine_attribute_t attribute,
4679 vm_machine_attribute_val_t* value) /* IN/OUT */
4680{
4681 kern_return_t ret;
4682
4683 if (address < vm_map_min(map)((map)->hdr.links.start) ||
4684 (address + size) > vm_map_max(map)((map)->hdr.links.end))
4685 return KERN_INVALID_ARGUMENT4;
4686
4687 vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
4688
4689 ret = pmap_attribute(map->pmap, address, size, attribute, value)(1);
4690
4691 vm_map_unlock(map)lock_done(&(map)->lock);
4692
4693 return ret;
4694}
4695
4696
4697#if MACH_KDB1
4698
4699#define printfdb_printf kdbprintfdb_printf
4700
4701/*
4702 * vm_map_print: [ debug ]
4703 */
4704void vm_map_print(vm_map_t map)
4705{
4706 vm_map_entry_t entry;
4707
4708 iprintf("Task map 0x%X: pmap=0x%X,",
4709 (vm_offset_t) map, (vm_offset_t) (map->pmap));
4710 printfdb_printf("ref=%d,nentries=%d,", map->ref_count, map->hdr.nentries);
4711 printfdb_printf("version=%d\n", map->timestamp);
4712 indent += 2;
4713 for (entry = vm_map_first_entry(map)((map)->hdr.links.next);
4714 entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links);
4715 entry = entry->vme_nextlinks.next) {
4716 static char *inheritance_name[3] = { "share", "copy", "none"};
4717
4718 iprintf("map entry 0x%X: ", (vm_offset_t) entry);
4719 printfdb_printf("start=0x%X, end=0x%X, ",
4720 (vm_offset_t) entry->vme_startlinks.start, (vm_offset_t) entry->vme_endlinks.end);
4721 printfdb_printf("prot=%X/%X/%s, ",
4722 entry->protection,
4723 entry->max_protection,
4724 inheritance_name[entry->inheritance]);
4725 if (entry->wired_count != 0) {
4726 printfdb_printf("wired(");
4727 if (entry->user_wired_count != 0)
4728 printfdb_printf("u");
4729 if (entry->wired_count >
4730 ((entry->user_wired_count == 0) ? 0 : 1))
4731 printfdb_printf("k");
4732 printfdb_printf(") ");
4733 }
4734 if (entry->in_transition) {
4735 printfdb_printf("in transition");
4736 if (entry->needs_wakeup)
4737 printfdb_printf("(wake request)");
4738 printfdb_printf(", ");
4739 }
4740 if (entry->is_sub_map) {
4741 printfdb_printf("submap=0x%X, offset=0x%X\n",
4742 (vm_offset_t) entry->object.sub_map,
4743 (vm_offset_t) entry->offset);
4744 } else {
4745 printfdb_printf("object=0x%X, offset=0x%X",
4746 (vm_offset_t) entry->object.vm_object,
4747 (vm_offset_t) entry->offset);
4748 if (entry->is_shared)
4749 printfdb_printf(", shared");
4750 if (entry->needs_copy)
4751 printfdb_printf(", copy needed");
4752 printfdb_printf("\n");
4753
4754 if ((entry->vme_prevlinks.prev == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) ||
4755 (entry->vme_prevlinks.prev->object.vm_object != entry->object.vm_object)) {
4756 indent += 2;
4757 vm_object_print(entry->object.vm_object);
4758 indent -= 2;
4759 }
4760 }
4761 }
4762 indent -= 2;
4763}
4764
4765/*
4766 * Routine: vm_map_copy_print
4767 * Purpose:
4768 * Pretty-print a copy object for ddb.
4769 */
4770
4771void vm_map_copy_print(copy)
4772 const vm_map_copy_t copy;
4773{
4774 int i, npages;
4775
4776 printfdb_printf("copy object 0x%x\n", copy);
4777
4778 indent += 2;
4779
4780 iprintf("type=%d", copy->type);
4781 switch (copy->type) {
4782 case VM_MAP_COPY_ENTRY_LIST1:
4783 printfdb_printf("[entry_list]");
4784 break;
4785
4786 case VM_MAP_COPY_OBJECT2:
4787 printfdb_printf("[object]");
4788 break;
4789
4790 case VM_MAP_COPY_PAGE_LIST3:
4791 printfdb_printf("[page_list]");
4792 break;
4793
4794 default:
4795 printfdb_printf("[bad type]");
4796 break;
4797 }
4798 printfdb_printf(", offset=0x%x", copy->offset);
4799 printfdb_printf(", size=0x%x\n", copy->size);
4800
4801 switch (copy->type) {
4802 case VM_MAP_COPY_ENTRY_LIST1:
4803 /* XXX add stuff here */
4804 break;
4805
4806 case VM_MAP_COPY_OBJECT2:
4807 iprintf("object=0x%x\n", copy->cpy_objectc_u.c_o.object);
4808 break;
4809
4810 case VM_MAP_COPY_PAGE_LIST3:
4811 iprintf("npages=%d", copy->cpy_npagesc_u.c_p.npages);
4812 printfdb_printf(", cont=%x", copy->cpy_contc_u.c_p.cont);
4813 printfdb_printf(", cont_args=%x\n", copy->cpy_cont_argsc_u.c_p.cont_args);
4814 if (copy->cpy_npagesc_u.c_p.npages < 0) {
4815 npages = 0;
4816 } else if (copy->cpy_npagesc_u.c_p.npages > VM_MAP_COPY_PAGE_LIST_MAX64) {
4817 npages = VM_MAP_COPY_PAGE_LIST_MAX64;
4818 } else {
4819 npages = copy->cpy_npagesc_u.c_p.npages;
4820 }
4821 iprintf("copy->cpy_page_list[0..%d] = {", npages);
4822 for (i = 0; i < npages - 1; i++) {
4823 printfdb_printf("0x%x, ", copy->cpy_page_listc_u.c_p.page_list[i]);
4824 }
4825 if (npages > 0) {
4826 printfdb_printf("0x%x", copy->cpy_page_listc_u.c_p.page_list[npages - 1]);
4827 }
4828 printfdb_printf("}\n");
4829 break;
4830 }
4831
4832 indent -= 2;
4833}
4834#endif /* MACH_KDB */