../vm/vm_map.c

Bug Summary

File:	obj-scan-build/../vm/vm_map.c
Location:	line 3390, column 4
Description:	Value stored to 'src_size' is never read

Annotated Source Code

1	/*
2	* Mach Operating System
3	* Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University.
4	* Copyright (c) 1993,1994 The University of Utah and
5	* the Computer Systems Laboratory (CSL).
6	* All rights reserved.
7	*
8	* Permission to use, copy, modify and distribute this software and its
9	* documentation is hereby granted, provided that both the copyright
10	* notice and this permission notice appear in all copies of the
11	* software, derivative works or modified versions, and any portions
12	* thereof, and that both notices appear in supporting documentation.
13	*
14	* CARNEGIE MELLON, THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF
15	* THIS SOFTWARE IN ITS "AS IS" CONDITION, AND DISCLAIM ANY LIABILITY
16	* OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF
17	* THIS SOFTWARE.
18	*
19	* Carnegie Mellon requests users of this software to return to
20	*
21	* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
22	* School of Computer Science
23	* Carnegie Mellon University
24	* Pittsburgh PA 15213-3890
25	*
26	* any improvements or extensions that they make and grant Carnegie Mellon
27	* the rights to redistribute these changes.
28	*/
29	/*
30	* File: vm/vm_map.c
31	* Author: Avadis Tevanian, Jr., Michael Wayne Young
32	* Date: 1985
33	*
34	* Virtual memory mapping module.
35	*/
36
37	#include <kern/printf.h>
38	#include <mach/kern_return.h>
39	#include <mach/port.h>
40	#include <mach/vm_attributes.h>
41	#include <mach/vm_param.h>
42	#include <kern/assert.h>
43	#include <kern/debug.h>
44	#include <kern/kalloc.h>
45	#include <kern/rbtree.h>
46	#include <kern/slab.h>
47	#include <vm/pmap.h>
48	#include <vm/vm_fault.h>
49	#include <vm/vm_map.h>
50	#include <vm/vm_object.h>
51	#include <vm/vm_page.h>
52	#include <vm/vm_resident.h>
53	#include <vm/vm_kern.h>
54	#include <ipc/ipc_port.h>
55
56	#if MACH_KDB0
57	#include <ddb/db_output.h>
58	#include <vm/vm_print.h>
59	#endif /* MACH_KDB */
60
61
62	/* Forward declarations */
63	kern_return_t vm_map_delete(
64	vm_map_t map,
65	vm_offset_t start,
66	vm_offset_t end);
67
68	kern_return_t vm_map_copyout_page_list(
69	vm_map_t dst_map,
70	vm_offset_t dst_addr, / OUT */
71	vm_map_copy_t copy);
72
73	void vm_map_copy_page_discard (vm_map_copy_t copy);
74
75	/*
76	* Macros to copy a vm_map_entry. We must be careful to correctly
77	* manage the wired page count. vm_map_entry_copy() creates a new
78	* map entry to the same memory - the wired count in the new entry
79	* must be set to zero. vm_map_entry_copy_full() creates a new
80	* entry that is identical to the old entry. This preserves the
81	* wire count; it's used for map splitting and cache changing in
82	* vm_map_copyout.
83	*/
84	#define vm_map_entry_copy(NEW,OLD)({ (NEW) = (OLD); (NEW)->is_shared = ((boolean_t) 0); (NEW )->needs_wakeup = ((boolean_t) 0); (NEW)->in_transition = ((boolean_t) 0); (NEW)->wired_count = 0; (NEW)->user_wired_count = 0; }) \({
85	MACRO_BEGIN({ \
86	(NEW) = (OLD); \
87	(NEW)->is_shared = FALSE((boolean_t) 0); \
88	(NEW)->needs_wakeup = FALSE((boolean_t) 0); \
89	(NEW)->in_transition = FALSE((boolean_t) 0); \
90	(NEW)->wired_count = 0; \
91	(NEW)->user_wired_count = 0; \})
92	MACRO_END})
93
94	#define vm_map_entry_copy_full(NEW,OLD)((NEW) = (OLD)) ((NEW) = (OLD))
95
96	/*
97	* Virtual memory maps provide for the mapping, protection,
98	* and sharing of virtual memory objects. In addition,
99	* this module provides for an efficient virtual copy of
100	* memory from one map to another.
101	*
102	* Synchronization is required prior to most operations.
103	*
104	* Maps consist of an ordered doubly-linked list of simple
105	* entries; a hint and a red-black tree are used to speed up lookups.
106	*
107	* Sharing maps have been deleted from this version of Mach.
108	* All shared objects are now mapped directly into the respective
109	* maps. This requires a change in the copy on write strategy;
110	* the asymmetric (delayed) strategy is used for shared temporary
111	* objects instead of the symmetric (shadow) strategy. This is
112	* selected by the (new) use_shared_copy bit in the object. See
113	* vm_object_copy_temporary in vm_object.c for details. All maps
114	* are now "top level" maps (either task map, kernel map or submap
115	* of the kernel map).
116	*
117	* Since portions of maps are specified by start/end addreses,
118	* which may not align with existing map entries, all
119	* routines merely "clip" entries to these start/end values.
120	* [That is, an entry is split into two, bordering at a
121	* start or end value.] Note that these clippings may not
122	* always be necessary (as the two resulting entries are then
123	* not changed); however, the clipping is done for convenience.
124	* No attempt is currently made to "glue back together" two
125	* abutting entries.
126	*
127	* The symmetric (shadow) copy strategy implements virtual copy
128	* by copying VM object references from one map to
129	* another, and then marking both regions as copy-on-write.
130	* It is important to note that only one writeable reference
131	* to a VM object region exists in any map when this strategy
132	* is used -- this means that shadow object creation can be
133	* delayed until a write operation occurs. The asymmetric (delayed)
134	* strategy allows multiple maps to have writeable references to
135	* the same region of a vm object, and hence cannot delay creating
136	* its copy objects. See vm_object_copy_temporary() in vm_object.c.
137	* Copying of permanent objects is completely different; see
138	* vm_object_copy_strategically() in vm_object.c.
139	*/
140
141	struct kmem_cache vm_map_cache; /* cache for vm_map structures */
142	struct kmem_cache vm_map_entry_cache; /* cache for vm_map_entry structures */
143	struct kmem_cache vm_map_kentry_cache; /* cache for kernel entry structures */
144	struct kmem_cache vm_map_copy_cache; /* cache for vm_map_copy structures */
145
146	boolean_t vm_map_lookup_entry(); /* forward declaration */
147
148	/*
149	* Placeholder object for submap operations. This object is dropped
150	* into the range by a call to vm_map_find, and removed when
151	* vm_map_submap creates the submap.
152	*/
153
154	static struct vm_object vm_submap_object_store;
155	vm_object_t vm_submap_object = &vm_submap_object_store;
156
157	/*
158	* vm_map_init:
159	*
160	* Initialize the vm_map module. Must be called before
161	* any other vm_map routines.
162	*
163	* Map and entry structures are allocated from caches -- we must
164	* initialize those caches.
165	*
166	* There are three caches of interest:
167	*
168	* vm_map_cache: used to allocate maps.
169	* vm_map_entry_cache: used to allocate map entries.
170	* vm_map_kentry_cache: used to allocate map entries for the kernel.
171	*
172	* Kernel map entries are allocated from a special cache, using a custom
173	* page allocation function to avoid recursion. It would be difficult
174	* (perhaps impossible) for the kernel to allocate more memory to an entry
175	* cache when it became empty since the very act of allocating memory
176	* implies the creation of a new entry.
177	*/
178
179	vm_offset_t kentry_data;
180	vm_size_t kentry_data_size = KENTRY_DATA_SIZE(256*(1 << 12));
181
182	static vm_offset_t kentry_pagealloc(vm_size_t size)
183	{
184	vm_offset_t result;
185
186	if (size > kentry_data_size)
187	panic("vm_map: kentry memory exhausted");
188
189	result = kentry_data;
190	kentry_data += size;
191	kentry_data_size -= size;
192	return result;
193	}
194
195	void vm_map_init(void)
196	{
197	kmem_cache_init(&vm_map_cache, "vm_map", sizeof(struct vm_map), 0,
198	NULL((void ) 0), NULL((void ) 0), NULL((void *) 0), 0);
199	kmem_cache_init(&vm_map_entry_cache, "vm_map_entry",
200	sizeof(struct vm_map_entry), 0, NULL((void ) 0), NULL((void ) 0), NULL((void *) 0), 0);
201	kmem_cache_init(&vm_map_kentry_cache, "vm_map_kentry",
202	sizeof(struct vm_map_entry), 0, NULL((void *) 0), kentry_pagealloc,
203	NULL((void *) 0), KMEM_CACHE_NOCPUPOOL0x1 \| KMEM_CACHE_NOOFFSLAB0x2
204	\| KMEM_CACHE_NORECLAIM0x4);
205	kmem_cache_init(&vm_map_copy_cache, "vm_map_copy",
206	sizeof(struct vm_map_copy), 0, NULL((void ) 0), NULL((void ) 0), NULL((void *) 0), 0);
207
208	/*
209	* Submap object is initialized by vm_object_init.
210	*/
211	}
212
213	void vm_map_setup(map, pmap, min, max, pageable)
214	vm_map_t map;
215	pmap_t pmap;
216	vm_offset_t min, max;
217	boolean_t pageable;
218	{
219	vm_map_first_entry(map)((map)->hdr.links.next) = vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links);
220	vm_map_last_entry(map)((map)->hdr.links.prev) = vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links);
221	map->hdr.nentries = 0;
222	map->hdr.entries_pageable = pageable;
223	rbtree_init(&map->hdr.tree);
224
225	map->size = 0;
226	map->ref_count = 1;
227	map->pmap = pmap;
228	map->min_offsethdr.links.start = min;
229	map->max_offsethdr.links.end = max;
230	map->wiring_required = FALSE((boolean_t) 0);
231	map->wait_for_space = FALSE((boolean_t) 0);
232	map->first_free = vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links);
233	map->hint = vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links);
234	vm_map_lock_init(map)({ lock_init(&(map)->lock, ((boolean_t) 1)); (map)-> timestamp = 0; });
235	simple_lock_init(&map->ref_lock);
236	simple_lock_init(&map->hint_lock);
237	}
238
239	/*
240	* vm_map_create:
241	*
242	* Creates and returns a new empty VM map with
243	* the given physical map structure, and having
244	* the given lower and upper address bounds.
245	*/
246	vm_map_t vm_map_create(pmap, min, max, pageable)
247	pmap_t pmap;
248	vm_offset_t min, max;
249	boolean_t pageable;
250	{
251	vm_map_t result;
252
253	result = (vm_map_t) kmem_cache_alloc(&vm_map_cache);
254	if (result == VM_MAP_NULL((vm_map_t) 0))
255	panic("vm_map_create");
256
257	vm_map_setup(result, pmap, min, max, pageable);
258
259	return(result);
260	}
261
262	/*
263	* vm_map_entry_create: [ internal use only ]
264	*
265	* Allocates a VM map entry for insertion in the
266	* given map (or map copy). No fields are filled.
267	*/
268	#define vm_map_entry_create(map)_vm_map_entry_create(&(map)->hdr) \
269	_vm_map_entry_create(&(map)->hdr)
270
271	#define vm_map_copy_entry_create(copy)_vm_map_entry_create(&(copy)->c_u.hdr) \
272	_vm_map_entry_create(&(copy)->cpy_hdrc_u.hdr)
273
274	vm_map_entry_t _vm_map_entry_create(map_header)
275	struct vm_map_header *map_header;
276	{
277	kmem_cache_t cache;
278	vm_map_entry_t entry;
279
280	if (map_header->entries_pageable)
281	cache = &vm_map_entry_cache;
282	else
283	cache = &vm_map_kentry_cache;
284
285	entry = (vm_map_entry_t) kmem_cache_alloc(cache);
286	if (entry == VM_MAP_ENTRY_NULL((vm_map_entry_t) 0))
287	panic("vm_map_entry_create");
288
289	return(entry);
290	}
291
292	/*
293	* vm_map_entry_dispose: [ internal use only ]
294	*
295	* Inverse of vm_map_entry_create.
296	*/
297	#define vm_map_entry_dispose(map, entry)_vm_map_entry_dispose(&(map)->hdr, (entry)) \
298	_vm_map_entry_dispose(&(map)->hdr, (entry))
299
300	#define vm_map_copy_entry_dispose(map, entry)_vm_map_entry_dispose(&(copy)->c_u.hdr, (entry)) \
301	_vm_map_entry_dispose(&(copy)->cpy_hdrc_u.hdr, (entry))
302
303	void _vm_map_entry_dispose(map_header, entry)
304	struct vm_map_header *map_header;
305	vm_map_entry_t entry;
306	{
307	kmem_cache_t cache;
308
309	if (map_header->entries_pageable)
310	cache = &vm_map_entry_cache;
311	else
312	cache = &vm_map_kentry_cache;
313
314	kmem_cache_free(cache, (vm_offset_t) entry);
315	}
316
317	/*
318	* Red-black tree lookup/insert comparison functions
319	*/
320	static inline int vm_map_entry_cmp_lookup(vm_offset_t addr,
321	const struct rbtree_node *node)
322	{
323	struct vm_map_entry *entry;
324
325	entry = rbtree_entry(node, struct vm_map_entry, tree_node)((struct vm_map_entry )((char )node - __builtin_offsetof (struct vm_map_entry, tree_node)));
326
327	if (addr < entry->vme_startlinks.start)
328	return -1;
329	else if (addr < entry->vme_endlinks.end)
330	return 0;
331	else
332	return 1;
333	}
334
335	static inline int vm_map_entry_cmp_insert(const struct rbtree_node *a,
336	const struct rbtree_node *b)
337	{
338	struct vm_map_entry *entry;
339
340	entry = rbtree_entry(a, struct vm_map_entry, tree_node)((struct vm_map_entry )((char )a - __builtin_offsetof (struct vm_map_entry, tree_node)));
341	return vm_map_entry_cmp_lookup(entry->vme_startlinks.start, b);
342	}
343
344	/*
345	* vm_map_entry_{un,}link:
346	*
347	* Insert/remove entries from maps (or map copies).
348	*
349	* The start and end addresses of the entries must be properly set
350	* before using these macros.
351	*/
352	#define vm_map_entry_link(map, after_where, entry)({ (&(map)->hdr)->nentries++; (entry)->links.prev = (after_where); (entry)->links.next = (after_where)-> links.next; (entry)->links.prev->links.next = (entry)-> links.next->links.prev = (entry); ({ struct rbtree_node ___cur , ___prev; int ___diff, ___index; ___prev = ((void ) 0); ___index = -1; ___cur = (&(&(map)->hdr)->tree)->root ; while (___cur != ((void ) 0)) { ___diff = vm_map_entry_cmp_insert (&(entry)->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c", 352); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur-> children[___index]; } rbtree_insert_rebalance(&(&(map )->hdr)->tree, ___prev, ___index, &(entry)->tree_node ); }); }) \
353	_vm_map_entry_link(&(map)->hdr, after_where, entry)({ (&(map)->hdr)->nentries++; (entry)->links.prev = (after_where); (entry)->links.next = (after_where)-> links.next; (entry)->links.prev->links.next = (entry)-> links.next->links.prev = (entry); ({ struct rbtree_node ___cur , ___prev; int ___diff, ___index; ___prev = ((void ) 0); ___index = -1; ___cur = (&(&(map)->hdr)->tree)->root ; while (___cur != ((void ) 0)) { ___diff = vm_map_entry_cmp_insert (&(entry)->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c", 353); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur-> children[___index]; } rbtree_insert_rebalance(&(&(map )->hdr)->tree, ___prev, ___index, &(entry)->tree_node ); }); })
354
355	#define vm_map_copy_entry_link(copy, after_where, entry)({ (&(copy)->c_u.hdr)->nentries++; (entry)->links .prev = (after_where); (entry)->links.next = (after_where) ->links.next; (entry)->links.prev->links.next = (entry )->links.next->links.prev = (entry); ({ struct rbtree_node ___cur, ___prev; int ___diff, ___index; ___prev = ((void * ) 0); ___index = -1; ___cur = (&(&(copy)->c_u.hdr) ->tree)->root; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert(&(entry)->tree_node, ___cur ); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c" , 355); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->children[___index]; } rbtree_insert_rebalance (&(&(copy)->c_u.hdr)->tree, ___prev, ___index, & (entry)->tree_node); }); }) \
356	_vm_map_entry_link(&(copy)->cpy_hdr, after_where, entry)({ (&(copy)->c_u.hdr)->nentries++; (entry)->links .prev = (after_where); (entry)->links.next = (after_where) ->links.next; (entry)->links.prev->links.next = (entry )->links.next->links.prev = (entry); ({ struct rbtree_node ___cur, ___prev; int ___diff, ___index; ___prev = ((void * ) 0); ___index = -1; ___cur = (&(&(copy)->c_u.hdr) ->tree)->root; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert(&(entry)->tree_node, ___cur ); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c" , 356); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->children[___index]; } rbtree_insert_rebalance (&(&(copy)->c_u.hdr)->tree, ___prev, ___index, & (entry)->tree_node); }); })
357
358	#define _vm_map_entry_link(hdr, after_where, entry)({ (hdr)->nentries++; (entry)->links.prev = (after_where ); (entry)->links.next = (after_where)->links.next; (entry )->links.prev->links.next = (entry)->links.next-> links.prev = (entry); ({ struct rbtree_node ___cur, ___prev ; int ___diff, ___index; ___prev = ((void ) 0); ___index = - 1; ___cur = (&(hdr)->tree)->root; while (___cur != ( (void ) 0)) { ___diff = vm_map_entry_cmp_insert(&(entry) ->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0" , "../vm/vm_map.c", 358); }); ___prev = ___cur; ___index = rbtree_d2i (___diff); ___cur = ___cur->children[___index]; } rbtree_insert_rebalance (&(hdr)->tree, ___prev, ___index, &(entry)->tree_node ); }); }) \
359	MACRO_BEGIN({ \
360	(hdr)->nentries++; \
361	(entry)->vme_prevlinks.prev = (after_where); \
362	(entry)->vme_nextlinks.next = (after_where)->vme_nextlinks.next; \
363	(entry)->vme_prevlinks.prev->vme_nextlinks.next = \
364	(entry)->vme_nextlinks.next->vme_prevlinks.prev = (entry); \
365	rbtree_insert(&(hdr)->tree, &(entry)->tree_node, \({ struct rbtree_node ___cur, ___prev; int ___diff, ___index ; ___prev = ((void ) 0); ___index = -1; ___cur = (&(hdr) ->tree)->root; while (___cur != ((void ) 0)) { ___diff = vm_map_entry_cmp_insert(&(entry)->tree_node, ___cur ); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c" , 366); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->children[___index]; } rbtree_insert_rebalance (&(hdr)->tree, ___prev, ___index, &(entry)->tree_node ); })
366	vm_map_entry_cmp_insert)({ struct rbtree_node ___cur, ___prev; int ___diff, ___index ; ___prev = ((void ) 0); ___index = -1; ___cur = (&(hdr) ->tree)->root; while (___cur != ((void ) 0)) { ___diff = vm_map_entry_cmp_insert(&(entry)->tree_node, ___cur ); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c" , 366); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->children[___index]; } rbtree_insert_rebalance (&(hdr)->tree, ___prev, ___index, &(entry)->tree_node ); }); \
367	MACRO_END})
368
369	#define vm_map_entry_unlink(map, entry)({ (&(map)->hdr)->nentries--; (entry)->links.next ->links.prev = (entry)->links.prev; (entry)->links.prev ->links.next = (entry)->links.next; rbtree_remove(& (&(map)->hdr)->tree, &(entry)->tree_node); } ) \
370	_vm_map_entry_unlink(&(map)->hdr, entry)({ (&(map)->hdr)->nentries--; (entry)->links.next ->links.prev = (entry)->links.prev; (entry)->links.prev ->links.next = (entry)->links.next; rbtree_remove(& (&(map)->hdr)->tree, &(entry)->tree_node); } )
371
372	#define vm_map_copy_entry_unlink(copy, entry)({ (&(copy)->c_u.hdr)->nentries--; (entry)->links .next->links.prev = (entry)->links.prev; (entry)->links .prev->links.next = (entry)->links.next; rbtree_remove( &(&(copy)->c_u.hdr)->tree, &(entry)->tree_node ); }) \
373	_vm_map_entry_unlink(&(copy)->cpy_hdr, entry)({ (&(copy)->c_u.hdr)->nentries--; (entry)->links .next->links.prev = (entry)->links.prev; (entry)->links .prev->links.next = (entry)->links.next; rbtree_remove( &(&(copy)->c_u.hdr)->tree, &(entry)->tree_node ); })
374
375	#define _vm_map_entry_unlink(hdr, entry)({ (hdr)->nentries--; (entry)->links.next->links.prev = (entry)->links.prev; (entry)->links.prev->links.next = (entry)->links.next; rbtree_remove(&(hdr)->tree, &(entry)->tree_node); }) \
376	MACRO_BEGIN({ \
377	(hdr)->nentries--; \
378	(entry)->vme_nextlinks.next->vme_prevlinks.prev = (entry)->vme_prevlinks.prev; \
379	(entry)->vme_prevlinks.prev->vme_nextlinks.next = (entry)->vme_nextlinks.next; \
380	rbtree_remove(&(hdr)->tree, &(entry)->tree_node); \
381	MACRO_END})
382
383	/*
384	* vm_map_reference:
385	*
386	* Creates another valid reference to the given map.
387	*
388	*/
389	void vm_map_reference(map)
390	vm_map_t map;
391	{
392	if (map == VM_MAP_NULL((vm_map_t) 0))
393	return;
394
395	simple_lock(&map->ref_lock);
396	map->ref_count++;
397	simple_unlock(&map->ref_lock);
398	}
399
400	/*
401	* vm_map_deallocate:
402	*
403	* Removes a reference from the specified map,
404	* destroying it if no references remain.
405	* The map should not be locked.
406	*/
407	void vm_map_deallocate(map)
408	vm_map_t map;
409	{
410	int c;
411
412	if (map == VM_MAP_NULL((vm_map_t) 0))
413	return;
414
415	simple_lock(&map->ref_lock);
416	c = --map->ref_count;
417	simple_unlock(&map->ref_lock);
418
419	if (c > 0) {
420	return;
421	}
422
423	projected_buffer_collect(map);
424	(void) vm_map_delete(map, map->min_offsethdr.links.start, map->max_offsethdr.links.end);
425
426	pmap_destroy(map->pmap);
427
428	kmem_cache_free(&vm_map_cache, (vm_offset_t) map);
429	}
430
431	/*
432	* SAVE_HINT:
433	*
434	* Saves the specified entry as the hint for
435	* future lookups. Performs necessary interlocks.
436	*/
437	#define SAVE_HINT(map,value); (map)->hint = (value); ; \
438	simple_lock(&(map)->hint_lock); \
439	(map)->hint = (value); \
440	simple_unlock(&(map)->hint_lock);
441
442	/*
443	* vm_map_lookup_entry: [ internal use only ]
444	*
445	* Finds the map entry containing (or
446	* immediately preceding) the specified address
447	* in the given map; the entry is returned
448	* in the "entry" parameter. The boolean
449	* result indicates whether the address is
450	* actually contained in the map.
451	*/
452	boolean_t vm_map_lookup_entry(map, address, entry)
453	vm_map_t map;
454	vm_offset_t address;
455	vm_map_entry_t entry; / OUT */
456	{
457	struct rbtree_node *node;
458	vm_map_entry_t hint;
459
460	/*
461	* First, make a quick check to see if we are already
462	* looking at the entry we want (which is often the case).
463	*/
464
465	simple_lock(&map->hint_lock);
466	hint = map->hint;
467	simple_unlock(&map->hint_lock);
468
469	if ((hint != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) && (address >= hint->vme_startlinks.start)) {
470	if (address < hint->vme_endlinks.end) {
471	*entry = hint;
472	return(TRUE((boolean_t) 1));
473	} else {
474	vm_map_entry_t next = hint->vme_nextlinks.next;
475
476	if ((next == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links))
477	\|\| (address < next->vme_startlinks.start)) {
478	*entry = hint;
479	return(FALSE((boolean_t) 0));
480	}
481	}
482	}
483
484	/*
485	* If the hint didn't help, use the red-black tree.
486	*/
487
488	node = rbtree_lookup_nearest(&map->hdr.tree, address,({ struct rbtree_node ___cur, ___prev; int ___diff, ___index ; ___prev = ((void ) 0); ___index = -1; ___cur = (&map-> hdr.tree)->root; while (___cur != ((void ) 0)) { ___diff = vm_map_entry_cmp_lookup(address, ___cur); if (___diff == 0) break ; ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur ->children[___index]; } if (___cur == ((void *) 0)) ___cur = rbtree_nearest(___prev, ___index, 0); ___cur; })
489	vm_map_entry_cmp_lookup, RBTREE_LEFT)({ struct rbtree_node ___cur, ___prev; int ___diff, ___index ; ___prev = ((void ) 0); ___index = -1; ___cur = (&map-> hdr.tree)->root; while (___cur != ((void ) 0)) { ___diff = vm_map_entry_cmp_lookup(address, ___cur); if (___diff == 0) break ; ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur ->children[___index]; } if (___cur == ((void *) 0)) ___cur = rbtree_nearest(___prev, ___index, 0); ___cur; });
490
491	if (node == NULL((void *) 0)) {
492	entry = vm_map_to_entry(map)((struct vm_map_entry ) &(map)->hdr.links);
493	SAVE_HINT(map, entry); (map)->hint = (entry); ;;
494	return(FALSE((boolean_t) 0));
495	} else {
496	entry = rbtree_entry(node, struct vm_map_entry, tree_node)((struct vm_map_entry )((char *)node - __builtin_offsetof (struct vm_map_entry, tree_node)));
497	SAVE_HINT(map, entry); (map)->hint = (entry); ;;
498	return((address < (*entry)->vme_endlinks.end) ? TRUE((boolean_t) 1) : FALSE((boolean_t) 0));
499	}
500	}
501
502	/*
503	* Routine: invalid_user_access
504	*
505	* Verifies whether user access is valid.
506	*/
507
508	boolean_t
509	invalid_user_access(map, start, end, prot)
510	vm_map_t map;
511	vm_offset_t start, end;
512	vm_prot_t prot;
513	{
514	vm_map_entry_t entry;
515
516	return (map == VM_MAP_NULL((vm_map_t) 0) \|\| map == kernel_map \|\|
517	!vm_map_lookup_entry(map, start, &entry) \|\|
518	entry->vme_endlinks.end < end \|\|
519	(prot & ~(entry->protection)));
520	}
521
522
523	/*
524	* Routine: vm_map_find_entry
525	* Purpose:
526	* Allocate a range in the specified virtual address map,
527	* returning the entry allocated for that range.
528	* Used by kmem_alloc, etc. Returns wired entries.
529	*
530	* The map must be locked.
531	*
532	* If an entry is allocated, the object/offset fields
533	* are initialized to zero. If an object is supplied,
534	* then an existing entry may be extended.
535	*/
536	kern_return_t vm_map_find_entry(map, address, size, mask, object, o_entry)
537	vm_map_t map;
538	vm_offset_t address; / OUT */
539	vm_size_t size;
540	vm_offset_t mask;
541	vm_object_t object;
542	vm_map_entry_t o_entry; / OUT */
543	{
544	vm_map_entry_t entry, new_entry;
545	vm_offset_t start;
546	vm_offset_t end;
547
548	/*
549	* Look for the first possible address;
550	* if there's already something at this
551	* address, we have to start after it.
552	*/
553
554	if ((entry = map->first_free) == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links))
555	start = map->min_offsethdr.links.start;
556	else
557	start = entry->vme_endlinks.end;
558
559	/*
560	* In any case, the "entry" always precedes
561	* the proposed new region throughout the loop:
562	*/
563
564	while (TRUE((boolean_t) 1)) {
565	vm_map_entry_t next;
566
567	/*
568	* Find the end of the proposed new region.
569	* Be sure we didn't go beyond the end, or
570	* wrap around the address.
571	*/
572
573	if (((start + mask) & ~mask) < start) {
574	printf_once("no more room for vm_map_find_entry in %p\n", map)({ static int __once = 0; if (!__once) { printf("no more room for vm_map_find_entry in %p\n" , map); __once = 1; } });
575	return(KERN_NO_SPACE3);
576	}
577	start = ((start + mask) & ~mask);
578	end = start + size;
579
580	if ((end > map->max_offsethdr.links.end) \|\| (end < start)) {
581	printf_once("no more room for vm_map_find_entry in %p\n", map)({ static int __once = 0; if (!__once) { printf("no more room for vm_map_find_entry in %p\n" , map); __once = 1; } });
582	return(KERN_NO_SPACE3);
583	}
584
585	/*
586	* If there are no more entries, we must win.
587	*/
588
589	next = entry->vme_nextlinks.next;
590	if (next == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links))
591	break;
592
593	/*
594	* If there is another entry, it must be
595	* after the end of the potential new region.
596	*/
597
598	if (next->vme_startlinks.start >= end)
599	break;
600
601	/*
602	* Didn't fit -- move to the next entry.
603	*/
604
605	entry = next;
606	start = entry->vme_endlinks.end;
607	}
608
609	/*
610	* At this point,
611	* "start" and "end" should define the endpoints of the
612	* available new range, and
613	* "entry" should refer to the region before the new
614	* range, and
615	*
616	* the map should be locked.
617	*/
618
619	*address = start;
620
621	/*
622	* See whether we can avoid creating a new entry by
623	* extending one of our neighbors. [So far, we only attempt to
624	* extend from below.]
625	*/
626
627	if ((object != VM_OBJECT_NULL((vm_object_t) 0)) &&
628	(entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
629	(entry->vme_endlinks.end == start) &&
630	(!entry->is_shared) &&
631	(!entry->is_sub_map) &&
632	(entry->object.vm_object == object) &&
633	(entry->needs_copy == FALSE((boolean_t) 0)) &&
634	(entry->inheritance == VM_INHERIT_DEFAULT((vm_inherit_t) 1)) &&
635	(entry->protection == VM_PROT_DEFAULT(((vm_prot_t) 0x01)\|((vm_prot_t) 0x02))) &&
636	(entry->max_protection == VM_PROT_ALL(((vm_prot_t) 0x01)\|((vm_prot_t) 0x02)\|((vm_prot_t) 0x04))) &&
637	(entry->wired_count == 1) &&
638	(entry->user_wired_count == 0) &&
639	(entry->projected_on == 0)) {
640	/*
641	* Because this is a special case,
642	* we don't need to use vm_object_coalesce.
643	*/
644
645	entry->vme_endlinks.end = end;
646	new_entry = entry;
647	} else {
648	new_entry = vm_map_entry_create(map)_vm_map_entry_create(&(map)->hdr);
649
650	new_entry->vme_startlinks.start = start;
651	new_entry->vme_endlinks.end = end;
652
653	new_entry->is_shared = FALSE((boolean_t) 0);
654	new_entry->is_sub_map = FALSE((boolean_t) 0);
655	new_entry->object.vm_object = VM_OBJECT_NULL((vm_object_t) 0);
656	new_entry->offset = (vm_offset_t) 0;
657
658	new_entry->needs_copy = FALSE((boolean_t) 0);
659
660	new_entry->inheritance = VM_INHERIT_DEFAULT((vm_inherit_t) 1);
661	new_entry->protection = VM_PROT_DEFAULT(((vm_prot_t) 0x01)\|((vm_prot_t) 0x02));
662	new_entry->max_protection = VM_PROT_ALL(((vm_prot_t) 0x01)\|((vm_prot_t) 0x02)\|((vm_prot_t) 0x04));
663	new_entry->wired_count = 1;
664	new_entry->user_wired_count = 0;
665
666	new_entry->in_transition = FALSE((boolean_t) 0);
667	new_entry->needs_wakeup = FALSE((boolean_t) 0);
668	new_entry->projected_on = 0;
669
670	/*
671	* Insert the new entry into the list
672	*/
673
674	vm_map_entry_link(map, entry, new_entry)({ (&(map)->hdr)->nentries++; (new_entry)->links .prev = (entry); (new_entry)->links.next = (entry)->links .next; (new_entry)->links.prev->links.next = (new_entry )->links.next->links.prev = (new_entry); ({ struct rbtree_node ___cur, ___prev; int ___diff, ___index; ___prev = ((void * ) 0); ___index = -1; ___cur = (&(&(map)->hdr)-> tree)->root; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert (&(new_entry)->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c", 674); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur-> children[___index]; } rbtree_insert_rebalance(&(&(map )->hdr)->tree, ___prev, ___index, &(new_entry)-> tree_node); }); });
675	}
676
677	map->size += size;
678
679	/*
680	* Update the free space hint and the lookup hint
681	*/
682
683	map->first_free = new_entry;
684	SAVE_HINT(map, new_entry); (map)->hint = (new_entry); ;;
685
686	*o_entry = new_entry;
687	return(KERN_SUCCESS0);
688	}
689
690	int vm_map_pmap_enter_print = FALSE((boolean_t) 0);
691	int vm_map_pmap_enter_enable = FALSE((boolean_t) 0);
692
693	/*
694	* Routine: vm_map_pmap_enter
695	*
696	* Description:
697	* Force pages from the specified object to be entered into
698	* the pmap at the specified address if they are present.
699	* As soon as a page not found in the object the scan ends.
700	*
701	* Returns:
702	* Nothing.
703	*
704	* In/out conditions:
705	* The source map should not be locked on entry.
706	*/
707	void
708	vm_map_pmap_enter(map, addr, end_addr, object, offset, protection)
709	vm_map_t map;
710	vm_offset_t addr;
711	vm_offset_t end_addr;
712	vm_object_t object;
713	vm_offset_t offset;
714	vm_prot_t protection;
715	{
716	while (addr < end_addr) {
717	vm_page_t m;
718
719	vm_object_lock(object);
720	vm_object_paging_begin(object)((object)->paging_in_progress++);
721
722	m = vm_page_lookup(object, offset);
723	if (m == VM_PAGE_NULL((vm_page_t) 0) \|\| m->absent) {
724	vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0" , "../vm/vm_map.c", 724); }); if (--(object)->paging_in_progress == 0) { ({ if ((object)->all_wanted & (1 << (2) )) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2) )), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 << (2)); }); } });
725	vm_object_unlock(object);
726	return;
727	}
728
729	if (vm_map_pmap_enter_print) {
730	printf("vm_map_pmap_enter:");
731	printf("map: %p, addr: %lx, object: %p, offset: %lx\n",
732	map, addr, object, offset);
733	}
734
735	m->busy = TRUE((boolean_t) 1);
736	vm_object_unlock(object);
737
738	PMAP_ENTER(map->pmap, addr, m,({ pmap_enter( (map->pmap), (addr), (m)->phys_addr, (protection ) & ~(m)->page_lock, (((boolean_t) 0)) ); })
739	protection, FALSE)({ pmap_enter( (map->pmap), (addr), (m)->phys_addr, (protection ) & ~(m)->page_lock, (((boolean_t) 0)) ); });
740
741	vm_object_lock(object);
742	PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)-> wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)), ((boolean_t) 0), 0); } });
743	vm_page_lock_queues();
744	if (!m->active && !m->inactive)
745	vm_page_activate(m);
746	vm_page_unlock_queues();
747	vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0" , "../vm/vm_map.c", 747); }); if (--(object)->paging_in_progress == 0) { ({ if ((object)->all_wanted & (1 << (2) )) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2) )), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 << (2)); }); } });
748	vm_object_unlock(object);
749
750	offset += PAGE_SIZE(1 << 12);
751	addr += PAGE_SIZE(1 << 12);
752	}
753	}
754
755	/*
756	* Routine: vm_map_enter
757	*
758	* Description:
759	* Allocate a range in the specified virtual address map.
760	* The resulting range will refer to memory defined by
761	* the given memory object and offset into that object.
762	*
763	* Arguments are as defined in the vm_map call.
764	*/
765	kern_return_t vm_map_enter(
766	map,
767	address, size, mask, anywhere,
768	object, offset, needs_copy,
769	cur_protection, max_protection, inheritance)
770	vm_map_t map;
771	vm_offset_t address; / IN/OUT */
772	vm_size_t size;
773	vm_offset_t mask;
774	boolean_t anywhere;
775	vm_object_t object;
776	vm_offset_t offset;
777	boolean_t needs_copy;
778	vm_prot_t cur_protection;
779	vm_prot_t max_protection;
780	vm_inherit_t inheritance;
781	{
782	vm_map_entry_t entry;
783	vm_offset_t start;
784	vm_offset_t end;
785	kern_return_t result = KERN_SUCCESS0;
786
787	#define RETURN(value) { result = value; goto BailOut; }
788
789	if (size == 0)
790	return KERN_INVALID_ARGUMENT4;
791
792	StartAgain: ;
793
794	start = *address;
795
796	if (anywhere) {
797	vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
798
799	/*
800	* Calculate the first possible address.
801	*/
802
803	if (start < map->min_offsethdr.links.start)
804	start = map->min_offsethdr.links.start;
805	if (start > map->max_offsethdr.links.end)
806	RETURN(KERN_NO_SPACE3);
807
808	/*
809	* Look for the first possible address;
810	* if there's already something at this
811	* address, we have to start after it.
812	*/
813
814	if (start == map->min_offsethdr.links.start) {
815	if ((entry = map->first_free) != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links))
816	start = entry->vme_endlinks.end;
817	} else {
818	vm_map_entry_t tmp_entry;
819	if (vm_map_lookup_entry(map, start, &tmp_entry))
820	start = tmp_entry->vme_endlinks.end;
821	entry = tmp_entry;
822	}
823
824	/*
825	* In any case, the "entry" always precedes
826	* the proposed new region throughout the
827	* loop:
828	*/
829
830	while (TRUE((boolean_t) 1)) {
831	vm_map_entry_t next;
832
833	/*
834	* Find the end of the proposed new region.
835	* Be sure we didn't go beyond the end, or
836	* wrap around the address.
837	*/
838
839	if (((start + mask) & ~mask) < start) {
840	printf_once("no more room for vm_map_enter in %p\n", map)({ static int __once = 0; if (!__once) { printf("no more room for vm_map_enter in %p\n" , map); __once = 1; } });
841	RETURN(KERN_NO_SPACE3);
842	}
843	start = ((start + mask) & ~mask);
844	end = start + size;
845
846	if ((end > map->max_offsethdr.links.end) \|\| (end < start)) {
847	if (map->wait_for_space) {
848	if (size <= (map->max_offsethdr.links.end -
849	map->min_offsethdr.links.start)) {
850	assert_wait((event_t) map, TRUE((boolean_t) 1));
851	vm_map_unlock(map)lock_done(&(map)->lock);
852	thread_block((void (*)()) 0);
853	goto StartAgain;
854	}
855	}
856
857	printf_once("no more room for vm_map_enter in %p\n", map)({ static int __once = 0; if (!__once) { printf("no more room for vm_map_enter in %p\n" , map); __once = 1; } });
858	RETURN(KERN_NO_SPACE3);
859	}
860
861	/*
862	* If there are no more entries, we must win.
863	*/
864
865	next = entry->vme_nextlinks.next;
866	if (next == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links))
867	break;
868
869	/*
870	* If there is another entry, it must be
871	* after the end of the potential new region.
872	*/
873
874	if (next->vme_startlinks.start >= end)
875	break;
876
877	/*
878	* Didn't fit -- move to the next entry.
879	*/
880
881	entry = next;
882	start = entry->vme_endlinks.end;
883	}
884	*address = start;
885	} else {
886	vm_map_entry_t temp_entry;
887
888	/*
889	* Verify that:
890	* the address doesn't itself violate
891	* the mask requirement.
892	*/
893
894	if ((start & mask) != 0)
895	return(KERN_NO_SPACE3);
896
897	vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
898
899	/*
900	* ... the address is within bounds
901	*/
902
903	end = start + size;
904
905	if ((start < map->min_offsethdr.links.start) \|\|
906	(end > map->max_offsethdr.links.end) \|\|
907	(start >= end)) {
908	RETURN(KERN_INVALID_ADDRESS1);
909	}
910
911	/*
912	* ... the starting address isn't allocated
913	*/
914
915	if (vm_map_lookup_entry(map, start, &temp_entry))
916	RETURN(KERN_NO_SPACE3);
917
918	entry = temp_entry;
919
920	/*
921	* ... the next region doesn't overlap the
922	* end point.
923	*/
924
925	if ((entry->vme_nextlinks.next != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
926	(entry->vme_nextlinks.next->vme_startlinks.start < end))
927	RETURN(KERN_NO_SPACE3);
928	}
929
930	/*
931	* At this point,
932	* "start" and "end" should define the endpoints of the
933	* available new range, and
934	* "entry" should refer to the region before the new
935	* range, and
936	*
937	* the map should be locked.
938	*/
939
940	/*
941	* See whether we can avoid creating a new entry (and object) by
942	* extending one of our neighbors. [So far, we only attempt to
943	* extend from below.]
944	*/
945
946	if ((object == VM_OBJECT_NULL((vm_object_t) 0)) &&
947	(entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
948	(entry->vme_endlinks.end == start) &&
949	(!entry->is_shared) &&
950	(!entry->is_sub_map) &&
951	(entry->inheritance == inheritance) &&
952	(entry->protection == cur_protection) &&
953	(entry->max_protection == max_protection) &&
954	(entry->wired_count == 0) && /* implies user_wired_count == 0 */
955	(entry->projected_on == 0)) {
956	if (vm_object_coalesce(entry->object.vm_object,
957	VM_OBJECT_NULL((vm_object_t) 0),
958	entry->offset,
959	(vm_offset_t) 0,
960	(vm_size_t)(entry->vme_endlinks.end - entry->vme_startlinks.start),
961	(vm_size_t)(end - entry->vme_endlinks.end))) {
962
963	/*
964	* Coalesced the two objects - can extend
965	* the previous map entry to include the
966	* new range.
967	*/
968	map->size += (end - entry->vme_endlinks.end);
969	entry->vme_endlinks.end = end;
970	RETURN(KERN_SUCCESS0);
971	}
972	}
973
974	/*
975	* Create a new entry
976	*/
977
978	/**/ {
979	vm_map_entry_t new_entry;
980
981	new_entry = vm_map_entry_create(map)_vm_map_entry_create(&(map)->hdr);
982
983	new_entry->vme_startlinks.start = start;
984	new_entry->vme_endlinks.end = end;
985
986	new_entry->is_shared = FALSE((boolean_t) 0);
987	new_entry->is_sub_map = FALSE((boolean_t) 0);
988	new_entry->object.vm_object = object;
989	new_entry->offset = offset;
990
991	new_entry->needs_copy = needs_copy;
992
993	new_entry->inheritance = inheritance;
994	new_entry->protection = cur_protection;
995	new_entry->max_protection = max_protection;
996	new_entry->wired_count = 0;
997	new_entry->user_wired_count = 0;
998
999	new_entry->in_transition = FALSE((boolean_t) 0);
1000	new_entry->needs_wakeup = FALSE((boolean_t) 0);
1001	new_entry->projected_on = 0;
1002
1003	/*
1004	* Insert the new entry into the list
1005	*/
1006
1007	vm_map_entry_link(map, entry, new_entry)({ (&(map)->hdr)->nentries++; (new_entry)->links .prev = (entry); (new_entry)->links.next = (entry)->links .next; (new_entry)->links.prev->links.next = (new_entry )->links.next->links.prev = (new_entry); ({ struct rbtree_node ___cur, ___prev; int ___diff, ___index; ___prev = ((void * ) 0); ___index = -1; ___cur = (&(&(map)->hdr)-> tree)->root; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert (&(new_entry)->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c", 1007); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur-> children[___index]; } rbtree_insert_rebalance(&(&(map )->hdr)->tree, ___prev, ___index, &(new_entry)-> tree_node); }); });
1008	map->size += size;
1009
1010	/*
1011	* Update the free space hint and the lookup hint
1012	*/
1013
1014	if ((map->first_free == entry) &&
1015	((entry == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links) ? map->min_offsethdr.links.start : entry->vme_endlinks.end)
1016	>= new_entry->vme_startlinks.start))
1017	map->first_free = new_entry;
1018
1019	SAVE_HINT(map, new_entry); (map)->hint = (new_entry); ;;
1020
1021	vm_map_unlock(map)lock_done(&(map)->lock);
1022
1023	if ((object != VM_OBJECT_NULL((vm_object_t) 0)) &&
1024	(vm_map_pmap_enter_enable) &&
1025	(!anywhere) &&
1026	(!needs_copy) &&
1027	(size < (128*1024))) {
1028	vm_map_pmap_enter(map, start, end,
1029	object, offset, cur_protection);
1030	}
1031
1032	return(result);
1033	/**/ }
1034
1035	BailOut: ;
1036
1037	vm_map_unlock(map)lock_done(&(map)->lock);
1038	return(result);
1039
1040	#undef RETURN
1041	}
1042
1043	/*
1044	* vm_map_clip_start: [ internal use only ]
1045	*
1046	* Asserts that the given entry begins at or after
1047	* the specified address; if necessary,
1048	* it splits the entry into two.
1049	*/
1050	void _vm_map_clip_start();
1051	#define vm_map_clip_start(map, entry, startaddr)({ if ((startaddr) > (entry)->links.start) _vm_map_clip_start (&(map)->hdr,(entry),(startaddr)); }) \
1052	MACRO_BEGIN({ \
1053	if ((startaddr) > (entry)->vme_startlinks.start) \
1054	_vm_map_clip_start(&(map)->hdr,(entry),(startaddr)); \
1055	MACRO_END})
1056
1057	void _vm_map_copy_clip_start();
1058	#define vm_map_copy_clip_start(copy, entry, startaddr)({ if ((startaddr) > (entry)->links.start) _vm_map_clip_start (&(copy)->c_u.hdr,(entry),(startaddr)); }) \
1059	MACRO_BEGIN({ \
1060	if ((startaddr) > (entry)->vme_startlinks.start) \
1061	_vm_map_clip_start(&(copy)->cpy_hdrc_u.hdr,(entry),(startaddr)); \
1062	MACRO_END})
1063
1064	/*
1065	* This routine is called only when it is known that
1066	* the entry must be split.
1067	*/
1068	void _vm_map_clip_start(map_header, entry, start)
1069	struct vm_map_header *map_header;
1070	vm_map_entry_t entry;
1071	vm_offset_t start;
1072	{
1073	vm_map_entry_t new_entry;
1074
1075	/*
1076	* Split off the front portion --
1077	* note that we must insert the new
1078	* entry BEFORE this one, so that
1079	* this entry has the specified starting
1080	* address.
1081	*/
1082
1083	new_entry = _vm_map_entry_create(map_header);
1084	vm_map_entry_copy_full(new_entry, entry)((new_entry) = (entry));
1085
1086	new_entry->vme_endlinks.end = start;
1087	entry->offset += (start - entry->vme_startlinks.start);
1088	entry->vme_startlinks.start = start;
1089
1090	_vm_map_entry_link(map_header, entry->vme_prev, new_entry)({ (map_header)->nentries++; (new_entry)->links.prev = ( entry->links.prev); (new_entry)->links.next = (entry-> links.prev)->links.next; (new_entry)->links.prev->links .next = (new_entry)->links.next->links.prev = (new_entry ); ({ struct rbtree_node ___cur, ___prev; int ___diff, ___index ; ___prev = ((void ) 0); ___index = -1; ___cur = (&(map_header )->tree)->root; while (___cur != ((void ) 0)) { ___diff = vm_map_entry_cmp_insert(&(new_entry)->tree_node, ___cur ); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c" , 1090); }); ___prev = ___cur; ___index = rbtree_d2i(___diff) ; ___cur = ___cur->children[___index]; } rbtree_insert_rebalance (&(map_header)->tree, ___prev, ___index, &(new_entry )->tree_node); }); });
1091
1092	if (entry->is_sub_map)
1093	vm_map_reference(new_entry->object.sub_map);
1094	else
1095	vm_object_reference(new_entry->object.vm_object);
1096	}
1097
1098	/*
1099	* vm_map_clip_end: [ internal use only ]
1100	*
1101	* Asserts that the given entry ends at or before
1102	* the specified address; if necessary,
1103	* it splits the entry into two.
1104	*/
1105	void _vm_map_clip_end();
1106	#define vm_map_clip_end(map, entry, endaddr)({ if ((endaddr) < (entry)->links.end) _vm_map_clip_end (&(map)->hdr,(entry),(endaddr)); }) \
1107	MACRO_BEGIN({ \
1108	if ((endaddr) < (entry)->vme_endlinks.end) \
1109	_vm_map_clip_end(&(map)->hdr,(entry),(endaddr)); \
1110	MACRO_END})
1111
1112	void _vm_map_copy_clip_end();
1113	#define vm_map_copy_clip_end(copy, entry, endaddr)({ if ((endaddr) < (entry)->links.end) _vm_map_clip_end (&(copy)->c_u.hdr,(entry),(endaddr)); }) \
1114	MACRO_BEGIN({ \
1115	if ((endaddr) < (entry)->vme_endlinks.end) \
1116	_vm_map_clip_end(&(copy)->cpy_hdrc_u.hdr,(entry),(endaddr)); \
1117	MACRO_END})
1118
1119	/*
1120	* This routine is called only when it is known that
1121	* the entry must be split.
1122	*/
1123	void _vm_map_clip_end(map_header, entry, end)
1124	struct vm_map_header *map_header;
1125	vm_map_entry_t entry;
1126	vm_offset_t end;
1127	{
1128	vm_map_entry_t new_entry;
1129
1130	/*
1131	* Create a new entry and insert it
1132	* AFTER the specified entry
1133	*/
1134
1135	new_entry = _vm_map_entry_create(map_header);
1136	vm_map_entry_copy_full(new_entry, entry)((new_entry) = (entry));
1137
1138	new_entry->vme_startlinks.start = entry->vme_endlinks.end = end;
1139	new_entry->offset += (end - entry->vme_startlinks.start);
1140
1141	_vm_map_entry_link(map_header, entry, new_entry)({ (map_header)->nentries++; (new_entry)->links.prev = ( entry); (new_entry)->links.next = (entry)->links.next; ( new_entry)->links.prev->links.next = (new_entry)->links .next->links.prev = (new_entry); ({ struct rbtree_node ___cur , ___prev; int ___diff, ___index; ___prev = ((void ) 0); ___index = -1; ___cur = (&(map_header)->tree)->root; while ( ___cur != ((void ) 0)) { ___diff = vm_map_entry_cmp_insert(& (new_entry)->tree_node, ___cur); ({ if (!(___diff != 0)) Assert ("___diff != 0", "../vm/vm_map.c", 1141); }); ___prev = ___cur ; ___index = rbtree_d2i(___diff); ___cur = ___cur->children [___index]; } rbtree_insert_rebalance(&(map_header)->tree , ___prev, ___index, &(new_entry)->tree_node); }); });
1142
1143	if (entry->is_sub_map)
1144	vm_map_reference(new_entry->object.sub_map);
1145	else
1146	vm_object_reference(new_entry->object.vm_object);
1147	}
1148
1149	/*
1150	* VM_MAP_RANGE_CHECK: [ internal use only ]
1151	*
1152	* Asserts that the starting and ending region
1153	* addresses fall within the valid range of the map.
1154	*/
1155	#define VM_MAP_RANGE_CHECK(map, start, end){ if (start < ((map)->hdr.links.start)) start = ((map)-> hdr.links.start); if (end > ((map)->hdr.links.end)) end = ((map)->hdr.links.end); if (start > end) start = end ; } \
1156	{ \
1157	if (start < vm_map_min(map)((map)->hdr.links.start)) \
1158	start = vm_map_min(map)((map)->hdr.links.start); \
1159	if (end > vm_map_max(map)((map)->hdr.links.end)) \
1160	end = vm_map_max(map)((map)->hdr.links.end); \
1161	if (start > end) \
1162	start = end; \
1163	}
1164
1165	/*
1166	* vm_map_submap: [ kernel use only ]
1167	*
1168	* Mark the given range as handled by a subordinate map.
1169	*
1170	* This range must have been created with vm_map_find using
1171	* the vm_submap_object, and no other operations may have been
1172	* performed on this range prior to calling vm_map_submap.
1173	*
1174	* Only a limited number of operations can be performed
1175	* within this rage after calling vm_map_submap:
1176	* vm_fault
1177	* [Don't try vm_map_copyin!]
1178	*
1179	* To remove a submapping, one must first remove the
1180	* range from the superior map, and then destroy the
1181	* submap (if desired). [Better yet, don't try it.]
1182	*/
1183	kern_return_t vm_map_submap(map, start, end, submap)
1184	vm_map_t map;
1185	vm_offset_t start;
1186	vm_offset_t end;
1187	vm_map_t submap;
1188	{
1189	vm_map_entry_t entry;
1190	kern_return_t result = KERN_INVALID_ARGUMENT4;
1191	vm_object_t object;
1192
1193	vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
1194
1195	VM_MAP_RANGE_CHECK(map, start, end){ if (start < ((map)->hdr.links.start)) start = ((map)-> hdr.links.start); if (end > ((map)->hdr.links.end)) end = ((map)->hdr.links.end); if (start > end) start = end ; };
1196
1197	if (vm_map_lookup_entry(map, start, &entry)) {
1198	vm_map_clip_start(map, entry, start)({ if ((start) > (entry)->links.start) _vm_map_clip_start (&(map)->hdr,(entry),(start)); });
1199	}
1200	else
1201	entry = entry->vme_nextlinks.next;
1202
1203	vm_map_clip_end(map, entry, end)({ if ((end) < (entry)->links.end) _vm_map_clip_end(& (map)->hdr,(entry),(end)); });
1204
1205	if ((entry->vme_startlinks.start == start) && (entry->vme_endlinks.end == end) &&
1206	(!entry->is_sub_map) &&
1207	((object = entry->object.vm_object) == vm_submap_object) &&
1208	(object->resident_page_count == 0) &&
1209	(object->copy == VM_OBJECT_NULL((vm_object_t) 0)) &&
1210	(object->shadow == VM_OBJECT_NULL((vm_object_t) 0)) &&
1211	(!object->pager_created)) {
1212	entry->object.vm_object = VM_OBJECT_NULL((vm_object_t) 0);
1213	vm_object_deallocate(object);
1214	entry->is_sub_map = TRUE((boolean_t) 1);
1215	vm_map_reference(entry->object.sub_map = submap);
1216	result = KERN_SUCCESS0;
1217	}
1218	vm_map_unlock(map)lock_done(&(map)->lock);
1219
1220	return(result);
1221	}
1222
1223	/*
1224	* vm_map_protect:
1225	*
1226	* Sets the protection of the specified address
1227	* region in the target map. If "set_max" is
1228	* specified, the maximum protection is to be set;
1229	* otherwise, only the current protection is affected.
1230	*/
1231	kern_return_t vm_map_protect(map, start, end, new_prot, set_max)
1232	vm_map_t map;
1233	vm_offset_t start;
1234	vm_offset_t end;
1235	vm_prot_t new_prot;
1236	boolean_t set_max;
1237	{
1238	vm_map_entry_t current;
1239	vm_map_entry_t entry;
1240
1241	vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
1242
1243	VM_MAP_RANGE_CHECK(map, start, end){ if (start < ((map)->hdr.links.start)) start = ((map)-> hdr.links.start); if (end > ((map)->hdr.links.end)) end = ((map)->hdr.links.end); if (start > end) start = end ; };
1244
1245	if (vm_map_lookup_entry(map, start, &entry)) {
1246	vm_map_clip_start(map, entry, start)({ if ((start) > (entry)->links.start) _vm_map_clip_start (&(map)->hdr,(entry),(start)); });
1247	}
1248	else
1249	entry = entry->vme_nextlinks.next;
1250
1251	/*
1252	* Make a first pass to check for protection
1253	* violations.
1254	*/
1255
1256	current = entry;
1257	while ((current != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
1258	(current->vme_startlinks.start < end)) {
1259
1260	if (current->is_sub_map) {
1261	vm_map_unlock(map)lock_done(&(map)->lock);
1262	return(KERN_INVALID_ARGUMENT4);
1263	}
1264	if ((new_prot & (VM_PROT_NOTIFY((vm_prot_t) 0x10) \| current->max_protection))
1265	!= new_prot) {
1266	vm_map_unlock(map)lock_done(&(map)->lock);
1267	return(KERN_PROTECTION_FAILURE2);
1268	}
1269
1270	current = current->vme_nextlinks.next;
1271	}
1272
1273	/*
1274	* Go back and fix up protections.
1275	* [Note that clipping is not necessary the second time.]
1276	*/
1277
1278	current = entry;
1279
1280	while ((current != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
1281	(current->vme_startlinks.start < end)) {
1282
1283	vm_prot_t old_prot;
1284
1285	vm_map_clip_end(map, current, end)({ if ((end) < (current)->links.end) _vm_map_clip_end(& (map)->hdr,(current),(end)); });
1286
1287	old_prot = current->protection;
1288	if (set_max)
1289	current->protection =
1290	(current->max_protection = new_prot) &
1291	old_prot;
1292	else
1293	current->protection = new_prot;
1294
1295	/*
1296	* Update physical map if necessary.
1297	*/
1298
1299	if (current->protection != old_prot) {
1300	pmap_protect(map->pmap, current->vme_startlinks.start,
1301	current->vme_endlinks.end,
1302	current->protection);
1303	}
1304	current = current->vme_nextlinks.next;
1305	}
1306
1307	vm_map_unlock(map)lock_done(&(map)->lock);
1308	return(KERN_SUCCESS0);
1309	}
1310
1311	/*
1312	* vm_map_inherit:
1313	*
1314	* Sets the inheritance of the specified address
1315	* range in the target map. Inheritance
1316	* affects how the map will be shared with
1317	* child maps at the time of vm_map_fork.
1318	*/
1319	kern_return_t vm_map_inherit(map, start, end, new_inheritance)
1320	vm_map_t map;
1321	vm_offset_t start;
1322	vm_offset_t end;
1323	vm_inherit_t new_inheritance;
1324	{
1325	vm_map_entry_t entry;
1326	vm_map_entry_t temp_entry;
1327
1328	vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
1329
1330	VM_MAP_RANGE_CHECK(map, start, end){ if (start < ((map)->hdr.links.start)) start = ((map)-> hdr.links.start); if (end > ((map)->hdr.links.end)) end = ((map)->hdr.links.end); if (start > end) start = end ; };
1331
1332	if (vm_map_lookup_entry(map, start, &temp_entry)) {
1333	entry = temp_entry;
1334	vm_map_clip_start(map, entry, start)({ if ((start) > (entry)->links.start) _vm_map_clip_start (&(map)->hdr,(entry),(start)); });
1335	}
1336	else
1337	entry = temp_entry->vme_nextlinks.next;
1338
1339	while ((entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) && (entry->vme_startlinks.start < end)) {
1340	vm_map_clip_end(map, entry, end)({ if ((end) < (entry)->links.end) _vm_map_clip_end(& (map)->hdr,(entry),(end)); });
1341
1342	entry->inheritance = new_inheritance;
1343
1344	entry = entry->vme_nextlinks.next;
1345	}
1346
1347	vm_map_unlock(map)lock_done(&(map)->lock);
1348	return(KERN_SUCCESS0);
1349	}
1350
1351	/*
1352	* vm_map_pageable_common:
1353	*
1354	* Sets the pageability of the specified address
1355	* range in the target map. Regions specified
1356	* as not pageable require locked-down physical
1357	* memory and physical page maps. access_type indicates
1358	* types of accesses that must not generate page faults.
1359	* This is checked against protection of memory being locked-down.
1360	* access_type of VM_PROT_NONE makes memory pageable.
1361	*
1362	* The map must not be locked, but a reference
1363	* must remain to the map throughout the call.
1364	*
1365	* Callers should use macros in vm/vm_map.h (i.e. vm_map_pageable,
1366	* or vm_map_pageable_user); don't call vm_map_pageable directly.
1367	*/
1368	kern_return_t vm_map_pageable_common(map, start, end, access_type, user_wire)
1369	vm_map_t map;
1370	vm_offset_t start;
1371	vm_offset_t end;
1372	vm_prot_t access_type;
1373	boolean_t user_wire;
1374	{
1375	vm_map_entry_t entry;
1376	vm_map_entry_t start_entry;
1377
1378	vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
1379
1380	VM_MAP_RANGE_CHECK(map, start, end){ if (start < ((map)->hdr.links.start)) start = ((map)-> hdr.links.start); if (end > ((map)->hdr.links.end)) end = ((map)->hdr.links.end); if (start > end) start = end ; };
1381
1382	if (vm_map_lookup_entry(map, start, &start_entry)) {
1383	entry = start_entry;
1384	/*
1385	* vm_map_clip_start will be done later.
1386	*/
1387	}
1388	else {
1389	/*
1390	* Start address is not in map; this is fatal.
1391	*/
1392	vm_map_unlock(map)lock_done(&(map)->lock);
1393	return(KERN_FAILURE5);
1394	}
1395
1396	/*
1397	* Actions are rather different for wiring and unwiring,
1398	* so we have two separate cases.
1399	*/
1400
1401	if (access_type == VM_PROT_NONE((vm_prot_t) 0x00)) {
1402
1403	vm_map_clip_start(map, entry, start)({ if ((start) > (entry)->links.start) _vm_map_clip_start (&(map)->hdr,(entry),(start)); });
1404
1405	/*
1406	* Unwiring. First ensure that the range to be
1407	* unwired is really wired down.
1408	*/
1409	while ((entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
1410	(entry->vme_startlinks.start < end)) {
1411
1412	if ((entry->wired_count == 0) \|\|
1413	((entry->vme_endlinks.end < end) &&
1414	((entry->vme_nextlinks.next == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) \|\|
1415	(entry->vme_nextlinks.next->vme_startlinks.start > entry->vme_endlinks.end))) \|\|
1416	(user_wire && (entry->user_wired_count == 0))) {
1417	vm_map_unlock(map)lock_done(&(map)->lock);
1418	return(KERN_INVALID_ARGUMENT4);
1419	}
1420	entry = entry->vme_nextlinks.next;
1421	}
1422
1423	/*
1424	* Now decrement the wiring count for each region.
1425	* If a region becomes completely unwired,
1426	* unwire its physical pages and mappings.
1427	*/
1428	entry = start_entry;
1429	while ((entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
1430	(entry->vme_startlinks.start < end)) {
1431	vm_map_clip_end(map, entry, end)({ if ((end) < (entry)->links.end) _vm_map_clip_end(& (map)->hdr,(entry),(end)); });
1432
1433	if (user_wire) {
1434	if (--(entry->user_wired_count) == 0)
1435	entry->wired_count--;
1436	}
1437	else {
1438	entry->wired_count--;
1439	}
1440
1441	if (entry->wired_count == 0)
1442	vm_fault_unwire(map, entry);
1443
1444	entry = entry->vme_nextlinks.next;
1445	}
1446	}
1447
1448	else {
1449	/*
1450	* Wiring. We must do this in two passes:
1451	*
1452	* 1. Holding the write lock, we create any shadow
1453	* or zero-fill objects that need to be created.
1454	* Then we clip each map entry to the region to be
1455	* wired and increment its wiring count. We
1456	* create objects before clipping the map entries
1457	* to avoid object proliferation.
1458	*
1459	* 2. We downgrade to a read lock, and call
1460	* vm_fault_wire to fault in the pages for any
1461	* newly wired area (wired_count is 1).
1462	*
1463	* Downgrading to a read lock for vm_fault_wire avoids
1464	* a possible deadlock with another thread that may have
1465	* faulted on one of the pages to be wired (it would mark
1466	* the page busy, blocking us, then in turn block on the
1467	* map lock that we hold). Because of problems in the
1468	* recursive lock package, we cannot upgrade to a write
1469	* lock in vm_map_lookup. Thus, any actions that require
1470	* the write lock must be done beforehand. Because we
1471	* keep the read lock on the map, the copy-on-write
1472	* status of the entries we modify here cannot change.
1473	*/
1474
1475	/*
1476	* Pass 1.
1477	*/
1478	while ((entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
1479	(entry->vme_startlinks.start < end)) {
1480	vm_map_clip_end(map, entry, end)({ if ((end) < (entry)->links.end) _vm_map_clip_end(& (map)->hdr,(entry),(end)); });
1481
1482	if (entry->wired_count == 0) {
1483
1484	/*
1485	* Perform actions of vm_map_lookup that need
1486	* the write lock on the map: create a shadow
1487	* object for a copy-on-write region, or an
1488	* object for a zero-fill region.
1489	*/
1490	if (entry->needs_copy &&
1491	((entry->protection & VM_PROT_WRITE((vm_prot_t) 0x02)) != 0)) {
1492
1493	vm_object_shadow(&entry->object.vm_object,
1494	&entry->offset,
1495	(vm_size_t)(entry->vme_endlinks.end
1496	- entry->vme_startlinks.start));
1497	entry->needs_copy = FALSE((boolean_t) 0);
1498	}
1499	if (entry->object.vm_object == VM_OBJECT_NULL((vm_object_t) 0)) {
1500	entry->object.vm_object =
1501	vm_object_allocate(
1502	(vm_size_t)(entry->vme_endlinks.end
1503	- entry->vme_startlinks.start));
1504	entry->offset = (vm_offset_t)0;
1505	}
1506	}
1507	vm_map_clip_start(map, entry, start)({ if ((start) > (entry)->links.start) _vm_map_clip_start (&(map)->hdr,(entry),(start)); });
1508	vm_map_clip_end(map, entry, end)({ if ((end) < (entry)->links.end) _vm_map_clip_end(& (map)->hdr,(entry),(end)); });
1509
1510	if (user_wire) {
1511	if ((entry->user_wired_count)++ == 0)
1512	entry->wired_count++;
1513	}
1514	else {
1515	entry->wired_count++;
1516	}
1517
1518	/*
1519	* Check for holes and protection mismatch.
1520	* Holes: Next entry should be contiguous unless
1521	* this is the end of the region.
1522	* Protection: Access requested must be allowed.
1523	*/
1524	if (((entry->vme_endlinks.end < end) &&
1525	((entry->vme_nextlinks.next == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) \|\|
1526	(entry->vme_nextlinks.next->vme_startlinks.start > entry->vme_endlinks.end))) \|\|
1527	((entry->protection & access_type) != access_type)) {
1528	/*
1529	* Found a hole or protection problem.
1530	* Object creation actions
1531	* do not need to be undone, but the
1532	* wired counts need to be restored.
1533	*/
1534	while ((entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
1535	(entry->vme_endlinks.end > start)) {
1536	if (user_wire) {
1537	if (--(entry->user_wired_count) == 0)
1538	entry->wired_count--;
1539	}
1540	else {
1541	entry->wired_count--;
1542	}
1543
1544	entry = entry->vme_prevlinks.prev;
1545	}
1546
1547	vm_map_unlock(map)lock_done(&(map)->lock);
1548	return(KERN_FAILURE5);
1549	}
1550	entry = entry->vme_nextlinks.next;
1551	}
1552
1553	/*
1554	* Pass 2.
1555	*/
1556
1557	/*
1558	* HACK HACK HACK HACK
1559	*
1560	* If we are wiring in the kernel map or a submap of it,
1561	* unlock the map to avoid deadlocks. We trust that the
1562	* kernel threads are well-behaved, and therefore will
1563	* not do anything destructive to this region of the map
1564	* while we have it unlocked. We cannot trust user threads
1565	* to do the same.
1566	*
1567	* HACK HACK HACK HACK
1568	*/
1569	if (vm_map_pmap(map)((map)->pmap) == kernel_pmap) {
1570	vm_map_unlock(map)lock_done(&(map)->lock); /* trust me ... */
1571	}
1572	else {
1573	vm_map_lock_set_recursive(map)lock_set_recursive(&(map)->lock);
1574	vm_map_lock_write_to_read(map)lock_write_to_read(&(map)->lock);
1575	}
1576
1577	entry = start_entry;
1578	while (entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links) &&
1579	entry->vme_startlinks.start < end) {
1580	/*
1581	* Wiring cases:
1582	* Kernel: wired == 1 && user_wired == 0
1583	* User: wired == 1 && user_wired == 1
1584	*
1585	* Don't need to wire if either is > 1. wired = 0 &&
1586	* user_wired == 1 can't happen.
1587	*/
1588
1589	/*
1590	* XXX This assumes that the faults always succeed.
1591	*/
1592	if ((entry->wired_count == 1) &&
1593	(entry->user_wired_count <= 1)) {
1594	vm_fault_wire(map, entry);
1595	}
1596	entry = entry->vme_nextlinks.next;
1597	}
1598
1599	if (vm_map_pmap(map)((map)->pmap) == kernel_pmap) {
1600	vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
1601	}
1602	else {
1603	vm_map_lock_clear_recursive(map)lock_clear_recursive(&(map)->lock);
1604	}
1605	}
1606
1607	vm_map_unlock(map)lock_done(&(map)->lock);
1608
1609	return(KERN_SUCCESS0);
1610	}
1611
1612	/*
1613	* vm_map_entry_delete: [ internal use only ]
1614	*
1615	* Deallocate the given entry from the target map.
1616	*/
1617	void vm_map_entry_delete(map, entry)
1618	vm_map_t map;
1619	vm_map_entry_t entry;
1620	{
1621	vm_offset_t s, e;
1622	vm_object_t object;
1623	extern vm_object_t kernel_object;
1624
1625	s = entry->vme_startlinks.start;
1626	e = entry->vme_endlinks.end;
1627
1628	/Check if projected buffer/
1629	if (map != kernel_map && entry->projected_on != 0) {
1630	/*Check if projected kernel entry is persistent;
1631	may only manipulate directly if it is*/
1632	if (entry->projected_on->projected_on == 0)
1633	entry->wired_count = 0; /Avoid unwire fault/
1634	else
1635	return;
1636	}
1637
1638	/*
1639	* Get the object. Null objects cannot have pmap entries.
1640	*/
1641
1642	if ((object = entry->object.vm_object) != VM_OBJECT_NULL((vm_object_t) 0)) {
1643
1644	/*
1645	* Unwire before removing addresses from the pmap;
1646	* otherwise, unwiring will put the entries back in
1647	* the pmap.
1648	*/
1649
1650	if (entry->wired_count != 0) {
1651	vm_fault_unwire(map, entry);
1652	entry->wired_count = 0;
1653	entry->user_wired_count = 0;
1654	}
1655
1656	/*
1657	* If the object is shared, we must remove
1658	* all references to this data, since we can't
1659	* find all of the physical maps which are sharing
1660	* it.
1661	*/
1662
1663	if (object == kernel_object) {
1664	vm_object_lock(object);
1665	vm_object_page_remove(object, entry->offset,
1666	entry->offset + (e - s));
1667	vm_object_unlock(object);
1668	} else if (entry->is_shared) {
1669	vm_object_pmap_remove(object,
1670	entry->offset,
1671	entry->offset + (e - s));
1672	}
1673	else {
1674	pmap_remove(map->pmap, s, e);
1675	}
1676	}
1677
1678	/*
1679	* Deallocate the object only after removing all
1680	* pmap entries pointing to its pages.
1681	*/
1682
1683	if (entry->is_sub_map)
1684	vm_map_deallocate(entry->object.sub_map);
1685	else
1686	vm_object_deallocate(entry->object.vm_object);
1687
1688	vm_map_entry_unlink(map, entry)({ (&(map)->hdr)->nentries--; (entry)->links.next ->links.prev = (entry)->links.prev; (entry)->links.prev ->links.next = (entry)->links.next; rbtree_remove(& (&(map)->hdr)->tree, &(entry)->tree_node); } );
1689	map->size -= e - s;
1690
1691	vm_map_entry_dispose(map, entry)_vm_map_entry_dispose(&(map)->hdr, (entry));
1692	}
1693
1694	/*
1695	* vm_map_delete: [ internal use only ]
1696	*
1697	* Deallocates the given address range from the target
1698	* map.
1699	*/
1700
1701	kern_return_t vm_map_delete(map, start, end)
1702	vm_map_t map;
1703	vm_offset_t start;
1704	vm_offset_t end;
1705	{
1706	vm_map_entry_t entry;
1707	vm_map_entry_t first_entry;
1708
1709	/*
1710	* Find the start of the region, and clip it
1711	*/
1712
1713	if (!vm_map_lookup_entry(map, start, &first_entry))
1714	entry = first_entry->vme_nextlinks.next;
1715	else {
1716	entry = first_entry;
1717	vm_map_clip_start(map, entry, start)({ if ((start) > (entry)->links.start) _vm_map_clip_start (&(map)->hdr,(entry),(start)); });
1718
1719	/*
1720	* Fix the lookup hint now, rather than each
1721	* time though the loop.
1722	*/
1723
1724	SAVE_HINT(map, entry->vme_prev); (map)->hint = (entry->links.prev); ;;
1725	}
1726
1727	/*
1728	* Save the free space hint
1729	*/
1730
1731	if (map->first_free->vme_startlinks.start >= start)
1732	map->first_free = entry->vme_prevlinks.prev;
1733
1734	/*
1735	* Step through all entries in this region
1736	*/
1737
1738	while ((entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) && (entry->vme_startlinks.start < end)) {
1739	vm_map_entry_t next;
1740
1741	vm_map_clip_end(map, entry, end)({ if ((end) < (entry)->links.end) _vm_map_clip_end(& (map)->hdr,(entry),(end)); });
1742
1743	/*
1744	* If the entry is in transition, we must wait
1745	* for it to exit that state. It could be clipped
1746	* while we leave the map unlocked.
1747	*/
1748	if(entry->in_transition) {
1749	/*
1750	* Say that we are waiting, and wait for entry.
1751	*/
1752	entry->needs_wakeup = TRUE((boolean_t) 1);
1753	vm_map_entry_wait(map, FALSE)({ assert_wait((event_t)&(map)->hdr, ((boolean_t) 0)); lock_done(&(map)->lock); thread_block((void (*)()) 0) ; });
1754	vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
1755
1756	/*
1757	* The entry could have been clipped or it
1758	* may not exist anymore. look it up again.
1759	*/
1760	if(!vm_map_lookup_entry(map, start, &entry)) {
1761	entry = entry->vme_nextlinks.next;
1762	}
1763	continue;
1764	}
1765
1766	next = entry->vme_nextlinks.next;
1767
1768	vm_map_entry_delete(map, entry);
1769	entry = next;
1770	}
1771
1772	if (map->wait_for_space)
1773	thread_wakeup((event_t) map)thread_wakeup_prim(((event_t) map), ((boolean_t) 0), 0);
1774
1775	return(KERN_SUCCESS0);
1776	}
1777
1778	/*
1779	* vm_map_remove:
1780	*
1781	* Remove the given address range from the target map.
1782	* This is the exported form of vm_map_delete.
1783	*/
1784	kern_return_t vm_map_remove(map, start, end)
1785	vm_map_t map;
1786	vm_offset_t start;
1787	vm_offset_t end;
1788	{
1789	kern_return_t result;
1790
1791	vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
1792	VM_MAP_RANGE_CHECK(map, start, end){ if (start < ((map)->hdr.links.start)) start = ((map)-> hdr.links.start); if (end > ((map)->hdr.links.end)) end = ((map)->hdr.links.end); if (start > end) start = end ; };
1793	result = vm_map_delete(map, start, end);
1794	vm_map_unlock(map)lock_done(&(map)->lock);
1795
1796	return(result);
1797	}
1798
1799
1800	/*
1801	* vm_map_copy_steal_pages:
1802	*
1803	* Steal all the pages from a vm_map_copy page_list by copying ones
1804	* that have not already been stolen.
1805	*/
1806	void
1807	vm_map_copy_steal_pages(copy)
1808	vm_map_copy_t copy;
1809	{
1810	vm_page_t m, new_m;
1811	int i;
1812	vm_object_t object;
1813
1814	for (i = 0; i < copy->cpy_npagesc_u.c_p.npages; i++) {
1815
1816	/*
1817	* If the page is not tabled, then it's already stolen.
1818	*/
1819	m = copy->cpy_page_listc_u.c_p.page_list[i];
1820	if (!m->tabled)
1821	continue;
1822
1823	/*
1824	* Page was not stolen, get a new
1825	* one and do the copy now.
1826	*/
1827	while ((new_m = vm_page_grab(FALSE((boolean_t) 0))) == VM_PAGE_NULL((vm_page_t) 0)) {
1828	VM_PAGE_WAIT((void()()) 0)vm_page_wait((void()()) 0);
1829	}
1830
1831	vm_page_copy(m, new_m);
1832
1833	object = m->object;
1834	vm_object_lock(object);
1835	vm_page_lock_queues();
1836	if (!m->active && !m->inactive)
1837	vm_page_activate(m);
1838	vm_page_unlock_queues();
1839	PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)-> wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)), ((boolean_t) 0), 0); } });
1840	vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0" , "../vm/vm_map.c", 1840); }); if (--(object)->paging_in_progress == 0) { ({ if ((object)->all_wanted & (1 << (2) )) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2) )), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 << (2)); }); } });
1841	vm_object_unlock(object);
1842
1843	copy->cpy_page_listc_u.c_p.page_list[i] = new_m;
1844	}
1845	}
1846
1847	/*
1848	* vm_map_copy_page_discard:
1849	*
1850	* Get rid of the pages in a page_list copy. If the pages are
1851	* stolen, they are freed. If the pages are not stolen, they
1852	* are unbusied, and associated state is cleaned up.
1853	*/
1854	void vm_map_copy_page_discard(copy)
1855	vm_map_copy_t copy;
1856	{
1857	while (copy->cpy_npagesc_u.c_p.npages > 0) {
1858	vm_page_t m;
1859
1860	if((m = copy->cpy_page_listc_u.c_p.page_list[--(copy->cpy_npagesc_u.c_p.npages)]) !=
1861	VM_PAGE_NULL((vm_page_t) 0)) {
1862
1863	/*
1864	* If it's not in the table, then it's
1865	* a stolen page that goes back
1866	* to the free list. Else it belongs
1867	* to some object, and we hold a
1868	* paging reference on that object.
1869	*/
1870	if (!m->tabled) {
1871	VM_PAGE_FREE(m)({ ; vm_page_free(m); ; });
1872	}
1873	else {
1874	vm_object_t object;
1875
1876	object = m->object;
1877
1878	vm_object_lock(object);
1879	vm_page_lock_queues();
1880	if (!m->active && !m->inactive)
1881	vm_page_activate(m);
1882	vm_page_unlock_queues();
1883
1884	PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)-> wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)), ((boolean_t) 0), 0); } });
1885	vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0" , "../vm/vm_map.c", 1885); }); if (--(object)->paging_in_progress == 0) { ({ if ((object)->all_wanted & (1 << (2) )) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2) )), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 << (2)); }); } });
1886	vm_object_unlock(object);
1887	}
1888	}
1889	}
1890	}
1891
1892	/*
1893	* Routine: vm_map_copy_discard
1894	*
1895	* Description:
1896	* Dispose of a map copy object (returned by
1897	* vm_map_copyin).
1898	*/
1899	void
1900	vm_map_copy_discard(copy)
1901	vm_map_copy_t copy;
1902	{
1903	free_next_copy:
1904	if (copy == VM_MAP_COPY_NULL((vm_map_copy_t) 0))
1905	return;
1906
1907	switch (copy->type) {
1908	case VM_MAP_COPY_ENTRY_LIST1:
1909	while (vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next) !=
1910	vm_map_copy_to_entry(copy)((struct vm_map_entry *) &(copy)->c_u.hdr.links)) {
1911	vm_map_entry_t entry = vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next);
1912
1913	vm_map_copy_entry_unlink(copy, entry)({ (&(copy)->c_u.hdr)->nentries--; (entry)->links .next->links.prev = (entry)->links.prev; (entry)->links .prev->links.next = (entry)->links.next; rbtree_remove( &(&(copy)->c_u.hdr)->tree, &(entry)->tree_node ); });
1914	vm_object_deallocate(entry->object.vm_object);
1915	vm_map_copy_entry_dispose(copy, entry)_vm_map_entry_dispose(&(copy)->c_u.hdr, (entry));
1916	}
1917	break;
1918	case VM_MAP_COPY_OBJECT2:
1919	vm_object_deallocate(copy->cpy_objectc_u.c_o.object);
1920	break;
1921	case VM_MAP_COPY_PAGE_LIST3:
1922
1923	/*
1924	* To clean this up, we have to unbusy all the pages
1925	* and release the paging references in their objects.
1926	*/
1927	if (copy->cpy_npagesc_u.c_p.npages > 0)
1928	vm_map_copy_page_discard(copy);
1929
1930	/*
1931	* If there's a continuation, abort it. The
1932	* abort routine releases any storage.
1933	*/
1934	if (vm_map_copy_has_cont(copy)(((copy)->c_u.c_p.cont) != (kern_return_t (*)()) 0)) {
1935
1936	/*
1937	* Special case: recognize
1938	* vm_map_copy_discard_cont and optimize
1939	* here to avoid tail recursion.
1940	*/
1941	if (copy->cpy_contc_u.c_p.cont == vm_map_copy_discard_cont) {
1942	vm_map_copy_t new_copy;
1943
1944	new_copy = (vm_map_copy_t) copy->cpy_cont_argsc_u.c_p.cont_args;
1945	kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1946	copy = new_copy;
1947	goto free_next_copy;
1948	}
1949	else {
1950	vm_map_copy_abort_cont(copy)({ vm_map_copy_page_discard(copy); (((copy)->c_u.c_p.cont ))((copy)->c_u.c_p.cont_args, (vm_map_copy_t ) 0); (copy) ->c_u.c_p.cont = (kern_return_t ()()) 0; (copy)->c_u.c_p .cont_args = (char ) 0; });
1951	}
1952	}
1953
1954	break;
1955	}
1956	kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
1957	}
1958
1959	/*
1960	* Routine: vm_map_copy_copy
1961	*
1962	* Description:
1963	* Move the information in a map copy object to
1964	* a new map copy object, leaving the old one
1965	* empty.
1966	*
1967	* This is used by kernel routines that need
1968	* to look at out-of-line data (in copyin form)
1969	* before deciding whether to return SUCCESS.
1970	* If the routine returns FAILURE, the original
1971	* copy object will be deallocated; therefore,
1972	* these routines must make a copy of the copy
1973	* object and leave the original empty so that
1974	* deallocation will not fail.
1975	*/
1976	vm_map_copy_t
1977	vm_map_copy_copy(copy)
1978	vm_map_copy_t copy;
1979	{
1980	vm_map_copy_t new_copy;
1981
1982	if (copy == VM_MAP_COPY_NULL((vm_map_copy_t) 0))
1983	return VM_MAP_COPY_NULL((vm_map_copy_t) 0);
1984
1985	/*
1986	* Allocate a new copy object, and copy the information
1987	* from the old one into it.
1988	*/
1989
1990	new_copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
1991	new_copy = copy;
1992
1993	if (copy->type == VM_MAP_COPY_ENTRY_LIST1) {
1994	/*
1995	* The links in the entry chain must be
1996	* changed to point to the new copy object.
1997	*/
1998	vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next)->vme_prevlinks.prev
1999	= vm_map_copy_to_entry(new_copy)((struct vm_map_entry *) &(new_copy)->c_u.hdr.links);
2000	vm_map_copy_last_entry(copy)((copy)->c_u.hdr.links.prev)->vme_nextlinks.next
2001	= vm_map_copy_to_entry(new_copy)((struct vm_map_entry *) &(new_copy)->c_u.hdr.links);
2002	}
2003
2004	/*
2005	* Change the old copy object into one that contains
2006	* nothing to be deallocated.
2007	*/
2008	copy->type = VM_MAP_COPY_OBJECT2;
2009	copy->cpy_objectc_u.c_o.object = VM_OBJECT_NULL((vm_object_t) 0);
2010
2011	/*
2012	* Return the new object.
2013	*/
2014	return new_copy;
2015	}
2016
2017	/*
2018	* Routine: vm_map_copy_discard_cont
2019	*
2020	* Description:
2021	* A version of vm_map_copy_discard that can be called
2022	* as a continuation from a vm_map_copy page list.
2023	*/
2024	kern_return_t vm_map_copy_discard_cont(cont_args, copy_result)
2025	vm_map_copyin_args_t cont_args;
2026	vm_map_copy_t copy_result; / OUT */
2027	{
2028	vm_map_copy_discard((vm_map_copy_t) cont_args);
2029	if (copy_result != (vm_map_copy_t *)0)
2030	*copy_result = VM_MAP_COPY_NULL((vm_map_copy_t) 0);
2031	return(KERN_SUCCESS0);
2032	}
2033
2034	/*
2035	* Routine: vm_map_copy_overwrite
2036	*
2037	* Description:
2038	* Copy the memory described by the map copy
2039	* object (copy; returned by vm_map_copyin) onto
2040	* the specified destination region (dst_map, dst_addr).
2041	* The destination must be writeable.
2042	*
2043	* Unlike vm_map_copyout, this routine actually
2044	* writes over previously-mapped memory. If the
2045	* previous mapping was to a permanent (user-supplied)
2046	* memory object, it is preserved.
2047	*
2048	* The attributes (protection and inheritance) of the
2049	* destination region are preserved.
2050	*
2051	* If successful, consumes the copy object.
2052	* Otherwise, the caller is responsible for it.
2053	*
2054	* Implementation notes:
2055	* To overwrite temporary virtual memory, it is
2056	* sufficient to remove the previous mapping and insert
2057	* the new copy. This replacement is done either on
2058	* the whole region (if no permanent virtual memory
2059	* objects are embedded in the destination region) or
2060	* in individual map entries.
2061	*
2062	* To overwrite permanent virtual memory, it is
2063	* necessary to copy each page, as the external
2064	* memory management interface currently does not
2065	* provide any optimizations.
2066	*
2067	* Once a page of permanent memory has been overwritten,
2068	* it is impossible to interrupt this function; otherwise,
2069	* the call would be neither atomic nor location-independent.
2070	* The kernel-state portion of a user thread must be
2071	* interruptible.
2072	*
2073	* It may be expensive to forward all requests that might
2074	* overwrite permanent memory (vm_write, vm_copy) to
2075	* uninterruptible kernel threads. This routine may be
2076	* called by interruptible threads; however, success is
2077	* not guaranteed -- if the request cannot be performed
2078	* atomically and interruptibly, an error indication is
2079	* returned.
2080	*/
2081	kern_return_t vm_map_copy_overwrite(dst_map, dst_addr, copy, interruptible)
2082	vm_map_t dst_map;
2083	vm_offset_t dst_addr;
2084	vm_map_copy_t copy;
2085	boolean_t interruptible;
2086	{
2087	vm_size_t size;
2088	vm_offset_t start;
2089	vm_map_entry_t tmp_entry;
2090	vm_map_entry_t entry;
2091
2092	boolean_t contains_permanent_objects = FALSE((boolean_t) 0);
2093
2094	interruptible = FALSE((boolean_t) 0); /* XXX */
2095
2096	/*
2097	* Check for null copy object.
2098	*/
2099
2100	if (copy == VM_MAP_COPY_NULL((vm_map_copy_t) 0))
2101	return(KERN_SUCCESS0);
2102
2103	/*
2104	* Only works for entry lists at the moment. Will
2105	* support page lists LATER.
2106	*/
2107
2108	assert(copy->type == VM_MAP_COPY_ENTRY_LIST)({ if (!(copy->type == 1)) Assert("copy->type == VM_MAP_COPY_ENTRY_LIST" , "../vm/vm_map.c", 2108); });
2109
2110	/*
2111	* Currently this routine only handles page-aligned
2112	* regions. Eventually, it should handle misalignments
2113	* by actually copying pages.
2114	*/
2115
2116	if (!page_aligned(copy->offset)((((vm_offset_t) (copy->offset)) & ((1 << 12)-1) ) == 0) \|\|
2117	!page_aligned(copy->size)((((vm_offset_t) (copy->size)) & ((1 << 12)-1)) == 0) \|\|
2118	!page_aligned(dst_addr)((((vm_offset_t) (dst_addr)) & ((1 << 12)-1)) == 0))
2119	return(KERN_INVALID_ARGUMENT4);
2120
2121	size = copy->size;
2122
2123	if (size == 0) {
2124	vm_map_copy_discard(copy);
2125	return(KERN_SUCCESS0);
2126	}
2127
2128	/*
2129	* Verify that the destination is all writeable
2130	* initially.
2131	*/
2132	start_pass_1:
2133	vm_map_lock(dst_map)({ lock_write(&(dst_map)->lock); (dst_map)->timestamp ++; });
2134	if (!vm_map_lookup_entry(dst_map, dst_addr, &tmp_entry)) {
2135	vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2136	return(KERN_INVALID_ADDRESS1);
2137	}
2138	vm_map_clip_start(dst_map, tmp_entry, dst_addr)({ if ((dst_addr) > (tmp_entry)->links.start) _vm_map_clip_start (&(dst_map)->hdr,(tmp_entry),(dst_addr)); });
2139	for (entry = tmp_entry;;) {
2140	vm_size_t sub_size = (entry->vme_endlinks.end - entry->vme_startlinks.start);
2141	vm_map_entry_t next = entry->vme_nextlinks.next;
2142
2143	if ( ! (entry->protection & VM_PROT_WRITE((vm_prot_t) 0x02))) {
2144	vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2145	return(KERN_PROTECTION_FAILURE2);
2146	}
2147
2148	/*
2149	* If the entry is in transition, we must wait
2150	* for it to exit that state. Anything could happen
2151	* when we unlock the map, so start over.
2152	*/
2153	if (entry->in_transition) {
2154
2155	/*
2156	* Say that we are waiting, and wait for entry.
2157	*/
2158	entry->needs_wakeup = TRUE((boolean_t) 1);
2159	vm_map_entry_wait(dst_map, FALSE)({ assert_wait((event_t)&(dst_map)->hdr, ((boolean_t) 0 )); lock_done(&(dst_map)->lock); thread_block((void (* )()) 0); });
2160
2161	goto start_pass_1;
2162	}
2163
2164	if (size <= sub_size)
2165	break;
2166
2167	if ((next == vm_map_to_entry(dst_map)((struct vm_map_entry *) &(dst_map)->hdr.links)) \|\|
2168	(next->vme_startlinks.start != entry->vme_endlinks.end)) {
2169	vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2170	return(KERN_INVALID_ADDRESS1);
2171	}
2172
2173
2174	/*
2175	* Check for permanent objects in the destination.
2176	*/
2177
2178	if ((entry->object.vm_object != VM_OBJECT_NULL((vm_object_t) 0)) &&
2179	!entry->object.vm_object->temporary)
2180	contains_permanent_objects = TRUE((boolean_t) 1);
2181
2182	size -= sub_size;
2183	entry = next;
2184	}
2185
2186	/*
2187	* If there are permanent objects in the destination, then
2188	* the copy cannot be interrupted.
2189	*/
2190
2191	if (interruptible && contains_permanent_objects) {
2192	vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2193	return(KERN_FAILURE5); /* XXX */
2194	}
2195
2196	/*
2197	* XXXO If there are no permanent objects in the destination,
2198	* XXXO and the source and destination map entry caches match,
2199	* XXXO and the destination map entry is not shared,
2200	* XXXO then the map entries can be deleted and replaced
2201	* XXXO with those from the copy. The following code is the
2202	* XXXO basic idea of what to do, but there are lots of annoying
2203	* XXXO little details about getting protection and inheritance
2204	* XXXO right. Should add protection, inheritance, and sharing checks
2205	* XXXO to the above pass and make sure that no wiring is involved.
2206	*/
2207	/*
2208	* if (!contains_permanent_objects &&
2209	* copy->cpy_hdr.entries_pageable == dst_map->hdr.entries_pageable) {
2210	*
2211	* *
2212	* * Run over copy and adjust entries. Steal code
2213	* * from vm_map_copyout() to do this.
2214	* *
2215	*
2216	* tmp_entry = tmp_entry->vme_prev;
2217	* vm_map_delete(dst_map, dst_addr, dst_addr + copy->size);
2218	* vm_map_copy_insert(dst_map, tmp_entry, copy);
2219	*
2220	* vm_map_unlock(dst_map);
2221	* vm_map_copy_discard(copy);
2222	* }
2223	*/
2224	/*
2225	*
2226	* Make a second pass, overwriting the data
2227	* At the beginning of each loop iteration,
2228	* the next entry to be overwritten is "tmp_entry"
2229	* (initially, the value returned from the lookup above),
2230	* and the starting address expected in that entry
2231	* is "start".
2232	*/
2233
2234	start = dst_addr;
2235
2236	while (vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next) != vm_map_copy_to_entry(copy)((struct vm_map_entry *) &(copy)->c_u.hdr.links)) {
2237	vm_map_entry_t copy_entry = vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next);
2238	vm_size_t copy_size = (copy_entry->vme_endlinks.end - copy_entry->vme_startlinks.start);
2239	vm_object_t object;
2240
2241	entry = tmp_entry;
2242	size = (entry->vme_endlinks.end - entry->vme_startlinks.start);
2243	/*
2244	* Make sure that no holes popped up in the
2245	* address map, and that the protection is
2246	* still valid, in case the map was unlocked
2247	* earlier.
2248	*/
2249
2250	if (entry->vme_startlinks.start != start) {
2251	vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2252	return(KERN_INVALID_ADDRESS1);
2253	}
2254	assert(entry != vm_map_to_entry(dst_map))({ if (!(entry != ((struct vm_map_entry *) &(dst_map)-> hdr.links))) Assert("entry != vm_map_to_entry(dst_map)", "../vm/vm_map.c" , 2254); });
2255
2256	/*
2257	* Check protection again
2258	*/
2259
2260	if ( ! (entry->protection & VM_PROT_WRITE((vm_prot_t) 0x02))) {
2261	vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2262	return(KERN_PROTECTION_FAILURE2);
2263	}
2264
2265	/*
2266	* Adjust to source size first
2267	*/
2268
2269	if (copy_size < size) {
2270	vm_map_clip_end(dst_map, entry, entry->vme_start + copy_size)({ if ((entry->links.start + copy_size) < (entry)->links .end) _vm_map_clip_end(&(dst_map)->hdr,(entry),(entry-> links.start + copy_size)); });
2271	size = copy_size;
2272	}
2273
2274	/*
2275	* Adjust to destination size
2276	*/
2277
2278	if (size < copy_size) {
2279	vm_map_copy_clip_end(copy, copy_entry,({ if ((copy_entry->links.start + size) < (copy_entry)-> links.end) _vm_map_clip_end(&(copy)->c_u.hdr,(copy_entry ),(copy_entry->links.start + size)); })
2280	copy_entry->vme_start + size)({ if ((copy_entry->links.start + size) < (copy_entry)-> links.end) _vm_map_clip_end(&(copy)->c_u.hdr,(copy_entry ),(copy_entry->links.start + size)); });
2281	copy_size = size;
2282	}
2283
2284	assert((entry->vme_end - entry->vme_start) == size)({ if (!((entry->links.end - entry->links.start) == size )) Assert("(entry->vme_end - entry->vme_start) == size" , "../vm/vm_map.c", 2284); });
2285	assert((tmp_entry->vme_end - tmp_entry->vme_start) == size)({ if (!((tmp_entry->links.end - tmp_entry->links.start ) == size)) Assert("(tmp_entry->vme_end - tmp_entry->vme_start) == size" , "../vm/vm_map.c", 2285); });
2286	assert((copy_entry->vme_end - copy_entry->vme_start) == size)({ if (!((copy_entry->links.end - copy_entry->links.start ) == size)) Assert("(copy_entry->vme_end - copy_entry->vme_start) == size" , "../vm/vm_map.c", 2286); });
2287
2288	/*
2289	* If the destination contains temporary unshared memory,
2290	* we can perform the copy by throwing it away and
2291	* installing the source data.
2292	*/
2293
2294	object = entry->object.vm_object;
2295	if (!entry->is_shared &&
2296	((object == VM_OBJECT_NULL((vm_object_t) 0)) \|\| object->temporary)) {
2297	vm_object_t old_object = entry->object.vm_object;
2298	vm_offset_t old_offset = entry->offset;
2299
2300	entry->object = copy_entry->object;
2301	entry->offset = copy_entry->offset;
2302	entry->needs_copy = copy_entry->needs_copy;
2303	entry->wired_count = 0;
2304	entry->user_wired_count = 0;
2305
2306	vm_map_copy_entry_unlink(copy, copy_entry)({ (&(copy)->c_u.hdr)->nentries--; (copy_entry)-> links.next->links.prev = (copy_entry)->links.prev; (copy_entry )->links.prev->links.next = (copy_entry)->links.next ; rbtree_remove(&(&(copy)->c_u.hdr)->tree, & (copy_entry)->tree_node); });
2307	vm_map_copy_entry_dispose(copy, copy_entry)_vm_map_entry_dispose(&(copy)->c_u.hdr, (copy_entry));
2308
2309	vm_object_pmap_protect(
2310	old_object,
2311	old_offset,
2312	size,
2313	dst_map->pmap,
2314	tmp_entry->vme_startlinks.start,
2315	VM_PROT_NONE((vm_prot_t) 0x00));
2316
2317	vm_object_deallocate(old_object);
2318
2319	/*
2320	* Set up for the next iteration. The map
2321	* has not been unlocked, so the next
2322	* address should be at the end of this
2323	* entry, and the next map entry should be
2324	* the one following it.
2325	*/
2326
2327	start = tmp_entry->vme_endlinks.end;
2328	tmp_entry = tmp_entry->vme_nextlinks.next;
2329	} else {
2330	vm_map_version_t version;
2331	vm_object_t dst_object = entry->object.vm_object;
2332	vm_offset_t dst_offset = entry->offset;
2333	kern_return_t r;
2334
2335	/*
2336	* Take an object reference, and record
2337	* the map version information so that the
2338	* map can be safely unlocked.
2339	*/
2340
2341	vm_object_reference(dst_object);
2342
2343	version.main_timestamp = dst_map->timestamp;
2344
2345	vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2346
2347	/*
2348	* Copy as much as possible in one pass
2349	*/
2350
2351	copy_size = size;
2352	r = vm_fault_copy(
2353	copy_entry->object.vm_object,
2354	copy_entry->offset,
2355	&copy_size,
2356	dst_object,
2357	dst_offset,
2358	dst_map,
2359	&version,
2360	FALSE((boolean_t) 0) /* XXX interruptible */ );
2361
2362	/*
2363	* Release the object reference
2364	*/
2365
2366	vm_object_deallocate(dst_object);
2367
2368	/*
2369	* If a hard error occurred, return it now
2370	*/
2371
2372	if (r != KERN_SUCCESS0)
2373	return(r);
2374
2375	if (copy_size != 0) {
2376	/*
2377	* Dispose of the copied region
2378	*/
2379
2380	vm_map_copy_clip_end(copy, copy_entry,({ if ((copy_entry->links.start + copy_size) < (copy_entry )->links.end) _vm_map_clip_end(&(copy)->c_u.hdr,(copy_entry ),(copy_entry->links.start + copy_size)); })
2381	copy_entry->vme_start + copy_size)({ if ((copy_entry->links.start + copy_size) < (copy_entry )->links.end) _vm_map_clip_end(&(copy)->c_u.hdr,(copy_entry ),(copy_entry->links.start + copy_size)); });
2382	vm_map_copy_entry_unlink(copy, copy_entry)({ (&(copy)->c_u.hdr)->nentries--; (copy_entry)-> links.next->links.prev = (copy_entry)->links.prev; (copy_entry )->links.prev->links.next = (copy_entry)->links.next ; rbtree_remove(&(&(copy)->c_u.hdr)->tree, & (copy_entry)->tree_node); });
2383	vm_object_deallocate(copy_entry->object.vm_object);
2384	vm_map_copy_entry_dispose(copy, copy_entry)_vm_map_entry_dispose(&(copy)->c_u.hdr, (copy_entry));
2385	}
2386
2387	/*
2388	* Pick up in the destination map where we left off.
2389	*
2390	* Use the version information to avoid a lookup
2391	* in the normal case.
2392	*/
2393
2394	start += copy_size;
2395	vm_map_lock(dst_map)({ lock_write(&(dst_map)->lock); (dst_map)->timestamp ++; });
2396	if ((version.main_timestamp + 1) == dst_map->timestamp) {
2397	/* We can safely use saved tmp_entry value */
2398
2399	vm_map_clip_end(dst_map, tmp_entry, start)({ if ((start) < (tmp_entry)->links.end) _vm_map_clip_end (&(dst_map)->hdr,(tmp_entry),(start)); });
2400	tmp_entry = tmp_entry->vme_nextlinks.next;
2401	} else {
2402	/* Must do lookup of tmp_entry */
2403
2404	if (!vm_map_lookup_entry(dst_map, start, &tmp_entry)) {
2405	vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2406	return(KERN_INVALID_ADDRESS1);
2407	}
2408	vm_map_clip_start(dst_map, tmp_entry, start)({ if ((start) > (tmp_entry)->links.start) _vm_map_clip_start (&(dst_map)->hdr,(tmp_entry),(start)); });
2409	}
2410	}
2411
2412	}
2413	vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2414
2415	/*
2416	* Throw away the vm_map_copy object
2417	*/
2418	vm_map_copy_discard(copy);
2419
2420	return(KERN_SUCCESS0);
2421	}
2422
2423	/*
2424	* Macro: vm_map_copy_insert
2425	*
2426	* Description:
2427	* Link a copy chain ("copy") into a map at the
2428	* specified location (after "where").
2429	* Side effects:
2430	* The copy chain is destroyed.
2431	* Warning:
2432	* The arguments are evaluated multiple times.
2433	*/
2434	#define vm_map_copy_insert(map, where, copy)({ struct rbtree_node node, tmp; for (node = rbtree_postwalk_deepest (&(copy)->c_u.hdr.tree), tmp = rbtree_postwalk_unlink( node); node != ((void ) 0); node = tmp, tmp = rbtree_postwalk_unlink (node)) ({ struct rbtree_node ___cur, ___prev; int ___diff, ___index; ___prev = ((void ) 0); ___index = -1; ___cur = (& (map)->hdr.tree)->root; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert(node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c", 2434); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur-> children[___index]; } rbtree_insert_rebalance(&(map)-> hdr.tree, ___prev, ___index, node); }); (((where)->links.next )->links.prev = ((copy)->c_u.hdr.links.prev)) ->links .next = ((where)->links.next); ((where)->links.next = ( (copy)->c_u.hdr.links.next)) ->links.prev = (where); (map )->hdr.nentries += (copy)->c_u.hdr.nentries; kmem_cache_free (&vm_map_copy_cache, (vm_offset_t) copy); }) \
2435	MACRO_BEGIN({ \
2436	struct rbtree_node node, tmp; \
2437	rbtree_for_each_remove(&(copy)->cpy_hdr.tree, node, tmp)for (node = rbtree_postwalk_deepest(&(copy)->c_u.hdr.tree ), tmp = rbtree_postwalk_unlink(node); node != ((void *) 0); node = tmp, tmp = rbtree_postwalk_unlink(node)) \
2438	rbtree_insert(&(map)->hdr.tree, node, \({ struct rbtree_node ___cur, ___prev; int ___diff, ___index ; ___prev = ((void ) 0); ___index = -1; ___cur = (&(map) ->hdr.tree)->root; while (___cur != ((void ) 0)) { ___diff = vm_map_entry_cmp_insert(node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c", 2439); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur-> children[___index]; } rbtree_insert_rebalance(&(map)-> hdr.tree, ___prev, ___index, node); })
2439	vm_map_entry_cmp_insert)({ struct rbtree_node ___cur, ___prev; int ___diff, ___index ; ___prev = ((void ) 0); ___index = -1; ___cur = (&(map) ->hdr.tree)->root; while (___cur != ((void ) 0)) { ___diff = vm_map_entry_cmp_insert(node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c", 2439); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur-> children[___index]; } rbtree_insert_rebalance(&(map)-> hdr.tree, ___prev, ___index, node); }); \
2440	(((where)->vme_nextlinks.next)->vme_prevlinks.prev = vm_map_copy_last_entry(copy)((copy)->c_u.hdr.links.prev)) \
2441	->vme_nextlinks.next = ((where)->vme_nextlinks.next); \
2442	((where)->vme_nextlinks.next = vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next)) \
2443	->vme_prevlinks.prev = (where); \
2444	(map)->hdr.nentries += (copy)->cpy_hdrc_u.hdr.nentries; \
2445	kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy); \
2446	MACRO_END})
2447
2448	/*
2449	* Routine: vm_map_copyout
2450	*
2451	* Description:
2452	* Copy out a copy chain ("copy") into newly-allocated
2453	* space in the destination map.
2454	*
2455	* If successful, consumes the copy object.
2456	* Otherwise, the caller is responsible for it.
2457	*/
2458	kern_return_t vm_map_copyout(dst_map, dst_addr, copy)
2459	vm_map_t dst_map;
2460	vm_offset_t dst_addr; / OUT */
2461	vm_map_copy_t copy;
2462	{
2463	vm_size_t size;
2464	vm_size_t adjustment;
2465	vm_offset_t start;
2466	vm_offset_t vm_copy_start;
2467	vm_map_entry_t last;
2468	vm_map_entry_t entry;
2469
2470	/*
2471	* Check for null copy object.
2472	*/
2473
2474	if (copy == VM_MAP_COPY_NULL((vm_map_copy_t) 0)) {
2475	*dst_addr = 0;
2476	return(KERN_SUCCESS0);
2477	}
2478
2479	/*
2480	* Check for special copy object, created
2481	* by vm_map_copyin_object.
2482	*/
2483
2484	if (copy->type == VM_MAP_COPY_OBJECT2) {
2485	vm_object_t object = copy->cpy_objectc_u.c_o.object;
2486	vm_size_t offset = copy->offset;
2487	vm_size_t tmp_size = copy->size;
2488	kern_return_t kr;
2489
2490	*dst_addr = 0;
2491	kr = vm_map_enter(dst_map, dst_addr, tmp_size,
2492	(vm_offset_t) 0, TRUE((boolean_t) 1),
2493	object, offset, FALSE((boolean_t) 0),
2494	VM_PROT_DEFAULT(((vm_prot_t) 0x01)\|((vm_prot_t) 0x02)), VM_PROT_ALL(((vm_prot_t) 0x01)\|((vm_prot_t) 0x02)\|((vm_prot_t) 0x04)),
2495	VM_INHERIT_DEFAULT((vm_inherit_t) 1));
2496	if (kr != KERN_SUCCESS0)
2497	return(kr);
2498	kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
2499	return(KERN_SUCCESS0);
2500	}
2501
2502	if (copy->type == VM_MAP_COPY_PAGE_LIST3)
2503	return(vm_map_copyout_page_list(dst_map, dst_addr, copy));
2504
2505	/*
2506	* Find space for the data
2507	*/
2508
2509	vm_copy_start = trunc_page(copy->offset)((vm_offset_t)(((vm_offset_t)(copy->offset)) & ~((1 << 12)-1)));
2510	size = round_page(copy->offset + copy->size)((vm_offset_t)((((vm_offset_t)(copy->offset + copy->size )) + ((1 << 12)-1)) & ~((1 << 12)-1))) - vm_copy_start;
2511
2512	StartAgain: ;
2513
2514	vm_map_lock(dst_map)({ lock_write(&(dst_map)->lock); (dst_map)->timestamp ++; });
2515	start = ((last = dst_map->first_free) == vm_map_to_entry(dst_map)((struct vm_map_entry *) &(dst_map)->hdr.links)) ?
2516	vm_map_min(dst_map)((dst_map)->hdr.links.start) : last->vme_endlinks.end;
2517
2518	while (TRUE((boolean_t) 1)) {
2519	vm_map_entry_t next = last->vme_nextlinks.next;
2520	vm_offset_t end = start + size;
2521
2522	if ((end > dst_map->max_offsethdr.links.end) \|\| (end < start)) {
2523	if (dst_map->wait_for_space) {
2524	if (size <= (dst_map->max_offsethdr.links.end - dst_map->min_offsethdr.links.start)) {
2525	assert_wait((event_t) dst_map, TRUE((boolean_t) 1));
2526	vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2527	thread_block((void (*)()) 0);
2528	goto StartAgain;
2529	}
2530	}
2531	vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2532	printf_once("no more room for vm_map_copyout in %p\n", dst_map)({ static int __once = 0; if (!__once) { printf("no more room for vm_map_copyout in %p\n" , dst_map); __once = 1; } });
2533	return(KERN_NO_SPACE3);
2534	}
2535
2536	if ((next == vm_map_to_entry(dst_map)((struct vm_map_entry *) &(dst_map)->hdr.links)) \|\|
2537	(next->vme_startlinks.start >= end))
2538	break;
2539
2540	last = next;
2541	start = last->vme_endlinks.end;
2542	}
2543
2544	/*
2545	* Since we're going to just drop the map
2546	* entries from the copy into the destination
2547	* map, they must come from the same pool.
2548	*/
2549
2550	if (copy->cpy_hdrc_u.hdr.entries_pageable != dst_map->hdr.entries_pageable) {
2551	/*
2552	* Mismatches occur when dealing with the default
2553	* pager.
2554	*/
2555	kmem_cache_t old_cache;
2556	vm_map_entry_t next, new;
2557
2558	/*
2559	* Find the cache that the copies were allocated from
2560	*/
2561	old_cache = (copy->cpy_hdrc_u.hdr.entries_pageable)
2562	? &vm_map_entry_cache
2563	: &vm_map_kentry_cache;
2564	entry = vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next);
2565
2566	/*
2567	* Reinitialize the copy so that vm_map_copy_entry_link
2568	* will work.
2569	*/
2570	copy->cpy_hdrc_u.hdr.nentries = 0;
2571	copy->cpy_hdrc_u.hdr.entries_pageable = dst_map->hdr.entries_pageable;
2572	vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next) =
2573	vm_map_copy_last_entry(copy)((copy)->c_u.hdr.links.prev) =
2574	vm_map_copy_to_entry(copy)((struct vm_map_entry *) &(copy)->c_u.hdr.links);
2575
2576	/*
2577	* Copy each entry.
2578	*/
2579	while (entry != vm_map_copy_to_entry(copy)((struct vm_map_entry *) &(copy)->c_u.hdr.links)) {
2580	new = vm_map_copy_entry_create(copy)_vm_map_entry_create(&(copy)->c_u.hdr);
2581	vm_map_entry_copy_full(new, entry)((new) = (entry));
2582	vm_map_copy_entry_link(copy,({ (&(copy)->c_u.hdr)->nentries++; (new)->links. prev = (((copy)->c_u.hdr.links.prev)); (new)->links.next = (((copy)->c_u.hdr.links.prev))->links.next; (new)-> links.prev->links.next = (new)->links.next->links.prev = (new); ({ struct rbtree_node ___cur, ___prev; int ___diff , ___index; ___prev = ((void ) 0); ___index = -1; ___cur = ( &(&(copy)->c_u.hdr)->tree)->root; while (___cur != ((void ) 0)) { ___diff = vm_map_entry_cmp_insert(&(new )->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0" , "../vm/vm_map.c", 2584); }); ___prev = ___cur; ___index = rbtree_d2i (___diff); ___cur = ___cur->children[___index]; } rbtree_insert_rebalance (&(&(copy)->c_u.hdr)->tree, ___prev, ___index, & (new)->tree_node); }); })
2583	vm_map_copy_last_entry(copy),({ (&(copy)->c_u.hdr)->nentries++; (new)->links. prev = (((copy)->c_u.hdr.links.prev)); (new)->links.next = (((copy)->c_u.hdr.links.prev))->links.next; (new)-> links.prev->links.next = (new)->links.next->links.prev = (new); ({ struct rbtree_node ___cur, ___prev; int ___diff , ___index; ___prev = ((void ) 0); ___index = -1; ___cur = ( &(&(copy)->c_u.hdr)->tree)->root; while (___cur != ((void ) 0)) { ___diff = vm_map_entry_cmp_insert(&(new )->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0" , "../vm/vm_map.c", 2584); }); ___prev = ___cur; ___index = rbtree_d2i (___diff); ___cur = ___cur->children[___index]; } rbtree_insert_rebalance (&(&(copy)->c_u.hdr)->tree, ___prev, ___index, & (new)->tree_node); }); })
2584	new)({ (&(copy)->c_u.hdr)->nentries++; (new)->links. prev = (((copy)->c_u.hdr.links.prev)); (new)->links.next = (((copy)->c_u.hdr.links.prev))->links.next; (new)-> links.prev->links.next = (new)->links.next->links.prev = (new); ({ struct rbtree_node ___cur, ___prev; int ___diff , ___index; ___prev = ((void ) 0); ___index = -1; ___cur = ( &(&(copy)->c_u.hdr)->tree)->root; while (___cur != ((void ) 0)) { ___diff = vm_map_entry_cmp_insert(&(new )->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0" , "../vm/vm_map.c", 2584); }); ___prev = ___cur; ___index = rbtree_d2i (___diff); ___cur = ___cur->children[___index]; } rbtree_insert_rebalance (&(&(copy)->c_u.hdr)->tree, ___prev, ___index, & (new)->tree_node); }); });
2585	next = entry->vme_nextlinks.next;
2586	kmem_cache_free(old_cache, (vm_offset_t) entry);
2587	entry = next;
2588	}
2589	}
2590
2591	/*
2592	* Adjust the addresses in the copy chain, and
2593	* reset the region attributes.
2594	*/
2595
2596	adjustment = start - vm_copy_start;
2597	for (entry = vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next);
2598	entry != vm_map_copy_to_entry(copy)((struct vm_map_entry *) &(copy)->c_u.hdr.links);
2599	entry = entry->vme_nextlinks.next) {
2600	entry->vme_startlinks.start += adjustment;
2601	entry->vme_endlinks.end += adjustment;
2602
2603	entry->inheritance = VM_INHERIT_DEFAULT((vm_inherit_t) 1);
2604	entry->protection = VM_PROT_DEFAULT(((vm_prot_t) 0x01)\|((vm_prot_t) 0x02));
2605	entry->max_protection = VM_PROT_ALL(((vm_prot_t) 0x01)\|((vm_prot_t) 0x02)\|((vm_prot_t) 0x04));
2606	entry->projected_on = 0;
2607
2608	/*
2609	* If the entry is now wired,
2610	* map the pages into the destination map.
2611	*/
2612	if (entry->wired_count != 0) {
2613	vm_offset_t va;
2614	vm_offset_t offset;
2615	vm_object_t object;
2616
2617	object = entry->object.vm_object;
2618	offset = entry->offset;
2619	va = entry->vme_startlinks.start;
2620
2621	pmap_pageable(dst_map->pmap,
2622	entry->vme_startlinks.start,
2623	entry->vme_endlinks.end,
2624	TRUE((boolean_t) 1));
2625
2626	while (va < entry->vme_endlinks.end) {
2627	vm_page_t m;
2628
2629	/*
2630	* Look up the page in the object.
2631	* Assert that the page will be found in the
2632	* top object:
2633	* either
2634	* the object was newly created by
2635	* vm_object_copy_slowly, and has
2636	* copies of all of the pages from
2637	* the source object
2638	* or
2639	* the object was moved from the old
2640	* map entry; because the old map
2641	* entry was wired, all of the pages
2642	* were in the top-level object.
2643	* (XXX not true if we wire pages for
2644	* reading)
2645	*/
2646	vm_object_lock(object);
2647	vm_object_paging_begin(object)((object)->paging_in_progress++);
2648
2649	m = vm_page_lookup(object, offset);
2650	if (m == VM_PAGE_NULL((vm_page_t) 0) \|\| m->wire_count == 0 \|\|
2651	m->absent)
2652	panic("vm_map_copyout: wiring 0x%x", m);
2653
2654	m->busy = TRUE((boolean_t) 1);
2655	vm_object_unlock(object);
2656
2657	PMAP_ENTER(dst_map->pmap, va, m,({ pmap_enter( (dst_map->pmap), (va), (m)->phys_addr, ( entry->protection) & ~(m)->page_lock, (((boolean_t) 1)) ); })
2658	entry->protection, TRUE)({ pmap_enter( (dst_map->pmap), (va), (m)->phys_addr, ( entry->protection) & ~(m)->page_lock, (((boolean_t) 1)) ); });
2659
2660	vm_object_lock(object);
2661	PAGE_WAKEUP_DONE(m)({ (m)->busy = ((boolean_t) 0); if ((m)->wanted) { (m)-> wanted = ((boolean_t) 0); thread_wakeup_prim((((event_t) m)), ((boolean_t) 0), 0); } });
2662	/* the page is wired, so we don't have to activate */
2663	vm_object_paging_end(object)({ ({ if (!((object)->paging_in_progress != 0)) Assert("(object)->paging_in_progress != 0" , "../vm/vm_map.c", 2663); }); if (--(object)->paging_in_progress == 0) { ({ if ((object)->all_wanted & (1 << (2) )) thread_wakeup_prim(((event_t)(((vm_offset_t) object) + (2) )), ((boolean_t) 0), 0); (object)->all_wanted &= ~(1 << (2)); }); } });
2664	vm_object_unlock(object);
2665
2666	offset += PAGE_SIZE(1 << 12);
2667	va += PAGE_SIZE(1 << 12);
2668	}
2669	}
2670
2671
2672	}
2673
2674	/*
2675	* Correct the page alignment for the result
2676	*/
2677
2678	*dst_addr = start + (copy->offset - vm_copy_start);
2679
2680	/*
2681	* Update the hints and the map size
2682	*/
2683
2684	if (dst_map->first_free == last)
2685	dst_map->first_free = vm_map_copy_last_entry(copy)((copy)->c_u.hdr.links.prev);
2686	SAVE_HINT(dst_map, vm_map_copy_last_entry(copy)); (dst_map)->hint = (((copy)->c_u.hdr.links.prev)); ;;
2687
2688	dst_map->size += size;
2689
2690	/*
2691	* Link in the copy
2692	*/
2693
2694	vm_map_copy_insert(dst_map, last, copy)({ struct rbtree_node node, tmp; for (node = rbtree_postwalk_deepest (&(copy)->c_u.hdr.tree), tmp = rbtree_postwalk_unlink( node); node != ((void ) 0); node = tmp, tmp = rbtree_postwalk_unlink (node)) ({ struct rbtree_node ___cur, ___prev; int ___diff, ___index; ___prev = ((void ) 0); ___index = -1; ___cur = (& (dst_map)->hdr.tree)->root; while (___cur != ((void *) 0 )) { ___diff = vm_map_entry_cmp_insert(node, ___cur); ({ if ( !(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c", 2694 ); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->children[___index]; } rbtree_insert_rebalance(& (dst_map)->hdr.tree, ___prev, ___index, node); }); (((last )->links.next)->links.prev = ((copy)->c_u.hdr.links. prev)) ->links.next = ((last)->links.next); ((last)-> links.next = ((copy)->c_u.hdr.links.next)) ->links.prev = (last); (dst_map)->hdr.nentries += (copy)->c_u.hdr.nentries ; kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy) ; });
2695
2696	vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2697
2698	/*
2699	* XXX If wiring_required, call vm_map_pageable
2700	*/
2701
2702	return(KERN_SUCCESS0);
2703	}
2704
2705	/*
2706	*
2707	* vm_map_copyout_page_list:
2708	*
2709	* Version of vm_map_copyout() for page list vm map copies.
2710	*
2711	*/
2712	kern_return_t vm_map_copyout_page_list(dst_map, dst_addr, copy)
2713	vm_map_t dst_map;
2714	vm_offset_t dst_addr; / OUT */
2715	vm_map_copy_t copy;
2716	{
2717	vm_size_t size;
2718	vm_offset_t start;
2719	vm_offset_t end;
2720	vm_offset_t offset;
2721	vm_map_entry_t last;
2722	vm_object_t object;
2723	vm_page_t *page_list, m;
2724	vm_map_entry_t entry;
2725	vm_offset_t old_last_offset;
2726	boolean_t cont_invoked, needs_wakeup = FALSE((boolean_t) 0);
2727	kern_return_t result = KERN_SUCCESS0;
2728	vm_map_copy_t orig_copy;
2729	vm_offset_t dst_offset;
2730	boolean_t must_wire;
2731
2732	/*
2733	* Make sure the pages are stolen, because we are
2734	* going to put them in a new object. Assume that
2735	* all pages are identical to first in this regard.
2736	*/
2737
2738	page_list = &copy->cpy_page_listc_u.c_p.page_list[0];
2739	if ((*page_list)->tabled)
2740	vm_map_copy_steal_pages(copy);
2741
2742	/*
2743	* Find space for the data
2744	*/
2745
2746	size = round_page(copy->offset + copy->size)((vm_offset_t)((((vm_offset_t)(copy->offset + copy->size )) + ((1 << 12)-1)) & ~((1 << 12)-1))) -
2747	trunc_page(copy->offset)((vm_offset_t)(((vm_offset_t)(copy->offset)) & ~((1 << 12)-1)));
2748	StartAgain:
2749	vm_map_lock(dst_map)({ lock_write(&(dst_map)->lock); (dst_map)->timestamp ++; });
2750	must_wire = dst_map->wiring_required;
2751
2752	last = dst_map->first_free;
2753	if (last == vm_map_to_entry(dst_map)((struct vm_map_entry *) &(dst_map)->hdr.links)) {
2754	start = vm_map_min(dst_map)((dst_map)->hdr.links.start);
2755	} else {
2756	start = last->vme_endlinks.end;
2757	}
2758
2759	while (TRUE((boolean_t) 1)) {
2760	vm_map_entry_t next = last->vme_nextlinks.next;
2761	end = start + size;
2762
2763	if ((end > dst_map->max_offsethdr.links.end) \|\| (end < start)) {
2764	if (dst_map->wait_for_space) {
2765	if (size <= (dst_map->max_offsethdr.links.end -
2766	dst_map->min_offsethdr.links.start)) {
2767	assert_wait((event_t) dst_map, TRUE((boolean_t) 1));
2768	vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2769	thread_block((void (*)()) 0);
2770	goto StartAgain;
2771	}
2772	}
2773	vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2774	printf_once("no more room for vm_map_copyout_page_list in %p\n", dst_map)({ static int __once = 0; if (!__once) { printf("no more room for vm_map_copyout_page_list in %p\n" , dst_map); __once = 1; } });
2775	return(KERN_NO_SPACE3);
2776	}
2777
2778	if ((next == vm_map_to_entry(dst_map)((struct vm_map_entry *) &(dst_map)->hdr.links)) \|\|
2779	(next->vme_startlinks.start >= end)) {
2780	break;
2781	}
2782
2783	last = next;
2784	start = last->vme_endlinks.end;
2785	}
2786
2787	/*
2788	* See whether we can avoid creating a new entry (and object) by
2789	* extending one of our neighbors. [So far, we only attempt to
2790	* extend from below.]
2791	*
2792	* The code path below here is a bit twisted. If any of the
2793	* extension checks fails, we branch to create_object. If
2794	* it all works, we fall out the bottom and goto insert_pages.
2795	*/
2796	if (last == vm_map_to_entry(dst_map)((struct vm_map_entry *) &(dst_map)->hdr.links) \|\|
2797	last->vme_endlinks.end != start \|\|
2798	last->is_shared != FALSE((boolean_t) 0) \|\|
2799	last->is_sub_map != FALSE((boolean_t) 0) \|\|
2800	last->inheritance != VM_INHERIT_DEFAULT((vm_inherit_t) 1) \|\|
2801	last->protection != VM_PROT_DEFAULT(((vm_prot_t) 0x01)\|((vm_prot_t) 0x02)) \|\|
2802	last->max_protection != VM_PROT_ALL(((vm_prot_t) 0x01)\|((vm_prot_t) 0x02)\|((vm_prot_t) 0x04)) \|\|
2803	(must_wire ? (last->wired_count != 1 \|\|
2804	last->user_wired_count != 1) :
2805	(last->wired_count != 0))) {
2806	goto create_object;
2807	}
2808
2809	/*
2810	* If this entry needs an object, make one.
2811	*/
2812	if (last->object.vm_object == VM_OBJECT_NULL((vm_object_t) 0)) {
2813	object = vm_object_allocate(
2814	(vm_size_t)(last->vme_endlinks.end - last->vme_startlinks.start + size));
2815	last->object.vm_object = object;
2816	last->offset = 0;
2817	vm_object_lock(object);
2818	}
2819	else {
2820	vm_offset_t prev_offset = last->offset;
2821	vm_size_t prev_size = start - last->vme_startlinks.start;
2822	vm_size_t new_size;
2823
2824	/*
2825	* This is basically vm_object_coalesce.
2826	*/
2827
2828	object = last->object.vm_object;
2829	vm_object_lock(object);
2830
2831	/*
2832	* Try to collapse the object first
2833	*/
2834	vm_object_collapse(object);
2835
2836	/*
2837	* Can't coalesce if pages not mapped to
2838	* last may be in use anyway:
2839	* . more than one reference
2840	* . paged out
2841	* . shadows another object
2842	* . has a copy elsewhere
2843	* . paging references (pages might be in page-list)
2844	*/
2845
2846	if ((object->ref_count > 1) \|\|
2847	object->pager_created \|\|
2848	(object->shadow != VM_OBJECT_NULL((vm_object_t) 0)) \|\|
2849	(object->copy != VM_OBJECT_NULL((vm_object_t) 0)) \|\|
2850	(object->paging_in_progress != 0)) {
2851	vm_object_unlock(object);
2852	goto create_object;
2853	}
2854
2855	/*
2856	* Extend the object if necessary. Don't have to call
2857	* vm_object_page_remove because the pages aren't mapped,
2858	* and vm_page_replace will free up any old ones it encounters.
2859	*/
2860	new_size = prev_offset + prev_size + size;
2861	if (new_size > object->size)
2862	object->size = new_size;
2863	}
2864
2865	/*
2866	* Coalesced the two objects - can extend
2867	* the previous map entry to include the
2868	* new range.
2869	*/
2870	dst_map->size += size;
2871	last->vme_endlinks.end = end;
2872
2873	SAVE_HINT(dst_map, last); (dst_map)->hint = (last); ;;
2874
2875	goto insert_pages;
2876
2877	create_object:
2878
2879	/*
2880	* Create object
2881	*/
2882	object = vm_object_allocate(size);
2883
2884	/*
2885	* Create entry
2886	*/
2887
2888	entry = vm_map_entry_create(dst_map)_vm_map_entry_create(&(dst_map)->hdr);
2889
2890	entry->object.vm_object = object;
2891	entry->offset = 0;
2892
2893	entry->is_shared = FALSE((boolean_t) 0);
2894	entry->is_sub_map = FALSE((boolean_t) 0);
2895	entry->needs_copy = FALSE((boolean_t) 0);
2896
2897	if (must_wire) {
2898	entry->wired_count = 1;
2899	entry->user_wired_count = 1;
2900	} else {
2901	entry->wired_count = 0;
2902	entry->user_wired_count = 0;
2903	}
2904
2905	entry->in_transition = TRUE((boolean_t) 1);
2906	entry->needs_wakeup = FALSE((boolean_t) 0);
2907
2908	entry->vme_startlinks.start = start;
2909	entry->vme_endlinks.end = start + size;
2910
2911	entry->inheritance = VM_INHERIT_DEFAULT((vm_inherit_t) 1);
2912	entry->protection = VM_PROT_DEFAULT(((vm_prot_t) 0x01)\|((vm_prot_t) 0x02));
2913	entry->max_protection = VM_PROT_ALL(((vm_prot_t) 0x01)\|((vm_prot_t) 0x02)\|((vm_prot_t) 0x04));
2914	entry->projected_on = 0;
2915
2916	vm_object_lock(object);
2917
2918	/*
2919	* Update the hints and the map size
2920	*/
2921	if (dst_map->first_free == last) {
2922	dst_map->first_free = entry;
2923	}
2924	SAVE_HINT(dst_map, entry); (dst_map)->hint = (entry); ;;
2925	dst_map->size += size;
2926
2927	/*
2928	* Link in the entry
2929	*/
2930	vm_map_entry_link(dst_map, last, entry)({ (&(dst_map)->hdr)->nentries++; (entry)->links .prev = (last); (entry)->links.next = (last)->links.next ; (entry)->links.prev->links.next = (entry)->links.next ->links.prev = (entry); ({ struct rbtree_node ___cur, ___prev ; int ___diff, ___index; ___prev = ((void ) 0); ___index = - 1; ___cur = (&(&(dst_map)->hdr)->tree)->root ; while (___cur != ((void ) 0)) { ___diff = vm_map_entry_cmp_insert (&(entry)->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c", 2930); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur-> children[___index]; } rbtree_insert_rebalance(&(&(dst_map )->hdr)->tree, ___prev, ___index, &(entry)->tree_node ); }); });
2931	last = entry;
2932
2933	/*
2934	* Transfer pages into new object.
2935	* Scan page list in vm_map_copy.
2936	*/
2937	insert_pages:
2938	dst_offset = copy->offset & PAGE_MASK((1 << 12)-1);
2939	cont_invoked = FALSE((boolean_t) 0);
2940	orig_copy = copy;
2941	last->in_transition = TRUE((boolean_t) 1);
2942	old_last_offset = last->offset
2943	+ (start - last->vme_startlinks.start);
2944
2945	vm_page_lock_queues();
2946
2947	for (offset = 0; offset < size; offset += PAGE_SIZE(1 << 12)) {
2948	m = *page_list;
2949	assert(m && !m->tabled)({ if (!(m && !m->tabled)) Assert("m && !m->tabled" , "../vm/vm_map.c", 2949); });
2950
2951	/*
2952	* Must clear busy bit in page before inserting it.
2953	* Ok to skip wakeup logic because nobody else
2954	* can possibly know about this page.
2955	* The page is dirty in its new object.
2956	*/
2957
2958	assert(!m->wanted)({ if (!(!m->wanted)) Assert("!m->wanted", "../vm/vm_map.c" , 2958); });
2959
2960	m->busy = FALSE((boolean_t) 0);
2961	m->dirty = TRUE((boolean_t) 1);
2962	vm_page_replace(m, object, old_last_offset + offset);
2963	if (must_wire) {
2964	vm_page_wire(m);
2965	PMAP_ENTER(dst_map->pmap,({ pmap_enter( (dst_map->pmap), (last->links.start + m-> offset - last->offset), (m)->phys_addr, (last->protection ) & ~(m)->page_lock, (((boolean_t) 1)) ); })
2966	last->vme_start + m->offset - last->offset,({ pmap_enter( (dst_map->pmap), (last->links.start + m-> offset - last->offset), (m)->phys_addr, (last->protection ) & ~(m)->page_lock, (((boolean_t) 1)) ); })
2967	m, last->protection, TRUE)({ pmap_enter( (dst_map->pmap), (last->links.start + m-> offset - last->offset), (m)->phys_addr, (last->protection ) & ~(m)->page_lock, (((boolean_t) 1)) ); });
2968	} else {
2969	vm_page_activate(m);
2970	}
2971
2972	*page_list++ = VM_PAGE_NULL((vm_page_t) 0);
2973	if (--(copy->cpy_npagesc_u.c_p.npages) == 0 &&
2974	vm_map_copy_has_cont(copy)(((copy)->c_u.c_p.cont) != (kern_return_t (*)()) 0)) {
2975	vm_map_copy_t new_copy;
2976
2977	/*
2978	* Ok to unlock map because entry is
2979	* marked in_transition.
2980	*/
2981	cont_invoked = TRUE((boolean_t) 1);
2982	vm_page_unlock_queues();
2983	vm_object_unlock(object);
2984	vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
2985	vm_map_copy_invoke_cont(copy, &new_copy, &result)({ vm_map_copy_page_discard(copy); &result = (((copy)-> c_u.c_p.cont))((copy)->c_u.c_p.cont_args, &new_copy); ( copy)->c_u.c_p.cont = (kern_return_t (*)()) 0; });
2986
2987	if (result == KERN_SUCCESS0) {
2988
2989	/*
2990	* If we got back a copy with real pages,
2991	* steal them now. Either all of the
2992	* pages in the list are tabled or none
2993	* of them are; mixtures are not possible.
2994	*
2995	* Save original copy for consume on
2996	* success logic at end of routine.
2997	*/
2998	if (copy != orig_copy)
2999	vm_map_copy_discard(copy);
3000
3001	if ((copy = new_copy) != VM_MAP_COPY_NULL((vm_map_copy_t) 0)) {
3002	page_list = &copy->cpy_page_listc_u.c_p.page_list[0];
3003	if ((*page_list)->tabled)
3004	vm_map_copy_steal_pages(copy);
3005	}
3006	}
3007	else {
3008	/*
3009	* Continuation failed.
3010	*/
3011	vm_map_lock(dst_map)({ lock_write(&(dst_map)->lock); (dst_map)->timestamp ++; });
3012	goto error;
3013	}
3014
3015	vm_map_lock(dst_map)({ lock_write(&(dst_map)->lock); (dst_map)->timestamp ++; });
3016	vm_object_lock(object);
3017	vm_page_lock_queues();
3018	}
3019	}
3020
3021	vm_page_unlock_queues();
3022	vm_object_unlock(object);
3023
3024	*dst_addr = start + dst_offset;
3025
3026	/*
3027	* Clear the in transition bits. This is easy if we
3028	* didn't have a continuation.
3029	*/
3030	error:
3031	if (!cont_invoked) {
3032	/*
3033	* We didn't unlock the map, so nobody could
3034	* be waiting.
3035	*/
3036	last->in_transition = FALSE((boolean_t) 0);
3037	assert(!last->needs_wakeup)({ if (!(!last->needs_wakeup)) Assert("!last->needs_wakeup" , "../vm/vm_map.c", 3037); });
3038	needs_wakeup = FALSE((boolean_t) 0);
3039	}
3040	else {
3041	if (!vm_map_lookup_entry(dst_map, start, &entry))
3042	panic("vm_map_copyout_page_list: missing entry");
3043
3044	/*
3045	* Clear transition bit for all constituent entries that
3046	* were in the original entry. Also check for waiters.
3047	*/
3048	while((entry != vm_map_to_entry(dst_map)((struct vm_map_entry *) &(dst_map)->hdr.links)) &&
3049	(entry->vme_startlinks.start < end)) {
3050	assert(entry->in_transition)({ if (!(entry->in_transition)) Assert("entry->in_transition" , "../vm/vm_map.c", 3050); });
3051	entry->in_transition = FALSE((boolean_t) 0);
3052	if(entry->needs_wakeup) {
3053	entry->needs_wakeup = FALSE((boolean_t) 0);
3054	needs_wakeup = TRUE((boolean_t) 1);
3055	}
3056	entry = entry->vme_nextlinks.next;
3057	}
3058	}
3059
3060	if (result != KERN_SUCCESS0)
3061	vm_map_delete(dst_map, start, end);
3062
3063	vm_map_unlock(dst_map)lock_done(&(dst_map)->lock);
3064
3065	if (needs_wakeup)
3066	vm_map_entry_wakeup(dst_map)thread_wakeup_prim(((event_t)&(dst_map)->hdr), ((boolean_t ) 0), 0);
3067
3068	/*
3069	* Consume on success logic.
3070	*/
3071	if (copy != orig_copy) {
3072	kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy);
3073	}
3074	if (result == KERN_SUCCESS0) {
3075	kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) orig_copy);
3076	}
3077
3078	return(result);
3079	}
3080
3081	/*
3082	* Routine: vm_map_copyin
3083	*
3084	* Description:
3085	* Copy the specified region (src_addr, len) from the
3086	* source address space (src_map), possibly removing
3087	* the region from the source address space (src_destroy).
3088	*
3089	* Returns:
3090	* A vm_map_copy_t object (copy_result), suitable for
3091	* insertion into another address space (using vm_map_copyout),
3092	* copying over another address space region (using
3093	* vm_map_copy_overwrite). If the copy is unused, it
3094	* should be destroyed (using vm_map_copy_discard).
3095	*
3096	* In/out conditions:
3097	* The source map should not be locked on entry.
3098	*/
3099	kern_return_t vm_map_copyin(src_map, src_addr, len, src_destroy, copy_result)
3100	vm_map_t src_map;
3101	vm_offset_t src_addr;
3102	vm_size_t len;
3103	boolean_t src_destroy;
3104	vm_map_copy_t copy_result; / OUT */
3105	{
3106	vm_map_entry_t tmp_entry; /* Result of last map lookup --
3107	* in multi-level lookup, this
3108	* entry contains the actual
3109	* vm_object/offset.
3110	*/
3111
3112	vm_offset_t src_start; /* Start of current entry --
3113	* where copy is taking place now
3114	*/
3115	vm_offset_t src_end; /* End of entire region to be
3116	* copied */
3117
3118	vm_map_copy_t copy; /* Resulting copy */
3119
3120	/*
3121	* Check for copies of zero bytes.
3122	*/
3123
3124	if (len == 0) {
3125	*copy_result = VM_MAP_COPY_NULL((vm_map_copy_t) 0);
3126	return(KERN_SUCCESS0);
3127	}
3128
3129	/*
3130	* Compute start and end of region
3131	*/
3132
3133	src_start = trunc_page(src_addr)((vm_offset_t)(((vm_offset_t)(src_addr)) & ~((1 << 12 )-1)));
3134	src_end = round_page(src_addr + len)((vm_offset_t)((((vm_offset_t)(src_addr + len)) + ((1 << 12)-1)) & ~((1 << 12)-1)));
3135
3136	/*
3137	* Check that the end address doesn't overflow
3138	*/
3139
3140	if (src_end <= src_start)
3141	if ((src_end < src_start) \|\| (src_start != 0))
3142	return(KERN_INVALID_ADDRESS1);
3143
3144	/*
3145	* Allocate a header element for the list.
3146	*
3147	* Use the start and end in the header to
3148	* remember the endpoints prior to rounding.
3149	*/
3150
3151	copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
3152	vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next) =
3153	vm_map_copy_last_entry(copy)((copy)->c_u.hdr.links.prev) = vm_map_copy_to_entry(copy)((struct vm_map_entry *) &(copy)->c_u.hdr.links);
3154	copy->type = VM_MAP_COPY_ENTRY_LIST1;
3155	copy->cpy_hdrc_u.hdr.nentries = 0;
3156	copy->cpy_hdrc_u.hdr.entries_pageable = TRUE((boolean_t) 1);
3157	rbtree_init(&copy->cpy_hdrc_u.hdr.tree);
3158
3159	copy->offset = src_addr;
3160	copy->size = len;
3161
3162	#define RETURN(x) \
3163	MACRO_BEGIN({ \
3164	vm_map_unlock(src_map)lock_done(&(src_map)->lock); \
3165	vm_map_copy_discard(copy); \
3166	MACRO_RETURNif (((boolean_t) 1)) return(x); \
3167	MACRO_END})
3168
3169	/*
3170	* Find the beginning of the region.
3171	*/
3172
3173	vm_map_lock(src_map)({ lock_write(&(src_map)->lock); (src_map)->timestamp ++; });
3174
3175	if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry))
3176	RETURN(KERN_INVALID_ADDRESS1);
3177	vm_map_clip_start(src_map, tmp_entry, src_start)({ if ((src_start) > (tmp_entry)->links.start) _vm_map_clip_start (&(src_map)->hdr,(tmp_entry),(src_start)); });
3178
3179	/*
3180	* Go through entries until we get to the end.
3181	*/
3182
3183	while (TRUE((boolean_t) 1)) {
3184	vm_map_entry_t src_entry = tmp_entry; /* Top-level entry */
3185	vm_size_t src_size; /* Size of source
3186	* map entry (in both
3187	* maps)
3188	*/
3189
3190	vm_object_t src_object; /* Object to copy */
3191	vm_offset_t src_offset;
3192
3193	boolean_t src_needs_copy; /* Should source map
3194	* be made read-only
3195	* for copy-on-write?
3196	*/
3197
3198	vm_map_entry_t new_entry; /* Map entry for copy */
3199	boolean_t new_entry_needs_copy; /* Will new entry be COW? */
3200
3201	boolean_t was_wired; /* Was source wired? */
3202	vm_map_version_t version; /* Version before locks
3203	* dropped to make copy
3204	*/
3205
3206	/*
3207	* Verify that the region can be read.
3208	*/
3209
3210	if (! (src_entry->protection & VM_PROT_READ((vm_prot_t) 0x01)))
3211	RETURN(KERN_PROTECTION_FAILURE2);
3212
3213	/*
3214	* Clip against the endpoints of the entire region.
3215	*/
3216
3217	vm_map_clip_end(src_map, src_entry, src_end)({ if ((src_end) < (src_entry)->links.end) _vm_map_clip_end (&(src_map)->hdr,(src_entry),(src_end)); });
3218
3219	src_size = src_entry->vme_endlinks.end - src_start;
3220	src_object = src_entry->object.vm_object;
3221	src_offset = src_entry->offset;
3222	was_wired = (src_entry->wired_count != 0);
3223
3224	/*
3225	* Create a new address map entry to
3226	* hold the result. Fill in the fields from
3227	* the appropriate source entries.
3228	*/
3229
3230	new_entry = vm_map_copy_entry_create(copy)_vm_map_entry_create(&(copy)->c_u.hdr);
3231	vm_map_entry_copy(new_entry, src_entry)({ (new_entry) = (src_entry); (new_entry)->is_shared = ( (boolean_t) 0); (new_entry)->needs_wakeup = ((boolean_t) 0 ); (new_entry)->in_transition = ((boolean_t) 0); (new_entry )->wired_count = 0; (new_entry)->user_wired_count = 0; } );
3232
3233	/*
3234	* Attempt non-blocking copy-on-write optimizations.
3235	*/
3236
3237	if (src_destroy &&
3238	(src_object == VM_OBJECT_NULL((vm_object_t) 0) \|\|
3239	(src_object->temporary && !src_object->use_shared_copy)))
3240	{
3241	/*
3242	* If we are destroying the source, and the object
3243	* is temporary, and not shared writable,
3244	* we can move the object reference
3245	* from the source to the copy. The copy is
3246	* copy-on-write only if the source is.
3247	* We make another reference to the object, because
3248	* destroying the source entry will deallocate it.
3249	*/
3250	vm_object_reference(src_object);
3251
3252	/*
3253	* Copy is always unwired. vm_map_copy_entry
3254	* set its wired count to zero.
3255	*/
3256
3257	goto CopySuccessful;
3258	}
3259
3260	if (!was_wired &&
3261	vm_object_copy_temporary(
3262	&new_entry->object.vm_object,
3263	&new_entry->offset,
3264	&src_needs_copy,
3265	&new_entry_needs_copy)) {
3266
3267	new_entry->needs_copy = new_entry_needs_copy;
3268
3269	/*
3270	* Handle copy-on-write obligations
3271	*/
3272
3273	if (src_needs_copy && !tmp_entry->needs_copy) {
3274	vm_object_pmap_protect(
3275	src_object,
3276	src_offset,
3277	src_size,
3278	(src_entry->is_shared ? PMAP_NULL((pmap_t) 0)
3279	: src_map->pmap),
3280	src_entry->vme_startlinks.start,
3281	src_entry->protection &
3282	~VM_PROT_WRITE((vm_prot_t) 0x02));
3283
3284	tmp_entry->needs_copy = TRUE((boolean_t) 1);
3285	}
3286
3287	/*
3288	* The map has never been unlocked, so it's safe to
3289	* move to the next entry rather than doing another
3290	* lookup.
3291	*/
3292
3293	goto CopySuccessful;
3294	}
3295
3296	new_entry->needs_copy = FALSE((boolean_t) 0);
3297
3298	/*
3299	* Take an object reference, so that we may
3300	* release the map lock(s).
3301	*/
3302
3303	assert(src_object != VM_OBJECT_NULL)({ if (!(src_object != ((vm_object_t) 0))) Assert("src_object != VM_OBJECT_NULL" , "../vm/vm_map.c", 3303); });
3304	vm_object_reference(src_object);
3305
3306	/*
3307	* Record the timestamp for later verification.
3308	* Unlock the map.
3309	*/
3310
3311	version.main_timestamp = src_map->timestamp;
3312	vm_map_unlock(src_map)lock_done(&(src_map)->lock);
3313
3314	/*
3315	* Perform the copy
3316	*/
3317
3318	if (was_wired) {
3319	vm_object_lock(src_object);
3320	(void) vm_object_copy_slowly(
3321	src_object,
3322	src_offset,
3323	src_size,
3324	FALSE((boolean_t) 0),
3325	&new_entry->object.vm_object);
3326	new_entry->offset = 0;
3327	new_entry->needs_copy = FALSE((boolean_t) 0);
3328	} else {
3329	kern_return_t result;
3330
3331	result = vm_object_copy_strategically(src_object,
3332	src_offset,
3333	src_size,
3334	&new_entry->object.vm_object,
3335	&new_entry->offset,
3336	&new_entry_needs_copy);
3337
3338	new_entry->needs_copy = new_entry_needs_copy;
3339
3340
3341	if (result != KERN_SUCCESS0) {
3342	vm_map_copy_entry_dispose(copy, new_entry)_vm_map_entry_dispose(&(copy)->c_u.hdr, (new_entry));
3343
3344	vm_map_lock(src_map)({ lock_write(&(src_map)->lock); (src_map)->timestamp ++; });
3345	RETURN(result);
3346	}
3347
3348	}
3349
3350	/*
3351	* Throw away the extra reference
3352	*/
3353
3354	vm_object_deallocate(src_object);
3355
3356	/*
3357	* Verify that the map has not substantially
3358	* changed while the copy was being made.
3359	*/
3360
3361	vm_map_lock(src_map)({ lock_write(&(src_map)->lock); (src_map)->timestamp ++; }); /* Increments timestamp once! */
3362
3363	if ((version.main_timestamp + 1) == src_map->timestamp)
3364	goto CopySuccessful;
3365
3366	/*
3367	* Simple version comparison failed.
3368	*
3369	* Retry the lookup and verify that the
3370	* same object/offset are still present.
3371	*
3372	* [Note: a memory manager that colludes with
3373	* the calling task can detect that we have
3374	* cheated. While the map was unlocked, the
3375	* mapping could have been changed and restored.]
3376	*/
3377
3378	if (!vm_map_lookup_entry(src_map, src_start, &tmp_entry)) {
3379	vm_map_copy_entry_dispose(copy, new_entry)_vm_map_entry_dispose(&(copy)->c_u.hdr, (new_entry));
3380	RETURN(KERN_INVALID_ADDRESS1);
3381	}
3382
3383	src_entry = tmp_entry;
3384	vm_map_clip_start(src_map, src_entry, src_start)({ if ((src_start) > (src_entry)->links.start) _vm_map_clip_start (&(src_map)->hdr,(src_entry),(src_start)); });
3385
3386	if ((src_entry->protection & VM_PROT_READ((vm_prot_t) 0x01)) == VM_PROT_NONE((vm_prot_t) 0x00))
3387	goto VerificationFailed;
3388
3389	if (src_entry->vme_endlinks.end < new_entry->vme_endlinks.end)
3390	src_size = (new_entry->vme_endlinks.end = src_entry->vme_endlinks.end) - src_start;
	Value stored to 'src_size' is never read
3391
3392	if ((src_entry->object.vm_object != src_object) \|\|
3393	(src_entry->offset != src_offset) ) {
3394
3395	/*
3396	* Verification failed.
3397	*
3398	* Start over with this top-level entry.
3399	*/
3400
3401	VerificationFailed: ;
3402
3403	vm_object_deallocate(new_entry->object.vm_object);
3404	vm_map_copy_entry_dispose(copy, new_entry)_vm_map_entry_dispose(&(copy)->c_u.hdr, (new_entry));
3405	tmp_entry = src_entry;
3406	continue;
3407	}
3408
3409	/*
3410	* Verification succeeded.
3411	*/
3412
3413	CopySuccessful: ;
3414
3415	/*
3416	* Link in the new copy entry.
3417	*/
3418
3419	vm_map_copy_entry_link(copy, vm_map_copy_last_entry(copy),({ (&(copy)->c_u.hdr)->nentries++; (new_entry)-> links.prev = (((copy)->c_u.hdr.links.prev)); (new_entry)-> links.next = (((copy)->c_u.hdr.links.prev))->links.next ; (new_entry)->links.prev->links.next = (new_entry)-> links.next->links.prev = (new_entry); ({ struct rbtree_node ___cur, ___prev; int ___diff, ___index; ___prev = ((void * ) 0); ___index = -1; ___cur = (&(&(copy)->c_u.hdr) ->tree)->root; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert(&(new_entry)->tree_node, ___cur ); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c" , 3420); }); ___prev = ___cur; ___index = rbtree_d2i(___diff) ; ___cur = ___cur->children[___index]; } rbtree_insert_rebalance (&(&(copy)->c_u.hdr)->tree, ___prev, ___index, & (new_entry)->tree_node); }); })
3420	new_entry)({ (&(copy)->c_u.hdr)->nentries++; (new_entry)-> links.prev = (((copy)->c_u.hdr.links.prev)); (new_entry)-> links.next = (((copy)->c_u.hdr.links.prev))->links.next ; (new_entry)->links.prev->links.next = (new_entry)-> links.next->links.prev = (new_entry); ({ struct rbtree_node ___cur, ___prev; int ___diff, ___index; ___prev = ((void * ) 0); ___index = -1; ___cur = (&(&(copy)->c_u.hdr) ->tree)->root; while (___cur != ((void *) 0)) { ___diff = vm_map_entry_cmp_insert(&(new_entry)->tree_node, ___cur ); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c" , 3420); }); ___prev = ___cur; ___index = rbtree_d2i(___diff) ; ___cur = ___cur->children[___index]; } rbtree_insert_rebalance (&(&(copy)->c_u.hdr)->tree, ___prev, ___index, & (new_entry)->tree_node); }); });
3421
3422	/*
3423	* Determine whether the entire region
3424	* has been copied.
3425	*/
3426	src_start = new_entry->vme_endlinks.end;
3427	if ((src_start >= src_end) && (src_end != 0))
3428	break;
3429
3430	/*
3431	* Verify that there are no gaps in the region
3432	*/
3433
3434	tmp_entry = src_entry->vme_nextlinks.next;
3435	if (tmp_entry->vme_startlinks.start != src_start)
3436	RETURN(KERN_INVALID_ADDRESS1);
3437	}
3438
3439	/*
3440	* If the source should be destroyed, do it now, since the
3441	* copy was successful.
3442	*/
3443	if (src_destroy)
3444	(void) vm_map_delete(src_map, trunc_page(src_addr)((vm_offset_t)(((vm_offset_t)(src_addr)) & ~((1 << 12 )-1))), src_end);
3445
3446	vm_map_unlock(src_map)lock_done(&(src_map)->lock);
3447
3448	*copy_result = copy;
3449	return(KERN_SUCCESS0);
3450
3451	#undef RETURN
3452	}
3453
3454	/*
3455	* vm_map_copyin_object:
3456	*
3457	* Create a copy object from an object.
3458	* Our caller donates an object reference.
3459	*/
3460
3461	kern_return_t vm_map_copyin_object(object, offset, size, copy_result)
3462	vm_object_t object;
3463	vm_offset_t offset; /* offset of region in object */
3464	vm_size_t size; /* size of region in object */
3465	vm_map_copy_t copy_result; / OUT */
3466	{
3467	vm_map_copy_t copy; /* Resulting copy */
3468
3469	/*
3470	* We drop the object into a special copy object
3471	* that contains the object directly. These copy objects
3472	* are distinguished by entries_pageable == FALSE
3473	* and null links.
3474	*/
3475
3476	copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
3477	vm_map_copy_first_entry(copy)((copy)->c_u.hdr.links.next) =
3478	vm_map_copy_last_entry(copy)((copy)->c_u.hdr.links.prev) = VM_MAP_ENTRY_NULL((vm_map_entry_t) 0);
3479	copy->type = VM_MAP_COPY_OBJECT2;
3480	copy->cpy_objectc_u.c_o.object = object;
3481	copy->offset = offset;
3482	copy->size = size;
3483
3484	*copy_result = copy;
3485	return(KERN_SUCCESS0);
3486	}
3487
3488	/*
3489	* vm_map_copyin_page_list_cont:
3490	*
3491	* Continuation routine for vm_map_copyin_page_list.
3492	*
3493	* If vm_map_copyin_page_list can't fit the entire vm range
3494	* into a single page list object, it creates a continuation.
3495	* When the target of the operation has used the pages in the
3496	* initial page list, it invokes the continuation, which calls
3497	* this routine. If an error happens, the continuation is aborted
3498	* (abort arg to this routine is TRUE). To avoid deadlocks, the
3499	* pages are discarded from the initial page list before invoking
3500	* the continuation.
3501	*
3502	* NOTE: This is not the same sort of continuation used by
3503	* the scheduler.
3504	*/
3505
3506	kern_return_t vm_map_copyin_page_list_cont(cont_args, copy_result)
3507	vm_map_copyin_args_t cont_args;
3508	vm_map_copy_t copy_result; / OUT */
3509	{
3510	kern_return_t result = 0; /* '=0' to quiet gcc warnings */
3511	boolean_t do_abort, src_destroy, src_destroy_only;
3512
3513	/*
3514	* Check for cases that only require memory destruction.
3515	*/
3516	do_abort = (copy_result == (vm_map_copy_t *) 0);
3517	src_destroy = (cont_args->destroy_len != (vm_size_t) 0);
3518	src_destroy_only = (cont_args->src_len == (vm_size_t) 0);
3519
3520	if (do_abort \|\| src_destroy_only) {
3521	if (src_destroy)
3522	result = vm_map_remove(cont_args->map,
3523	cont_args->destroy_addr,
3524	cont_args->destroy_addr + cont_args->destroy_len);
3525	if (!do_abort)
3526	*copy_result = VM_MAP_COPY_NULL((vm_map_copy_t) 0);
3527	}
3528	else {
3529	result = vm_map_copyin_page_list(cont_args->map,
3530	cont_args->src_addr, cont_args->src_len, src_destroy,
3531	cont_args->steal_pages, copy_result, TRUE((boolean_t) 1));
3532
3533	if (src_destroy && !cont_args->steal_pages &&
3534	vm_map_copy_has_cont(copy_result)(((copy_result)->c_u.c_p.cont) != (kern_return_t (*)()) 0 )) {
3535	vm_map_copyin_args_t new_args;
3536	/*
3537	* Transfer old destroy info.
3538	*/
3539	new_args = (vm_map_copyin_args_t)
3540	(*copy_result)->cpy_cont_argsc_u.c_p.cont_args;
3541	new_args->destroy_addr = cont_args->destroy_addr;
3542	new_args->destroy_len = cont_args->destroy_len;
3543	}
3544	}
3545
3546	vm_map_deallocate(cont_args->map);
3547	kfree((vm_offset_t)cont_args, sizeof(vm_map_copyin_args_data_t));
3548
3549	return(result);
3550	}
3551
3552	/*
3553	* vm_map_copyin_page_list:
3554	*
3555	* This is a variant of vm_map_copyin that copies in a list of pages.
3556	* If steal_pages is TRUE, the pages are only in the returned list.
3557	* If steal_pages is FALSE, the pages are busy and still in their
3558	* objects. A continuation may be returned if not all the pages fit:
3559	* the recipient of this copy_result must be prepared to deal with it.
3560	*/
3561
3562	kern_return_t vm_map_copyin_page_list(src_map, src_addr, len, src_destroy,
3563	steal_pages, copy_result, is_cont)
3564	vm_map_t src_map;
3565	vm_offset_t src_addr;
3566	vm_size_t len;
3567	boolean_t src_destroy;
3568	boolean_t steal_pages;
3569	vm_map_copy_t copy_result; / OUT */
3570	boolean_t is_cont;
3571	{
3572	vm_map_entry_t src_entry;
3573	vm_page_t m;
3574	vm_offset_t src_start;
3575	vm_offset_t src_end;
3576	vm_size_t src_size;
3577	vm_object_t src_object;
3578	vm_offset_t src_offset;
3579	vm_offset_t src_last_offset;
3580	vm_map_copy_t copy; /* Resulting copy */
3581	kern_return_t result = KERN_SUCCESS0;
3582	boolean_t need_map_lookup;
3583	vm_map_copyin_args_t cont_args;
3584
3585	/*
3586	* If steal_pages is FALSE, this leaves busy pages in
3587	* the object. A continuation must be used if src_destroy
3588	* is true in this case (!steal_pages && src_destroy).
3589	*
3590	* XXX Still have a more general problem of what happens
3591	* XXX if the same page occurs twice in a list. Deadlock
3592	* XXX can happen if vm_fault_page was called. A
3593	* XXX possible solution is to use a continuation if vm_fault_page
3594	* XXX is called and we cross a map entry boundary.
3595	*/
3596
3597	/*
3598	* Check for copies of zero bytes.
3599	*/
3600
3601	if (len == 0) {
3602	*copy_result = VM_MAP_COPY_NULL((vm_map_copy_t) 0);
3603	return(KERN_SUCCESS0);
3604	}
3605
3606	/*
3607	* Compute start and end of region
3608	*/
3609
3610	src_start = trunc_page(src_addr)((vm_offset_t)(((vm_offset_t)(src_addr)) & ~((1 << 12 )-1)));
3611	src_end = round_page(src_addr + len)((vm_offset_t)((((vm_offset_t)(src_addr + len)) + ((1 << 12)-1)) & ~((1 << 12)-1)));
3612
3613	/*
3614	* Check that the end address doesn't overflow
3615	*/
3616
3617	if (src_end <= src_start && (src_end < src_start \|\| src_start != 0)) {
3618	return KERN_INVALID_ADDRESS1;
3619	}
3620
3621	/*
3622	* Allocate a header element for the page list.
3623	*
3624	* Record original offset and size, as caller may not
3625	* be page-aligned.
3626	*/
3627
3628	copy = (vm_map_copy_t) kmem_cache_alloc(&vm_map_copy_cache);
3629	copy->type = VM_MAP_COPY_PAGE_LIST3;
3630	copy->cpy_npagesc_u.c_p.npages = 0;
3631	copy->offset = src_addr;
3632	copy->size = len;
3633	copy->cpy_contc_u.c_p.cont = ((kern_return_t (*)()) 0);
3634	copy->cpy_cont_argsc_u.c_p.cont_args = (char *) VM_MAP_COPYIN_ARGS_NULL((vm_map_copyin_args_t) 0);
3635
3636	/*
3637	* Find the beginning of the region.
3638	*/
3639
3640	do_map_lookup:
3641
3642	vm_map_lock(src_map)({ lock_write(&(src_map)->lock); (src_map)->timestamp ++; });
3643
3644	if (!vm_map_lookup_entry(src_map, src_start, &src_entry)) {
3645	result = KERN_INVALID_ADDRESS1;
3646	goto error;
3647	}
3648	need_map_lookup = FALSE((boolean_t) 0);
3649
3650	/*
3651	* Go through entries until we get to the end.
3652	*/
3653
3654	while (TRUE((boolean_t) 1)) {
3655
3656	if (! (src_entry->protection & VM_PROT_READ((vm_prot_t) 0x01))) {
3657	result = KERN_PROTECTION_FAILURE2;
3658	goto error;
3659	}
3660
3661	if (src_end > src_entry->vme_endlinks.end)
3662	src_size = src_entry->vme_endlinks.end - src_start;
3663	else
3664	src_size = src_end - src_start;
3665
3666	src_object = src_entry->object.vm_object;
3667	src_offset = src_entry->offset +
3668	(src_start - src_entry->vme_startlinks.start);
3669
3670	/*
3671	* If src_object is NULL, allocate it now;
3672	* we're going to fault on it shortly.
3673	*/
3674	if (src_object == VM_OBJECT_NULL((vm_object_t) 0)) {
3675	src_object = vm_object_allocate((vm_size_t)
3676	src_entry->vme_endlinks.end -
3677	src_entry->vme_startlinks.start);
3678	src_entry->object.vm_object = src_object;
3679	}
3680
3681	/*
3682	* Iterate over pages. Fault in ones that aren't present.
3683	*/
3684	src_last_offset = src_offset + src_size;
3685	for (; (src_offset < src_last_offset && !need_map_lookup);
3686	src_offset += PAGE_SIZE(1 << 12), src_start += PAGE_SIZE(1 << 12)) {
3687
3688	if (copy->cpy_npagesc_u.c_p.npages == VM_MAP_COPY_PAGE_LIST_MAX64) {
3689	make_continuation:
3690	/*
3691	* At this point we have the max number of
3692	* pages busy for this thread that we're
3693	* willing to allow. Stop here and record
3694	* arguments for the remainder. Note:
3695	* this means that this routine isn't atomic,
3696	* but that's the breaks. Note that only
3697	* the first vm_map_copy_t that comes back
3698	* from this routine has the right offset
3699	* and size; those from continuations are
3700	* page rounded, and short by the amount
3701	* already done.
3702	*
3703	* Reset src_end so the src_destroy
3704	* code at the bottom doesn't do
3705	* something stupid.
3706	*/
3707
3708	cont_args = (vm_map_copyin_args_t)
3709	kalloc(sizeof(vm_map_copyin_args_data_t));
3710	cont_args->map = src_map;
3711	vm_map_reference(src_map);
3712	cont_args->src_addr = src_start;
3713	cont_args->src_len = len - (src_start - src_addr);
3714	if (src_destroy) {
3715	cont_args->destroy_addr = cont_args->src_addr;
3716	cont_args->destroy_len = cont_args->src_len;
3717	}
3718	else {
3719	cont_args->destroy_addr = (vm_offset_t) 0;
3720	cont_args->destroy_len = (vm_offset_t) 0;
3721	}
3722	cont_args->steal_pages = steal_pages;
3723
3724	copy->cpy_cont_argsc_u.c_p.cont_args = (char *) cont_args;
3725	copy->cpy_contc_u.c_p.cont = vm_map_copyin_page_list_cont;
3726
3727	src_end = src_start;
3728	vm_map_clip_end(src_map, src_entry, src_end)({ if ((src_end) < (src_entry)->links.end) _vm_map_clip_end (&(src_map)->hdr,(src_entry),(src_end)); });
3729	break;
3730	}
3731
3732	/*
3733	* Try to find the page of data.
3734	*/
3735	vm_object_lock(src_object);
3736	vm_object_paging_begin(src_object)((src_object)->paging_in_progress++);
3737	if (((m = vm_page_lookup(src_object, src_offset)) !=
3738	VM_PAGE_NULL((vm_page_t) 0)) && !m->busy && !m->fictitious &&
3739	!m->absent && !m->error) {
3740
3741	/*
3742	* This is the page. Mark it busy
3743	* and keep the paging reference on
3744	* the object whilst we do our thing.
3745	*/
3746	m->busy = TRUE((boolean_t) 1);
3747
3748	/*
3749	* Also write-protect the page, so
3750	* that the map`s owner cannot change
3751	* the data. The busy bit will prevent
3752	* faults on the page from succeeding
3753	* until the copy is released; after
3754	* that, the page can be re-entered
3755	* as writable, since we didn`t alter
3756	* the map entry. This scheme is a
3757	* cheap copy-on-write.
3758	*
3759	* Don`t forget the protection and
3760	* the page_lock value!
3761	*
3762	* If the source is being destroyed
3763	* AND not shared writable, we don`t
3764	* have to protect the page, since
3765	* we will destroy the (only)
3766	* writable mapping later.
3767	*/
3768	if (!src_destroy \|\|
3769	src_object->use_shared_copy)
3770	{
3771	pmap_page_protect(m->phys_addr,
3772	src_entry->protection
3773	& ~m->page_lock
3774	& ~VM_PROT_WRITE((vm_prot_t) 0x02));
3775	}
3776
3777	}
3778	else {
3779	vm_prot_t result_prot;
3780	vm_page_t top_page;
3781	kern_return_t kr;
3782
3783	/*
3784	* Have to fault the page in; must
3785	* unlock the map to do so. While
3786	* the map is unlocked, anything
3787	* can happen, we must lookup the
3788	* map entry before continuing.
3789	*/
3790	vm_map_unlock(src_map)lock_done(&(src_map)->lock);
3791	need_map_lookup = TRUE((boolean_t) 1);
3792	retry:
3793	result_prot = VM_PROT_READ((vm_prot_t) 0x01);
3794
3795	kr = vm_fault_page(src_object, src_offset,
3796	VM_PROT_READ((vm_prot_t) 0x01), FALSE((boolean_t) 0), FALSE((boolean_t) 0),
3797	&result_prot, &m, &top_page,
3798	FALSE((boolean_t) 0), (void (*)()) 0);
3799	/*
3800	* Cope with what happened.
3801	*/
3802	switch (kr) {
3803	case VM_FAULT_SUCCESS0:
3804	break;
3805	case VM_FAULT_INTERRUPTED2: /* ??? */
3806	case VM_FAULT_RETRY1:
3807	vm_object_lock(src_object);
3808	vm_object_paging_begin(src_object)((src_object)->paging_in_progress++);
3809	goto retry;
3810	case VM_FAULT_MEMORY_SHORTAGE3:
3811	VM_PAGE_WAIT((void ()()) 0)vm_page_wait((void ()()) 0);
3812	vm_object_lock(src_object);
3813	vm_object_paging_begin(src_object)((src_object)->paging_in_progress++);
3814	goto retry;
3815	case VM_FAULT_FICTITIOUS_SHORTAGE4:
3816	vm_page_more_fictitious();
3817	vm_object_lock(src_object);
3818	vm_object_paging_begin(src_object)((src_object)->paging_in_progress++);
3819	goto retry;
3820	case VM_FAULT_MEMORY_ERROR5:
3821	/*
3822	* Something broke. If this
3823	* is a continuation, return
3824	* a partial result if possible,
3825	* else fail the whole thing.
3826	* In the continuation case, the
3827	* next continuation call will
3828	* get this error if it persists.
3829	*/
3830	vm_map_lock(src_map)({ lock_write(&(src_map)->lock); (src_map)->timestamp ++; });
3831	if (is_cont &&
3832	copy->cpy_npagesc_u.c_p.npages != 0)
3833	goto make_continuation;
3834
3835	result = KERN_MEMORY_ERROR10;
3836	goto error;
3837	}
3838
3839	if (top_page != VM_PAGE_NULL((vm_page_t) 0)) {
3840	vm_object_lock(src_object);
3841	VM_PAGE_FREE(top_page)({ ; vm_page_free(top_page); ; });
3842	vm_object_paging_end(src_object)({ ({ if (!((src_object)->paging_in_progress != 0)) Assert ("(src_object)->paging_in_progress != 0", "../vm/vm_map.c" , 3842); }); if (--(src_object)->paging_in_progress == 0) { ({ if ((src_object)->all_wanted & (1 << (2))) thread_wakeup_prim (((event_t)(((vm_offset_t) src_object) + (2))), ((boolean_t) 0 ), 0); (src_object)->all_wanted &= ~(1 << (2)); } ); } });
3843	vm_object_unlock(src_object);
3844	}
3845
3846	/*
3847	* We do not need to write-protect
3848	* the page, since it cannot have
3849	* been in the pmap (and we did not
3850	* enter it above). The busy bit
3851	* will protect the page from being
3852	* entered as writable until it is
3853	* unlocked.
3854	*/
3855
3856	}
3857
3858	/*
3859	* The page is busy, its object is locked, and
3860	* we have a paging reference on it. Either
3861	* the map is locked, or need_map_lookup is
3862	* TRUE.
3863	*
3864	* Put the page in the page list.
3865	*/
3866	copy->cpy_page_listc_u.c_p.page_list[copy->cpy_npagesc_u.c_p.npages++] = m;
3867	vm_object_unlock(m->object);
3868	}
3869
3870	/*
3871	* DETERMINE whether the entire region
3872	* has been copied.
3873	*/
3874	if (src_start >= src_end && src_end != 0) {
3875	if (need_map_lookup)
3876	vm_map_lock(src_map)({ lock_write(&(src_map)->lock); (src_map)->timestamp ++; });
3877	break;
3878	}
3879
3880	/*
3881	* If need_map_lookup is TRUE, have to start over with
3882	* another map lookup. Note that we dropped the map
3883	* lock (to call vm_fault_page) above only in this case.
3884	*/
3885	if (need_map_lookup)
3886	goto do_map_lookup;
3887
3888	/*
3889	* Verify that there are no gaps in the region
3890	*/
3891
3892	src_start = src_entry->vme_endlinks.end;
3893	src_entry = src_entry->vme_nextlinks.next;
3894	if (src_entry->vme_startlinks.start != src_start) {
3895	result = KERN_INVALID_ADDRESS1;
3896	goto error;
3897	}
3898	}
3899
3900	/*
3901	* If steal_pages is true, make sure all
3902	* pages in the copy are not in any object
3903	* We try to remove them from the original
3904	* object, but we may have to copy them.
3905	*
3906	* At this point every page in the list is busy
3907	* and holds a paging reference to its object.
3908	* When we're done stealing, every page is busy,
3909	* and in no object (m->tabled == FALSE).
3910	*/
3911	src_start = trunc_page(src_addr)((vm_offset_t)(((vm_offset_t)(src_addr)) & ~((1 << 12 )-1)));
3912	if (steal_pages) {
3913	int i;
3914	vm_offset_t unwire_end;
3915
3916	unwire_end = src_start;
3917	for (i = 0; i < copy->cpy_npagesc_u.c_p.npages; i++) {
3918
3919	/*
3920	* Remove the page from its object if it
3921	* can be stolen. It can be stolen if:
3922	*
3923	* (1) The source is being destroyed,
3924	* the object is temporary, and
3925	* not shared.
3926	* (2) The page is not precious.
3927	*
3928	* The not shared check consists of two
3929	* parts: (a) there are no objects that
3930	* shadow this object. (b) it is not the
3931	* object in any shared map entries (i.e.,
3932	* use_shared_copy is not set).
3933	*
3934	* The first check (a) means that we can't
3935	* steal pages from objects that are not
3936	* at the top of their shadow chains. This
3937	* should not be a frequent occurrence.
3938	*
3939	* Stealing wired pages requires telling the
3940	* pmap module to let go of them.
3941	*
3942	* NOTE: stealing clean pages from objects
3943	* whose mappings survive requires a call to
3944	* the pmap module. Maybe later.
3945	*/
3946	m = copy->cpy_page_listc_u.c_p.page_list[i];
3947	src_object = m->object;
3948	vm_object_lock(src_object);
3949
3950	if (src_destroy &&
3951	src_object->temporary &&
3952	(!src_object->shadowed) &&
3953	(!src_object->use_shared_copy) &&
3954	!m->precious) {
3955	vm_offset_t page_vaddr;
3956
3957	page_vaddr = src_start + (i * PAGE_SIZE(1 << 12));
3958	if (m->wire_count > 0) {
3959
3960	assert(m->wire_count == 1)({ if (!(m->wire_count == 1)) Assert("m->wire_count == 1" , "../vm/vm_map.c", 3960); });
3961	/*
3962	* In order to steal a wired
3963	* page, we have to unwire it
3964	* first. We do this inline
3965	* here because we have the page.
3966	*
3967	* Step 1: Unwire the map entry.
3968	* Also tell the pmap module
3969	* that this piece of the
3970	* pmap is pageable.
3971	*/
3972	vm_object_unlock(src_object);
3973	if (page_vaddr >= unwire_end) {
3974	if (!vm_map_lookup_entry(src_map,
3975	page_vaddr, &src_entry))
3976	panic("vm_map_copyin_page_list: missing wired map entry");
3977
3978	vm_map_clip_start(src_map, src_entry,({ if ((page_vaddr) > (src_entry)->links.start) _vm_map_clip_start (&(src_map)->hdr,(src_entry),(page_vaddr)); })
3979	page_vaddr)({ if ((page_vaddr) > (src_entry)->links.start) _vm_map_clip_start (&(src_map)->hdr,(src_entry),(page_vaddr)); });
3980	vm_map_clip_end(src_map, src_entry,({ if ((src_start + src_size) < (src_entry)->links.end) _vm_map_clip_end(&(src_map)->hdr,(src_entry),(src_start + src_size)); })
3981	src_start + src_size)({ if ((src_start + src_size) < (src_entry)->links.end) _vm_map_clip_end(&(src_map)->hdr,(src_entry),(src_start + src_size)); });
3982
3983	assert(src_entry->wired_count > 0)({ if (!(src_entry->wired_count > 0)) Assert("src_entry->wired_count > 0" , "../vm/vm_map.c", 3983); });
3984	src_entry->wired_count = 0;
3985	src_entry->user_wired_count = 0;
3986	unwire_end = src_entry->vme_endlinks.end;
3987	pmap_pageable(vm_map_pmap(src_map)((src_map)->pmap),
3988	page_vaddr, unwire_end, TRUE((boolean_t) 1));
3989	}
3990
3991	/*
3992	* Step 2: Unwire the page.
3993	* pmap_remove handles this for us.
3994	*/
3995	vm_object_lock(src_object);
3996	}
3997
3998	/*
3999	* Don't need to remove the mapping;
4000	* vm_map_delete will handle it.
4001	*
4002	* Steal the page. Setting the wire count
4003	* to zero is vm_page_unwire without
4004	* activating the page.
4005	*/
4006	vm_page_lock_queues();
4007	vm_page_remove(m);
4008	if (m->wire_count > 0) {
4009	m->wire_count = 0;
4010	vm_page_wire_count--;
4011	} else {
4012	VM_PAGE_QUEUES_REMOVE(m)({ if (m->active) { { queue_entry_t next, prev; next = (m) ->pageq.next; prev = (m)->pageq.prev; if ((&vm_page_queue_active ) == next) (&vm_page_queue_active)->prev = prev; else ( (vm_page_t)next)->pageq.prev = prev; if ((&vm_page_queue_active ) == prev) (&vm_page_queue_active)->next = next; else ( (vm_page_t)prev)->pageq.next = next; }; m->active = ((boolean_t ) 0); vm_page_active_count--; } if (m->inactive) { { queue_entry_t next, prev; next = (m)->pageq.next; prev = (m)->pageq. prev; if ((&vm_page_queue_inactive) == next) (&vm_page_queue_inactive )->prev = prev; else ((vm_page_t)next)->pageq.prev = prev ; if ((&vm_page_queue_inactive) == prev) (&vm_page_queue_inactive )->next = next; else ((vm_page_t)prev)->pageq.next = next ; }; m->inactive = ((boolean_t) 0); vm_page_inactive_count --; } });
4013	}
4014	vm_page_unlock_queues();
4015	}
4016	else {
4017	/*
4018	* Have to copy this page. Have to
4019	* unlock the map while copying,
4020	* hence no further page stealing.
4021	* Hence just copy all the pages.
4022	* Unlock the map while copying;
4023	* This means no further page stealing.
4024	*/
4025	vm_object_unlock(src_object);
4026	vm_map_unlock(src_map)lock_done(&(src_map)->lock);
4027
4028	vm_map_copy_steal_pages(copy);
4029
4030	vm_map_lock(src_map)({ lock_write(&(src_map)->lock); (src_map)->timestamp ++; });
4031	break;
4032	}
4033
4034	vm_object_paging_end(src_object)({ ({ if (!((src_object)->paging_in_progress != 0)) Assert ("(src_object)->paging_in_progress != 0", "../vm/vm_map.c" , 4034); }); if (--(src_object)->paging_in_progress == 0) { ({ if ((src_object)->all_wanted & (1 << (2))) thread_wakeup_prim (((event_t)(((vm_offset_t) src_object) + (2))), ((boolean_t) 0 ), 0); (src_object)->all_wanted &= ~(1 << (2)); } ); } });
4035	vm_object_unlock(src_object);
4036	}
4037
4038	/*
4039	* If the source should be destroyed, do it now, since the
4040	* copy was successful.
4041	*/
4042
4043	if (src_destroy) {
4044	(void) vm_map_delete(src_map, src_start, src_end);
4045	}
4046	}
4047	else {
4048	/*
4049	* !steal_pages leaves busy pages in the map.
4050	* This will cause src_destroy to hang. Use
4051	* a continuation to prevent this.
4052	*/
4053	if (src_destroy && !vm_map_copy_has_cont(copy)(((copy)->c_u.c_p.cont) != (kern_return_t (*)()) 0)) {
4054	cont_args = (vm_map_copyin_args_t)
4055	kalloc(sizeof(vm_map_copyin_args_data_t));
4056	vm_map_reference(src_map);
4057	cont_args->map = src_map;
4058	cont_args->src_addr = (vm_offset_t) 0;
4059	cont_args->src_len = (vm_size_t) 0;
4060	cont_args->destroy_addr = src_start;
4061	cont_args->destroy_len = src_end - src_start;
4062	cont_args->steal_pages = FALSE((boolean_t) 0);
4063
4064	copy->cpy_cont_argsc_u.c_p.cont_args = (char *) cont_args;
4065	copy->cpy_contc_u.c_p.cont = vm_map_copyin_page_list_cont;
4066	}
4067
4068	}
4069
4070	vm_map_unlock(src_map)lock_done(&(src_map)->lock);
4071
4072	*copy_result = copy;
4073	return(result);
4074
4075	error:
4076	vm_map_unlock(src_map)lock_done(&(src_map)->lock);
4077	vm_map_copy_discard(copy);
4078	return(result);
4079	}
4080
4081	/*
4082	* vm_map_fork:
4083	*
4084	* Create and return a new map based on the old
4085	* map, according to the inheritance values on the
4086	* regions in that map.
4087	*
4088	* The source map must not be locked.
4089	*/
4090	vm_map_t vm_map_fork(old_map)
4091	vm_map_t old_map;
4092	{
4093	vm_map_t new_map;
4094	vm_map_entry_t old_entry;
4095	vm_map_entry_t new_entry;
4096	pmap_t new_pmap = pmap_create((vm_size_t) 0);
4097	vm_size_t new_size = 0;
4098	vm_size_t entry_size;
4099	vm_object_t object;
4100
4101	vm_map_lock(old_map)({ lock_write(&(old_map)->lock); (old_map)->timestamp ++; });
4102
4103	new_map = vm_map_create(new_pmap,
4104	old_map->min_offsethdr.links.start,
4105	old_map->max_offsethdr.links.end,
4106	old_map->hdr.entries_pageable);
4107
4108	for (
4109	old_entry = vm_map_first_entry(old_map)((old_map)->hdr.links.next);
4110	old_entry != vm_map_to_entry(old_map)((struct vm_map_entry *) &(old_map)->hdr.links);
4111	) {
4112	if (old_entry->is_sub_map)
4113	panic("vm_map_fork: encountered a submap");
4114
4115	entry_size = (old_entry->vme_endlinks.end - old_entry->vme_startlinks.start);
4116
4117	switch (old_entry->inheritance) {
4118	case VM_INHERIT_NONE((vm_inherit_t) 2):
4119	break;
4120
4121	case VM_INHERIT_SHARE((vm_inherit_t) 0):
4122	/*
4123	* New sharing code. New map entry
4124	* references original object. Temporary
4125	* objects use asynchronous copy algorithm for
4126	* future copies. First make sure we have
4127	* the right object. If we need a shadow,
4128	* or someone else already has one, then
4129	* make a new shadow and share it.
4130	*/
4131
4132	object = old_entry->object.vm_object;
4133	if (object == VM_OBJECT_NULL((vm_object_t) 0)) {
4134	object = vm_object_allocate(
4135	(vm_size_t)(old_entry->vme_endlinks.end -
4136	old_entry->vme_startlinks.start));
4137	old_entry->offset = 0;
4138	old_entry->object.vm_object = object;
4139	assert(!old_entry->needs_copy)({ if (!(!old_entry->needs_copy)) Assert("!old_entry->needs_copy" , "../vm/vm_map.c", 4139); });
4140	}
4141	else if (old_entry->needs_copy \|\| object->shadowed \|\|
4142	(object->temporary && !old_entry->is_shared &&
4143	object->size > (vm_size_t)(old_entry->vme_endlinks.end -
4144	old_entry->vme_startlinks.start))) {
4145
4146	assert(object->temporary)({ if (!(object->temporary)) Assert("object->temporary" , "../vm/vm_map.c", 4146); });
4147	assert(!(object->shadowed && old_entry->is_shared))({ if (!(!(object->shadowed && old_entry->is_shared ))) Assert("!(object->shadowed && old_entry->is_shared)" , "../vm/vm_map.c", 4147); });
4148	vm_object_shadow(
4149	&old_entry->object.vm_object,
4150	&old_entry->offset,
4151	(vm_size_t) (old_entry->vme_endlinks.end -
4152	old_entry->vme_startlinks.start));
4153
4154	/*
4155	* If we're making a shadow for other than
4156	* copy on write reasons, then we have
4157	* to remove write permission.
4158	*/
4159
4160	if (!old_entry->needs_copy &&
4161	(old_entry->protection & VM_PROT_WRITE((vm_prot_t) 0x02))) {
4162	pmap_protect(vm_map_pmap(old_map)((old_map)->pmap),
4163	old_entry->vme_startlinks.start,
4164	old_entry->vme_endlinks.end,
4165	old_entry->protection &
4166	~VM_PROT_WRITE((vm_prot_t) 0x02));
4167	}
4168	old_entry->needs_copy = FALSE((boolean_t) 0);
4169	object = old_entry->object.vm_object;
4170	}
4171
4172	/*
4173	* Set use_shared_copy to indicate that
4174	* object must use shared (delayed) copy-on
4175	* write. This is ignored for permanent objects.
4176	* Bump the reference count for the new entry
4177	*/
4178
4179	vm_object_lock(object);
4180	object->use_shared_copy = TRUE((boolean_t) 1);
4181	object->ref_count++;
4182	vm_object_unlock(object);
4183
4184	new_entry = vm_map_entry_create(new_map)_vm_map_entry_create(&(new_map)->hdr);
4185
4186	if (old_entry->projected_on != 0) {
4187	/*
4188	* If entry is projected buffer, clone the
4189	* entry exactly.
4190	*/
4191
4192	vm_map_entry_copy_full(new_entry, old_entry)((new_entry) = (old_entry));
4193
4194	} else {
4195	/*
4196	* Clone the entry, using object ref from above.
4197	* Mark both entries as shared.
4198	*/
4199
4200	vm_map_entry_copy(new_entry, old_entry)({ (new_entry) = (old_entry); (new_entry)->is_shared = ( (boolean_t) 0); (new_entry)->needs_wakeup = ((boolean_t) 0 ); (new_entry)->in_transition = ((boolean_t) 0); (new_entry )->wired_count = 0; (new_entry)->user_wired_count = 0; } );
4201	old_entry->is_shared = TRUE((boolean_t) 1);
4202	new_entry->is_shared = TRUE((boolean_t) 1);
4203	}
4204
4205	/*
4206	* Insert the entry into the new map -- we
4207	* know we're inserting at the end of the new
4208	* map.
4209	*/
4210
4211	vm_map_entry_link(({ (&(new_map)->hdr)->nentries++; (new_entry)->links .prev = (((new_map)->hdr.links.prev)); (new_entry)->links .next = (((new_map)->hdr.links.prev))->links.next; (new_entry )->links.prev->links.next = (new_entry)->links.next-> links.prev = (new_entry); ({ struct rbtree_node ___cur, ___prev ; int ___diff, ___index; ___prev = ((void ) 0); ___index = - 1; ___cur = (&(&(new_map)->hdr)->tree)->root ; while (___cur != ((void ) 0)) { ___diff = vm_map_entry_cmp_insert (&(new_entry)->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c", 4214); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur-> children[___index]; } rbtree_insert_rebalance(&(&(new_map )->hdr)->tree, ___prev, ___index, &(new_entry)-> tree_node); }); })
4212	new_map,({ (&(new_map)->hdr)->nentries++; (new_entry)->links .prev = (((new_map)->hdr.links.prev)); (new_entry)->links .next = (((new_map)->hdr.links.prev))->links.next; (new_entry )->links.prev->links.next = (new_entry)->links.next-> links.prev = (new_entry); ({ struct rbtree_node ___cur, ___prev ; int ___diff, ___index; ___prev = ((void ) 0); ___index = - 1; ___cur = (&(&(new_map)->hdr)->tree)->root ; while (___cur != ((void ) 0)) { ___diff = vm_map_entry_cmp_insert (&(new_entry)->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c", 4214); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur-> children[___index]; } rbtree_insert_rebalance(&(&(new_map )->hdr)->tree, ___prev, ___index, &(new_entry)-> tree_node); }); })
4213	vm_map_last_entry(new_map),({ (&(new_map)->hdr)->nentries++; (new_entry)->links .prev = (((new_map)->hdr.links.prev)); (new_entry)->links .next = (((new_map)->hdr.links.prev))->links.next; (new_entry )->links.prev->links.next = (new_entry)->links.next-> links.prev = (new_entry); ({ struct rbtree_node ___cur, ___prev ; int ___diff, ___index; ___prev = ((void ) 0); ___index = - 1; ___cur = (&(&(new_map)->hdr)->tree)->root ; while (___cur != ((void ) 0)) { ___diff = vm_map_entry_cmp_insert (&(new_entry)->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c", 4214); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur-> children[___index]; } rbtree_insert_rebalance(&(&(new_map )->hdr)->tree, ___prev, ___index, &(new_entry)-> tree_node); }); })
4214	new_entry)({ (&(new_map)->hdr)->nentries++; (new_entry)->links .prev = (((new_map)->hdr.links.prev)); (new_entry)->links .next = (((new_map)->hdr.links.prev))->links.next; (new_entry )->links.prev->links.next = (new_entry)->links.next-> links.prev = (new_entry); ({ struct rbtree_node ___cur, ___prev ; int ___diff, ___index; ___prev = ((void ) 0); ___index = - 1; ___cur = (&(&(new_map)->hdr)->tree)->root ; while (___cur != ((void ) 0)) { ___diff = vm_map_entry_cmp_insert (&(new_entry)->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c", 4214); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur-> children[___index]; } rbtree_insert_rebalance(&(&(new_map )->hdr)->tree, ___prev, ___index, &(new_entry)-> tree_node); }); });
4215
4216	/*
4217	* Update the physical map
4218	*/
4219
4220	pmap_copy(new_map->pmap, old_map->pmap,
4221	new_entry->vme_start,
4222	entry_size,
4223	old_entry->vme_start);
4224
4225	new_size += entry_size;
4226	break;
4227
4228	case VM_INHERIT_COPY((vm_inherit_t) 1):
4229	if (old_entry->wired_count == 0) {
4230	boolean_t src_needs_copy;
4231	boolean_t new_entry_needs_copy;
4232
4233	new_entry = vm_map_entry_create(new_map)_vm_map_entry_create(&(new_map)->hdr);
4234	vm_map_entry_copy(new_entry, old_entry)({ (new_entry) = (old_entry); (new_entry)->is_shared = ( (boolean_t) 0); (new_entry)->needs_wakeup = ((boolean_t) 0 ); (new_entry)->in_transition = ((boolean_t) 0); (new_entry )->wired_count = 0; (new_entry)->user_wired_count = 0; } );
4235
4236	if (vm_object_copy_temporary(
4237	&new_entry->object.vm_object,
4238	&new_entry->offset,
4239	&src_needs_copy,
4240	&new_entry_needs_copy)) {
4241
4242	/*
4243	* Handle copy-on-write obligations
4244	*/
4245
4246	if (src_needs_copy && !old_entry->needs_copy) {
4247	vm_object_pmap_protect(
4248	old_entry->object.vm_object,
4249	old_entry->offset,
4250	entry_size,
4251	(old_entry->is_shared ?
4252	PMAP_NULL((pmap_t) 0) :
4253	old_map->pmap),
4254	old_entry->vme_startlinks.start,
4255	old_entry->protection &
4256	~VM_PROT_WRITE((vm_prot_t) 0x02));
4257
4258	old_entry->needs_copy = TRUE((boolean_t) 1);
4259	}
4260
4261	new_entry->needs_copy = new_entry_needs_copy;
4262
4263	/*
4264	* Insert the entry at the end
4265	* of the map.
4266	*/
4267
4268	vm_map_entry_link(new_map,({ (&(new_map)->hdr)->nentries++; (new_entry)->links .prev = (((new_map)->hdr.links.prev)); (new_entry)->links .next = (((new_map)->hdr.links.prev))->links.next; (new_entry )->links.prev->links.next = (new_entry)->links.next-> links.prev = (new_entry); ({ struct rbtree_node ___cur, ___prev ; int ___diff, ___index; ___prev = ((void ) 0); ___index = - 1; ___cur = (&(&(new_map)->hdr)->tree)->root ; while (___cur != ((void ) 0)) { ___diff = vm_map_entry_cmp_insert (&(new_entry)->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c", 4270); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur-> children[___index]; } rbtree_insert_rebalance(&(&(new_map )->hdr)->tree, ___prev, ___index, &(new_entry)-> tree_node); }); })
4269	vm_map_last_entry(new_map),({ (&(new_map)->hdr)->nentries++; (new_entry)->links .prev = (((new_map)->hdr.links.prev)); (new_entry)->links .next = (((new_map)->hdr.links.prev))->links.next; (new_entry )->links.prev->links.next = (new_entry)->links.next-> links.prev = (new_entry); ({ struct rbtree_node ___cur, ___prev ; int ___diff, ___index; ___prev = ((void ) 0); ___index = - 1; ___cur = (&(&(new_map)->hdr)->tree)->root ; while (___cur != ((void ) 0)) { ___diff = vm_map_entry_cmp_insert (&(new_entry)->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c", 4270); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur-> children[___index]; } rbtree_insert_rebalance(&(&(new_map )->hdr)->tree, ___prev, ___index, &(new_entry)-> tree_node); }); })
4270	new_entry)({ (&(new_map)->hdr)->nentries++; (new_entry)->links .prev = (((new_map)->hdr.links.prev)); (new_entry)->links .next = (((new_map)->hdr.links.prev))->links.next; (new_entry )->links.prev->links.next = (new_entry)->links.next-> links.prev = (new_entry); ({ struct rbtree_node ___cur, ___prev ; int ___diff, ___index; ___prev = ((void ) 0); ___index = - 1; ___cur = (&(&(new_map)->hdr)->tree)->root ; while (___cur != ((void ) 0)) { ___diff = vm_map_entry_cmp_insert (&(new_entry)->tree_node, ___cur); ({ if (!(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c", 4270); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur-> children[___index]; } rbtree_insert_rebalance(&(&(new_map )->hdr)->tree, ___prev, ___index, &(new_entry)-> tree_node); }); });
4271
4272
4273	new_size += entry_size;
4274	break;
4275	}
4276
4277	vm_map_entry_dispose(new_map, new_entry)_vm_map_entry_dispose(&(new_map)->hdr, (new_entry));
4278	}
4279
4280	/* INNER BLOCK (copy cannot be optimized) */ {
4281
4282	vm_offset_t start = old_entry->vme_startlinks.start;
4283	vm_map_copy_t copy;
4284	vm_map_entry_t last = vm_map_last_entry(new_map)((new_map)->hdr.links.prev);
4285
4286	vm_map_unlock(old_map)lock_done(&(old_map)->lock);
4287	if (vm_map_copyin(old_map,
4288	start,
4289	entry_size,
4290	FALSE((boolean_t) 0),
4291	&copy)
4292	!= KERN_SUCCESS0) {
4293	vm_map_lock(old_map)({ lock_write(&(old_map)->lock); (old_map)->timestamp ++; });
4294	if (!vm_map_lookup_entry(old_map, start, &last))
4295	last = last->vme_nextlinks.next;
4296	old_entry = last;
4297	/*
4298	* For some error returns, want to
4299	* skip to the next element.
4300	*/
4301
4302	continue;
4303	}
4304
4305	/*
4306	* Insert the copy into the new map
4307	*/
4308
4309	vm_map_copy_insert(new_map, last, copy)({ struct rbtree_node node, tmp; for (node = rbtree_postwalk_deepest (&(copy)->c_u.hdr.tree), tmp = rbtree_postwalk_unlink( node); node != ((void ) 0); node = tmp, tmp = rbtree_postwalk_unlink (node)) ({ struct rbtree_node ___cur, ___prev; int ___diff, ___index; ___prev = ((void ) 0); ___index = -1; ___cur = (& (new_map)->hdr.tree)->root; while (___cur != ((void *) 0 )) { ___diff = vm_map_entry_cmp_insert(node, ___cur); ({ if ( !(___diff != 0)) Assert("___diff != 0", "../vm/vm_map.c", 4309 ); }); ___prev = ___cur; ___index = rbtree_d2i(___diff); ___cur = ___cur->children[___index]; } rbtree_insert_rebalance(& (new_map)->hdr.tree, ___prev, ___index, node); }); (((last )->links.next)->links.prev = ((copy)->c_u.hdr.links. prev)) ->links.next = ((last)->links.next); ((last)-> links.next = ((copy)->c_u.hdr.links.next)) ->links.prev = (last); (new_map)->hdr.nentries += (copy)->c_u.hdr.nentries ; kmem_cache_free(&vm_map_copy_cache, (vm_offset_t) copy) ; });
4310	new_size += entry_size;
4311
4312	/*
4313	* Pick up the traversal at the end of
4314	* the copied region.
4315	*/
4316
4317	vm_map_lock(old_map)({ lock_write(&(old_map)->lock); (old_map)->timestamp ++; });
4318	start += entry_size;
4319	if (!vm_map_lookup_entry(old_map, start, &last))
4320	last = last->vme_nextlinks.next;
4321	else
4322	vm_map_clip_start(old_map, last, start)({ if ((start) > (last)->links.start) _vm_map_clip_start (&(old_map)->hdr,(last),(start)); });
4323	old_entry = last;
4324
4325	continue;
4326	/* INNER BLOCK (copy cannot be optimized) */ }
4327	}
4328	old_entry = old_entry->vme_nextlinks.next;
4329	}
4330
4331	new_map->size = new_size;
4332	vm_map_unlock(old_map)lock_done(&(old_map)->lock);
4333
4334	return(new_map);
4335	}
4336
4337	/*
4338	* vm_map_lookup:
4339	*
4340	* Finds the VM object, offset, and
4341	* protection for a given virtual address in the
4342	* specified map, assuming a page fault of the
4343	* type specified.
4344	*
4345	* Returns the (object, offset, protection) for
4346	* this address, whether it is wired down, and whether
4347	* this map has the only reference to the data in question.
4348	* In order to later verify this lookup, a "version"
4349	* is returned.
4350	*
4351	* The map should not be locked; it will not be
4352	* locked on exit. In order to guarantee the
4353	* existence of the returned object, it is returned
4354	* locked.
4355	*
4356	* If a lookup is requested with "write protection"
4357	* specified, the map may be changed to perform virtual
4358	* copying operations, although the data referenced will
4359	* remain the same.
4360	*/
4361	kern_return_t vm_map_lookup(var_map, vaddr, fault_type, out_version,
4362	object, offset, out_prot, wired)
4363	vm_map_t var_map; / IN/OUT */
4364	vm_offset_t vaddr;
4365	vm_prot_t fault_type;
4366
4367	vm_map_version_t out_version; / OUT */
4368	vm_object_t object; / OUT */
4369	vm_offset_t offset; / OUT */
4370	vm_prot_t out_prot; / OUT */
4371	boolean_t wired; / OUT */
4372	{
4373	vm_map_entry_t entry;
4374	vm_map_t map = *var_map;
4375	vm_prot_t prot;
4376
4377	RetryLookup: ;
4378
4379	/*
4380	* Lookup the faulting address.
4381	*/
4382
4383	vm_map_lock_read(map)lock_read(&(map)->lock);
4384
4385	#define RETURN(why) \
4386	{ \
4387	vm_map_unlock_read(map)lock_done(&(map)->lock); \
4388	return(why); \
4389	}
4390
4391	/*
4392	* If the map has an interesting hint, try it before calling
4393	* full blown lookup routine.
4394	*/
4395
4396	simple_lock(&map->hint_lock);
4397	entry = map->hint;
4398	simple_unlock(&map->hint_lock);
4399
4400	if ((entry == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) \|\|
4401	(vaddr < entry->vme_startlinks.start) \|\| (vaddr >= entry->vme_endlinks.end)) {
4402	vm_map_entry_t tmp_entry;
4403
4404	/*
4405	* Entry was either not a valid hint, or the vaddr
4406	* was not contained in the entry, so do a full lookup.
4407	*/
4408	if (!vm_map_lookup_entry(map, vaddr, &tmp_entry))
4409	RETURN(KERN_INVALID_ADDRESS1);
4410
4411	entry = tmp_entry;
4412	}
4413
4414	/*
4415	* Handle submaps.
4416	*/
4417
4418	if (entry->is_sub_map) {
4419	vm_map_t old_map = map;
4420
4421	*var_map = map = entry->object.sub_map;
4422	vm_map_unlock_read(old_map)lock_done(&(old_map)->lock);
4423	goto RetryLookup;
4424	}
4425
4426	/*
4427	* Check whether this task is allowed to have
4428	* this page.
4429	*/
4430
4431	prot = entry->protection;
4432
4433	if ((fault_type & (prot)) != fault_type) {
4434	if ((prot & VM_PROT_NOTIFY((vm_prot_t) 0x10)) && (fault_type & VM_PROT_WRITE((vm_prot_t) 0x02))) {
4435	RETURN(KERN_WRITE_PROTECTION_FAILURE24);
4436	} else {
4437	RETURN(KERN_PROTECTION_FAILURE2);
4438	}
4439	}
4440
4441	/*
4442	* If this page is not pageable, we have to get
4443	* it for all possible accesses.
4444	*/
4445
4446	if ((*wired = (entry->wired_count != 0)))
4447	prot = fault_type = entry->protection;
4448
4449	/*
4450	* If the entry was copy-on-write, we either ...
4451	*/
4452
4453	if (entry->needs_copy) {
4454	/*
4455	* If we want to write the page, we may as well
4456	* handle that now since we've got the map locked.
4457	*
4458	* If we don't need to write the page, we just
4459	* demote the permissions allowed.
4460	*/
4461
4462	if (fault_type & VM_PROT_WRITE((vm_prot_t) 0x02)) {
4463	/*
4464	* Make a new object, and place it in the
4465	* object chain. Note that no new references
4466	* have appeared -- one just moved from the
4467	* map to the new object.
4468	*/
4469
4470	if (vm_map_lock_read_to_write(map)(lock_read_to_write(&(map)->lock) \|\| (((map)->timestamp ++), 0))) {
4471	goto RetryLookup;
4472	}
4473	map->timestamp++;
4474
4475	vm_object_shadow(
4476	&entry->object.vm_object,
4477	&entry->offset,
4478	(vm_size_t) (entry->vme_endlinks.end - entry->vme_startlinks.start));
4479
4480	entry->needs_copy = FALSE((boolean_t) 0);
4481
4482	vm_map_lock_write_to_read(map)lock_write_to_read(&(map)->lock);
4483	}
4484	else {
4485	/*
4486	* We're attempting to read a copy-on-write
4487	* page -- don't allow writes.
4488	*/
4489
4490	prot &= (~VM_PROT_WRITE((vm_prot_t) 0x02));
4491	}
4492	}
4493
4494	/*
4495	* Create an object if necessary.
4496	*/
4497	if (entry->object.vm_object == VM_OBJECT_NULL((vm_object_t) 0)) {
4498
4499	if (vm_map_lock_read_to_write(map)(lock_read_to_write(&(map)->lock) \|\| (((map)->timestamp ++), 0))) {
4500	goto RetryLookup;
4501	}
4502
4503	entry->object.vm_object = vm_object_allocate(
4504	(vm_size_t)(entry->vme_endlinks.end - entry->vme_startlinks.start));
4505	entry->offset = 0;
4506	vm_map_lock_write_to_read(map)lock_write_to_read(&(map)->lock);
4507	}
4508
4509	/*
4510	* Return the object/offset from this entry. If the entry
4511	* was copy-on-write or empty, it has been fixed up. Also
4512	* return the protection.
4513	*/
4514
4515	*offset = (vaddr - entry->vme_startlinks.start) + entry->offset;
4516	*object = entry->object.vm_object;
4517	*out_prot = prot;
4518
4519	/*
4520	* Lock the object to prevent it from disappearing
4521	*/
4522
4523	vm_object_lock(*object);
4524
4525	/*
4526	* Save the version number and unlock the map.
4527	*/
4528
4529	out_version->main_timestamp = map->timestamp;
4530
4531	RETURN(KERN_SUCCESS0);
4532
4533	#undef RETURN
4534	}
4535
4536	/*
4537	* vm_map_verify:
4538	*
4539	* Verifies that the map in question has not changed
4540	* since the given version. If successful, the map
4541	* will not change until vm_map_verify_done() is called.
4542	*/
4543	boolean_t vm_map_verify(map, version)
4544	vm_map_t map;
4545	vm_map_version_t version; / REF */
4546	{
4547	boolean_t result;
4548
4549	vm_map_lock_read(map)lock_read(&(map)->lock);
4550	result = (map->timestamp == version->main_timestamp);
4551
4552	if (!result)
4553	vm_map_unlock_read(map)lock_done(&(map)->lock);
4554
4555	return(result);
4556	}
4557
4558	/*
4559	* vm_map_verify_done:
4560	*
4561	* Releases locks acquired by a vm_map_verify.
4562	*
4563	* This is now a macro in vm/vm_map.h. It does a
4564	* vm_map_unlock_read on the map.
4565	*/
4566
4567	/*
4568	* vm_region:
4569	*
4570	* User call to obtain information about a region in
4571	* a task's address map.
4572	*/
4573
4574	kern_return_t vm_region(map, address, size,
4575	protection, max_protection,
4576	inheritance, is_shared,
4577	object_name, offset_in_object)
4578	vm_map_t map;
4579	vm_offset_t address; / IN/OUT */
4580	vm_size_t size; / OUT */
4581	vm_prot_t protection; / OUT */
4582	vm_prot_t max_protection; / OUT */
4583	vm_inherit_t inheritance; / OUT */
4584	boolean_t is_shared; / OUT */
4585	ipc_port_t object_name; / OUT */
4586	vm_offset_t offset_in_object; / OUT */
4587	{
4588	vm_map_entry_t tmp_entry;
4589	vm_map_entry_t entry;
4590	vm_offset_t tmp_offset;
4591	vm_offset_t start;
4592
4593	if (map == VM_MAP_NULL((vm_map_t) 0))
4594	return(KERN_INVALID_ARGUMENT4);
4595
4596	start = *address;
4597
4598	vm_map_lock_read(map)lock_read(&(map)->lock);
4599	if (!vm_map_lookup_entry(map, start, &tmp_entry)) {
4600	if ((entry = tmp_entry->vme_nextlinks.next) == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) {
4601	vm_map_unlock_read(map)lock_done(&(map)->lock);
4602	return(KERN_NO_SPACE3);
4603	}
4604	} else {
4605	entry = tmp_entry;
4606	}
4607
4608	start = entry->vme_startlinks.start;
4609	*protection = entry->protection;
4610	*max_protection = entry->max_protection;
4611	*inheritance = entry->inheritance;
4612	*address = start;
4613	*size = (entry->vme_endlinks.end - start);
4614
4615	tmp_offset = entry->offset;
4616
4617
4618	if (entry->is_sub_map) {
4619	*is_shared = FALSE((boolean_t) 0);
4620	*object_name = IP_NULL((ipc_port_t) ((ipc_object_t) 0));
4621	*offset_in_object = tmp_offset;
4622	} else {
4623	*is_shared = entry->is_shared;
4624	*object_name = vm_object_name(entry->object.vm_object);
4625	*offset_in_object = tmp_offset;
4626	}
4627
4628	vm_map_unlock_read(map)lock_done(&(map)->lock);
4629
4630	return(KERN_SUCCESS0);
4631	}
4632
4633	/*
4634	* Routine: vm_map_simplify
4635	*
4636	* Description:
4637	* Attempt to simplify the map representation in
4638	* the vicinity of the given starting address.
4639	* Note:
4640	* This routine is intended primarily to keep the
4641	* kernel maps more compact -- they generally don't
4642	* benefit from the "expand a map entry" technology
4643	* at allocation time because the adjacent entry
4644	* is often wired down.
4645	*/
4646	void vm_map_simplify(map, start)
4647	vm_map_t map;
4648	vm_offset_t start;
4649	{
4650	vm_map_entry_t this_entry;
4651	vm_map_entry_t prev_entry;
4652
4653	vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
4654	if (
4655	(vm_map_lookup_entry(map, start, &this_entry)) &&
4656	((prev_entry = this_entry->vme_prevlinks.prev) != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) &&
4657
4658	(prev_entry->vme_endlinks.end == start) &&
4659
4660	(prev_entry->is_shared == FALSE((boolean_t) 0)) &&
4661	(prev_entry->is_sub_map == FALSE((boolean_t) 0)) &&
4662
4663	(this_entry->is_shared == FALSE((boolean_t) 0)) &&
4664	(this_entry->is_sub_map == FALSE((boolean_t) 0)) &&
4665
4666	(prev_entry->inheritance == this_entry->inheritance) &&
4667	(prev_entry->protection == this_entry->protection) &&
4668	(prev_entry->max_protection == this_entry->max_protection) &&
4669	(prev_entry->wired_count == this_entry->wired_count) &&
4670	(prev_entry->user_wired_count == this_entry->user_wired_count) &&
4671
4672	(prev_entry->needs_copy == this_entry->needs_copy) &&
4673
4674	(prev_entry->object.vm_object == this_entry->object.vm_object) &&
4675	((prev_entry->offset + (prev_entry->vme_endlinks.end - prev_entry->vme_startlinks.start))
4676	== this_entry->offset) &&
4677	(prev_entry->projected_on == 0) &&
4678	(this_entry->projected_on == 0)
4679	) {
4680	if (map->first_free == this_entry)
4681	map->first_free = prev_entry;
4682
4683	SAVE_HINT(map, prev_entry); (map)->hint = (prev_entry); ;;
4684	vm_map_entry_unlink(map, this_entry)({ (&(map)->hdr)->nentries--; (this_entry)->links .next->links.prev = (this_entry)->links.prev; (this_entry )->links.prev->links.next = (this_entry)->links.next ; rbtree_remove(&(&(map)->hdr)->tree, &(this_entry )->tree_node); });
4685	prev_entry->vme_endlinks.end = this_entry->vme_endlinks.end;
4686	vm_object_deallocate(this_entry->object.vm_object);
4687	vm_map_entry_dispose(map, this_entry)_vm_map_entry_dispose(&(map)->hdr, (this_entry));
4688	}
4689	vm_map_unlock(map)lock_done(&(map)->lock);
4690	}
4691
4692
4693	/*
4694	* Routine: vm_map_machine_attribute
4695	* Purpose:
4696	* Provide machine-specific attributes to mappings,
4697	* such as cachability etc. for machines that provide
4698	* them. NUMA architectures and machines with big/strange
4699	* caches will use this.
4700	* Note:
4701	* Responsibilities for locking and checking are handled here,
4702	* everything else in the pmap module. If any non-volatile
4703	* information must be kept, the pmap module should handle
4704	* it itself. [This assumes that attributes do not
4705	* need to be inherited, which seems ok to me]
4706	*/
4707	kern_return_t vm_map_machine_attribute(map, address, size, attribute, value)
4708	vm_map_t map;
4709	vm_offset_t address;
4710	vm_size_t size;
4711	vm_machine_attribute_t attribute;
4712	vm_machine_attribute_val_t* value; /* IN/OUT */
4713	{
4714	kern_return_t ret;
4715
4716	if (address < vm_map_min(map)((map)->hdr.links.start) \|\|
4717	(address + size) > vm_map_max(map)((map)->hdr.links.end))
4718	return KERN_INVALID_ARGUMENT4;
4719
4720	vm_map_lock(map)({ lock_write(&(map)->lock); (map)->timestamp++; });
4721
4722	ret = pmap_attribute(map->pmap, address, size, attribute, value)(1);
4723
4724	vm_map_unlock(map)lock_done(&(map)->lock);
4725
4726	return ret;
4727	}
4728
4729
4730	#if MACH_KDB0
4731
4732	#define printf kdbprintf
4733
4734	/*
4735	* vm_map_print: [ debug ]
4736	*/
4737	void vm_map_print(map)
4738	vm_map_t map;
4739	{
4740	vm_map_entry_t entry;
4741
4742	iprintf("Task map 0x%X: pmap=0x%X,",
4743	(vm_offset_t) map, (vm_offset_t) (map->pmap));
4744	printf("ref=%d,nentries=%d,", map->ref_count, map->hdr.nentries);
4745	printf("version=%d\n", map->timestamp);
4746	indent += 2;
4747	for (entry = vm_map_first_entry(map)((map)->hdr.links.next);
4748	entry != vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links);
4749	entry = entry->vme_nextlinks.next) {
4750	static char *inheritance_name[3] = { "share", "copy", "none"};
4751
4752	iprintf("map entry 0x%X: ", (vm_offset_t) entry);
4753	printf("start=0x%X, end=0x%X, ",
4754	(vm_offset_t) entry->vme_startlinks.start, (vm_offset_t) entry->vme_endlinks.end);
4755	printf("prot=%X/%X/%s, ",
4756	entry->protection,
4757	entry->max_protection,
4758	inheritance_name[entry->inheritance]);
4759	if (entry->wired_count != 0) {
4760	printf("wired(");
4761	if (entry->user_wired_count != 0)
4762	printf("u");
4763	if (entry->wired_count >
4764	((entry->user_wired_count == 0) ? 0 : 1))
4765	printf("k");
4766	printf(") ");
4767	}
4768	if (entry->in_transition) {
4769	printf("in transition");
4770	if (entry->needs_wakeup)
4771	printf("(wake request)");
4772	printf(", ");
4773	}
4774	if (entry->is_sub_map) {
4775	printf("submap=0x%X, offset=0x%X\n",
4776	(vm_offset_t) entry->object.sub_map,
4777	(vm_offset_t) entry->offset);
4778	} else {
4779	printf("object=0x%X, offset=0x%X",
4780	(vm_offset_t) entry->object.vm_object,
4781	(vm_offset_t) entry->offset);
4782	if (entry->is_shared)
4783	printf(", shared");
4784	if (entry->needs_copy)
4785	printf(", copy needed");
4786	printf("\n");
4787
4788	if ((entry->vme_prevlinks.prev == vm_map_to_entry(map)((struct vm_map_entry *) &(map)->hdr.links)) \|\|
4789	(entry->vme_prevlinks.prev->object.vm_object != entry->object.vm_object)) {
4790	indent += 2;
4791	vm_object_print(entry->object.vm_object);
4792	indent -= 2;
4793	}
4794	}
4795	}
4796	indent -= 2;
4797	}
4798
4799	/*
4800	* Routine: vm_map_copy_print
4801	* Purpose:
4802	* Pretty-print a copy object for ddb.
4803	*/
4804
4805	void vm_map_copy_print(copy)
4806	vm_map_copy_t copy;
4807	{
4808	int i, npages;
4809
4810	printf("copy object 0x%x\n", copy);
4811
4812	indent += 2;
4813
4814	iprintf("type=%d", copy->type);
4815	switch (copy->type) {
4816	case VM_MAP_COPY_ENTRY_LIST1:
4817	printf("[entry_list]");
4818	break;
4819
4820	case VM_MAP_COPY_OBJECT2:
4821	printf("[object]");
4822	break;
4823
4824	case VM_MAP_COPY_PAGE_LIST3:
4825	printf("[page_list]");
4826	break;
4827
4828	default:
4829	printf("[bad type]");
4830	break;
4831	}
4832	printf(", offset=0x%x", copy->offset);
4833	printf(", size=0x%x\n", copy->size);
4834
4835	switch (copy->type) {
4836	case VM_MAP_COPY_ENTRY_LIST1:
4837	/* XXX add stuff here */
4838	break;
4839
4840	case VM_MAP_COPY_OBJECT2:
4841	iprintf("object=0x%x\n", copy->cpy_objectc_u.c_o.object);
4842	break;
4843
4844	case VM_MAP_COPY_PAGE_LIST3:
4845	iprintf("npages=%d", copy->cpy_npagesc_u.c_p.npages);
4846	printf(", cont=%x", copy->cpy_contc_u.c_p.cont);
4847	printf(", cont_args=%x\n", copy->cpy_cont_argsc_u.c_p.cont_args);
4848	if (copy->cpy_npagesc_u.c_p.npages < 0) {
4849	npages = 0;
4850	} else if (copy->cpy_npagesc_u.c_p.npages > VM_MAP_COPY_PAGE_LIST_MAX64) {
4851	npages = VM_MAP_COPY_PAGE_LIST_MAX64;
4852	} else {
4853	npages = copy->cpy_npagesc_u.c_p.npages;
4854	}
4855	iprintf("copy->cpy_page_list[0..%d] = {", npages);
4856	for (i = 0; i < npages - 1; i++) {
4857	printf("0x%x, ", copy->cpy_page_listc_u.c_p.page_list[i]);
4858	}
4859	if (npages > 0) {
4860	printf("0x%x", copy->cpy_page_listc_u.c_p.page_list[npages - 1]);
4861	}
4862	printf("}\n");
4863	break;
4864	}
4865
4866	indent -= 2;
4867	}
4868	#endif /* MACH_KDB */