../../mach-defpager/default

Bug Summary

File:	obj-scan-build/mach-defpager/../../mach-defpager/default_pager.c
Location:	line 887, column 3
Description:	Value stored to 'offset' is never read

Annotated Source Code

1	/*
2	* Mach Operating System
3	* Copyright (c) 1993-1989 Carnegie Mellon University
4	* All Rights Reserved.
5	*
6	* Permission to use, copy, modify and distribute this software and its
7	* documentation is hereby granted, provided that both the copyright
8	* notice and this permission notice appear in all copies of the
9	* software, derivative works or modified versions, and any portions
10	* thereof, and that both notices appear in supporting documentation.
11	*
12	* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
13	* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
14	* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
15	*
16	* Carnegie Mellon requests users of this software to return to
17	*
18	* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
19	* School of Computer Science
20	* Carnegie Mellon University
21	* Pittsburgh PA 15213-3890
22	*
23	* any improvements or extensions that they make and grant Carnegie Mellon
24	* the rights to redistribute these changes.
25	*/
26	/*
27	* Default pager. Pages to paging partition.
28	*
29	* MUST BE ABLE TO ALLOCATE WIRED-DOWN MEMORY!!!
30	*/
31
32	#include <mach.h>
33	#include <mach/message.h>
34	#include <mach/notify.h>
35	#include <mach/mig_errors.h>
36	#include <mach/thread_switch.h>
37	#include <mach/task_info.h>
38	#include <mach/default_pager_types.h>
39
40	#include <pthread.h>
41
42	#include <device/device_types.h>
43	#include <device/device.h>
44
45	#include <queue.h>
46	#include <wiring.h>
47	#include <kalloc.h>
48	#include <default_pager.h>
49
50	#include <assert.h>
51	#include <errno(*__errno_location ()).h>
52	#include <stdio.h>
53	#include <string.h>
54
55	#include <file_io.h>
56
57	#include "default_pager_S.h"
58
59	#define debug0 0
60
61	static char my_name[] = "(default pager):";
62
63	static pthread_mutex_t printf_lock = PTHREAD_MUTEX_INITIALIZER{ ((__pthread_spinlock_t) 0), ((__pthread_spinlock_t) 0), 0, 0 , 0, 0, 0, 0 };
64
65	#if 0
66	#define dprintf(f, x...) \
67	({ pthread_mutex_lock (&printf_lock); \
68	printf (f , ##x); \
69	fflush (stdoutstdout); \
70	pthread_mutex_unlock (&printf_lock); })
71	#else
72	#define dprintf(f, x...)
73	#endif
74
75	#if 0
76	#define ddprintf(f, x...) \
77	({ pthread_mutex_lock (&printf_lock); \
78	printf (f , ##x); \
79	fflush (stdoutstdout); \
80	pthread_mutex_unlock (&printf_lock); })
81	#else
82	#define ddprintf(f, x...)
83	#endif
84
85	/*
86	* parallel vs serial switch
87	*/
88	#define PARALLEL1 1
89
90	#if 0
91	#define CHECKSUM 1
92	#endif
93
94	#define USE_PRECIOUS1 1
95
96	#define ptoa(p)((p)vm_page_size) ((p)vm_page_size)
97	#define atop(a)((a)/vm_page_size) ((a)/vm_page_size)
98
99	/*
100
101	*/
102	/*
103	* Bitmap allocation.
104	*/
105	typedef unsigned int bm_entry_t;
106	#define NB_BM32 32
107	#define BM_MASK0xffffffff 0xffffffff
108
109	#define howmany(a,b)(((a) + (b) - 1)/(b)) (((a) + (b) - 1)/(b))
110
111	/*
112	* Value to indicate no block assigned
113	*/
114	#define NO_BLOCK((vm_offset_t)-1) ((vm_offset_t)-1)
115
116	/*
117	* 'Partition' structure for each paging area.
118	* Controls allocation of blocks within paging area.
119	*/
120	struct part {
121	pthread_mutex_t p_lock; /* for bitmap/free */
122	vm_size_t total_size; /* total number of blocks */
123	vm_size_t free; /* number of blocks free */
124	unsigned int id; /* named lookup */
125	bm_entry_t bitmap; / allocation map */
126	boolean_t going_away; /* destroy attempt in progress */
127	struct file_direct file; / file paged to */
128	};
129	typedef struct part *partition_t;
130
131	struct {
132	pthread_mutex_t lock;
133	int n_partitions;
134	partition_t partition_list;/ array, for quick mapping */
135	} all_partitions; /* list of all such */
136
137	typedef unsigned char p_index_t;
138
139	#define P_INDEX_INVALID((p_index_t)-1) ((p_index_t)-1)
140
141	#define no_partition(x)((x) == ((p_index_t)-1)) ((x) == P_INDEX_INVALID((p_index_t)-1))
142
143	partition_t partition_of(x)
144	int x;
145	{
146	if (x >= all_partitions.n_partitions \|\| x < 0)
147	panic("partition_of x%x", x);
148	return all_partitions.partition_list[x];
149	}
150
151	void set_partition_of(x, p)
152	int x;
153	partition_t p;
154	{
155	if (x >= all_partitions.n_partitions \|\| x < 0)
156	panic("set_partition_of x%x", x);
157	all_partitions.partition_list[x] = p;
158	}
159
160	/*
161	* Simple mapping from (file)NAME to id
162	* Saves space, filenames can be long.
163	*/
164	unsigned int
165	part_id(const char *name)
166	{
167	register unsigned int id, xorid;
168	size_t len;
169
170	len = strlen(name);
171	id = xorid = 0;
172	while (len--) {
173	xorid ^= *name;
174	id += *name++;
175	}
176	return (id << 8) \| xorid;
177	}
178
179	void
180	partition_init()
181	{
182	pthread_mutex_init(&all_partitions.lock, NULL((void*)0));
183	all_partitions.n_partitions = 0;
184	}
185
186	static partition_t
187	new_partition (const char name, struct file_direct fdp,
188	int check_linux_signature)
189	{
190	register partition_t part;
191	register vm_size_t size, bmsize;
192	vm_offset_t raddr;
193	mach_msg_type_number_t rsize;
194	int rc;
195	unsigned int id = part_id(name);
196
197	pthread_mutex_lock(&all_partitions.lock);
198	{
199	unsigned int i;
200	for (i = 0; i < all_partitions.n_partitions; i++)
201	{
202	part = partition_of(i);
203	if (part && part->id == id)
204	{
205	printf ("(default pager): Already paging to partition %s!\n",
206	name);
207	pthread_mutex_unlock(&all_partitions.lock);
208	return 0;
209	}
210	}
211	}
212	pthread_mutex_unlock(&all_partitions.lock);
213
214	size = atop(fdp->fd_size * fdp->fd_bsize)((fdp->fd_size * fdp->fd_bsize)/vm_page_size);
215	bmsize = howmany(size, NB_BM)(((size) + (32) - 1)/(32)) * sizeof(bm_entry_t);
216
217	part = (partition_t) kalloc(sizeof(struct part));
218	pthread_mutex_init(&part->p_lock, NULL((void*)0));
219	part->total_size = size;
220	part->free = size;
221	part->id = id;
222	part->bitmap = (bm_entry_t *)kalloc(bmsize);
223	part->going_away= FALSE((boolean_t) 0);
224	part->file = fdp;
225
226	bzero((char *)part->bitmap, bmsize);
227
228	if (check_linux_signature < 0)
229	{
230	if (check_linux_signature != -3)
231	printf("(default pager): "
232	"Paging to raw partition %s (%uk paging space)\n",
233	name, part->total_size * (vm_page_size / 1024));
234	return part;
235	}
236
237	#define LINUX_PAGE_SIZE4096 4096 /* size of pages in Linux swap partitions */
238	rc = page_read_file_direct(part->file,
239	0, LINUX_PAGE_SIZE4096,
240	&raddr,
241	&rsize);
242	if (rc)
243	panic("(default pager): cannot read first page of %s! rc=%#x\n",
244	name, rc);
245	while (rsize < LINUX_PAGE_SIZE4096)
246	{
247	/* Filesystem block size is smaller than page size,
248	so we must do several reads to get the whole page. */
249	vm_address_t baddr, bsize;
250	rc = page_read_file_direct(part->file,
251	rsize, LINUX_PAGE_SIZE4096-rsize,
252	&baddr,
253	&bsize);
254	if (rc)
255	panic("(default pager): "
256	"cannot read first page of %s! rc=%#x at %#x\n",
257	name, rc, rsize);
258
259	memcpy ((char ) raddr + rsize, (void ) baddr, bsize);
260	rsize += bsize;
261	vm_deallocate (mach_task_self ()((__mach_task_self_ + 0)), baddr, bsize);
262	}
263
264	if (!memcmp("SWAP-SPACE", (char *) raddr + LINUX_PAGE_SIZE4096-10, 10))
265	{
266	/* The partition's first page has a Linux swap signature.
267	This means the beginning of the page contains a bitmap
268	of good pages, and all others are bad. */
269	unsigned int i, j, bad, max;
270	int waste;
271
272	printf("(default pager): Found Linux 2.0 swap signature in %s\n",
273	name);
274
275	/* The first page, and the pages corresponding to the bits
276	occupied by the signature in the final 10 bytes of the page,
277	are always unavailable ("bad"). */
278	(u_int32_t )raddr &= ~(u_int32_t) 1;
279	memset((char *) raddr + LINUX_PAGE_SIZE4096-10, 0, 10);
280
281	max = LINUX_PAGE_SIZE4096 / sizeof(u_int32_t);
282	if (max > (part->total_size + 31) / 32)
283	max = (part->total_size + 31) / 32;
284
285	bad = 0;
286	for (i = 0; i < max; ++i)
287	{
288	u_int32_t bm = ((u_int32_t *) raddr)[i];
289	if (bm == ~(u_int32_t) 0)
290	continue;
291	/* There are some zero bits in this word. */
292	for (j = 0; j < 32; ++j)
293	if ((bm & (1 << j)) == 0)
294	{
295	unsigned int p = i*32 + j;
296	if (p >= part->total_size)
297	break;
298	++bad;
299	part->bitmap[p / NB_BM32] \|= 1 << (p % NB_BM32);
300	}
301	}
302	part->free -= bad;
303
304	--bad; /* Don't complain about first page. */
305	waste = part->total_size - (8 * (LINUX_PAGE_SIZE4096-10));
306	if (waste > 0)
307	{
308	/* The wasted pages were already marked "bad". */
309	bad -= waste;
310	if (bad > 0)
311	printf("\
312	(default pager): Paging to %s, %dk swap-space (%dk bad, %dk wasted at end)\n",
313	name,
314	part->free * (LINUX_PAGE_SIZE4096 / 1024),
315	bad * (LINUX_PAGE_SIZE4096 / 1024),
316	waste * (LINUX_PAGE_SIZE4096 / 1024));
317	else
318	printf("\
319	(default pager): Paging to %s, %dk swap-space (%dk wasted at end)\n",
320	name,
321	part->free * (LINUX_PAGE_SIZE4096 / 1024),
322	waste * (LINUX_PAGE_SIZE4096 / 1024));
323	}
324	else if (bad > 0)
325	printf("\
326	(default pager): Paging to %s, %dk swap-space (excludes %dk marked bad)\n",
327	name,
328	part->free * (LINUX_PAGE_SIZE4096 / 1024),
329	bad * (LINUX_PAGE_SIZE4096 / 1024));
330	else
331	printf("\
332	(default pager): Paging to %s, %dk swap-space\n",
333	name,
334	part->free * (LINUX_PAGE_SIZE4096 / 1024));
335	}
336	else if (!memcmp("SWAPSPACE2",
337	(char *) raddr + LINUX_PAGE_SIZE4096-10, 10))
338	{
339	struct
340	{
341	u_int8_t bootbits[1024];
342	u_int32_t version;
343	u_int32_t last_page;
344	u_int32_t nr_badpages;
345	u_int32_t padding[125];
346	u_int32_t badpages[1];
347	} hdr = (void ) raddr;
348
349	printf("\
350	(default pager): Found Linux 2.2 swap signature (v%u) in %s...",
351	hdr->version, name);
352
353	part->bitmap[0] \|= 1; /* first page unusable */
354	part->free--;
355
356	switch (hdr->version)
357	{
358	default:
359	if (check_linux_signature)
360	{
361	printf ("version %u unknown! SKIPPING %s!\n",
362	hdr->version,
363	name);
364	vm_deallocate(mach_task_self()((__mach_task_self_ + 0)), raddr, rsize);
365	kfree(part->bitmap, bmsize);
366	kfree(part, sizeof *part);
367	return 0;
368	}
369	else
370	printf ("version %u unknown! IGNORING SIGNATURE PAGE!"
371	" %dk swap-space\n",
372	hdr->version,
373	part->free * (LINUX_PAGE_SIZE4096 / 1024));
374	break;
375
376	case 1:
377	{
378	unsigned int waste, i;
379	if (hdr->last_page > part->total_size)
380	{
381	printf ("signature says %uk, partition has only %uk! ",
382	hdr->last_page * (LINUX_PAGE_SIZE4096 / 1024),
383	part->total_size * (LINUX_PAGE_SIZE4096 / 1024));
384	waste = 0;
385	}
386	else
387	{
388	waste = part->total_size - hdr->last_page;
389	part->total_size = hdr->last_page;
390	part->free = part->total_size - 1;
391	}
392	for (i = 0; i < hdr->nr_badpages; ++i)
393	{
394	const u_int32_t bad = hdr->badpages[i];
395	part->bitmap[bad / NB_BM32] \|= 1 << (bad % NB_BM32);
396	part->free--;
397	}
398	printf ("%uk swap-space",
399	part->free * (LINUX_PAGE_SIZE4096 / 1024));
400	if (hdr->nr_badpages != 0)
401	printf (" (excludes %uk marked bad)",
402	hdr->nr_badpages * (LINUX_PAGE_SIZE4096 / 1024));
403	if (waste != 0)
404	printf (" (excludes %uk at end of partition)",
405	waste * (LINUX_PAGE_SIZE4096 / 1024));
406	printf ("\n");
407	}
408	}
409	}
410	else if (check_linux_signature)
411	{
412	printf ("(default pager): "
413	"Cannot find Linux swap signature page! "
414	"SKIPPING %s (%uk partition)!",
415	name, part->total_size * (vm_page_size / 1024));
416	kfree(part->bitmap, bmsize);
417	kfree(part, sizeof *part);
418	part = 0;
419	}
420	else
421	printf("(default pager): "
422	"Paging to raw partition %s (%uk paging space)\n",
423	name, part->total_size * (vm_page_size / 1024));
424
425	vm_deallocate(mach_task_self()((__mach_task_self_ + 0)), raddr, rsize);
426
427	return part;
428	}
429
430	/*
431	* Create a partition descriptor,
432	* add it to the list of all such.
433	* size is in BYTES.
434	*/
435	void
436	create_paging_partition(const char *name,
437	struct file_direct *fdp, int isa_file,
438	int linux_signature)
439	{
440	register partition_t part;
441
442	part = new_partition (name, fdp, linux_signature);
443	if (!part)
444	return;
445
446	pthread_mutex_lock(&all_partitions.lock);
447	{
448	register int i;
449
450	for (i = 0; i < all_partitions.n_partitions; i++)
451	if (partition_of(i) == 0) break;
452
453	if (i == all_partitions.n_partitions) {
454	register partition_t new_list, old_list;
455	register int n;
456
457	n = i ? (i<<1) : 2;
458	new_list = (partition_t *)
459	kalloc( n * sizeof(partition_t) );
460	if (new_list == 0) no_paging_space(TRUE((boolean_t) 1));
461	bzero(new_list, n*sizeof(partition_t));
462	if (i) {
463	old_list = all_partitions.partition_list;
464	bcopy(old_list, new_list, i*sizeof(partition_t));
465	}
466	all_partitions.partition_list = new_list;
467	all_partitions.n_partitions = n;
468	if (i) kfree(old_list, i*sizeof(partition_t));
469	}
470	set_partition_of(i, part);
471	}
472	pthread_mutex_unlock(&all_partitions.lock);
473
474	#if 0
475	dprintf("%s Added paging %s %s\n", my_name,
476	(isa_file) ? "file" : "device", name);
477	#endif
478	overcommitted(TRUE((boolean_t) 1), part->free);
479	}
480
481	/*
482	* Choose the most appropriate default partition
483	* for an object of SIZE bytes.
484	* Return the partition locked, unless
485	* the object has no CUR_PARTition.
486	*/
487	p_index_t
488	choose_partition(size, cur_part)
489	unsigned int size;
490	register p_index_t cur_part;
491	{
492	register partition_t part;
493	register boolean_t found = FALSE((boolean_t) 0);
494	register int i;
495
496	pthread_mutex_lock(&all_partitions.lock);
497	for (i = 0; i < all_partitions.n_partitions; i++) {
498
499	/* the undesirable one ? */
500	if (i == cur_part)
501	continue;
502
503	ddprintf ("choose_partition(%x,%d,%d)\n",size,cur_part,i);
504	/* one that was removed ? */
505	if ((part = partition_of(i)) == 0)
506	continue;
507
508	/* one that is being removed ? */
509	if (part->going_away)
510	continue;
511
512	/* is it big enough ? */
513	pthread_mutex_lock(&part->p_lock);
514	if (ptoa(part->free)((part->free)*vm_page_size) >= size) {
515	if (cur_part != P_INDEX_INVALID((p_index_t)-1)) {
516	pthread_mutex_unlock(&all_partitions.lock);
517	return (p_index_t)i;
518	} else
519	found = TRUE((boolean_t) 1);
520	}
521	pthread_mutex_unlock(&part->p_lock);
522
523	if (found) break;
524	}
525	pthread_mutex_unlock(&all_partitions.lock);
526	return (found) ? (p_index_t)i : P_INDEX_INVALID((p_index_t)-1);
527	}
528
529	/*
530	* Allocate a page in a paging partition
531	* The partition is returned unlocked.
532	*/
533	vm_offset_t
534	pager_alloc_page(pindex, lock_it)
535	p_index_t pindex;
536	boolean_t lock_it;
537	{
538	register int bm_e;
539	register int bit;
540	register int limit;
541	register bm_entry_t *bm;
542	partition_t part;
543	static char here[] = "%spager_alloc_page";
544
545	if (no_partition(pindex)((pindex) == ((p_index_t)-1)))
546	return (NO_BLOCK((vm_offset_t)-1));
547	ddprintf ("pager_alloc_page(%d,%d)\n",pindex,lock_it);
548	part = partition_of(pindex);
549
550	/* unlikely, but possible deadlock against destroy_partition */
551	if (!part \|\| part->going_away)
552	return (NO_BLOCK((vm_offset_t)-1));
553
554	if (lock_it)
555	pthread_mutex_lock(&part->p_lock);
556
557	if (part->free == 0) {
558	/* out of paging space */
559	pthread_mutex_unlock(&part->p_lock);
560	return (NO_BLOCK((vm_offset_t)-1));
561	}
562
563	limit = howmany(part->total_size, NB_BM)(((part->total_size) + (32) - 1)/(32));
564	bm = part->bitmap;
565	for (bm_e = 0; bm_e < limit; bm_e++, bm++)
566	if (*bm != BM_MASK0xffffffff)
567	break;
568
569	if (bm_e == limit)
570	panic(here,my_name);
571
572	/*
573	* Find and set the proper bit
574	*/
575	{
576	register bm_entry_t b = *bm;
577
578	for (bit = 0; bit < NB_BM32; bit++)
579	if ((b & (1<<bit)) == 0)
580	break;
581	if (bit == NB_BM32)
582	panic(here,my_name);
583
584	*bm = b \| (1<<bit);
585	part->free--;
586
587	}
588
589	pthread_mutex_unlock(&part->p_lock);
590
591	return (bm_e*NB_BM32+bit);
592	}
593
594	/*
595	* Deallocate a page in a paging partition
596	*/
597	void
598	pager_dealloc_page(pindex, page, lock_it)
599	p_index_t pindex;
600	register vm_offset_t page;
601	boolean_t lock_it;
602	{
603	register partition_t part;
604	register int bit, bm_e;
605
606	/* be paranoid */
607	if (no_partition(pindex)((pindex) == ((p_index_t)-1)))
608	panic("%sdealloc_page",my_name);
609	ddprintf ("pager_dealloc_page(%d,%x,%d)\n",pindex,page,lock_it);
610	part = partition_of(pindex);
611
612	if (page >= part->total_size)
613	panic("%sdealloc_page",my_name);
614
615	bm_e = page / NB_BM32;
616	bit = page % NB_BM32;
617
618	if (lock_it)
619	pthread_mutex_lock(&part->p_lock);
620
621	part->bitmap[bm_e] &= ~(1<<bit);
622	part->free++;
623
624	if (lock_it)
625	pthread_mutex_unlock(&part->p_lock);
626	}
627
628	/*
629
630	*/
631	/*
632	* Allocation info for each paging object.
633	*
634	* Most operations, even pager_write_offset and pager_put_checksum,
635	* just need a read lock. Higher-level considerations prevent
636	* conflicting operations on a single page. The lock really protects
637	* the underlying size and block map memory, so pager_extend needs a
638	* write lock.
639	*
640	* An object can now span multiple paging partitions. The allocation
641	* info we keep is a pair (offset,p_index) where the index is in the
642	* array of all partition ptrs, and the offset is partition-relative.
643	* Size wise we are doing ok fitting the pair into a single integer:
644	* the offset really is in pages so we have vm_page_size bits available
645	* for the partition index.
646	*/
647	#define DEBUG_READER_CONFLICTS0 0
648
649	#if DEBUG_READER_CONFLICTS0
650	int default_pager_read_conflicts = 0;
651	#endif
652
653	union dp_map {
654
655	struct {
656	unsigned int p_offset : 24,
657	p_index : 8;
658	} block;
659
660	union dp_map *indirect;
661	};
662	typedef union dp_map *dp_map_t;
663
664	/* quick check for part==block==invalid */
665	#define no_block(e)((e).indirect == (dp_map_t)((vm_offset_t)-1)) ((e).indirect == (dp_map_t)NO_BLOCK((vm_offset_t)-1))
666	#define invalidate_block(e)((e).indirect = (dp_map_t)((vm_offset_t)-1)) ((e).indirect = (dp_map_t)NO_BLOCK((vm_offset_t)-1))
667
668	struct dpager {
669	pthread_mutex_t lock; /* lock for extending block map */
670	/* XXX should be read-write lock */
671	#if DEBUG_READER_CONFLICTS0
672	int readers;
673	boolean_t writer;
674	#endif
675	dp_map_t map; /* block map */
676	vm_size_t size; /* size of paging object, in pages */
677	vm_size_t limit; /* limit (bytes) allowed to grow to */
678	vm_size_t byte_limit; /* limit, which wasn't
679	rounded to page boundary */
680	p_index_t cur_partition;
681	#ifdef CHECKSUM
682	vm_offset_t checksum; / checksum - parallel to block map */
683	#define NO_CHECKSUM ((vm_offset_t)-1)
684	#endif /* CHECKSUM */
685	};
686	typedef struct dpager *dpager_t;
687
688	/*
689	* A paging object uses either a one- or a two-level map of offsets
690	* into a paging partition.
691	*/
692	#define PAGEMAP_ENTRIES64 64
693	/* number of pages in a second-level map */
694	#define PAGEMAP_SIZE(npgs)((npgs)sizeof(vm_offset_t)) ((npgs)sizeof(vm_offset_t))
695
696	#define INDIRECT_PAGEMAP_ENTRIES(npgs)((((npgs)-1)/64) + 1) \
697	((((npgs)-1)/PAGEMAP_ENTRIES64) + 1)
698	#define INDIRECT_PAGEMAP_SIZE(npgs)(((((npgs)-1)/64) + 1) * sizeof(vm_offset_t *)) \
699	(INDIRECT_PAGEMAP_ENTRIES(npgs)((((npgs)-1)/64) + 1) * sizeof(vm_offset_t *))
700	#define INDIRECT_PAGEMAP(size)(size > 64) \
701	(size > PAGEMAP_ENTRIES64)
702
703	#define ROUNDUP_TO_PAGEMAP(npgs)(((npgs) + 64 - 1) & ~(64 - 1)) \
704	(((npgs) + PAGEMAP_ENTRIES64 - 1) & ~(PAGEMAP_ENTRIES64 - 1))
705
706	/*
707	* Object sizes are rounded up to the next power of 2,
708	* unless they are bigger than a given maximum size.
709	*/
710	vm_size_t max_doubled_size = 4 * 1024 * 1024; /* 4 meg */
711
712	/*
713	* Return first level map for pager.
714	* If there is no such map, than allocate it.
715	*/
716	dp_map_t pager_get_direct_map(pager)
717	register dpager_t pager;
718	{
719	register dp_map_t mapptr, emapptr;
720	register vm_size_t size = pager->size;
721
722	if (pager->map)
723	return pager->map;
724	/*
725	* Allocate and initialize the block map
726	*/
727	{
728	register vm_size_t alloc_size;
729	dp_map_t init_value;
730
731	if (INDIRECT_PAGEMAP(size)(size > 64)) {
732	alloc_size = INDIRECT_PAGEMAP_SIZE(size)(((((size)-1)/64) + 1) * sizeof(vm_offset_t *));
733	init_value = (dp_map_t)0;
734	} else {
735	alloc_size = PAGEMAP_SIZE(size)((size)*sizeof(vm_offset_t));
736	init_value = (dp_map_t)NO_BLOCK((vm_offset_t)-1);
737	}
738
739	mapptr = (dp_map_t) kalloc(alloc_size);
740	for (emapptr = &mapptr[(alloc_size-1) / sizeof(vm_offset_t)];
741	emapptr >= mapptr;
742	emapptr--)
743	emapptr->indirect = init_value;
744	}
745	pager->map = mapptr;
746	return mapptr;
747	}
748
749	/*
750	* Attach a new paging object to a paging partition
751	*/
752	void
753	pager_alloc(pager, part, size)
754	register dpager_t pager;
755	p_index_t part;
756	register vm_size_t size; /* in BYTES */
757	{
758	register int i;
759	register dp_map_t mapptr, emapptr;
760
761	pthread_mutex_init(&pager->lock, NULL((void*)0));
762	#if DEBUG_READER_CONFLICTS0
763	pager->readers = 0;
764	pager->writer = FALSE((boolean_t) 0);
765	#endif
766	pager->cur_partition = part;
767
768	/*
769	* Convert byte size to number of pages, then increase to the nearest
770	* power of 2.
771	*/
772	size = atop(size)((size)/vm_page_size);
773	if (size <= atop(max_doubled_size)((max_doubled_size)/vm_page_size)) {
774	i = 1;
775	while (i < size)
776	i <<= 1;
777	size = i;
778	} else
779	size = ROUNDUP_TO_PAGEMAP(size)(((size) + 64 - 1) & ~(64 - 1));
780
781	pager->map = NULL((void*)0);
782	pager->size = size;
783	pager->limit = (vm_size_t)-1;
784
785	#ifdef CHECKSUM
786	if (INDIRECT_PAGEMAP(size)(size > 64)) {
787	mapptr = (vm_offset_t *)
788	kalloc(INDIRECT_PAGEMAP_SIZE(size)(((((size)-1)/64) + 1) * sizeof(vm_offset_t *)));
789	for (i = INDIRECT_PAGEMAP_ENTRIES(size)((((size)-1)/64) + 1); --i >= 0;)
790	mapptr[i] = 0;
791	} else {
792	mapptr = (vm_offset_t ) kalloc(PAGEMAP_SIZE(size)((size)sizeof(vm_offset_t)));
793	for (i = 0; i < size; i++)
794	mapptr[i] = NO_CHECKSUM;
795	}
796	pager->checksum = mapptr;
797	#endif /* CHECKSUM */
798	}
799
800	/*
801	* Return size (in bytes) of space actually allocated to this pager.
802	* The pager is read-locked.
803	*/
804
805	vm_size_t
806	pager_allocated(pager)
807	register dpager_t pager;
808	{
809	vm_size_t size;
810	register dp_map_t map, emap;
811	vm_size_t asize;
812
813	size = pager->size; /* in pages */
814	asize = 0; /* allocated, in pages */
815	map = pager_get_direct_map(pager);
816
817	if (INDIRECT_PAGEMAP(size)(size > 64)) {
818	for (emap = &map[INDIRECT_PAGEMAP_ENTRIES(size)((((size)-1)/64) + 1)];
819	map < emap; map++) {
820
821	register dp_map_t map2, emap2;
822
823	if ((map2 = map->indirect) == 0)
824	continue;
825
826	for (emap2 = &map2[PAGEMAP_ENTRIES64];
827	map2 < emap2; map2++)
828	if ( ! no_block(map2)((map2).indirect == (dp_map_t)((vm_offset_t)-1)) )
829	asize++;
830
831	}
832	} else {
833	for (emap = &map[size]; map < emap; map++)
834	if ( ! no_block(map)((map).indirect == (dp_map_t)((vm_offset_t)-1)) )
835	asize++;
836	}
837
838	return ptoa(asize)((asize)*vm_page_size);
839	}
840
841	/*
842	* Find offsets (in the object) of pages actually allocated to this pager.
843	* Returns the number of allocated pages, whether or not they all fit.
844	* The pager is read-locked.
845	*/
846
847	unsigned int
848	pager_pages(pager, pages, numpages)
849	dpager_t pager;
850	register default_pager_page_t *pages;
851	unsigned int numpages;
852	{
853	vm_size_t size;
854	dp_map_t map, emap;
855	unsigned int actual;
856	vm_offset_t offset;
857
858	size = pager->size; /* in pages */
859	map = pager_get_direct_map(pager);
860	actual = 0;
861	offset = 0;
862
863	if (INDIRECT_PAGEMAP(size)(size > 64)) {
864	for (emap = &map[INDIRECT_PAGEMAP_ENTRIES(size)((((size)-1)/64) + 1)];
865	map < emap; map++) {
866
867	register dp_map_t map2, emap2;
868
869	if ((map2 = map->indirect) == 0) {
870	offset += vm_page_size * PAGEMAP_ENTRIES64;
871	continue;
872	}
873	for (emap2 = &map2[PAGEMAP_ENTRIES64];
874	map2 < emap2; map2++)
875	if ( ! no_block(map2)((map2).indirect == (dp_map_t)((vm_offset_t)-1)) ) {
876	if (actual++ < numpages)
877	pages++->dpp_offset = offset;
878	}
879	offset += vm_page_size;
880	}
881	} else {
882	for (emap = &map[size]; map < emap; map++)
883	if ( ! no_block(map)((map).indirect == (dp_map_t)((vm_offset_t)-1)) ) {
884	if (actual++ < numpages)
885	pages++->dpp_offset = offset;
886	}
887	offset += vm_page_size;
	Value stored to 'offset' is never read
888	}
889	return actual;
890	}
891
892	/*
893	* Extend the map for a paging object.
894	*
895	* XXX This implementation can allocate an arbitrary large amount
896	* of wired memory when extending a big block map. Because vm-privileged
897	* threads call pager_extend, this can crash the system by exhausting
898	* system memory.
899	*/
900	void
901	pager_extend(pager, new_size)
902	register dpager_t pager;
903	register vm_size_t new_size; /* in pages */
904	{
905	register dp_map_t new_mapptr;
906	register dp_map_t old_mapptr;
907	register int i;
908	register vm_size_t old_size;
909
910	pthread_mutex_lock(&pager->lock); /* XXX lock_write */
911	#if DEBUG_READER_CONFLICTS0
912	pager->writer = TRUE((boolean_t) 1);
913	#endif
914	/*
915	* Double current size until we cover new size.
916	* If object is 'too big' just use new size.
917	*/
918	old_size = pager->size;
919
920	if (new_size <= atop(max_doubled_size)((max_doubled_size)/vm_page_size)) {
921	/* New size cannot be less than 1 */
922	i = old_size ? old_size : 1;
923	while (i < new_size)
924	i <<= 1;
925	new_size = i;
926	} else
927	new_size = ROUNDUP_TO_PAGEMAP(new_size)(((new_size) + 64 - 1) & ~(64 - 1));
928
929	if (INDIRECT_PAGEMAP(old_size)(old_size > 64)) {
930	/*
931	* Pager already uses two levels. Allocate
932	* a larger indirect block.
933	*/
934	new_mapptr = (dp_map_t)
935	kalloc(INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
936	old_mapptr = pager_get_direct_map(pager);
937	for (i = 0; i < INDIRECT_PAGEMAP_ENTRIES(old_size)((((old_size)-1)/64) + 1); i++)
938	new_mapptr[i] = old_mapptr[i];
939	for (; i < INDIRECT_PAGEMAP_ENTRIES(new_size)((((new_size)-1)/64) + 1); i++)
940	new_mapptr[i].indirect = (dp_map_t)0;
941	kfree((char )old_mapptr, INDIRECT_PAGEMAP_SIZE(old_size)(((((old_size)-1)/64) + 1) sizeof(vm_offset_t *)));
942	pager->map = new_mapptr;
943	pager->size = new_size;
944	#ifdef CHECKSUM
945	new_mapptr = (vm_offset_t *)
946	kalloc(INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
947	old_mapptr = pager->checksum;
948	for (i = 0; i < INDIRECT_PAGEMAP_ENTRIES(old_size)((((old_size)-1)/64) + 1); i++)
949	new_mapptr[i] = old_mapptr[i];
950	for (; i < INDIRECT_PAGEMAP_ENTRIES(new_size)((((new_size)-1)/64) + 1); i++)
951	new_mapptr[i] = 0;
952	kfree((char )old_mapptr, INDIRECT_PAGEMAP_SIZE(old_size)(((((old_size)-1)/64) + 1) sizeof(vm_offset_t *)));
953	pager->checksum = new_mapptr;
954	#endif /* CHECKSUM */
955	#if DEBUG_READER_CONFLICTS0
956	pager->writer = FALSE((boolean_t) 0);
957	#endif
958	pthread_mutex_unlock(&pager->lock);
959	#if 0
960	ddprintf ("pager_extend 1 mapptr %x [3b] = %x\n", new_mapptr,
961	new_mapptr[0x3b]);
962	if (new_mapptr[0x3b].indirect > 0x10000
963	&& new_mapptr[0x3b].indirect != NO_BLOCK((vm_offset_t)-1))
964	panic ("debug panic");
965	#endif
966	return;
967	}
968
969	if (INDIRECT_PAGEMAP(new_size)(new_size > 64)) {
970	/*
971	* Changing from direct map to indirect map.
972	* Allocate both indirect and direct map blocks,
973	* since second-level (direct) block must be
974	* full size (PAGEMAP_SIZE(PAGEMAP_ENTRIES)).
975	*/
976
977	/*
978	* Allocate new second-level map first.
979	*/
980	new_mapptr = (dp_map_t) kalloc(PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
981	old_mapptr = pager_get_direct_map(pager);
982	for (i = 0; i < old_size; i++)
983	new_mapptr[i] = old_mapptr[i];
984	for (; i < PAGEMAP_ENTRIES64; i++)
985	invalidate_block(new_mapptr[i])((new_mapptr[i]).indirect = (dp_map_t)((vm_offset_t)-1));
986	kfree((char )old_mapptr, PAGEMAP_SIZE(old_size)((old_size)sizeof(vm_offset_t)));
987	old_mapptr = new_mapptr;
988
989	#if 0
990	ddprintf ("pager_extend 2 mapptr %x [3b] = %x\n", new_mapptr,
991	new_mapptr[0x3b]);
992	if (new_mapptr[0x3b].indirect > 0x10000
993	&& new_mapptr[0x3b].indirect != NO_BLOCK((vm_offset_t)-1))
994	panic ("debug panic");
995	#endif
996
997	/*
998	* Now allocate indirect map.
999	*/
1000	new_mapptr = (dp_map_t)
1001	kalloc(INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1002	new_mapptr[0].indirect = old_mapptr;
1003	for (i = 1; i < INDIRECT_PAGEMAP_ENTRIES(new_size)((((new_size)-1)/64) + 1); i++)
1004	new_mapptr[i].indirect = 0;
1005	pager->map = new_mapptr;
1006	pager->size = new_size;
1007	#ifdef CHECKSUM
1008	/*
1009	* Allocate new second-level map first.
1010	*/
1011	new_mapptr = (vm_offset_t )kalloc(PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)sizeof(vm_offset_t)));
1012	old_mapptr = pager->checksum;
1013	for (i = 0; i < old_size; i++)
1014	new_mapptr[i] = old_mapptr[i];
1015	for (; i < PAGEMAP_ENTRIES64; i++)
1016	new_mapptr[i] = NO_CHECKSUM;
1017	kfree((char )old_mapptr, PAGEMAP_SIZE(old_size)((old_size)sizeof(vm_offset_t)));
1018	old_mapptr = new_mapptr;
1019
1020	/*
1021	* Now allocate indirect map.
1022	*/
1023	new_mapptr = (vm_offset_t *)
1024	kalloc(INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1025	new_mapptr[0] = (vm_offset_t) old_mapptr;
1026	for (i = 1; i < INDIRECT_PAGEMAP_ENTRIES(new_size)((((new_size)-1)/64) + 1); i++)
1027	new_mapptr[i] = 0;
1028	pager->checksum = new_mapptr;
1029	#endif /* CHECKSUM */
1030	#if DEBUG_READER_CONFLICTS0
1031	pager->writer = FALSE((boolean_t) 0);
1032	#endif
1033	pthread_mutex_unlock(&pager->lock);
1034	return;
1035	}
1036	/*
1037	* Enlarging a direct block.
1038	*/
1039	new_mapptr = (dp_map_t) kalloc(PAGEMAP_SIZE(new_size)((new_size)*sizeof(vm_offset_t)));
1040	old_mapptr = pager_get_direct_map(pager);
1041	for (i = 0; i < old_size; i++)
1042	new_mapptr[i] = old_mapptr[i];
1043	for (; i < new_size; i++)
1044	invalidate_block(new_mapptr[i])((new_mapptr[i]).indirect = (dp_map_t)((vm_offset_t)-1));
1045	kfree((char )old_mapptr, PAGEMAP_SIZE(old_size)((old_size)sizeof(vm_offset_t)));
1046	pager->map = new_mapptr;
1047	pager->size = new_size;
1048	#ifdef CHECKSUM
1049	new_mapptr = (vm_offset_t *)
1050	kalloc(PAGEMAP_SIZE(new_size)((new_size)*sizeof(vm_offset_t)));
1051	old_mapptr = pager->checksum;
1052	for (i = 0; i < old_size; i++)
1053	new_mapptr[i] = old_mapptr[i];
1054	for (; i < new_size; i++)
1055	new_mapptr[i] = NO_CHECKSUM;
1056	kfree((char )old_mapptr, PAGEMAP_SIZE(old_size)((old_size)sizeof(vm_offset_t)));
1057	pager->checksum = new_mapptr;
1058	#endif /* CHECKSUM */
1059	#if DEBUG_READER_CONFLICTS0
1060	pager->writer = FALSE((boolean_t) 0);
1061	#endif
1062	pthread_mutex_unlock(&pager->lock);
1063	}
1064
1065	/* This deallocates the pages necessary to truncate a direct map
1066	previously of size NEW_SIZE to the smaller size OLD_SIZE. */
1067	static void
1068	dealloc_direct (dp_map_t mapptr,
1069	vm_size_t old_size, vm_size_t new_size)
1070	{
1071	vm_size_t i;
1072
1073	if (!mapptr)
1074	return;
1075
1076	for (i = new_size; i < old_size; ++i)
1077	{
1078	const union dp_map entry = mapptr[i];
1079	if (!no_block(entry)((entry).indirect == (dp_map_t)((vm_offset_t)-1)))
1080	{
1081	pager_dealloc_page(entry.block.p_index, entry.block.p_offset,
1082	TRUE((boolean_t) 1));
1083	invalidate_block(mapptr[i])((mapptr[i]).indirect = (dp_map_t)((vm_offset_t)-1));
1084	}
1085	}
1086	}
1087
1088	/* Truncate a memory object. First, any pages between the new size
1089	and the (larger) old size are deallocated. Then, the size of
1090	the pagemap may be reduced, an indirect map may be turned into
1091	a direct map.
1092
1093	The pager must be locked by the caller. */
1094	static void
1095	pager_truncate(dpager_t pager, vm_size_t new_size) /* in pages */
1096	{
1097	dp_map_t new_mapptr;
1098	dp_map_t old_mapptr;
1099	int i;
1100	vm_size_t old_size;
1101
1102	pthread_mutex_lock(&pager->lock); /* XXX lock_write */
1103
1104	if (!pager->map)
1105	goto done;
1106
1107	old_size = pager->size;
1108
1109	if (INDIRECT_PAGEMAP(old_size)(old_size > 64))
1110	{
1111	/* First handle the entire second-levels blocks that are being freed. */
1112	for (i = INDIRECT_PAGEMAP_ENTRIES(new_size)((((new_size)-1)/64) + 1);
1113	i < INDIRECT_PAGEMAP_ENTRIES(old_size)((((old_size)-1)/64) + 1);
1114	++i)
1115	{
1116	const dp_map_t mapptr = pager->map[i].indirect;
1117	pager->map[i].indirect = (dp_map_t)0;
1118	dealloc_direct (mapptr, PAGEMAP_ENTRIES64, 0);
1119	kfree ((char )mapptr, PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)sizeof(vm_offset_t)));
1120	}
1121
1122	/* Now truncate what's now the final nonempty direct block. */
1123	dealloc_direct (pager->map[(new_size - 1) / PAGEMAP_ENTRIES64].indirect,
1124	old_size & (PAGEMAP_ENTRIES64 - 1),
1125	new_size & (PAGEMAP_ENTRIES64 - 1));
1126
1127	if (INDIRECT_PAGEMAP (new_size)(new_size > 64))
1128	{
1129	const dp_map_t old_mapptr = pager->map;
1130	pager->map = (dp_map_t) kalloc (INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1131	memcpy (pager->map, old_mapptr, INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1132	kfree ((char ) old_mapptr, INDIRECT_PAGEMAP_SIZE (old_size)(((((old_size)-1)/64) + 1) sizeof(vm_offset_t *)));
1133	}
1134	else
1135	{
1136	/* We are truncating to a size small enough that it goes to using
1137	a one-level map. We already have that map, as the first and only
1138	nonempty element in our indirect map. */
1139	const dp_map_t mapptr = pager->map[0].indirect;
1140	kfree((char )pager->map, INDIRECT_PAGEMAP_SIZE(old_size)(((((old_size)-1)/64) + 1) sizeof(vm_offset_t *)));
1141	pager->map = mapptr;
1142	}
1143	}
1144
1145	if (! INDIRECT_PAGEMAP(old_size)(old_size > 64))
1146	{
1147	/* First deallocate pages in the truncated region. */
1148	dealloc_direct (pager->map, old_size, new_size);
1149	/* Now reduce the size of the direct map itself. We don't bother
1150	with kalloc/kfree if it's not shrinking enough that kalloc.c
1151	would actually use less. */
1152	if (PAGEMAP_SIZE (new_size)((new_size)sizeof(vm_offset_t)) <= PAGEMAP_SIZE (old_size)((old_size)sizeof(vm_offset_t)) / 2)
1153	{
1154	const dp_map_t old_mapptr = pager->map;
1155	pager->map = (dp_map_t) kalloc (PAGEMAP_SIZE (new_size)((new_size)*sizeof(vm_offset_t)));
1156	memcpy (pager->map, old_mapptr, PAGEMAP_SIZE (new_size)((new_size)*sizeof(vm_offset_t)));
1157	kfree ((char ) old_mapptr, PAGEMAP_SIZE (old_size)((old_size)sizeof(vm_offset_t)));
1158	}
1159	}
1160
1161	done:
1162	pager->size = new_size;
1163	pthread_mutex_unlock(&pager->lock);
1164
1165	#ifdef CHECKSUM
1166	#error write me
1167	#endif /* CHECKSUM */
1168	}
1169
1170
1171	/*
1172	* Given an offset within a paging object, find the
1173	* corresponding block within the paging partition.
1174	* Return NO_BLOCK if none allocated.
1175	*/
1176	union dp_map
1177	pager_read_offset(pager, offset)
1178	register dpager_t pager;
1179	vm_offset_t offset;
1180	{
1181	register vm_offset_t f_page;
1182	union dp_map pager_offset;
1183
1184	f_page = atop(offset)((offset)/vm_page_size);
1185
1186	#if DEBUG_READER_CONFLICTS0
1187	if (pager->readers > 0)
1188	default_pager_read_conflicts++; /* would have proceeded with
1189	read/write lock */
1190	#endif
1191	pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1192	#if DEBUG_READER_CONFLICTS0
1193	pager->readers++;
1194	#endif
1195	if (f_page >= pager->size)
1196	{
1197	ddprintf ("%spager_read_offset pager %x: bad page %d >= size %d",
1198	my_name, pager, f_page, pager->size);
1199	pthread_mutex_unlock(&pager->lock);
1200	return (union dp_map) (union dp_map *) NO_BLOCK((vm_offset_t)-1);
1201	#if 0
1202	panic("%spager_read_offset",my_name);
1203	#endif
1204	}
1205
1206	invalidate_block(pager_offset)((pager_offset).indirect = (dp_map_t)((vm_offset_t)-1));
1207	if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1208	register dp_map_t mapptr;
1209
1210	if (pager->map) {
1211	mapptr = pager->map[f_page/PAGEMAP_ENTRIES64].indirect;
1212	if (mapptr)
1213	pager_offset = mapptr[f_page%PAGEMAP_ENTRIES64];
1214	}
1215	}
1216	else {
1217	if (pager->map)
1218	pager_offset = pager->map[f_page];
1219	}
1220
1221	#if DEBUG_READER_CONFLICTS0
1222	pager->readers--;
1223	#endif
1224	pthread_mutex_unlock(&pager->lock);
1225	return (pager_offset);
1226	}
1227
1228	#if USE_PRECIOUS1
1229	/*
1230	* Release a single disk block.
1231	*/
1232	void pager_release_offset(pager, offset)
1233	register dpager_t pager;
1234	vm_offset_t offset;
1235	{
1236	register union dp_map entry;
1237
1238	offset = atop(offset)((offset)/vm_page_size);
1239
1240	pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1241
1242	assert (pager->map)((pager->map) ? (void) (0) : __assert_fail ("pager->map" , "../../mach-defpager/default_pager.c", 1242, __PRETTY_FUNCTION__ ));
1243	if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1244	register dp_map_t mapptr;
1245
1246	mapptr = pager->map[offset / PAGEMAP_ENTRIES64].indirect;
1247	entry = mapptr[offset % PAGEMAP_ENTRIES64];
1248	invalidate_block(mapptr[offset % PAGEMAP_ENTRIES])((mapptr[offset % 64]).indirect = (dp_map_t)((vm_offset_t)-1) );
1249	} else {
1250	entry = pager->map[offset];
1251	invalidate_block(pager->map[offset])((pager->map[offset]).indirect = (dp_map_t)((vm_offset_t)- 1));
1252	}
1253
1254	pthread_mutex_unlock(&pager->lock);
1255
1256	pager_dealloc_page(entry.block.p_index, entry.block.p_offset, TRUE((boolean_t) 1));
1257	}
1258	#endif /USE_PRECIOUS/
1259
1260
1261	/*
1262	* Move a page from one partition to another
1263	* New partition is locked, old partition is
1264	* locked unless LOCK_OLD sez otherwise.
1265	*/
1266	union dp_map
1267	pager_move_page(block)
1268	union dp_map block;
1269	{
1270	partition_t old_part, new_part;
1271	p_index_t old_pindex, new_pindex;
1272	union dp_map ret;
1273	vm_size_t size;
1274	vm_offset_t raddr, offset, new_offset;
1275	kern_return_t rc;
1276	static char here[] = "%spager_move_page";
1277
1278	old_pindex = block.block.p_index;
1279	invalidate_block(ret)((ret).indirect = (dp_map_t)((vm_offset_t)-1));
1280
1281	/* See if we have room to put it anywhere else */
1282	new_pindex = choose_partition( ptoa(1)((1)*vm_page_size), old_pindex);
1283	if (no_partition(new_pindex)((new_pindex) == ((p_index_t)-1)))
1284	return ret;
1285
1286	/* this unlocks the new partition */
1287	new_offset = pager_alloc_page(new_pindex, FALSE((boolean_t) 0));
1288	if (new_offset == NO_BLOCK((vm_offset_t)-1))
1289	panic(here,my_name);
1290
1291	/*
1292	* Got the resources, now move the data
1293	*/
1294	ddprintf ("pager_move_page(%x,%d,%d)\n",block.block.p_offset,old_pindex,new_pindex);
1295	old_part = partition_of(old_pindex);
1296	offset = ptoa(block.block.p_offset)((block.block.p_offset)*vm_page_size);
1297	rc = page_read_file_direct (old_part->file,
1298	offset,
1299	vm_page_size,
1300	&raddr,
1301	&size);
1302	if (rc != 0)
1303	panic(here,my_name);
1304
1305	/* release old */
1306	pager_dealloc_page(old_pindex, block.block.p_offset, FALSE((boolean_t) 0));
1307
1308	new_part = partition_of(new_pindex);
1309	offset = ptoa(new_offset)((new_offset)*vm_page_size);
1310	rc = page_write_file_direct (new_part->file,
1311	offset,
1312	raddr,
1313	size,
1314	&size);
1315	if (rc != 0)
1316	panic(here,my_name);
1317
1318	(void) vm_deallocate( mach_task_self()((__mach_task_self_ + 0)), raddr, size);
1319
1320	ret.block.p_offset = new_offset;
1321	ret.block.p_index = new_pindex;
1322
1323	return ret;
1324	}
1325
1326	#ifdef CHECKSUM
1327	/*
1328	* Return the checksum for a block.
1329	*/
1330	int
1331	pager_get_checksum(pager, offset)
1332	register dpager_t pager;
1333	vm_offset_t offset;
1334	{
1335	register vm_offset_t f_page;
1336	int checksum;
1337
1338	f_page = atop(offset)((offset)/vm_page_size);
1339
1340	pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1341	if (f_page >= pager->size)
1342	panic("%spager_get_checksum",my_name);
1343
1344	if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1345	register vm_offset_t *mapptr;
1346
1347	mapptr = (vm_offset_t *)pager->checksum[f_page/PAGEMAP_ENTRIES64];
1348	if (mapptr == 0)
1349	checksum = NO_CHECKSUM;
1350	else
1351	checksum = mapptr[f_page%PAGEMAP_ENTRIES64];
1352	}
1353	else {
1354	checksum = pager->checksum[f_page];
1355	}
1356
1357	pthread_mutex_unlock(&pager->lock);
1358	return (checksum);
1359	}
1360
1361	/*
1362	* Remember the checksum for a block.
1363	*/
1364	int
1365	pager_put_checksum(pager, offset, checksum)
1366	register dpager_t pager;
1367	vm_offset_t offset;
1368	int checksum;
1369	{
1370	register vm_offset_t f_page;
1371	static char here[] = "%spager_put_checksum";
1372
1373	f_page = atop(offset)((offset)/vm_page_size);
1374
1375	pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1376	if (f_page >= pager->size)
1377	panic(here,my_name);
1378
1379	if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1380	register vm_offset_t *mapptr;
1381
1382	mapptr = (vm_offset_t *)pager->checksum[f_page/PAGEMAP_ENTRIES64];
1383	if (mapptr == 0)
1384	panic(here,my_name);
1385
1386	mapptr[f_page%PAGEMAP_ENTRIES64] = checksum;
1387	}
1388	else {
1389	pager->checksum[f_page] = checksum;
1390	}
1391	pthread_mutex_unlock(&pager->lock);
1392	}
1393
1394	/*
1395	* Compute a checksum - XOR each 32-bit word.
1396	*/
1397	int
1398	compute_checksum(addr, size)
1399	vm_offset_t addr;
1400	vm_size_t size;
1401	{
1402	register int checksum = NO_CHECKSUM;
1403	register int *ptr;
1404	register int count;
1405
1406	ptr = (int *)addr;
1407	count = size / sizeof(int);
1408
1409	while (--count >= 0)
1410	checksum ^= *ptr++;
1411
1412	return (checksum);
1413	}
1414	#endif /* CHECKSUM */
1415
1416	/*
1417	* Given an offset within a paging object, find the
1418	* corresponding block within the paging partition.
1419	* Allocate a new block if necessary.
1420	*
1421	* WARNING: paging objects apparently may be extended
1422	* without notice!
1423	*/
1424	union dp_map
1425	pager_write_offset(pager, offset)
1426	register dpager_t pager;
1427	vm_offset_t offset;
1428	{
1429	register vm_offset_t f_page;
1430	register dp_map_t mapptr;
1431	register union dp_map block;
1432
1433	invalidate_block(block)((block).indirect = (dp_map_t)((vm_offset_t)-1));
1434
1435	f_page = atop(offset)((offset)/vm_page_size);
1436
1437	#if DEBUG_READER_CONFLICTS0
1438	if (pager->readers > 0)
1439	default_pager_read_conflicts++; /* would have proceeded with
1440	read/write lock */
1441	#endif
1442	pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1443	#if DEBUG_READER_CONFLICTS0
1444	pager->readers++;
1445	#endif
1446
1447	/* Catch the case where we had no initial fit partition
1448	for this object, but one was added later on */
1449	if (no_partition(pager->cur_partition)((pager->cur_partition) == ((p_index_t)-1))) {
1450	p_index_t new_part;
1451	vm_size_t size;
1452
1453	size = (f_page > pager->size) ? f_page : pager->size;
1454	new_part = choose_partition(ptoa(size)((size)*vm_page_size), P_INDEX_INVALID((p_index_t)-1));
1455	if (no_partition(new_part)((new_part) == ((p_index_t)-1)))
1456	new_part = choose_partition(ptoa(1)((1)*vm_page_size), P_INDEX_INVALID((p_index_t)-1));
1457	if (no_partition(new_part)((new_part) == ((p_index_t)-1)))
1458	/* give up right now to avoid confusion */
1459	goto out;
1460	else
1461	pager->cur_partition = new_part;
1462	}
1463
1464	while (f_page >= pager->size) {
1465	ddprintf ("pager_write_offset: extending: %x %x\n", f_page, pager->size);
1466
1467	/*
1468	* Paging object must be extended.
1469	* Remember that offset is 0-based, but size is 1-based.
1470	*/
1471	#if DEBUG_READER_CONFLICTS0
1472	pager->readers--;
1473	#endif
1474	pthread_mutex_unlock(&pager->lock);
1475	pager_extend(pager, f_page + 1);
1476	#if DEBUG_READER_CONFLICTS0
1477	if (pager->readers > 0)
1478	default_pager_read_conflicts++; /* would have proceeded with
1479	read/write lock */
1480	#endif
1481	pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1482	#if DEBUG_READER_CONFLICTS0
1483	pager->readers++;
1484	#endif
1485	ddprintf ("pager_write_offset: done extending: %x %x\n", f_page, pager->size);
1486	}
1487
1488	if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1489	ddprintf ("pager_write_offset: indirect\n");
1490	mapptr = pager_get_direct_map(pager);
1491	mapptr = mapptr[f_page/PAGEMAP_ENTRIES64].indirect;
1492	if (mapptr == 0) {
1493	/*
1494	* Allocate the indirect block
1495	*/
1496	register int i;
1497	ddprintf ("pager_write_offset: allocating indirect\n");
1498
1499	mapptr = (dp_map_t) kalloc(PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
1500	if (mapptr == 0) {
1501	/* out of space! */
1502	no_paging_space(TRUE((boolean_t) 1));
1503	goto out;
1504	}
1505	pager->map[f_page/PAGEMAP_ENTRIES64].indirect = mapptr;
1506	for (i = 0; i < PAGEMAP_ENTRIES64; i++)
1507	invalidate_block(mapptr[i])((mapptr[i]).indirect = (dp_map_t)((vm_offset_t)-1));
1508	#ifdef CHECKSUM
1509	{
1510	register vm_offset_t *cksumptr;
1511	register int j;
1512
1513	cksumptr = (vm_offset_t *)
1514	kalloc(PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
1515	if (cksumptr == 0) {
1516	/* out of space! */
1517	no_paging_space(TRUE((boolean_t) 1));
1518	goto out;
1519	}
1520	pager->checksum[f_page/PAGEMAP_ENTRIES64]
1521	= (vm_offset_t)cksumptr;
1522	for (j = 0; j < PAGEMAP_ENTRIES64; j++)
1523	cksumptr[j] = NO_CHECKSUM;
1524	}
1525	#endif /* CHECKSUM */
1526	}
1527	f_page %= PAGEMAP_ENTRIES64;
1528	}
1529	else {
1530	mapptr = pager_get_direct_map(pager);
1531	}
1532
1533	block = mapptr[f_page];
1534	ddprintf ("pager_write_offset: block starts as %x[%x] %x\n", mapptr, f_page, block);
1535	if (no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1))) {
1536	vm_offset_t off;
1537
1538	/* get room now */
1539	off = pager_alloc_page(pager->cur_partition, TRUE((boolean_t) 1));
1540	if (off == NO_BLOCK((vm_offset_t)-1)) {
1541	/*
1542	* Before giving up, try all other partitions.
1543	*/
1544	p_index_t new_part;
1545
1546	ddprintf ("pager_write_offset: could not allocate block\n");
1547	/* returns it locked (if any one is non-full) */
1548	new_part = choose_partition( ptoa(1)((1)*vm_page_size), pager->cur_partition);
1549	if ( ! no_partition(new_part)((new_part) == ((p_index_t)-1)) ) {
1550
1551	#if debug0
1552	dprintf("%s partition %x filled,", my_name, pager->cur_partition);
1553	dprintf("extending object %x (size %x) to %x.\n",
1554	pager, pager->size, new_part);
1555	#endif
1556
1557	/* this one tastes better */
1558	pager->cur_partition = new_part;
1559
1560	/* this unlocks the partition too */
1561	off = pager_alloc_page(pager->cur_partition, FALSE((boolean_t) 0));
1562
1563	}
1564
1565	if (off == NO_BLOCK((vm_offset_t)-1)) {
1566	/*
1567	* Oh well.
1568	*/
1569	overcommitted(FALSE((boolean_t) 0), 1);
1570	goto out;
1571	}
1572	ddprintf ("pager_write_offset: decided to allocate block\n");
1573	}
1574	block.block.p_offset = off;
1575	block.block.p_index = pager->cur_partition;
1576	mapptr[f_page] = block;
1577	}
1578
1579	out:
1580
1581	#if DEBUG_READER_CONFLICTS0
1582	pager->readers--;
1583	#endif
1584	pthread_mutex_unlock(&pager->lock);
1585	return (block);
1586	}
1587
1588	/*
1589	* Deallocate all of the blocks belonging to a paging object.
1590	* No locking needed because no other operations can be in progress.
1591	*/
1592	void
1593	pager_dealloc(pager)
1594	register dpager_t pager;
1595	{
1596	register int i, j;
1597	register dp_map_t mapptr;
1598	register union dp_map block;
1599
1600	if (!pager->map)
1601	return;
1602
1603	if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1604	for (i = INDIRECT_PAGEMAP_ENTRIES(pager->size)((((pager->size)-1)/64) + 1); --i >= 0; ) {
1605	mapptr = pager->map[i].indirect;
1606	if (mapptr != 0) {
1607	for (j = 0; j < PAGEMAP_ENTRIES64; j++) {
1608	block = mapptr[j];
1609	if ( ! no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)) )
1610	pager_dealloc_page(block.block.p_index,
1611	block.block.p_offset, TRUE((boolean_t) 1));
1612	}
1613	kfree((char )mapptr, PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)sizeof(vm_offset_t)));
1614	pager->map[i].indirect = (dp_map_t) 0;
1615	}
1616	}
1617	kfree((char )pager->map, INDIRECT_PAGEMAP_SIZE(pager->size)(((((pager->size)-1)/64) + 1) sizeof(vm_offset_t *)));
1618	pager->map = (dp_map_t) 0;
1619	#ifdef CHECKSUM
1620	for (i = INDIRECT_PAGEMAP_ENTRIES(pager->size)((((pager->size)-1)/64) + 1); --i >= 0; ) {
1621	mapptr = (vm_offset_t *)pager->checksum[i];
1622	if (mapptr) {
1623	kfree((char )mapptr, PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)sizeof(vm_offset_t)));
1624	}
1625	}
1626	kfree((char *)pager->checksum,
1627	INDIRECT_PAGEMAP_SIZE(pager->size)(((((pager->size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1628	#endif /* CHECKSUM */
1629	}
1630	else {
1631	mapptr = pager->map;
1632	for (i = 0; i < pager->size; i++ ) {
1633	block = mapptr[i];
1634	if ( ! no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)) )
1635	pager_dealloc_page(block.block.p_index,
1636	block.block.p_offset, TRUE((boolean_t) 1));
1637	}
1638	kfree((char )pager->map, PAGEMAP_SIZE(pager->size)((pager->size)sizeof(vm_offset_t)));
1639	pager->map = (dp_map_t) 0;
1640	#ifdef CHECKSUM
1641	kfree((char )pager->checksum, PAGEMAP_SIZE(pager->size)((pager->size)sizeof(vm_offset_t)));
1642	#endif /* CHECKSUM */
1643	}
1644	}
1645
1646	/*
1647	* Move all the pages of a PAGER that live in a
1648	* partition PINDEX somewhere else.
1649	* Pager should be write-locked, partition too.
1650	* Returns FALSE if it could not do it, but
1651	* some pages might have been moved nonetheless.
1652	*/
1653	boolean_t
1654	pager_realloc(pager, pindex)
1655	register dpager_t pager;
1656	p_index_t pindex;
1657	{
1658	register dp_map_t map, emap;
1659	vm_size_t size;
1660	union dp_map block;
1661
1662	if (!pager->map)
1663	return TRUE((boolean_t) 1);
1664
1665	size = pager->size; /* in pages */
1666	map = pager->map;
1667
1668	if (INDIRECT_PAGEMAP(size)(size > 64)) {
1669	for (emap = &map[INDIRECT_PAGEMAP_ENTRIES(size)((((size)-1)/64) + 1)];
1670	map < emap; map++) {
1671
1672	register dp_map_t map2, emap2;
1673
1674	if ((map2 = map->indirect) == 0)
1675	continue;
1676
1677	for (emap2 = &map2[PAGEMAP_ENTRIES64];
1678	map2 < emap2; map2++)
1679	if ( map2->block.p_index == pindex) {
1680
1681	block = pager_move_page(*map2);
1682	if (!no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)))
1683	*map2 = block;
1684	else
1685	return FALSE((boolean_t) 0);
1686	}
1687
1688	}
1689	goto ok;
1690	}
1691
1692	/* A small one */
1693	for (emap = &map[size]; map < emap; map++)
1694	if (map->block.p_index == pindex) {
1695	block = pager_move_page(*map);
1696	if (!no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)))
1697	*map = block;
1698	else
1699	return FALSE((boolean_t) 0);
1700	}
1701	ok:
1702	pager->cur_partition = choose_partition(0, P_INDEX_INVALID((p_index_t)-1));
1703	return TRUE((boolean_t) 1);
1704	}
1705
1706	/*
1707
1708	*/
1709
1710	/*
1711	* Read/write routines.
1712	*/
1713	#define PAGER_SUCCESS0 0
1714	#define PAGER_ABSENT1 1
1715	#define PAGER_ERROR2 2
1716
1717	/*
1718	* Read data from a default pager. Addr is the address of a buffer
1719	* to fill. Out_addr returns the buffer that contains the data;
1720	* if it is different from <addr>, it must be deallocated after use.
1721	*/
1722	int
1723	default_read(ds, addr, size, offset, out_addr, deallocate, external)
1724	register dpager_t ds;
1725	vm_offset_t addr; /* pointer to block to fill */
1726	register vm_size_t size;
1727	register vm_offset_t offset;
1728	vm_offset_t *out_addr;
1729	/* returns pointer to data */
1730	boolean_t deallocate;
1731	boolean_t external;
1732	{
1733	register union dp_map block;
1734	vm_offset_t raddr;
1735	vm_size_t rsize;
1736	register int rc;
1737	boolean_t first_time;
1738	register partition_t part;
1739	#ifdef CHECKSUM
1740	vm_size_t original_size = size;
1741	#endif /* CHECKSUM */
1742	vm_offset_t original_offset = offset;
1743
1744	/*
1745	* Find the block in the paging partition
1746	*/
1747	block = pager_read_offset(ds, offset);
1748	if ( no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)) ) {
1749	if (external) {
1750	/*
1751	* An external object is requesting unswapped data,
1752	* zero fill the page and return.
1753	*/
1754	bzero((char *) addr, vm_page_size);
1755	*out_addr = addr;
1756	return (PAGER_SUCCESS0);
1757	}
1758	return (PAGER_ABSENT1);
1759	}
1760
1761	/*
1762	* Read it, trying for the entire page.
1763	*/
1764	offset = ptoa(block.block.p_offset)((block.block.p_offset)*vm_page_size);
1765	ddprintf ("default_read(%x,%x,%x,%d)\n",addr,size,offset,block.block.p_index);
1766	part = partition_of(block.block.p_index);
1767	first_time = TRUE((boolean_t) 1);
1768	*out_addr = addr;
1769
1770	do {
1771	rc = page_read_file_direct(part->file,
1772	offset,
1773	size,
1774	&raddr,
1775	&rsize);
1776	if (rc != 0)
1777	return (PAGER_ERROR2);
1778
1779	/*
1780	* If we got the entire page on the first read, return it.
1781	*/
1782	if (first_time && rsize == size) {
1783	*out_addr = raddr;
1784	break;
1785	}
1786	/*
1787	* Otherwise, copy the data into the
1788	* buffer we were passed, and try for
1789	* the next piece.
1790	*/
1791	first_time = FALSE((boolean_t) 0);
1792	bcopy((char )raddr, (char )addr, rsize);
1793	addr += rsize;
1794	offset += rsize;
1795	size -= rsize;
1796	} while (size != 0);
1797
1798	#if USE_PRECIOUS1
1799	if (deallocate)
1800	pager_release_offset(ds, original_offset);
1801	#endif /USE_PRECIOUS/
1802
1803	#ifdef CHECKSUM
1804	{
1805	int write_checksum,
1806	read_checksum;
1807
1808	write_checksum = pager_get_checksum(ds, original_offset);
1809	read_checksum = compute_checksum(*out_addr, original_size);
1810	if (write_checksum != read_checksum) {
1811	panic(
1812	"PAGER CHECKSUM ERROR: offset 0x%x, written 0x%x, read 0x%x",
1813	original_offset, write_checksum, read_checksum);
1814	}
1815	}
1816	#endif /* CHECKSUM */
1817	return (PAGER_SUCCESS0);
1818	}
1819
1820	int
1821	default_write(ds, addr, size, offset)
1822	register dpager_t ds;
1823	register vm_offset_t addr;
1824	register vm_size_t size;
1825	register vm_offset_t offset;
1826	{
1827	register union dp_map block;
1828	partition_t part;
1829	vm_size_t wsize;
1830	register int rc;
1831
1832	ddprintf ("default_write: pager offset %x\n", offset);
1833
1834	/*
1835	* Find block in paging partition
1836	*/
1837	block = pager_write_offset(ds, offset);
1838	if ( no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)) )
1839	return (PAGER_ERROR2);
1840
1841	#ifdef CHECKSUM
1842	/*
1843	* Save checksum
1844	*/
1845	{
1846	int checksum;
1847
1848	checksum = compute_checksum(addr, size);
1849	pager_put_checksum(ds, offset, checksum);
1850	}
1851	#endif /* CHECKSUM */
1852	offset = ptoa(block.block.p_offset)((block.block.p_offset)*vm_page_size);
1853	ddprintf ("default_write(%x,%x,%x,%d)\n",addr,size,offset,block.block.p_index);
1854	part = partition_of(block.block.p_index);
1855
1856	/*
1857	* There are various assumptions made here,we
1858	* will not get into the next disk 'block' by
1859	* accident. It might well be non-contiguous.
1860	*/
1861	do {
1862	rc = page_write_file_direct(part->file,
1863	offset,
1864	addr,
1865	size,
1866	&wsize);
1867	if (rc != 0) {
1868	dprintf("*** PAGER ERROR: default_write: ");
1869	dprintf("ds=0x%x addr=0x%x size=0x%x offset=0x%x resid=0x%x\n",
1870	ds, addr, size, offset, wsize);
1871	return (PAGER_ERROR2);
1872	}
1873	addr += wsize;
1874	offset += wsize;
1875	size -= wsize;
1876	} while (size != 0);
1877	return (PAGER_SUCCESS0);
1878	}
1879
1880	boolean_t
1881	default_has_page(ds, offset)
1882	dpager_t ds;
1883	vm_offset_t offset;
1884	{
1885	return ( ! no_block(pager_read_offset(ds, offset))((pager_read_offset(ds, offset)).indirect == (dp_map_t)((vm_offset_t )-1)) );
1886	}
1887	/*
1888
1889	*/
1890
1891	/*
1892	* Mapping between pager port and paging object.
1893	*/
1894	struct dstruct {
1895	queue_chain_t links; /* Link in pager-port list */
1896
1897	pthread_mutex_t lock; /* Lock for the structure */
1898	pthread_cond_t
1899	waiting_seqno, /* someone waiting on seqno */
1900	waiting_read, /* someone waiting on readers */
1901	waiting_write, /* someone waiting on writers */
1902	waiting_refs; /* someone waiting on refs */
1903
1904	memory_object_t pager; /* Pager port */
1905	mach_port_seqno_t seqno; /* Pager port sequence number */
1906	mach_port_t pager_request; /* Request port */
1907	mach_port_urefs_t request_refs; /* Request port user-refs */
1908	mach_port_t pager_name; /* Name port */
1909	mach_port_urefs_t name_refs; /* Name port user-refs */
1910	boolean_t external; /* Is an external object? */
1911
1912	unsigned int readers; /* Reads in progress */
1913	unsigned int writers; /* Writes in progress */
1914
1915	/* This is the reply port of an outstanding
1916	default_pager_object_set_size call. */
1917	mach_port_t lock_request;
1918
1919	unsigned int errors; /* Pageout error count */
1920	struct dpager dpager; /* Actual pager */
1921	};
1922	typedef struct dstruct * default_pager_t;
1923	#define DEFAULT_PAGER_NULL((default_pager_t)0) ((default_pager_t)0)
1924
1925	#if PARALLEL1
1926	#define dstruct_lock_init(ds)pthread_mutex_init(&ds->lock, ((void)0)) pthread_mutex_init(&ds->lock, NULL((void)0))
1927	#define dstruct_lock(ds)pthread_mutex_lock(&ds->lock) pthread_mutex_lock(&ds->lock)
1928	#define dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock) pthread_mutex_unlock(&ds->lock)
1929	#else /* PARALLEL */
1930	#define dstruct_lock_init(ds)pthread_mutex_init(&ds->lock, ((void*)0))
1931	#define dstruct_lock(ds)pthread_mutex_lock(&ds->lock)
1932	#define dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock)
1933	#endif /* PARALLEL */
1934
1935	/*
1936	* List of all pagers. A specific pager is
1937	* found directly via its port, this list is
1938	* only used for monitoring purposes by the
1939	* default_pager_object* calls
1940	*/
1941	struct pager_port {
1942	queue_head_t queue;
1943	pthread_mutex_t lock;
1944	int count; /* saves code */
1945	queue_head_t leak_queue;
1946	} all_pagers;
1947
1948	#define pager_port_list_init(){ pthread_mutex_init(&all_pagers.lock, ((void*)0)); ((& all_pagers.queue)->next = (&all_pagers.queue)->prev = &all_pagers.queue); ((&all_pagers.leak_queue)-> next = (&all_pagers.leak_queue)->prev = &all_pagers .leak_queue); all_pagers.count = 0; } \
1949	{ \
1950	pthread_mutex_init(&all_pagers.lock, NULL((void*)0)); \
1951	queue_init(&all_pagers.queue)((&all_pagers.queue)->next = (&all_pagers.queue)-> prev = &all_pagers.queue); \
1952	queue_init(&all_pagers.leak_queue)((&all_pagers.leak_queue)->next = (&all_pagers.leak_queue )->prev = &all_pagers.leak_queue); \
1953	all_pagers.count = 0; \
1954	}
1955
1956	void pager_port_list_insert(port, ds)
1957	mach_port_t port;
1958	default_pager_t ds;
1959	{
1960	pthread_mutex_lock(&all_pagers.lock);
1961	queue_enter(&all_pagers.queue, ds, default_pager_t, links){ register queue_entry_t prev; prev = (&all_pagers.queue) ->prev; if ((&all_pagers.queue) == prev) { (&all_pagers .queue)->next = (queue_entry_t) (ds); } else { ((default_pager_t )prev)->links.next = (queue_entry_t)(ds); } (ds)->links .prev = prev; (ds)->links.next = &all_pagers.queue; (& all_pagers.queue)->prev = (queue_entry_t) ds; };
1962	all_pagers.count++;
1963	pthread_mutex_unlock(&all_pagers.lock);
1964	}
1965
1966	/* given a data structure return a good port-name to associate it to */
1967	#define pnameof(_x_)(((vm_offset_t)(_x_))+1) (((vm_offset_t)(_x_))+1)
1968	/* reverse, assumes no-odd-pointers */
1969	#define dnameof(_x_)(((vm_offset_t)(_x_))&~1) (((vm_offset_t)(_x_))&~1)
1970
1971	/* The magic typecast */
1972	#define pager_port_lookup(_port_)((! (((_port_) != ((mach_port_t) 0)) && ((_port_) != ( (mach_port_t) ~0))) \|\| ((default_pager_t)(((vm_offset_t)(_port_ ))&~1))->pager != (_port_)) ? ((default_pager_t)0) : ( default_pager_t)(((vm_offset_t)(_port_))&~1)) \
1973	((! MACH_PORT_VALID(_port_)(((_port_) != ((mach_port_t) 0)) && ((_port_) != ((mach_port_t ) ~0))) \|\| \
1974	((default_pager_t)dnameof(_port_)(((vm_offset_t)(_port_))&~1))->pager != (_port_)) ? \
1975	DEFAULT_PAGER_NULL((default_pager_t)0) : (default_pager_t)dnameof(_port_)(((vm_offset_t)(_port_))&~1))
1976
1977	void pager_port_list_delete(ds)
1978	default_pager_t ds;
1979	{
1980	pthread_mutex_lock(&all_pagers.lock);
1981	queue_remove(&all_pagers.queue, ds, default_pager_t, links){ register queue_entry_t next, prev; next = (ds)->links.next ; prev = (ds)->links.prev; if ((&all_pagers.queue) == next ) (&all_pagers.queue)->prev = prev; else ((default_pager_t )next)->links.prev = prev; if ((&all_pagers.queue) == prev ) (&all_pagers.queue)->next = next; else ((default_pager_t )prev)->links.next = next; };
1982	all_pagers.count--;
1983	pthread_mutex_unlock(&all_pagers.lock);
1984	}
1985
1986	/*
1987	* Destroy a paging partition.
1988	* XXX this is not re-entrant XXX
1989	*/
1990	kern_return_t
1991	destroy_paging_partition(name, pp_private)
1992	char *name;
1993	void **pp_private;
1994	{
1995	register unsigned int id = part_id(name);
1996	register partition_t part;
1997	boolean_t all_ok = TRUE((boolean_t) 1);
1998	default_pager_t entry;
1999	int pindex;
2000
2001	/*
2002	* Find and take partition out of list
2003	* This prevents choose_partition from
2004	* getting in the way.
2005	*/
2006	pthread_mutex_lock(&all_partitions.lock);
2007	for (pindex = 0; pindex < all_partitions.n_partitions; pindex++) {
2008	part = partition_of(pindex);
2009	if (part && (part->id == id)) break;
2010	}
2011	if (pindex == all_partitions.n_partitions) {
2012	pthread_mutex_unlock(&all_partitions.lock);
2013	return KERN_INVALID_ARGUMENT4;
2014	}
2015	part->going_away = TRUE((boolean_t) 1);
2016	pthread_mutex_unlock(&all_partitions.lock);
2017
2018	/*
2019	* This might take a while..
2020	*/
2021	all_over_again:
2022	#if debug0
2023	dprintf("Partition x%x (id x%x) for %s, all_ok %d\n", part, id, name, all_ok);
2024	#endif
2025	all_ok = TRUE((boolean_t) 1);
2026	pthread_mutex_lock(&part->p_lock);
2027
2028	pthread_mutex_lock(&all_pagers.lock);
2029	queue_iterate(&all_pagers.queue, entry, default_pager_t, links)for ((entry) = (default_pager_t) ((&all_pagers.queue)-> next); !(((&all_pagers.queue)) == ((queue_entry_t)(entry) )); (entry) = (default_pager_t) ((&(entry)->links)-> next)) {
2030
2031	dstruct_lock(entry)pthread_mutex_lock(&entry->lock);
2032
2033	if (pthread_mutex_trylock(&entry->dpager.lock)) {
2034
2035	dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
2036	pthread_mutex_unlock(&all_pagers.lock);
2037	pthread_mutex_unlock(&part->p_lock);
2038
2039	/* yield the processor */
2040	(void) thread_switch(MACH_PORT_NULL((mach_port_t) 0),
2041	SWITCH_OPTION_NONE0, 0);
2042
2043	goto all_over_again;
2044
2045	}
2046
2047	/*
2048	* See if we can relocate all the pages of this object
2049	* currently on this partition on some other partition
2050	*/
2051	all_ok = pager_realloc(&entry->dpager, pindex);
2052
2053	pthread_mutex_unlock(&entry->dpager.lock);
2054	dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
2055
2056	if (!all_ok) break;
2057
2058	}
2059	pthread_mutex_unlock(&all_pagers.lock);
2060
2061	if (all_ok) {
2062	/* No need to unlock partition, there are no refs left */
2063
2064	set_partition_of(pindex, 0);
2065	*pp_private = part->file;
2066	kfree(part->bitmap, howmany(part->total_size, NB_BM)(((part->total_size) + (32) - 1)/(32)) * sizeof(bm_entry_t));
2067	kfree(part, sizeof(struct part));
2068	dprintf("%s Removed paging partition %s\n", my_name, name);
2069	return KERN_SUCCESS0;
2070	}
2071
2072	/*
2073	* Put partition back in.
2074	*/
2075	part->going_away = FALSE((boolean_t) 0);
2076
2077	return KERN_FAILURE5;
2078	}
2079
2080
2081	/*
2082	* We use the sequence numbers on requests to regulate
2083	* our parallelism. In general, we allow multiple reads and writes
2084	* to proceed in parallel, with the exception that reads must
2085	* wait for previous writes to finish. (Because the kernel might
2086	* generate a data-request for a page on the heels of a data-write
2087	* for the same page, and we must avoid returning stale data.)
2088	* terminate requests wait for proceeding reads and writes to finish.
2089	*/
2090
2091	unsigned int default_pager_total = 0; /* debugging */
2092	unsigned int default_pager_wait_seqno = 0; /* debugging */
2093	unsigned int default_pager_wait_read = 0; /* debugging */
2094	unsigned int default_pager_wait_write = 0; /* debugging */
2095	unsigned int default_pager_wait_refs = 0; /* debugging */
2096
2097	#if PARALLEL1
2098	/*
2099	* Waits for correct sequence number. Leaves pager locked.
2100	*/
2101	void pager_port_lock(ds, seqno)
2102	default_pager_t ds;
2103	mach_port_seqno_t seqno;
2104	{
2105	default_pager_total++;
2106	dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2107	while (ds->seqno != seqno) {
2108	default_pager_wait_seqno++;
2109	pthread_cond_wait(&ds->waiting_seqno, &ds->lock);
2110	}
2111	}
2112
2113	/*
2114	* Increments sequence number and unlocks pager.
2115	*/
2116	void pager_port_unlock(ds)
2117	default_pager_t ds;
2118	{
2119	ds->seqno++;
2120	dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2121	pthread_cond_broadcast(&ds->waiting_seqno);
2122	}
2123
2124	/*
2125	* Start a read - one more reader. Pager must be locked.
2126	*/
2127	void pager_port_start_read(ds)
2128	default_pager_t ds;
2129	{
2130	ds->readers++;
2131	}
2132
2133	/*
2134	* Wait for readers. Unlocks and relocks pager if wait needed.
2135	*/
2136	void pager_port_wait_for_readers(ds)
2137	default_pager_t ds;
2138	{
2139	while (ds->readers != 0) {
2140	default_pager_wait_read++;
2141	pthread_cond_wait(&ds->waiting_read, &ds->lock);
2142	}
2143	}
2144
2145	/*
2146	* Finish a read. Pager is unlocked and returns unlocked.
2147	*/
2148	void pager_port_finish_read(ds)
2149	default_pager_t ds;
2150	{
2151	dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2152	if (--ds->readers == 0) {
2153	dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2154	pthread_cond_broadcast(&ds->waiting_read);
2155	}
2156	else {
2157	dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2158	}
2159	}
2160
2161	/*
2162	* Start a write - one more writer. Pager must be locked.
2163	*/
2164	void pager_port_start_write(ds)
2165	default_pager_t ds;
2166	{
2167	ds->writers++;
2168	}
2169
2170	/*
2171	* Wait for writers. Unlocks and relocks pager if wait needed.
2172	*/
2173	void pager_port_wait_for_writers(ds)
2174	default_pager_t ds;
2175	{
2176	while (ds->writers != 0) {
2177	default_pager_wait_write++;
2178	pthread_cond_wait(&ds->waiting_write, &ds->lock);
2179	}
2180	}
2181
2182	/*
2183	* Finish a write. Pager is unlocked and returns unlocked.
2184	*/
2185	void pager_port_finish_write(ds)
2186	default_pager_t ds;
2187	{
2188	dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2189	if (--ds->writers == 0) {
2190	dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2191	pthread_cond_broadcast(&ds->waiting_write);
2192	}
2193	else {
2194	dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2195	}
2196	}
2197
2198	/*
2199	* Wait for concurrent default_pager_objects.
2200	* Unlocks and relocks pager if wait needed.
2201	*/
2202	void pager_port_wait_for_refs(ds)
2203	default_pager_t ds;
2204	{
2205	while (ds->name_refs == 0) {
2206	default_pager_wait_refs++;
2207	pthread_cond_wait(&ds->waiting_refs, &ds->lock);
2208	}
2209	}
2210
2211	/*
2212	* Finished creating name refs - wake up waiters.
2213	*/
2214	void pager_port_finish_refs(ds)
2215	default_pager_t ds;
2216	{
2217	pthread_cond_broadcast(&ds->waiting_refs);
2218	}
2219
2220	#else /* PARALLEL */
2221
2222	#define pager_port_lock(ds,seqno)
2223	#define pager_port_unlock(ds)
2224	#define pager_port_start_read(ds)
2225	#define pager_port_wait_for_readers(ds)
2226	#define pager_port_finish_read(ds)
2227	#define pager_port_start_write(ds)
2228	#define pager_port_wait_for_writers(ds)
2229	#define pager_port_finish_write(ds)
2230	#define pager_port_wait_for_refs(ds)
2231	#define pager_port_finish_refs(ds)
2232
2233	#endif /* PARALLEL */
2234
2235	/*
2236	* Default pager.
2237	*/
2238	task_t default_pager_self; /* Our task port. */
2239
2240	mach_port_t default_pager_default_port; /* Port for memory_object_create. */
2241
2242	/* We catch exceptions on ourself & startup using this port. */
2243	mach_port_t default_pager_exception_port;
2244
2245	mach_port_t default_pager_internal_set; /* Port set for internal objects. */
2246	mach_port_t default_pager_external_set; /* Port set for external objects. */
2247	mach_port_t default_pager_default_set; /* Port set for "default" thread. */
2248
2249	typedef struct default_pager_thread {
2250	pthread_t dpt_thread; /* Server thread. */
2251	vm_offset_t dpt_buffer; /* Read buffer. */
2252	boolean_t dpt_internal; /* Do we handle internal objects? */
2253	} default_pager_thread_t;
2254
2255	#if PARALLEL1
2256	/* determine number of threads at run time */
2257	#define DEFAULT_PAGER_INTERNAL_COUNT(0) (0)
2258
2259	#else /* PARALLEL */
2260	#define DEFAULT_PAGER_INTERNAL_COUNT(0) (1)
2261	#endif /* PARALLEL */
2262
2263	/* Memory created by default_pager_object_create should mostly be resident. */
2264	#define DEFAULT_PAGER_EXTERNAL_COUNT(1) (1)
2265
2266	unsigned int default_pager_internal_count = DEFAULT_PAGER_INTERNAL_COUNT(0);
2267	/* Number of "internal" threads. */
2268	unsigned int default_pager_external_count = DEFAULT_PAGER_EXTERNAL_COUNT(1);
2269	/* Number of "external" threads. */
2270
2271	default_pager_t pager_port_alloc(size)
2272	vm_size_t size;
2273	{
2274	default_pager_t ds;
2275	p_index_t part;
2276
2277	ds = (default_pager_t) kalloc(sizeof *ds);
2278	if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2279	panic("%spager_port_alloc",my_name);
2280	bzero((char ) ds, sizeof ds);
2281
2282	dstruct_lock_init(ds)pthread_mutex_init(&ds->lock, ((void*)0));
2283
2284	/*
2285	* Get a suitable partition. If none big enough
2286	* just pick one and overcommit. If no partitions
2287	* at all.. well just fake one so that we will
2288	* kill specific objects on pageouts rather than
2289	* panicing the system now.
2290	*/
2291	part = choose_partition(size, P_INDEX_INVALID((p_index_t)-1));
2292	if (no_partition(part)((part) == ((p_index_t)-1))) {
2293	overcommitted(FALSE((boolean_t) 0), atop(size)((size)/vm_page_size));
2294	part = choose_partition(0,P_INDEX_INVALID((p_index_t)-1));
2295	#if debug0
2296	if (no_partition(part)((part) == ((p_index_t)-1)))
2297	dprintf("%s No paging space at all !!\n", my_name);
2298	#endif
2299	}
2300	pager_alloc(&ds->dpager, part, size);
2301
2302	return ds;
2303	}
2304
2305	mach_port_urefs_t default_pager_max_urefs = 10000;
2306
2307	/*
2308	* Check user reference count on pager_request port.
2309	* Pager must be locked.
2310	* Unlocks and re-locks pager if needs to call kernel.
2311	*/
2312	void pager_port_check_request(ds, pager_request)
2313	default_pager_t ds;
2314	mach_port_t pager_request;
2315	{
2316	mach_port_delta_t delta;
2317	kern_return_t kr;
2318
2319	assert(ds->pager_request == pager_request)((ds->pager_request == pager_request) ? (void) (0) : __assert_fail ("ds->pager_request == pager_request", "../../mach-defpager/default_pager.c" , 2319, __PRETTY_FUNCTION__));
2320
2321	if (++ds->request_refs > default_pager_max_urefs) {
2322	delta = 1 - ds->request_refs;
2323	ds->request_refs = 1;
2324
2325	dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2326
2327	/*
2328	* Deallocate excess user references.
2329	*/
2330
2331	kr = mach_port_mod_refs(default_pager_self, pager_request,
2332	MACH_PORT_RIGHT_SEND((mach_port_right_t) 0), delta);
2333	if (kr != KERN_SUCCESS0)
2334	panic("%spager_port_check_request",my_name);
2335
2336	dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2337	}
2338	}
2339
2340	void default_pager_add(ds, internal)
2341	default_pager_t ds;
2342	boolean_t internal;
2343	{
2344	mach_port_t pager = ds->pager;
2345	mach_port_t pset;
2346	mach_port_mscount_t sync;
2347	mach_port_t previous;
2348	kern_return_t kr;
2349	static char here[] = "%sdefault_pager_add";
2350
2351	/*
2352	* The port currently has a make-send count of zero,
2353	* because either we just created the port or we just
2354	* received the port in a memory_object_create request.
2355	*/
2356
2357	if (internal) {
2358	/* possibly generate an immediate no-senders notification */
2359	sync = 0;
2360	pset = default_pager_internal_set;
2361	ds->external = FALSE((boolean_t) 0);
2362	} else {
2363	/* delay notification till send right is created */
2364	sync = 1;
2365	pset = default_pager_external_set;
2366	ds->external = TRUE((boolean_t) 1);
2367	}
2368
2369	kr = mach_port_request_notification(default_pager_self, pager,
2370	MACH_NOTIFY_NO_SENDERS(0100 + 006), sync,
2371	pager, MACH_MSG_TYPE_MAKE_SEND_ONCE21,
2372	&previous);
2373	if ((kr != KERN_SUCCESS0) \|\| (previous != MACH_PORT_NULL((mach_port_t) 0)))
2374	panic(here,my_name);
2375
2376	kr = mach_port_move_member(default_pager_self, pager, pset);
2377	if (kr != KERN_SUCCESS0)
2378	panic(here,my_name);
2379	}
2380
2381	/*
2382	* Routine: memory_object_create
2383	* Purpose:
2384	* Handle requests for memory objects from the
2385	* kernel.
2386	* Notes:
2387	* Because we only give out the default memory
2388	* manager port to the kernel, we don't have to
2389	* be so paranoid about the contents.
2390	*/
2391	kern_return_t
2392	seqnos_memory_object_create(old_pager, seqno, new_pager, new_size,
2393	new_pager_request, new_pager_name, new_page_size)
2394	mach_port_t old_pager;
2395	mach_port_seqno_t seqno;
2396	mach_port_t new_pager;
2397	vm_size_t new_size;
2398	mach_port_t new_pager_request;
2399	mach_port_t new_pager_name;
2400	vm_size_t new_page_size;
2401	{
2402	register default_pager_t ds;
2403	kern_return_t kr;
2404
2405	assert(old_pager == default_pager_default_port)((old_pager == default_pager_default_port) ? (void) (0) : __assert_fail ("old_pager == default_pager_default_port", "../../mach-defpager/default_pager.c" , 2405, __PRETTY_FUNCTION__));
2406	assert(MACH_PORT_VALID(new_pager_request))(((((new_pager_request) != ((mach_port_t) 0)) && ((new_pager_request ) != ((mach_port_t) ~0)))) ? (void) (0) : __assert_fail ("(((new_pager_request) != ((mach_port_t) 0)) && ((new_pager_request) != ((mach_port_t) ~0)))" , "../../mach-defpager/default_pager.c", 2406, __PRETTY_FUNCTION__ ));
2407	assert(MACH_PORT_VALID(new_pager_name))(((((new_pager_name) != ((mach_port_t) 0)) && ((new_pager_name ) != ((mach_port_t) ~0)))) ? (void) (0) : __assert_fail ("(((new_pager_name) != ((mach_port_t) 0)) && ((new_pager_name) != ((mach_port_t) ~0)))" , "../../mach-defpager/default_pager.c", 2407, __PRETTY_FUNCTION__ ));
2408	assert(new_page_size == vm_page_size)((new_page_size == vm_page_size) ? (void) (0) : __assert_fail ("new_page_size == vm_page_size", "../../mach-defpager/default_pager.c" , 2408, __PRETTY_FUNCTION__));
2409
2410	ds = pager_port_alloc(new_size);
2411	rename_it:
2412	kr = mach_port_rename( default_pager_self,
2413	new_pager, (mach_port_t)pnameof(ds)(((vm_offset_t)(ds))+1));
2414	if (kr != KERN_SUCCESS0) {
2415	default_pager_t ds1;
2416
2417	if (kr != KERN_NAME_EXISTS13)
2418	panic("%s m_o_create", my_name);
2419	ds1 = (default_pager_t) kalloc(sizeof *ds1);
2420	ds1 = ds;
2421	pthread_mutex_lock(&all_pagers.lock);
2422	queue_enter(&all_pagers.leak_queue, ds, default_pager_t, links){ register queue_entry_t prev; prev = (&all_pagers.leak_queue )->prev; if ((&all_pagers.leak_queue) == prev) { (& all_pagers.leak_queue)->next = (queue_entry_t) (ds); } else { ((default_pager_t)prev)->links.next = (queue_entry_t)(ds ); } (ds)->links.prev = prev; (ds)->links.next = &all_pagers .leak_queue; (&all_pagers.leak_queue)->prev = (queue_entry_t ) ds; };
2423	pthread_mutex_unlock(&all_pagers.lock);
2424	ds = ds1;
2425	goto rename_it;
2426	}
2427
2428	new_pager = (mach_port_t) pnameof(ds)(((vm_offset_t)(ds))+1);
2429
2430	/*
2431	* Set up associations between these ports
2432	* and this default_pager structure
2433	*/
2434
2435	ds->pager = new_pager;
2436	ds->pager_request = new_pager_request;
2437	ds->request_refs = 1;
2438	ds->pager_name = new_pager_name;
2439	ds->name_refs = 1;
2440
2441	/*
2442	* After this, other threads might receive requests
2443	* for this memory object or find it in the port list.
2444	*/
2445
2446	pager_port_list_insert(new_pager, ds);
2447	default_pager_add(ds, TRUE((boolean_t) 1));
2448
2449	return(KERN_SUCCESS0);
2450	}
2451
2452	memory_object_copy_strategy_t default_pager_copy_strategy =
2453	MEMORY_OBJECT_COPY_DELAY2;
2454
2455	kern_return_t
2456	seqnos_memory_object_init(pager, seqno, pager_request, pager_name,
2457	pager_page_size)
2458	mach_port_t pager;
2459	mach_port_seqno_t seqno;
2460	mach_port_t pager_request;
2461	mach_port_t pager_name;
2462	vm_size_t pager_page_size;
2463	{
2464	register default_pager_t ds;
2465	kern_return_t kr;
2466	static char here[] = "%sinit";
2467
2468	assert(MACH_PORT_VALID(pager_request))(((((pager_request) != ((mach_port_t) 0)) && ((pager_request ) != ((mach_port_t) ~0)))) ? (void) (0) : __assert_fail ("(((pager_request) != ((mach_port_t) 0)) && ((pager_request) != ((mach_port_t) ~0)))" , "../../mach-defpager/default_pager.c", 2468, __PRETTY_FUNCTION__ ));
2469	assert(MACH_PORT_VALID(pager_name))(((((pager_name) != ((mach_port_t) 0)) && ((pager_name ) != ((mach_port_t) ~0)))) ? (void) (0) : __assert_fail ("(((pager_name) != ((mach_port_t) 0)) && ((pager_name) != ((mach_port_t) ~0)))" , "../../mach-defpager/default_pager.c", 2469, __PRETTY_FUNCTION__ ));
2470	assert(pager_page_size == vm_page_size)((pager_page_size == vm_page_size) ? (void) (0) : __assert_fail ("pager_page_size == vm_page_size", "../../mach-defpager/default_pager.c" , 2470, __PRETTY_FUNCTION__));
2471
2472	ds = pager_port_lookup(pager)((! (((pager) != ((mach_port_t) 0)) && ((pager) != (( mach_port_t) ~0))) \|\| ((default_pager_t)(((vm_offset_t)(pager ))&~1))->pager != (pager)) ? ((default_pager_t)0) : (default_pager_t )(((vm_offset_t)(pager))&~1));
2473	if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2474	panic(here, my_name);
2475	pager_port_lock(ds, seqno);
2476
2477	if (ds->pager_request != MACH_PORT_NULL((mach_port_t) 0))
2478	panic(here, my_name);
2479
2480	ds->pager_request = pager_request;
2481	ds->request_refs = 1;
2482	ds->pager_name = pager_name;
2483	ds->name_refs = 1;
2484
2485	/*
2486	* Even if the kernel immediately terminates the object,
2487	* the pager_request port won't be destroyed until
2488	* we process the terminate request, which won't happen
2489	* until we unlock the object.
2490	*/
2491
2492	kr = memory_object_ready(pager_request,
2493	FALSE((boolean_t) 0), /* Do not cache */
2494	default_pager_copy_strategy);
2495	if (kr != KERN_SUCCESS0)
2496	panic(here, my_name);
2497
2498	pager_port_unlock(ds);
2499
2500	return(KERN_SUCCESS0);
2501	}
2502
2503	kern_return_t
2504	seqnos_memory_object_terminate(pager, seqno, pager_request, pager_name)
2505	mach_port_t pager;
2506	mach_port_seqno_t seqno;
2507	mach_port_t pager_request;
2508	mach_port_t pager_name;
2509	{
2510	register default_pager_t ds;
2511	kern_return_t kr;
2512	static char here[] = "%sterminate";
2513
2514	/*
2515	* pager_request and pager_name are receive rights,
2516	* not send rights.
2517	*/
2518
2519	ds = pager_port_lookup(pager)((! (((pager) != ((mach_port_t) 0)) && ((pager) != (( mach_port_t) ~0))) \|\| ((default_pager_t)(((vm_offset_t)(pager ))&~1))->pager != (pager)) ? ((default_pager_t)0) : (default_pager_t )(((vm_offset_t)(pager))&~1));
2520	if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2521	panic(here, my_name);
2522	ddprintf ("seqnos_memory_object_terminate <%p>: pager_port_lock: <%p>[s:%d,r:%d,w:%d,l:%d], %d\n",
2523	&kr, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held, seqno);
2524	pager_port_lock(ds, seqno);
2525
2526	/*
2527	* Wait for read and write requests to terminate.
2528	*/
2529
2530	pager_port_wait_for_readers(ds);
2531	pager_port_wait_for_writers(ds);
2532
2533	/*
2534	* After memory_object_terminate both memory_object_init
2535	* and a no-senders notification are possible, so we need
2536	* to clean up the request and name ports but leave
2537	* the pager port.
2538	*
2539	* A concurrent default_pager_objects might be allocating
2540	* more references for the name port. In this case,
2541	* we must first wait for it to finish.
2542	*/
2543
2544	pager_port_wait_for_refs(ds);
2545
2546	if (ds->external)
2547	pager_request = ds->pager_request;
2548	ds->pager_request = MACH_PORT_NULL((mach_port_t) 0);
2549	ds->request_refs = 0;
2550	assert(ds->pager_name == pager_name)((ds->pager_name == pager_name) ? (void) (0) : __assert_fail ("ds->pager_name == pager_name", "../../mach-defpager/default_pager.c" , 2550, __PRETTY_FUNCTION__));
2551	ds->pager_name = MACH_PORT_NULL((mach_port_t) 0);
2552	ds->name_refs = 0;
2553	ddprintf ("seqnos_memory_object_terminate <%p>: pager_port_unlock: <%p>[s:%d,r:%d,w:%d,l:%d]\n",
2554	&kr, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held);
2555	pager_port_unlock(ds);
2556
2557	/*
2558	* Now we destroy our port rights.
2559	*/
2560
2561	mach_port_destroy(mach_task_self()((__mach_task_self_ + 0)), pager_request);
2562	mach_port_destroy(mach_task_self()((__mach_task_self_ + 0)), pager_name);
2563
2564	return (KERN_SUCCESS0);
2565	}
2566
2567	void default_pager_no_senders(pager, seqno, mscount)
2568	memory_object_t pager;
2569	mach_port_seqno_t seqno;
2570	mach_port_mscount_t mscount;
2571	{
2572	register default_pager_t ds;
2573	kern_return_t kr;
2574	static char here[] = "%sno_senders";
2575
2576	/*
2577	* Because we don't give out multiple send rights
2578	* for a memory object, there can't be a race
2579	* between getting a no-senders notification
2580	* and creating a new send right for the object.
2581	* Hence we don't keep track of mscount.
2582	*/
2583
2584
2585	ds = pager_port_lookup(pager)((! (((pager) != ((mach_port_t) 0)) && ((pager) != (( mach_port_t) ~0))) \|\| ((default_pager_t)(((vm_offset_t)(pager ))&~1))->pager != (pager)) ? ((default_pager_t)0) : (default_pager_t )(((vm_offset_t)(pager))&~1));
2586	if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2587	panic(here,my_name);
2588	pager_port_lock(ds, seqno);
2589
2590	/*
2591	* We shouldn't get a no-senders notification
2592	* when the kernel has the object cached.
2593	*/
2594
2595	if (ds->pager_request != MACH_PORT_NULL((mach_port_t) 0))
2596	panic(here,my_name);
2597
2598	/*
2599	* Unlock the pager (though there should be no one
2600	* waiting for it).
2601	*/
2602	dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2603
2604	/*
2605	* Remove the memory object port association, and then
2606	* the destroy the port itself. We must remove the object
2607	* from the port list before deallocating the pager,
2608	* because of default_pager_objects.
2609	*/
2610
2611	pager_port_list_delete(ds);
2612	pager_dealloc(&ds->dpager);
2613
2614	kr = mach_port_mod_refs(default_pager_self, pager,
2615	MACH_PORT_RIGHT_RECEIVE((mach_port_right_t) 1), -1);
2616	if (kr != KERN_SUCCESS0)
2617	panic(here,my_name);
2618
2619	/*
2620	* Do this after deallocating the port name
2621	*/
2622	kfree((char ) ds, sizeof(ds));
2623
2624	/*
2625	* Recover memory that we might have wasted because
2626	* of name conflicts
2627	*/
2628	pthread_mutex_lock(&all_pagers.lock);
2629
2630	while (!queue_empty(&all_pagers.leak_queue)(((&all_pagers.leak_queue)) == (((&all_pagers.leak_queue )->next)))) {
2631
2632	ds = (default_pager_t) queue_first(&all_pagers.leak_queue)((&all_pagers.leak_queue)->next);
2633	queue_remove_first(&all_pagers.leak_queue, ds, default_pager_t, links){ register queue_entry_t next; (ds) = (default_pager_t) ((& all_pagers.leak_queue)->next); next = (ds)->links.next; if ((&all_pagers.leak_queue) == next) (&all_pagers.leak_queue )->prev = (&all_pagers.leak_queue); else ((default_pager_t )(next))->links.prev = (&all_pagers.leak_queue); (& all_pagers.leak_queue)->next = next; };
2634	kfree((char ) ds, sizeof(ds));
2635	}
2636
2637	pthread_mutex_unlock(&all_pagers.lock);
2638	}
2639
2640	int default_pager_pagein_count = 0;
2641	int default_pager_pageout_count = 0;
2642
2643	static __thread default_pager_thread_t *dpt;
2644
2645	kern_return_t
2646	seqnos_memory_object_data_request(pager, seqno, reply_to, offset,
2647	length, protection_required)
2648	memory_object_t pager;
2649	mach_port_seqno_t seqno;
2650	mach_port_t reply_to;
2651	vm_offset_t offset;
2652	vm_size_t length;
2653	vm_prot_t protection_required;
2654	{
2655	default_pager_t ds;
2656	vm_offset_t addr;
2657	unsigned int errors;
2658	kern_return_t rc;
2659	static char here[] = "%sdata_request";
2660
2661	if (length != vm_page_size)
2662	panic(here,my_name);
2663
2664	ds = pager_port_lookup(pager)((! (((pager) != ((mach_port_t) 0)) && ((pager) != (( mach_port_t) ~0))) \|\| ((default_pager_t)(((vm_offset_t)(pager ))&~1))->pager != (pager)) ? ((default_pager_t)0) : (default_pager_t )(((vm_offset_t)(pager))&~1));
2665	if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2666	panic(here,my_name);
2667	ddprintf ("seqnos_memory_object_data_request <%p>: pager_port_lock: <%p>[s:%d,r:%d,w:%d,l:%d], %d\n",
2668	&ds, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held, seqno);
2669	pager_port_lock(ds, seqno);
2670	pager_port_check_request(ds, reply_to);
2671	pager_port_wait_for_writers(ds);
2672	pager_port_start_read(ds);
2673
2674	/*
2675	* Get error count while pager locked.
2676	*/
2677	errors = ds->errors;
2678
2679	ddprintf ("seqnos_memory_object_data_request <%p>: pager_port_unlock: <%p>[s:%d,r:%d,w:%d,l:%d]\n",
2680	&ds, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held);
2681	pager_port_unlock(ds);
2682
2683	if (errors) {
2684	dprintf("%s %s\n", my_name,
2685	"dropping data_request because of previous paging errors");
2686	(void) memory_object_data_error(reply_to,
2687	offset, vm_page_size,
2688	KERN_FAILURE5);
2689	goto done;
2690	}
2691
2692	if (offset >= ds->dpager.limit)
2693	rc = PAGER_ERROR2;
2694	else
2695	rc = default_read(&ds->dpager, dpt->dpt_buffer,
2696	vm_page_size, offset,
2697	&addr, protection_required & VM_PROT_WRITE((vm_prot_t) 0x02),
2698	ds->external);
2699
2700	switch (rc) {
2701	case PAGER_SUCCESS0:
2702	if (addr != dpt->dpt_buffer) {
2703	/*
2704	* Deallocates data buffer
2705	*/
2706	(void) memory_object_data_supply(
2707	reply_to, offset,
2708	addr, vm_page_size, TRUE((boolean_t) 1),
2709	VM_PROT_NONE((vm_prot_t) 0x00),
2710	FALSE((boolean_t) 0), MACH_PORT_NULL((mach_port_t) 0));
2711	} else {
2712	(void) memory_object_data_supply(
2713	reply_to, offset,
2714	addr, vm_page_size, FALSE((boolean_t) 0),
2715	VM_PROT_NONE((vm_prot_t) 0x00),
2716	FALSE((boolean_t) 0), MACH_PORT_NULL((mach_port_t) 0));
2717	}
2718	break;
2719
2720	case PAGER_ABSENT1:
2721	(void) memory_object_data_unavailable(
2722	reply_to,
2723	offset,
2724	vm_page_size);
2725	break;
2726
2727	case PAGER_ERROR2:
2728	(void) memory_object_data_error(
2729	reply_to,
2730	offset,
2731	vm_page_size,
2732	KERN_FAILURE5);
2733	break;
2734	}
2735
2736	default_pager_pagein_count++;
2737
2738	done:
2739	pager_port_finish_read(ds);
2740	return(KERN_SUCCESS0);
2741	}
2742
2743	/*
2744	* memory_object_data_initialize: check whether we already have each page, and
2745	* write it if we do not. The implementation is far from optimized, and
2746	* also assumes that the default_pager is single-threaded.
2747	*/
2748	kern_return_t
2749	seqnos_memory_object_data_initialize(pager, seqno, pager_request,
2750	offset, addr, data_cnt)
2751	memory_object_t pager;
2752	mach_port_seqno_t seqno;
2753	mach_port_t pager_request;
2754	register
2755	vm_offset_t offset;
2756	register
2757	pointer_t addr;
2758	vm_size_t data_cnt;
2759	{
2760	vm_offset_t amount_sent;
2761	default_pager_t ds;
2762	static char here[] = "%sdata_initialize";
2763
2764	#ifdef lint
2765	pager_request++;
2766	#endif /* lint */
2767
2768	ds = pager_port_lookup(pager)((! (((pager) != ((mach_port_t) 0)) && ((pager) != (( mach_port_t) ~0))) \|\| ((default_pager_t)(((vm_offset_t)(pager ))&~1))->pager != (pager)) ? ((default_pager_t)0) : (default_pager_t )(((vm_offset_t)(pager))&~1));
2769	if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2770	panic(here,my_name);
2771	ddprintf ("seqnos_memory_object_data_initialize <%p>: pager_port_lock: <%p>[s:%d,r:%d,w:%d,l:%d], %d\n",
2772	&ds, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held, seqno);
2773	pager_port_lock(ds, seqno);
2774	pager_port_check_request(ds, pager_request);
2775	pager_port_start_write(ds);
2776	ddprintf ("seqnos_memory_object_data_initialize <%p>: pager_port_unlock: <%p>[s:%d,r:%d,w:%d,l:%d]\n",
2777	&ds, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held);
2778	pager_port_unlock(ds);
2779
2780	for (amount_sent = 0;
2781	amount_sent < data_cnt;
2782	amount_sent += vm_page_size) {
2783
2784	if (!default_has_page(&ds->dpager, offset + amount_sent)) {
2785	if (default_write(&ds->dpager,
2786	addr + amount_sent,
2787	vm_page_size,
2788	offset + amount_sent)
2789	!= PAGER_SUCCESS0) {
2790	dprintf("%s%s write error\n", my_name, here);
2791	dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2792	ds->errors++;
2793	dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2794	}
2795	}
2796	}
2797
2798	pager_port_finish_write(ds);
2799	if (vm_deallocate(default_pager_self, addr, data_cnt) != KERN_SUCCESS0)
2800	panic(here,my_name);
2801
2802	return(KERN_SUCCESS0);
2803	}
2804
2805	/*
2806	* memory_object_data_write: split up the stuff coming in from
2807	* a memory_object_data_write call
2808	* into individual pages and pass them off to default_write.
2809	*/
2810	kern_return_t
2811	seqnos_memory_object_data_write(pager, seqno, pager_request,
2812	offset, addr, data_cnt)
2813	memory_object_t pager;
2814	mach_port_seqno_t seqno;
2815	mach_port_t pager_request;
2816	register
2817	vm_offset_t offset;
2818	register
2819	pointer_t addr;
2820	vm_size_t data_cnt;
2821	{
2822	register
2823	vm_size_t amount_sent;
2824	default_pager_t ds;
2825	static char here[] = "%sdata_write";
2826	int err;
2827
2828	#ifdef lint
2829	pager_request++;
2830	#endif /* lint */
2831
2832	if ((data_cnt % vm_page_size) != 0)
2833	panic(here,my_name);
2834
2835	ds = pager_port_lookup(pager)((! (((pager) != ((mach_port_t) 0)) && ((pager) != (( mach_port_t) ~0))) \|\| ((default_pager_t)(((vm_offset_t)(pager ))&~1))->pager != (pager)) ? ((default_pager_t)0) : (default_pager_t )(((vm_offset_t)(pager))&~1));
2836	if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2837	panic(here,my_name);
2838
2839	pager_port_lock(ds, seqno);
2840	pager_port_start_write(ds);
2841
2842	vm_size_t limit = ds->dpager.byte_limit;
2843	pager_port_unlock(ds);
2844	if ((limit != round_page(limit)((((vm_offset_t) (limit) + __vm_page_size - 1) / __vm_page_size ) * __vm_page_size)) && (trunc_page(limit)((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size) == offset)) {
2845	assert(trunc_page(limit) == offset)((((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size ) == offset) ? (void) (0) : __assert_fail ("((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size) == offset" , "../../mach-defpager/default_pager.c", 2845, __PRETTY_FUNCTION__ ));
2846	assert(data_cnt == vm_page_size)((data_cnt == vm_page_size) ? (void) (0) : __assert_fail ("data_cnt == vm_page_size" , "../../mach-defpager/default_pager.c", 2846, __PRETTY_FUNCTION__ ));
2847
2848	vm_offset_t tail = addr + limit - trunc_page(limit)((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size);
2849	vm_size_t tail_size = round_page(limit)((((vm_offset_t) (limit) + __vm_page_size - 1) / __vm_page_size ) * __vm_page_size) - limit;
2850	memset((void *) tail, 0, tail_size);
2851
2852	unsigned arr = (unsigned )addr;
2853	memory_object_data_supply(pager_request, trunc_page(limit)((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size), addr,
2854	vm_page_size, TRUE((boolean_t) 1), VM_PROT_NONE((vm_prot_t) 0x00),
2855	TRUE((boolean_t) 1), MACH_PORT_NULL((mach_port_t) 0));
2856	dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2857	ds->dpager.byte_limit = round_page(limit)((((vm_offset_t) (limit) + __vm_page_size - 1) / __vm_page_size ) * __vm_page_size);
2858	dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2859	pager_port_finish_write(ds);
2860
2861	return(KERN_SUCCESS0);
2862	}
2863
2864	for (amount_sent = 0;
2865	amount_sent < data_cnt;
2866	amount_sent += vm_page_size) {
2867
2868	register int result;
2869
2870	result = default_write(&ds->dpager,
2871	addr + amount_sent,
2872	vm_page_size,
2873	offset + amount_sent);
2874	if (result != KERN_SUCCESS0) {
2875	dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2876	ds->errors++;
2877	dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2878	}
2879	default_pager_pageout_count++;
2880	}
2881
2882	pager_port_finish_write(ds);
2883	err = vm_deallocate(default_pager_self, addr, data_cnt);
2884	if (err != KERN_SUCCESS0)
2885	{
2886	panic(here,my_name);
2887	}
2888
2889	return(KERN_SUCCESS0);
2890	}
2891
2892	/ARGSUSED/
2893	kern_return_t
2894	seqnos_memory_object_copy(old_memory_object, seqno, old_memory_control,
2895	offset, length, new_memory_object)
2896	memory_object_t old_memory_object;
2897	mach_port_seqno_t seqno;
2898	memory_object_control_t
2899	old_memory_control;
2900	vm_offset_t offset;
2901	vm_size_t length;
2902	memory_object_t new_memory_object;
2903	{
2904	panic("%scopy", my_name);
2905	return KERN_FAILURE5;
2906	}
2907
2908	/* We get this when our memory_object_lock_request has completed
2909	after we truncated an object. */
2910	kern_return_t
2911	seqnos_memory_object_lock_completed (memory_object_t pager,
2912	mach_port_seqno_t seqno,
2913	mach_port_t pager_request,
2914	vm_offset_t offset,
2915	vm_size_t length)
2916	{
2917	panic("%slock_completed",my_name);
2918	return KERN_FAILURE5;
2919	}
2920
2921	kern_return_t
2922	seqnos_memory_object_data_unlock(pager, seqno, pager_request,
2923	offset, addr, data_cnt)
2924	memory_object_t pager;
2925	mach_port_seqno_t seqno;
2926	mach_port_t pager_request;
2927	vm_offset_t offset;
2928	pointer_t addr;
2929	vm_size_t data_cnt;
2930	{
2931	panic("%sdata_unlock",my_name);
2932	return(KERN_FAILURE5);
2933	}
2934
2935	kern_return_t
2936	seqnos_memory_object_supply_completed(pager, seqno, pager_request,
2937	offset, length,
2938	result, error_offset)
2939	memory_object_t pager;
2940	mach_port_seqno_t seqno;
2941	mach_port_t pager_request;
2942	vm_offset_t offset;
2943	vm_size_t length;
2944	kern_return_t result;
2945	vm_offset_t error_offset;
2946	{
2947	panic("%ssupply_completed",my_name);
2948	return(KERN_FAILURE5);
2949	}
2950
2951	/*
2952	* memory_object_data_return: split up the stuff coming in from
2953	* a memory_object_data_write call
2954	* into individual pages and pass them off to default_write.
2955	*/
2956	kern_return_t
2957	seqnos_memory_object_data_return(pager, seqno, pager_request,
2958	offset, addr, data_cnt,
2959	dirty, kernel_copy)
2960	memory_object_t pager;
2961	mach_port_seqno_t seqno;
2962	mach_port_t pager_request;
2963	vm_offset_t offset;
2964	pointer_t addr;
2965	vm_size_t data_cnt;
2966	boolean_t dirty;
2967	boolean_t kernel_copy;
2968	{
2969
2970	return seqnos_memory_object_data_write (pager, seqno, pager_request,
2971	offset, addr, data_cnt);
2972	}
2973
2974	kern_return_t
2975	seqnos_memory_object_change_completed(pager, seqno, may_cache, copy_strategy)
2976	memory_object_t pager;
2977	mach_port_seqno_t seqno;
2978	boolean_t may_cache;
2979	memory_object_copy_strategy_t copy_strategy;
2980	{
2981	panic("%schange_completed",my_name);
2982	return(KERN_FAILURE5);
2983	}
2984
2985
2986	boolean_t default_pager_notify_server(in, out)
2987	mach_msg_header_t in, out;
2988	{
2989	register mach_no_senders_notification_t *n =
2990	(mach_no_senders_notification_t *) in;
2991
2992	/*
2993	* The only send-once rights we create are for
2994	* receiving no-more-senders notifications.
2995	* Hence, if we receive a message directed to
2996	* a send-once right, we can assume it is
2997	* a genuine no-senders notification from the kernel.
2998	*/
2999
3000	if ((n->not_header.msgh_bits !=
3001	MACH_MSGH_BITS(0, MACH_MSG_TYPE_PORT_SEND_ONCE)((0) \| ((18) << 8))) \|\|
3002	(n->not_header.msgh_id != MACH_NOTIFY_NO_SENDERS(0100 + 006)))
3003	return FALSE((boolean_t) 0);
3004
3005	assert(n->not_header.msgh_size == sizeof n)((n->not_header.msgh_size == sizeof n) ? (void) (0) : __assert_fail ("n->not_header.msgh_size == sizeof *n", "../../mach-defpager/default_pager.c" , 3005, __PRETTY_FUNCTION__));
3006	assert(n->not_header.msgh_remote_port == MACH_PORT_NULL)((n->not_header.msgh_remote_port == ((mach_port_t) 0)) ? ( void) (0) : __assert_fail ("n->not_header.msgh_remote_port == ((mach_port_t) 0)" , "../../mach-defpager/default_pager.c", 3006, __PRETTY_FUNCTION__ ));
3007
3008	assert(n->not_type.msgt_name == MACH_MSG_TYPE_INTEGER_32)((n->not_type.msgt_name == 2) ? (void) (0) : __assert_fail ("n->not_type.msgt_name == 2", "../../mach-defpager/default_pager.c" , 3008, __PRETTY_FUNCTION__));
3009	assert(n->not_type.msgt_size == 32)((n->not_type.msgt_size == 32) ? (void) (0) : __assert_fail ("n->not_type.msgt_size == 32", "../../mach-defpager/default_pager.c" , 3009, __PRETTY_FUNCTION__));
3010	assert(n->not_type.msgt_number == 1)((n->not_type.msgt_number == 1) ? (void) (0) : __assert_fail ("n->not_type.msgt_number == 1", "../../mach-defpager/default_pager.c" , 3010, __PRETTY_FUNCTION__));
3011	assert(n->not_type.msgt_inline)((n->not_type.msgt_inline) ? (void) (0) : __assert_fail ("n->not_type.msgt_inline" , "../../mach-defpager/default_pager.c", 3011, __PRETTY_FUNCTION__ ));
3012	assert(! n->not_type.msgt_longform)((! n->not_type.msgt_longform) ? (void) (0) : __assert_fail ("! n->not_type.msgt_longform", "../../mach-defpager/default_pager.c" , 3012, __PRETTY_FUNCTION__));
3013
3014	default_pager_no_senders(n->not_header.msgh_local_port,
3015	n->not_header.msgh_seqno, n->not_count);
3016
3017	out->msgh_remote_port = MACH_PORT_NULL((mach_port_t) 0);
3018	return TRUE((boolean_t) 1);
3019	}
3020
3021	extern boolean_t seqnos_memory_object_server();
3022	extern boolean_t seqnos_memory_object_default_server();
3023	extern boolean_t default_pager_server();
3024	extern boolean_t exc_server();
3025	extern boolean_t bootstrap_server();
3026	extern void bootstrap_compat();
3027
3028	mach_msg_size_t default_pager_msg_size_object = 128;
3029
3030	boolean_t
3031	default_pager_demux_object(in, out)
3032	mach_msg_header_t *in;
3033	mach_msg_header_t *out;
3034	{
3035	/*
3036	* We receive memory_object_data_initialize messages in
3037	* the memory_object_default interface.
3038	*/
3039
3040	int rval;
3041	ddprintf ("DPAGER DEMUX OBJECT <%p>: %d\n", in, in->msgh_id);
3042	rval =
3043	(seqnos_memory_object_server(in, out) \|\|
3044	seqnos_memory_object_default_server(in, out) \|\|
3045	default_pager_notify_server(in, out) \|\|
3046	default_pager_server(in, out));
3047	ddprintf ("DPAGER DEMUX OBJECT DONE <%p>: %d\n", in, in->msgh_id);
3048	return rval;
3049	}
3050
3051	mach_msg_size_t default_pager_msg_size_default = 8 * 1024;
3052
3053	boolean_t
3054	default_pager_demux_default(in, out)
3055	mach_msg_header_t *in;
3056	mach_msg_header_t *out;
3057	{
3058	if (in->msgh_local_port == default_pager_default_port) {
3059	/*
3060	* We receive memory_object_create messages in
3061	* the memory_object_default interface.
3062	*/
3063
3064	int rval;
3065	ddprintf ("DPAGER DEMUX DEFAULT <%p>: %d\n", in, in->msgh_id);
3066	rval =
3067	(seqnos_memory_object_default_server(in, out) \|\|
3068	default_pager_server(in, out));
3069	ddprintf ("DPAGER DEMUX DEFAULT DONE <%p>: %d\n", in, in->msgh_id);
3070	return rval;
3071	} else if (in->msgh_local_port == default_pager_exception_port) {
3072	/*
3073	* We receive exception messages for
3074	* ourself and the startup task.
3075	*/
3076
3077	return exc_server(in, out);
3078	} else {
3079	panic(my_name);
3080	return FALSE((boolean_t) 0);
3081	}
3082	}
3083
3084	/*
3085	* We use multiple threads, for two reasons.
3086	*
3087	* First, memory objects created by default_pager_object_create
3088	* are "external", instead of "internal". This means the kernel
3089	* sends data (memory_object_data_write) to the object pageable.
3090	* To prevent deadlocks, the external and internal objects must
3091	* be managed by different threads.
3092	*
3093	* Second, the default pager uses synchronous IO operations.
3094	* Spreading requests across multiple threads should
3095	* recover some of the performance loss from synchronous IO.
3096	*
3097	* We have 3+ threads.
3098	* One receives memory_object_create and
3099	* default_pager_object_create requests.
3100	* One or more manage internal objects.
3101	* One or more manage external objects.
3102	*/
3103
3104	void
3105	default_pager_thread_privileges()
3106	{
3107	/*
3108	* Set thread privileges.
3109	*/
3110	wire_thread(); /* grab a kernel stack and memory allocation
3111	privileges */
3112	}
3113
3114	void *
3115	default_pager_default_thread(void *arg)
3116	{
3117	kern_return_t kr;
3118	default_pager_thread_privileges ();
3119	for (;;) {
3120	kr = mach_msg_server(default_pager_demux_default,
3121	default_pager_msg_size_default,
3122	default_pager_default_set);
3123	panic(my_name, kr);
3124	}
3125	}
3126
3127
3128
3129	void *
3130	default_pager_thread(void *arg)
3131	{
3132	mach_port_t pset;
3133	kern_return_t kr;
3134
3135	dpt = (default_pager_thread_t *) arg;
3136
3137	/*
3138	* Threads handling external objects cannot have
3139	* privileges. Otherwise a burst of data-requests for an
3140	* external object could empty the free-page queue,
3141	* because the fault code only reserves real pages for
3142	* requests sent to internal objects.
3143	*/
3144
3145	if (dpt->dpt_internal) {
3146	default_pager_thread_privileges();
3147	pset = default_pager_internal_set;
3148	} else {
3149	pset = default_pager_external_set;
3150	}
3151
3152	for (;;) {
3153	kr = mach_msg_server(default_pager_demux_object,
3154	default_pager_msg_size_object,
3155	pset);
3156	panic(my_name, kr);
3157	}
3158	}
3159
3160	void
3161	start_default_pager_thread(internal)
3162	boolean_t internal;
3163	{
3164	default_pager_thread_t *ndpt;
3165	kern_return_t kr;
3166	error_t err;
3167
3168	ndpt = (default_pager_thread_t ) kalloc(sizeof ndpt);
3169	if (ndpt == 0)
3170	panic(my_name);
3171
3172	ndpt->dpt_internal = internal;
3173
3174	kr = vm_allocate(default_pager_self, &ndpt->dpt_buffer,
3175	vm_page_size, TRUE((boolean_t) 1));
3176	if (kr != KERN_SUCCESS0)
3177	panic(my_name);
3178	wire_memory(ndpt->dpt_buffer, vm_page_size,
3179	VM_PROT_READ((vm_prot_t) 0x01)\|VM_PROT_WRITE((vm_prot_t) 0x02));
3180
3181	err = pthread_create(&ndpt->dpt_thread, NULL((void*)0), default_pager_thread,
3182	ndpt);
3183	if (!err)
3184	pthread_detach (ndpt->dpt_thread);
3185	else {
3186	errno(*__errno_location ()) = err;
3187	perror ("pthread_create");
3188	}
3189	}
3190
3191	void
3192	default_pager_initialize(host_port)
3193	mach_port_t host_port;
3194	{
3195	memory_object_t DMM;
3196	kern_return_t kr;
3197
3198	/*
3199	* This task will become the default pager.
3200	*/
3201	default_pager_self = mach_task_self()((__mach_task_self_ + 0));
3202
3203	/*
3204	* Initialize the "default pager" port.
3205	*/
3206	kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_RECEIVE((mach_port_right_t) 1),
3207	&default_pager_default_port);
3208	if (kr != KERN_SUCCESS0)
3209	panic(my_name);
3210
3211	DMM = default_pager_default_port;
3212	kr = vm_set_default_memory_manager(host_port, &DMM);
3213	if ((kr != KERN_SUCCESS0) \|\| MACH_PORT_VALID(DMM)(((DMM) != ((mach_port_t) 0)) && ((DMM) != ((mach_port_t ) ~0))))
3214	panic(my_name);
3215
3216	/*
3217	* Initialize the exception port.
3218	*/
3219	kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_RECEIVE((mach_port_right_t) 1),
3220	&default_pager_exception_port);
3221	if (kr != KERN_SUCCESS0)
3222	panic(my_name);
3223
3224	/*
3225	* Arrange for wiring privileges.
3226	*/
3227	wire_setup(host_port);
3228
3229	/*
3230	* Find out how many CPUs we have, to determine the number
3231	* of threads to create.
3232	*/
3233	if (default_pager_internal_count == 0) {
3234	host_basic_info_data_t h_info;
3235	natural_t h_info_count;
3236
3237	h_info_count = HOST_BASIC_INFO_COUNT(sizeof(host_basic_info_data_t)/sizeof(integer_t));
3238	(void) host_info(host_port, HOST_BASIC_INFO1,
3239	(host_info_t)&h_info, &h_info_count);
3240
3241	/*
3242	* Random computation to get more parallelism on
3243	* multiprocessors.
3244	*/
3245	default_pager_internal_count =
3246	(h_info.avail_cpus > 32 ? 32 : h_info.avail_cpus) / 4 + 3;
3247	}
3248	}
3249
3250	/*
3251	* Initialize and Run the default pager
3252	*/
3253	void
3254	default_pager()
3255	{
3256	kern_return_t kr;
3257	int i;
3258
3259	default_pager_thread_privileges();
3260
3261	/*
3262	* Wire down code, data, stack
3263	*/
3264	wire_all_memory();
3265
3266
3267	/*
3268	* Initialize the list of all pagers.
3269	*/
3270	pager_port_list_init(){ pthread_mutex_init(&all_pagers.lock, ((void*)0)); ((& all_pagers.queue)->next = (&all_pagers.queue)->prev = &all_pagers.queue); ((&all_pagers.leak_queue)-> next = (&all_pagers.leak_queue)->prev = &all_pagers .leak_queue); all_pagers.count = 0; };
3271
3272	kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_PORT_SET((mach_port_right_t) 3),
3273	&default_pager_internal_set);
3274	if (kr != KERN_SUCCESS0)
3275	panic(my_name);
3276
3277	kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_PORT_SET((mach_port_right_t) 3),
3278	&default_pager_external_set);
3279	if (kr != KERN_SUCCESS0)
3280	panic(my_name);
3281
3282	kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_PORT_SET((mach_port_right_t) 3),
3283	&default_pager_default_set);
3284	if (kr != KERN_SUCCESS0)
3285	panic(my_name);
3286
3287	kr = mach_port_move_member(default_pager_self,
3288	default_pager_default_port,
3289	default_pager_default_set);
3290	if (kr != KERN_SUCCESS0)
3291	panic(my_name);
3292
3293	kr = mach_port_move_member(default_pager_self,
3294	default_pager_exception_port,
3295	default_pager_default_set);
3296	if (kr != KERN_SUCCESS0)
3297	panic(my_name);
3298
3299	/*
3300	* Now we create the threads that will actually
3301	* manage objects.
3302	*/
3303
3304	for (i = 0; i < default_pager_internal_count; i++)
3305	start_default_pager_thread(TRUE((boolean_t) 1));
3306
3307	for (i = 0; i < default_pager_external_count; i++)
3308	start_default_pager_thread(FALSE((boolean_t) 0));
3309
3310	default_pager_default_thread(0); /* Become the default_pager server */
3311	#if 0
3312	cthread_fork (default_pager_default_thread, 0);
3313	/* cthread_exit (cthread_self ()); */
3314	thread_suspend (mach_thread_self ());
3315	#endif
3316	}
3317
3318	/*
3319	* Create an external object.
3320	*/
3321	kern_return_t
3322	S_default_pager_object_create (mach_port_t pager,
3323	mach_port_t *mem_obj,
3324	vm_size_t size)
3325	{
3326	default_pager_t ds;
3327	mach_port_t port;
3328	kern_return_t result;
3329
3330	if (pager != default_pager_default_port)
3331	return KERN_INVALID_ARGUMENT4;
3332
3333	ds = pager_port_alloc(size);
3334	rename_it:
3335	port = (mach_port_t) pnameof(ds)(((vm_offset_t)(ds))+1);
3336	result = mach_port_allocate_name(default_pager_self,
3337	MACH_PORT_RIGHT_RECEIVE((mach_port_right_t) 1), port);
3338	if (result != KERN_SUCCESS0) {
3339	default_pager_t ds1;
3340
3341	if (result != KERN_NAME_EXISTS13) return (result);
3342
3343	ds1 = (default_pager_t) kalloc(sizeof *ds1);
3344	ds1 = ds;
3345	pthread_mutex_lock(&all_pagers.lock);
3346	queue_enter(&all_pagers.leak_queue, ds, default_pager_t, links){ register queue_entry_t prev; prev = (&all_pagers.leak_queue )->prev; if ((&all_pagers.leak_queue) == prev) { (& all_pagers.leak_queue)->next = (queue_entry_t) (ds); } else { ((default_pager_t)prev)->links.next = (queue_entry_t)(ds ); } (ds)->links.prev = prev; (ds)->links.next = &all_pagers .leak_queue; (&all_pagers.leak_queue)->prev = (queue_entry_t ) ds; };
3347	pthread_mutex_unlock(&all_pagers.lock);
3348	ds = ds1;
3349	goto rename_it;
3350	}
3351
3352	/*
3353	* Set up associations between these ports
3354	* and this default_pager structure
3355	*/
3356
3357	ds->pager = port;
3358	ds->dpager.limit = size;
3359	pager_port_list_insert(port, ds);
3360	default_pager_add(ds, FALSE((boolean_t) 0));
3361
3362	*mem_obj = port;
3363	return (KERN_SUCCESS0);
3364	}
3365
3366	kern_return_t
3367	S_default_pager_info (mach_port_t pager,
3368	default_pager_info_t *infop)
3369	{
3370	vm_size_t total, free;
3371
3372	if (pager != default_pager_default_port)
3373	return KERN_INVALID_ARGUMENT4;
3374
3375	pthread_mutex_lock(&all_partitions.lock);
3376	paging_space_info(&total, &free);
3377	pthread_mutex_unlock(&all_partitions.lock);
3378
3379	infop->dpi_total_space = ptoa(total)((total)*vm_page_size);
3380	infop->dpi_free_space = ptoa(free)((free)*vm_page_size);
3381	infop->dpi_page_size = vm_page_size;
3382	return KERN_SUCCESS0;
3383	}
3384
3385	kern_return_t
3386	S_default_pager_objects (mach_port_t pager,
3387	default_pager_object_array_t *objectsp,
3388	natural_t *ocountp,
3389	mach_port_array_t *portsp,
3390	natural_t *pcountp)
3391	{
3392	vm_offset_t oaddr; /* memory for objects */
3393	vm_size_t osize; /* current size */
3394	default_pager_object_t *objects;
3395	natural_t opotential;
3396
3397	vm_offset_t paddr; /* memory for ports */
3398	vm_size_t psize; /* current size */
3399	mach_port_t *ports;
3400	natural_t ppotential;
3401
3402	unsigned int actual;
3403	unsigned int num_pagers;
3404	kern_return_t kr;
3405	default_pager_t entry;
3406
3407	if (pager != default_pager_default_port)
3408	return KERN_INVALID_ARGUMENT4;
3409
3410	/* start with the inline memory */
3411
3412	num_pagers = 0;
3413
3414	objects = *objectsp;
3415	opotential = *ocountp;
3416
3417	ports = *portsp;
3418	ppotential = *pcountp;
3419
3420	pthread_mutex_lock(&all_pagers.lock);
3421	/*
3422	* We will send no more than this many
3423	*/
3424	actual = all_pagers.count;
3425	pthread_mutex_unlock(&all_pagers.lock);
3426
3427	if (opotential < actual) {
3428	vm_offset_t newaddr;
3429	vm_size_t newsize;
3430
3431	newsize = 2 * round_page(actual * sizeof objects)((((vm_offset_t) (actual sizeof objects) + __vm_page_size - 1) / __vm_page_size) __vm_page_size);
3432
3433	kr = vm_allocate(default_pager_self, &newaddr, newsize, TRUE((boolean_t) 1));
3434	if (kr != KERN_SUCCESS0)
3435	goto nomemory;
3436
3437	oaddr = newaddr;
3438	osize = newsize;
3439	opotential = osize/sizeof *objects;
3440	objects = (default_pager_object_t *) oaddr;
3441	}
3442
3443	if (ppotential < actual) {
3444	vm_offset_t newaddr;
3445	vm_size_t newsize;
3446
3447	newsize = 2 * round_page(actual * sizeof ports)((((vm_offset_t) (actual sizeof ports) + __vm_page_size - 1 ) / __vm_page_size) __vm_page_size);
3448
3449	kr = vm_allocate(default_pager_self, &newaddr, newsize, TRUE((boolean_t) 1));
3450	if (kr != KERN_SUCCESS0)
3451	goto nomemory;
3452
3453	paddr = newaddr;
3454	psize = newsize;
3455	ppotential = psize/sizeof *ports;
3456	ports = (mach_port_t *) paddr;
3457	}
3458
3459	/*
3460	* Now scan the list.
3461	*/
3462
3463	pthread_mutex_lock(&all_pagers.lock);
3464
3465	num_pagers = 0;
3466	queue_iterate(&all_pagers.queue, entry, default_pager_t, links)for ((entry) = (default_pager_t) ((&all_pagers.queue)-> next); !(((&all_pagers.queue)) == ((queue_entry_t)(entry) )); (entry) = (default_pager_t) ((&(entry)->links)-> next)) {
3467
3468	mach_port_t port;
3469	vm_size_t size;
3470
3471	if ((num_pagers >= opotential) \|\|
3472	(num_pagers >= ppotential)) {
3473	/*
3474	* This should be rare. In any case,
3475	* we will only miss recent objects,
3476	* because they are added at the end.
3477	*/
3478	break;
3479	}
3480
3481	/*
3482	* Avoid interfering with normal operations
3483	*/
3484	if (pthread_mutex_trylock(&entry->dpager.lock))
3485	goto not_this_one;
3486	size = pager_allocated(&entry->dpager);
3487	pthread_mutex_unlock(&entry->dpager.lock);
3488
3489	dstruct_lock(entry)pthread_mutex_lock(&entry->lock);
3490
3491	port = entry->pager_name;
3492	if (port == MACH_PORT_NULL((mach_port_t) 0)) {
3493	/*
3494	* The object is waiting for no-senders
3495	* or memory_object_init.
3496	*/
3497	dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3498	goto not_this_one;
3499	}
3500
3501	/*
3502	* We need a reference for the reply message.
3503	* While we are unlocked, the bucket queue
3504	* can change and the object might be terminated.
3505	* memory_object_terminate will wait for us,
3506	* preventing deallocation of the entry.
3507	*/
3508
3509	if (--entry->name_refs == 0) {
3510	dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3511
3512	/* keep the list locked, wont take long */
3513
3514	kr = mach_port_mod_refs(default_pager_self,
3515	port, MACH_PORT_RIGHT_SEND((mach_port_right_t) 0),
3516	default_pager_max_urefs);
3517	if (kr != KERN_SUCCESS0)
3518	panic("%sdefault_pager_objects",my_name);
3519
3520	dstruct_lock(entry)pthread_mutex_lock(&entry->lock);
3521
3522	entry->name_refs += default_pager_max_urefs;
3523	pager_port_finish_refs(entry);
3524	}
3525	dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3526
3527	/* the arrays are wired, so no deadlock worries */
3528
3529	objects[num_pagers].dpo_object = (vm_offset_t) entry;
3530	objects[num_pagers].dpo_size = size;
3531	ports [num_pagers++] = port;
3532	continue;
3533	not_this_one:
3534	/*
3535	* Do not return garbage
3536	*/
3537	objects[num_pagers].dpo_object = (vm_offset_t) 0;
3538	objects[num_pagers].dpo_size = 0;
3539	ports [num_pagers++] = MACH_PORT_NULL((mach_port_t) 0);
3540
3541	}
3542
3543	pthread_mutex_unlock(&all_pagers.lock);
3544
3545	/*
3546	* Deallocate and clear unused memory.
3547	* (Returned memory will automagically become pageable.)
3548	*/
3549
3550	if (objects == *objectsp) {
3551	/*
3552	* Our returned information fit inline.
3553	* Nothing to deallocate.
3554	*/
3555
3556	*ocountp = num_pagers;
3557	} else if (actual == 0) {
3558	(void) vm_deallocate(default_pager_self, oaddr, osize);
3559
3560	/* return zero items inline */
3561	*ocountp = 0;
3562	} else {
3563	vm_offset_t used;
3564
3565	used = round_page(actual * sizeof objects)((((vm_offset_t) (actual sizeof objects) + __vm_page_size - 1) / __vm_page_size) __vm_page_size);
3566
3567	if (used != osize)
3568	(void) vm_deallocate(default_pager_self,
3569	oaddr + used, osize - used);
3570
3571	*objectsp = objects;
3572	*ocountp = num_pagers;
3573	}
3574
3575	if (ports == *portsp) {
3576	/*
3577	* Our returned information fit inline.
3578	* Nothing to deallocate.
3579	*/
3580
3581	*pcountp = num_pagers;
3582	} else if (actual == 0) {
3583	(void) vm_deallocate(default_pager_self, paddr, psize);
3584
3585	/* return zero items inline */
3586	*pcountp = 0;
3587	} else {
3588	vm_offset_t used;
3589
3590	used = round_page(actual * sizeof ports)((((vm_offset_t) (actual sizeof ports) + __vm_page_size - 1 ) / __vm_page_size) __vm_page_size);
3591
3592	if (used != psize)
3593	(void) vm_deallocate(default_pager_self,
3594	paddr + used, psize - used);
3595
3596	*portsp = ports;
3597	*pcountp = num_pagers;
3598	}
3599
3600	return KERN_SUCCESS0;
3601
3602	nomemory:
3603
3604	{
3605	register int i;
3606	for (i = 0; i < num_pagers; i++)
3607	(void) mach_port_deallocate(default_pager_self, ports[i]);
3608	}
3609
3610	if (objects != *objectsp)
3611	(void) vm_deallocate(default_pager_self, oaddr, osize);
3612
3613	if (ports != *portsp)
3614	(void) vm_deallocate(default_pager_self, paddr, psize);
3615
3616	return KERN_RESOURCE_SHORTAGE6;
3617	}
3618
3619
3620	kern_return_t
3621	S_default_pager_object_pages (mach_port_t pager,
3622	mach_port_t object,
3623	default_pager_page_array_t *pagesp,
3624	natural_t *countp)
3625	{
3626	vm_offset_t addr; /* memory for page offsets */
3627	vm_size_t size; /* current memory size */
3628	default_pager_page_t *pages;
3629	natural_t potential, actual;
3630	kern_return_t kr;
3631
3632	if (pager != default_pager_default_port)
3633	return KERN_INVALID_ARGUMENT4;
3634
3635	/* we start with the inline space */
3636
3637	pages = *pagesp;
3638	potential = *countp;
3639
3640	for (;;) {
3641	default_pager_t entry;
3642
3643	pthread_mutex_lock(&all_pagers.lock);
3644	queue_iterate(&all_pagers.queue, entry, default_pager_t, links)for ((entry) = (default_pager_t) ((&all_pagers.queue)-> next); !(((&all_pagers.queue)) == ((queue_entry_t)(entry) )); (entry) = (default_pager_t) ((&(entry)->links)-> next)) {
3645	dstruct_lock(entry)pthread_mutex_lock(&entry->lock);
3646	if (entry->pager_name == object) {
3647	pthread_mutex_unlock(&all_pagers.lock);
3648	goto found_object;
3649	}
3650	dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3651	}
3652	pthread_mutex_unlock(&all_pagers.lock);
3653
3654	/* did not find the object */
3655
3656	if (pages != *pagesp)
3657	(void) vm_deallocate(default_pager_self, addr, size);
3658	return KERN_INVALID_ARGUMENT4;
3659
3660	found_object:
3661
3662	if (pthread_mutex_trylock(&entry->dpager.lock)) {
3663	/* oh well bad luck */
3664
3665	dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3666
3667	/* yield the processor */
3668	(void) thread_switch(MACH_PORT_NULL((mach_port_t) 0),
3669	SWITCH_OPTION_NONE0, 0);
3670	continue;
3671	}
3672
3673	actual = pager_pages(&entry->dpager, pages, potential);
3674	pthread_mutex_unlock(&entry->dpager.lock);
3675	dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3676
3677	if (actual <= potential)
3678	break;
3679
3680	/* allocate more memory */
3681
3682	if (pages != *pagesp)
3683	(void) vm_deallocate(default_pager_self, addr, size);
3684	size = round_page(actual * sizeof pages)((((vm_offset_t) (actual sizeof pages) + __vm_page_size - 1 ) / __vm_page_size) __vm_page_size);
3685	kr = vm_allocate(default_pager_self, &addr, size, TRUE((boolean_t) 1));
3686	if (kr != KERN_SUCCESS0)
3687	return kr;
3688	pages = (default_pager_page_t *) addr;
3689	potential = size/sizeof *pages;
3690	}
3691
3692	/*
3693	* Deallocate and clear unused memory.
3694	* (Returned memory will automagically become pageable.)
3695	*/
3696
3697	if (pages == *pagesp) {
3698	/*
3699	* Our returned information fit inline.
3700	* Nothing to deallocate.
3701	*/
3702
3703	*countp = actual;
3704	} else if (actual == 0) {
3705	(void) vm_deallocate(default_pager_self, addr, size);
3706
3707	/* return zero items inline */
3708	*countp = 0;
3709	} else {
3710	vm_offset_t used;
3711
3712	used = round_page(actual * sizeof pages)((((vm_offset_t) (actual sizeof pages) + __vm_page_size - 1 ) / __vm_page_size) __vm_page_size);
3713
3714	if (used != size)
3715	(void) vm_deallocate(default_pager_self,
3716	addr + used, size - used);
3717
3718	*pagesp = pages;
3719	*countp = actual;
3720	}
3721	return KERN_SUCCESS0;
3722	}
3723
3724
3725	kern_return_t
3726	S_default_pager_object_set_size (mach_port_t pager,
3727	mach_port_seqno_t seqno,
3728	vm_size_t limit)
3729	{
3730	kern_return_t kr;
3731	default_pager_t ds;
3732
3733	ds = pager_port_lookup(pager)((! (((pager) != ((mach_port_t) 0)) && ((pager) != (( mach_port_t) ~0))) \|\| ((default_pager_t)(((vm_offset_t)(pager ))&~1))->pager != (pager)) ? ((default_pager_t)0) : (default_pager_t )(((vm_offset_t)(pager))&~1));
3734	if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
3735	return KERN_INVALID_ARGUMENT4;
3736
3737	pager_port_lock(ds, seqno);
3738	pager_port_wait_for_readers(ds);
3739	pager_port_wait_for_writers(ds);
3740
3741	vm_size_t rounded_limit = round_page (limit)((((vm_offset_t) (limit) + __vm_page_size - 1) / __vm_page_size ) * __vm_page_size);
3742	vm_size_t trunc_limit = trunc_page (limit)((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size);
3743
3744
3745	if (ds->dpager.limit < rounded_limit)
3746	{
3747	/* The limit has not been exceeded heretofore. Just change it. */
3748	ds->dpager.limit = rounded_limit;
3749
3750	/* Byte limit is used for truncation of file, which aren't rounded to
3751	page boundary. But by enlarging of file we are free to increase this value*/
3752	ds->dpager.byte_limit = rounded_limit;
3753	kr = memory_object_lock_request(ds->pager_request, 0,
3754	rounded_limit,
3755	MEMORY_OBJECT_RETURN_NONE0, FALSE((boolean_t) 0),
3756	VM_PROT_NONE((vm_prot_t) 0x00), MACH_PORT_NULL((mach_port_t) 0));
3757	if (kr != KERN_SUCCESS0)
3758	panic ("memory_object_lock_request: %d", kr);
3759	}
3760	else
3761	{
3762	if (ds->dpager.limit != rounded_limit)
3763	{
3764	kr = memory_object_lock_request(ds->pager_request, rounded_limit,
3765	ds->dpager.limit - rounded_limit,
3766	MEMORY_OBJECT_RETURN_NONE0, TRUE((boolean_t) 1),
3767	VM_PROT_ALL(((vm_prot_t) 0x01)\|((vm_prot_t) 0x02)\|((vm_prot_t) 0x04)), MACH_PORT_NULL((mach_port_t) 0));
3768	if (kr != KERN_SUCCESS0)
3769	panic ("memory_object_lock_request: %d", kr);
3770
3771	ds->dpager.limit = rounded_limit;
3772	}
3773
3774	/* Deallocate the old backing store pages and shrink the page map. */
3775	if (ds->dpager.size > ds->dpager.limit / vm_page_size)
3776	pager_truncate (&ds->dpager, ds->dpager.limit / vm_page_size);
3777
3778	/* If memory object size isn't page aligned, fill the tail
3779	of last page with zeroes */
3780	if ((limit != rounded_limit) && (ds->dpager.limit > limit))
3781	{
3782	/* Clean part of last page which isn't part of file.
3783	For file sizes that aren't multiple of vm_page_size */
3784	ds->dpager.byte_limit = limit;
3785	kr = memory_object_lock_request(ds->pager_request, trunc_limit,
3786	vm_page_size,
3787	MEMORY_OBJECT_RETURN_ALL2, TRUE((boolean_t) 1),
3788	VM_PROT_NONE((vm_prot_t) 0x00), MACH_PORT_NULL((mach_port_t) 0));
3789	}
3790	}
3791
3792	pager_port_unlock(ds);
3793
3794	return kr;
3795	}
3796
3797	/*
3798	* Add/remove extra paging space
3799	*/
3800
3801	extern mach_port_t bootstrap_master_device_port;
3802	extern mach_port_t bootstrap_master_host_port;
3803
3804	kern_return_t
3805	S_default_pager_paging_file (pager, mdport, file_name, add)
3806	mach_port_t pager;
3807	mach_port_t mdport;
3808	default_pager_filename_t file_name;
3809	boolean_t add;
3810	{
3811	kern_return_t kr;
3812
3813	if (pager != default_pager_default_port)
3814	return KERN_INVALID_ARGUMENT4;
3815
3816	#if 0
3817	dprintf("bmd %x md %x\n", bootstrap_master_device_port, mdport);
3818	#endif
3819	if (add) {
3820	kr = add_paging_file(bootstrap_master_device_port,
3821	file_name, 0);
3822	} else {
3823	kr = remove_paging_file(file_name);
3824	}
3825
3826	/* XXXX more code needed */
3827	if (mdport != bootstrap_master_device_port)
3828	mach_port_deallocate( mach_task_self()((__mach_task_self_ + 0)), mdport);
3829
3830	return kr;
3831	}
3832
3833	kern_return_t
3834	default_pager_register_fileserver(pager, fileserver)
3835	mach_port_t pager;
3836	mach_port_t fileserver;
3837	{
3838	if (pager != default_pager_default_port)
3839	return KERN_INVALID_ARGUMENT4;
3840	#if notyet
3841	mach_port_deallocate(mach_task_self()((__mach_task_self_ + 0)), fileserver);
3842	if (0) dp_helper_paging_space(0,0,0);/just linkit/
3843	#endif
3844	return KERN_SUCCESS0;
3845	}
3846
3847	/*
3848	* When things do not quite workout...
3849	*/
3850	void no_paging_space(out_of_memory)
3851	boolean_t out_of_memory;
3852	{
3853	static char here[] = "%s * NOT ENOUGH PAGING SPACE *";
3854
3855	if (out_of_memory)
3856	dprintf("* OUT OF MEMORY * ");
3857	panic(here, my_name);
3858	}
3859
3860	void overcommitted(got_more_space, space)
3861	boolean_t got_more_space;
3862	vm_size_t space; /* in pages */
3863	{
3864	vm_size_t pages_free, pages_total;
3865
3866	static boolean_t user_warned = FALSE((boolean_t) 0);
3867	static vm_size_t pages_shortage = 0;
3868
3869	paging_space_info(&pages_total, &pages_free);
3870
3871	/*
3872	* If user added more space, see if it is enough
3873	*/
3874	if (got_more_space) {
3875	pages_free -= pages_shortage;
3876	if (pages_free > 0) {
3877	pages_shortage = 0;
3878	if (user_warned)
3879	dprintf("%s paging space ok now.\n", my_name);
3880	} else
3881	pages_shortage = pages_free;
3882	user_warned = FALSE((boolean_t) 0);
3883	return;
3884	}
3885	/*
3886	* We ran out of gas, let user know.
3887	*/
3888	pages_free -= space;
3889	pages_shortage = (pages_free > 0) ? 0 : -pages_free;
3890	if (!user_warned && pages_shortage) {
3891	user_warned = TRUE((boolean_t) 1);
3892	dprintf("%s paging space over-committed.\n", my_name);
3893	}
3894	#if debug0
3895	user_warned = FALSE((boolean_t) 0);
3896	dprintf("%s paging space over-committed [+%d (%d) pages].\n",
3897	my_name, space, pages_shortage);
3898	#endif
3899	}
3900
3901	void paging_space_info(totp, freep)
3902	vm_size_t totp, freep;
3903	{
3904	register vm_size_t total, free;
3905	register partition_t part;
3906	register int i;
3907
3908	total = free = 0;
3909	for (i = 0; i < all_partitions.n_partitions; i++) {
3910
3911	if ((part = partition_of(i)) == 0) continue;
3912
3913	/* no need to lock: by the time this data
3914	gets back to any remote requestor it
3915	will be obsolete anyways */
3916	total += part->total_size;
3917	free += part->free;
3918	#if debug0
3919	dprintf("Partition %d: x%x total, x%x free\n",
3920	i, part->total_size, part->free);
3921	#endif
3922	}
3923	*totp = total;
3924	*freep = free;
3925	}
3926
3927	/*
3928	* Catch exceptions.
3929	*/
3930
3931	kern_return_t
3932	catch_exception_raise(exception_port, thread, task, exception, code, subcode)
3933	mach_port_t exception_port;
3934	mach_port_t thread, task;
3935	int exception, code, subcode;
3936	{
3937	ddprintf ("(default_pager)catch_exception_raise(%d,%d,%d)\n",
3938	exception, code, subcode);
3939	panic(my_name);
3940
3941	/* mach_msg_server will deallocate thread/task for us */
3942
3943	return KERN_FAILURE5;
3944	}