Bug Summary

File:obj-scan-build/mach-defpager/../../mach-defpager/default_pager.c
Location:line 2015, column 2
Description:Dereference of undefined pointer value

Annotated Source Code

1/*
2 * Mach Operating System
3 * Copyright (c) 1993-1989 Carnegie Mellon University
4 * All Rights Reserved.
5 *
6 * Permission to use, copy, modify and distribute this software and its
7 * documentation is hereby granted, provided that both the copyright
8 * notice and this permission notice appear in all copies of the
9 * software, derivative works or modified versions, and any portions
10 * thereof, and that both notices appear in supporting documentation.
11 *
12 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
13 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
14 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
15 *
16 * Carnegie Mellon requests users of this software to return to
17 *
18 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
19 * School of Computer Science
20 * Carnegie Mellon University
21 * Pittsburgh PA 15213-3890
22 *
23 * any improvements or extensions that they make and grant Carnegie Mellon
24 * the rights to redistribute these changes.
25 */
26/*
27 * Default pager. Pages to paging partition.
28 *
29 * MUST BE ABLE TO ALLOCATE WIRED-DOWN MEMORY!!!
30 */
31
32#include <mach.h>
33#include <mach/message.h>
34#include <mach/notify.h>
35#include <mach/mig_errors.h>
36#include <mach/thread_switch.h>
37#include <mach/task_info.h>
38#include <mach/default_pager_types.h>
39
40#include <pthread.h>
41
42#include <device/device_types.h>
43#include <device/device.h>
44
45#include <queue.h>
46#include <wiring.h>
47#include <kalloc.h>
48#include <default_pager.h>
49
50#include <assert.h>
51#include <errno(*__errno_location ()).h>
52#include <stdio.h>
53#include <string.h>
54
55#include <file_io.h>
56
57#include "default_pager_S.h"
58
59#define debug0 0
60
61static char my_name[] = "(default pager):";
62
63static pthread_mutex_t printf_lock = PTHREAD_MUTEX_INITIALIZER{ ((__pthread_spinlock_t) 0), ((__pthread_spinlock_t) 0), 0, 0
, 0, 0, 0, 0 };
64
65#if 0
66#define dprintf(f, x...) \
67 ({ pthread_mutex_lock (&printf_lock); \
68 printf (f , ##x); \
69 fflush (stdoutstdout); \
70 pthread_mutex_unlock (&printf_lock); })
71#else
72#define dprintf(f, x...)
73#endif
74
75#if 0
76#define ddprintf(f, x...) \
77 ({ pthread_mutex_lock (&printf_lock); \
78 printf (f , ##x); \
79 fflush (stdoutstdout); \
80 pthread_mutex_unlock (&printf_lock); })
81#else
82#define ddprintf(f, x...)
83#endif
84
85/*
86 * parallel vs serial switch
87 */
88#define PARALLEL1 1
89
90#if 0
91#define CHECKSUM 1
92#endif
93
94#define USE_PRECIOUS1 1
95
96#define ptoa(p)((p)*vm_page_size) ((p)*vm_page_size)
97#define atop(a)((a)/vm_page_size) ((a)/vm_page_size)
98
99/*
100
101 */
102/*
103 * Bitmap allocation.
104 */
105typedef unsigned int bm_entry_t;
106#define NB_BM32 32
107#define BM_MASK0xffffffff 0xffffffff
108
109#define howmany(a,b)(((a) + (b) - 1)/(b)) (((a) + (b) - 1)/(b))
110
111/*
112 * Value to indicate no block assigned
113 */
114#define NO_BLOCK((vm_offset_t)-1) ((vm_offset_t)-1)
115
116/*
117 * 'Partition' structure for each paging area.
118 * Controls allocation of blocks within paging area.
119 */
120struct part {
121 pthread_mutex_t p_lock; /* for bitmap/free */
122 vm_size_t total_size; /* total number of blocks */
123 vm_size_t free; /* number of blocks free */
124 unsigned int id; /* named lookup */
125 bm_entry_t *bitmap; /* allocation map */
126 boolean_t going_away; /* destroy attempt in progress */
127 struct file_direct *file; /* file paged to */
128};
129typedef struct part *partition_t;
130
131struct {
132 pthread_mutex_t lock;
133 int n_partitions;
134 partition_t *partition_list;/* array, for quick mapping */
135} all_partitions; /* list of all such */
136
137typedef unsigned char p_index_t;
138
139#define P_INDEX_INVALID((p_index_t)-1) ((p_index_t)-1)
140
141#define no_partition(x)((x) == ((p_index_t)-1)) ((x) == P_INDEX_INVALID((p_index_t)-1))
142
143partition_t partition_of(x)
144 int x;
145{
146 if (x >= all_partitions.n_partitions || x < 0)
147 panic("partition_of x%x", x);
148 return all_partitions.partition_list[x];
149}
150
151void set_partition_of(x, p)
152 int x;
153 partition_t p;
154{
155 if (x >= all_partitions.n_partitions || x < 0)
156 panic("set_partition_of x%x", x);
157 all_partitions.partition_list[x] = p;
158}
159
160/*
161 * Simple mapping from (file)NAME to id
162 * Saves space, filenames can be long.
163 */
164unsigned int
165part_id(const char *name)
166{
167 register unsigned int id, xorid;
168 size_t len;
169
170 len = strlen(name);
171 id = xorid = 0;
172 while (len--) {
173 xorid ^= *name;
174 id += *name++;
175 }
176 return (id << 8) | xorid;
177}
178
179void
180partition_init()
181{
182 pthread_mutex_init(&all_partitions.lock, NULL((void*)0));
183 all_partitions.n_partitions = 0;
184}
185
186static partition_t
187new_partition (const char *name, struct file_direct *fdp,
188 int check_linux_signature)
189{
190 register partition_t part;
191 register vm_size_t size, bmsize;
192 vm_offset_t raddr;
193 mach_msg_type_number_t rsize;
194 int rc;
195 unsigned int id = part_id(name);
196
197 pthread_mutex_lock(&all_partitions.lock);
198 {
199 unsigned int i;
200 for (i = 0; i < all_partitions.n_partitions; i++)
201 {
202 part = partition_of(i);
203 if (part && part->id == id)
204 {
205 printf ("(default pager): Already paging to partition %s!\n",
206 name);
207 pthread_mutex_unlock(&all_partitions.lock);
208 return 0;
209 }
210 }
211 }
212 pthread_mutex_unlock(&all_partitions.lock);
213
214 size = atop(fdp->fd_size * fdp->fd_bsize)((fdp->fd_size * fdp->fd_bsize)/vm_page_size);
215 bmsize = howmany(size, NB_BM)(((size) + (32) - 1)/(32)) * sizeof(bm_entry_t);
216
217 part = (partition_t) kalloc(sizeof(struct part));
218 pthread_mutex_init(&part->p_lock, NULL((void*)0));
219 part->total_size = size;
220 part->free = size;
221 part->id = id;
222 part->bitmap = (bm_entry_t *)kalloc(bmsize);
223 part->going_away= FALSE((boolean_t) 0);
224 part->file = fdp;
225
226 bzero((char *)part->bitmap, bmsize);
227
228 if (check_linux_signature < 0)
229 {
230 if (check_linux_signature != -3)
231 printf("(default pager): "
232 "Paging to raw partition %s (%uk paging space)\n",
233 name, part->total_size * (vm_page_size / 1024));
234 return part;
235 }
236
237#define LINUX_PAGE_SIZE4096 4096 /* size of pages in Linux swap partitions */
238 rc = page_read_file_direct(part->file,
239 0, LINUX_PAGE_SIZE4096,
240 &raddr,
241 &rsize);
242 if (rc)
243 panic("(default pager): cannot read first page of %s! rc=%#x\n",
244 name, rc);
245 while (rsize < LINUX_PAGE_SIZE4096)
246 {
247 /* Filesystem block size is smaller than page size,
248 so we must do several reads to get the whole page. */
249 vm_address_t baddr, bsize;
250 rc = page_read_file_direct(part->file,
251 rsize, LINUX_PAGE_SIZE4096-rsize,
252 &baddr,
253 &bsize);
254 if (rc)
255 panic("(default pager): "
256 "cannot read first page of %s! rc=%#x at %#x\n",
257 name, rc, rsize);
258
259 memcpy ((char *) raddr + rsize, (void *) baddr, bsize);
260 rsize += bsize;
261 vm_deallocate (mach_task_self ()((__mach_task_self_ + 0)), baddr, bsize);
262 }
263
264 if (!memcmp("SWAP-SPACE", (char *) raddr + LINUX_PAGE_SIZE4096-10, 10))
265 {
266 /* The partition's first page has a Linux swap signature.
267 This means the beginning of the page contains a bitmap
268 of good pages, and all others are bad. */
269 unsigned int i, j, bad, max;
270 int waste;
271
272 printf("(default pager): Found Linux 2.0 swap signature in %s\n",
273 name);
274
275 /* The first page, and the pages corresponding to the bits
276 occupied by the signature in the final 10 bytes of the page,
277 are always unavailable ("bad"). */
278 *(u_int32_t *)raddr &= ~(u_int32_t) 1;
279 memset((char *) raddr + LINUX_PAGE_SIZE4096-10, 0, 10);
280
281 max = LINUX_PAGE_SIZE4096 / sizeof(u_int32_t);
282 if (max > (part->total_size + 31) / 32)
283 max = (part->total_size + 31) / 32;
284
285 bad = 0;
286 for (i = 0; i < max; ++i)
287 {
288 u_int32_t bm = ((u_int32_t *) raddr)[i];
289 if (bm == ~(u_int32_t) 0)
290 continue;
291 /* There are some zero bits in this word. */
292 for (j = 0; j < 32; ++j)
293 if ((bm & (1 << j)) == 0)
294 {
295 unsigned int p = i*32 + j;
296 if (p >= part->total_size)
297 break;
298 ++bad;
299 part->bitmap[p / NB_BM32] |= 1 << (p % NB_BM32);
300 }
301 }
302 part->free -= bad;
303
304 --bad; /* Don't complain about first page. */
305 waste = part->total_size - (8 * (LINUX_PAGE_SIZE4096-10));
306 if (waste > 0)
307 {
308 /* The wasted pages were already marked "bad". */
309 bad -= waste;
310 if (bad > 0)
311 printf("\
312(default pager): Paging to %s, %dk swap-space (%dk bad, %dk wasted at end)\n",
313 name,
314 part->free * (LINUX_PAGE_SIZE4096 / 1024),
315 bad * (LINUX_PAGE_SIZE4096 / 1024),
316 waste * (LINUX_PAGE_SIZE4096 / 1024));
317 else
318 printf("\
319(default pager): Paging to %s, %dk swap-space (%dk wasted at end)\n",
320 name,
321 part->free * (LINUX_PAGE_SIZE4096 / 1024),
322 waste * (LINUX_PAGE_SIZE4096 / 1024));
323 }
324 else if (bad > 0)
325 printf("\
326(default pager): Paging to %s, %dk swap-space (excludes %dk marked bad)\n",
327 name,
328 part->free * (LINUX_PAGE_SIZE4096 / 1024),
329 bad * (LINUX_PAGE_SIZE4096 / 1024));
330 else
331 printf("\
332(default pager): Paging to %s, %dk swap-space\n",
333 name,
334 part->free * (LINUX_PAGE_SIZE4096 / 1024));
335 }
336 else if (!memcmp("SWAPSPACE2",
337 (char *) raddr + LINUX_PAGE_SIZE4096-10, 10))
338 {
339 struct
340 {
341 u_int8_t bootbits[1024];
342 u_int32_t version;
343 u_int32_t last_page;
344 u_int32_t nr_badpages;
345 u_int32_t padding[125];
346 u_int32_t badpages[1];
347 } *hdr = (void *) raddr;
348
349 printf("\
350(default pager): Found Linux 2.2 swap signature (v%u) in %s...",
351 hdr->version, name);
352
353 part->bitmap[0] |= 1; /* first page unusable */
354 part->free--;
355
356 switch (hdr->version)
357 {
358 default:
359 if (check_linux_signature)
360 {
361 printf ("version %u unknown! SKIPPING %s!\n",
362 hdr->version,
363 name);
364 vm_deallocate(mach_task_self()((__mach_task_self_ + 0)), raddr, rsize);
365 kfree(part->bitmap, bmsize);
366 kfree(part, sizeof *part);
367 return 0;
368 }
369 else
370 printf ("version %u unknown! IGNORING SIGNATURE PAGE!"
371 " %dk swap-space\n",
372 hdr->version,
373 part->free * (LINUX_PAGE_SIZE4096 / 1024));
374 break;
375
376 case 1:
377 {
378 unsigned int waste, i;
379 if (hdr->last_page > part->total_size)
380 {
381 printf ("signature says %uk, partition has only %uk! ",
382 hdr->last_page * (LINUX_PAGE_SIZE4096 / 1024),
383 part->total_size * (LINUX_PAGE_SIZE4096 / 1024));
384 waste = 0;
385 }
386 else
387 {
388 waste = part->total_size - hdr->last_page;
389 part->total_size = hdr->last_page;
390 part->free = part->total_size - 1;
391 }
392 for (i = 0; i < hdr->nr_badpages; ++i)
393 {
394 const u_int32_t bad = hdr->badpages[i];
395 part->bitmap[bad / NB_BM32] |= 1 << (bad % NB_BM32);
396 part->free--;
397 }
398 printf ("%uk swap-space",
399 part->free * (LINUX_PAGE_SIZE4096 / 1024));
400 if (hdr->nr_badpages != 0)
401 printf (" (excludes %uk marked bad)",
402 hdr->nr_badpages * (LINUX_PAGE_SIZE4096 / 1024));
403 if (waste != 0)
404 printf (" (excludes %uk at end of partition)",
405 waste * (LINUX_PAGE_SIZE4096 / 1024));
406 printf ("\n");
407 }
408 }
409 }
410 else if (check_linux_signature)
411 {
412 printf ("(default pager): "
413 "Cannot find Linux swap signature page! "
414 "SKIPPING %s (%uk partition)!",
415 name, part->total_size * (vm_page_size / 1024));
416 kfree(part->bitmap, bmsize);
417 kfree(part, sizeof *part);
418 part = 0;
419 }
420 else
421 printf("(default pager): "
422 "Paging to raw partition %s (%uk paging space)\n",
423 name, part->total_size * (vm_page_size / 1024));
424
425 vm_deallocate(mach_task_self()((__mach_task_self_ + 0)), raddr, rsize);
426
427 return part;
428}
429
430/*
431 * Create a partition descriptor,
432 * add it to the list of all such.
433 * size is in BYTES.
434 */
435void
436create_paging_partition(const char *name,
437 struct file_direct *fdp, int isa_file,
438 int linux_signature)
439{
440 register partition_t part;
441
442 part = new_partition (name, fdp, linux_signature);
443 if (!part)
444 return;
445
446 pthread_mutex_lock(&all_partitions.lock);
447 {
448 register int i;
449
450 for (i = 0; i < all_partitions.n_partitions; i++)
451 if (partition_of(i) == 0) break;
452
453 if (i == all_partitions.n_partitions) {
454 register partition_t *new_list, *old_list;
455 register int n;
456
457 n = i ? (i<<1) : 2;
458 new_list = (partition_t *)
459 kalloc( n * sizeof(partition_t) );
460 if (new_list == 0) no_paging_space(TRUE((boolean_t) 1));
461 bzero(new_list, n*sizeof(partition_t));
462 if (i) {
463 old_list = all_partitions.partition_list;
464 bcopy(old_list, new_list, i*sizeof(partition_t));
465 }
466 all_partitions.partition_list = new_list;
467 all_partitions.n_partitions = n;
468 if (i) kfree(old_list, i*sizeof(partition_t));
469 }
470 set_partition_of(i, part);
471 }
472 pthread_mutex_unlock(&all_partitions.lock);
473
474#if 0
475 dprintf("%s Added paging %s %s\n", my_name,
476 (isa_file) ? "file" : "device", name);
477#endif
478 overcommitted(TRUE((boolean_t) 1), part->free);
479}
480
481/*
482 * Choose the most appropriate default partition
483 * for an object of SIZE bytes.
484 * Return the partition locked, unless
485 * the object has no CUR_PARTition.
486 */
487p_index_t
488choose_partition(size, cur_part)
489 unsigned int size;
490 register p_index_t cur_part;
491{
492 register partition_t part;
493 register boolean_t found = FALSE((boolean_t) 0);
494 register int i;
495
496 pthread_mutex_lock(&all_partitions.lock);
497 for (i = 0; i < all_partitions.n_partitions; i++) {
498
499 /* the undesirable one ? */
500 if (i == cur_part)
501 continue;
502
503ddprintf ("choose_partition(%x,%d,%d)\n",size,cur_part,i);
504 /* one that was removed ? */
505 if ((part = partition_of(i)) == 0)
506 continue;
507
508 /* one that is being removed ? */
509 if (part->going_away)
510 continue;
511
512 /* is it big enough ? */
513 pthread_mutex_lock(&part->p_lock);
514 if (ptoa(part->free)((part->free)*vm_page_size) >= size) {
515 if (cur_part != P_INDEX_INVALID((p_index_t)-1)) {
516 pthread_mutex_unlock(&all_partitions.lock);
517 return (p_index_t)i;
518 } else
519 found = TRUE((boolean_t) 1);
520 }
521 pthread_mutex_unlock(&part->p_lock);
522
523 if (found) break;
524 }
525 pthread_mutex_unlock(&all_partitions.lock);
526 return (found) ? (p_index_t)i : P_INDEX_INVALID((p_index_t)-1);
527}
528
529/*
530 * Allocate a page in a paging partition
531 * The partition is returned unlocked.
532 */
533vm_offset_t
534pager_alloc_page(pindex, lock_it)
535 p_index_t pindex;
536 boolean_t lock_it;
537{
538 register int bm_e;
539 register int bit;
540 register int limit;
541 register bm_entry_t *bm;
542 partition_t part;
543 static char here[] = "%spager_alloc_page";
544
545 if (no_partition(pindex)((pindex) == ((p_index_t)-1)))
546 return (NO_BLOCK((vm_offset_t)-1));
547ddprintf ("pager_alloc_page(%d,%d)\n",pindex,lock_it);
548 part = partition_of(pindex);
549
550 /* unlikely, but possible deadlock against destroy_partition */
551 if (!part || part->going_away)
552 return (NO_BLOCK((vm_offset_t)-1));
553
554 if (lock_it)
555 pthread_mutex_lock(&part->p_lock);
556
557 if (part->free == 0) {
558 /* out of paging space */
559 pthread_mutex_unlock(&part->p_lock);
560 return (NO_BLOCK((vm_offset_t)-1));
561 }
562
563 limit = howmany(part->total_size, NB_BM)(((part->total_size) + (32) - 1)/(32));
564 bm = part->bitmap;
565 for (bm_e = 0; bm_e < limit; bm_e++, bm++)
566 if (*bm != BM_MASK0xffffffff)
567 break;
568
569 if (bm_e == limit)
570 panic(here,my_name);
571
572 /*
573 * Find and set the proper bit
574 */
575 {
576 register bm_entry_t b = *bm;
577
578 for (bit = 0; bit < NB_BM32; bit++)
579 if ((b & (1<<bit)) == 0)
580 break;
581 if (bit == NB_BM32)
582 panic(here,my_name);
583
584 *bm = b | (1<<bit);
585 part->free--;
586
587 }
588
589 pthread_mutex_unlock(&part->p_lock);
590
591 return (bm_e*NB_BM32+bit);
592}
593
594/*
595 * Deallocate a page in a paging partition
596 */
597void
598pager_dealloc_page(pindex, page, lock_it)
599 p_index_t pindex;
600 register vm_offset_t page;
601 boolean_t lock_it;
602{
603 register partition_t part;
604 register int bit, bm_e;
605
606 /* be paranoid */
607 if (no_partition(pindex)((pindex) == ((p_index_t)-1)))
608 panic("%sdealloc_page",my_name);
609ddprintf ("pager_dealloc_page(%d,%x,%d)\n",pindex,page,lock_it);
610 part = partition_of(pindex);
611
612 if (page >= part->total_size)
613 panic("%sdealloc_page",my_name);
614
615 bm_e = page / NB_BM32;
616 bit = page % NB_BM32;
617
618 if (lock_it)
619 pthread_mutex_lock(&part->p_lock);
620
621 part->bitmap[bm_e] &= ~(1<<bit);
622 part->free++;
623
624 if (lock_it)
625 pthread_mutex_unlock(&part->p_lock);
626}
627
628/*
629
630 */
631/*
632 * Allocation info for each paging object.
633 *
634 * Most operations, even pager_write_offset and pager_put_checksum,
635 * just need a read lock. Higher-level considerations prevent
636 * conflicting operations on a single page. The lock really protects
637 * the underlying size and block map memory, so pager_extend needs a
638 * write lock.
639 *
640 * An object can now span multiple paging partitions. The allocation
641 * info we keep is a pair (offset,p_index) where the index is in the
642 * array of all partition ptrs, and the offset is partition-relative.
643 * Size wise we are doing ok fitting the pair into a single integer:
644 * the offset really is in pages so we have vm_page_size bits available
645 * for the partition index.
646 */
647#define DEBUG_READER_CONFLICTS0 0
648
649#if DEBUG_READER_CONFLICTS0
650int default_pager_read_conflicts = 0;
651#endif
652
653union dp_map {
654
655 struct {
656 unsigned int p_offset : 24,
657 p_index : 8;
658 } block;
659
660 union dp_map *indirect;
661};
662typedef union dp_map *dp_map_t;
663
664/* quick check for part==block==invalid */
665#define no_block(e)((e).indirect == (dp_map_t)((vm_offset_t)-1)) ((e).indirect == (dp_map_t)NO_BLOCK((vm_offset_t)-1))
666#define invalidate_block(e)((e).indirect = (dp_map_t)((vm_offset_t)-1)) ((e).indirect = (dp_map_t)NO_BLOCK((vm_offset_t)-1))
667
668struct dpager {
669 pthread_mutex_t lock; /* lock for extending block map */
670 /* XXX should be read-write lock */
671#if DEBUG_READER_CONFLICTS0
672 int readers;
673 boolean_t writer;
674#endif
675 dp_map_t map; /* block map */
676 vm_size_t size; /* size of paging object, in pages */
677 vm_size_t limit; /* limit (bytes) allowed to grow to */
678 vm_size_t byte_limit; /* limit, which wasn't
679 rounded to page boundary */
680 p_index_t cur_partition;
681#ifdef CHECKSUM
682 vm_offset_t *checksum; /* checksum - parallel to block map */
683#define NO_CHECKSUM ((vm_offset_t)-1)
684#endif /* CHECKSUM */
685};
686typedef struct dpager *dpager_t;
687
688/*
689 * A paging object uses either a one- or a two-level map of offsets
690 * into a paging partition.
691 */
692#define PAGEMAP_ENTRIES64 64
693 /* number of pages in a second-level map */
694#define PAGEMAP_SIZE(npgs)((npgs)*sizeof(vm_offset_t)) ((npgs)*sizeof(vm_offset_t))
695
696#define INDIRECT_PAGEMAP_ENTRIES(npgs)((((npgs)-1)/64) + 1) \
697 ((((npgs)-1)/PAGEMAP_ENTRIES64) + 1)
698#define INDIRECT_PAGEMAP_SIZE(npgs)(((((npgs)-1)/64) + 1) * sizeof(vm_offset_t *)) \
699 (INDIRECT_PAGEMAP_ENTRIES(npgs)((((npgs)-1)/64) + 1) * sizeof(vm_offset_t *))
700#define INDIRECT_PAGEMAP(size)(size > 64) \
701 (size > PAGEMAP_ENTRIES64)
702
703#define ROUNDUP_TO_PAGEMAP(npgs)(((npgs) + 64 - 1) & ~(64 - 1)) \
704 (((npgs) + PAGEMAP_ENTRIES64 - 1) & ~(PAGEMAP_ENTRIES64 - 1))
705
706/*
707 * Object sizes are rounded up to the next power of 2,
708 * unless they are bigger than a given maximum size.
709 */
710vm_size_t max_doubled_size = 4 * 1024 * 1024; /* 4 meg */
711
712/*
713 * Return first level map for pager.
714 * If there is no such map, than allocate it.
715 */
716dp_map_t pager_get_direct_map(pager)
717 register dpager_t pager;
718{
719 register dp_map_t mapptr, emapptr;
720 register vm_size_t size = pager->size;
721
722 if (pager->map)
723 return pager->map;
724 /*
725 * Allocate and initialize the block map
726 */
727 {
728 register vm_size_t alloc_size;
729 dp_map_t init_value;
730
731 if (INDIRECT_PAGEMAP(size)(size > 64)) {
732 alloc_size = INDIRECT_PAGEMAP_SIZE(size)(((((size)-1)/64) + 1) * sizeof(vm_offset_t *));
733 init_value = (dp_map_t)0;
734 } else {
735 alloc_size = PAGEMAP_SIZE(size)((size)*sizeof(vm_offset_t));
736 init_value = (dp_map_t)NO_BLOCK((vm_offset_t)-1);
737 }
738
739 mapptr = (dp_map_t) kalloc(alloc_size);
740 for (emapptr = &mapptr[(alloc_size-1) / sizeof(vm_offset_t)];
741 emapptr >= mapptr;
742 emapptr--)
743 emapptr->indirect = init_value;
744 }
745 pager->map = mapptr;
746 return mapptr;
747}
748
749/*
750 * Attach a new paging object to a paging partition
751 */
752void
753pager_alloc(pager, part, size)
754 register dpager_t pager;
755 p_index_t part;
756 register vm_size_t size; /* in BYTES */
757{
758 register int i;
759 register dp_map_t mapptr, emapptr;
760
761 pthread_mutex_init(&pager->lock, NULL((void*)0));
762#if DEBUG_READER_CONFLICTS0
763 pager->readers = 0;
764 pager->writer = FALSE((boolean_t) 0);
765#endif
766 pager->cur_partition = part;
767
768 /*
769 * Convert byte size to number of pages, then increase to the nearest
770 * power of 2.
771 */
772 size = atop(size)((size)/vm_page_size);
773 if (size <= atop(max_doubled_size)((max_doubled_size)/vm_page_size)) {
774 i = 1;
775 while (i < size)
776 i <<= 1;
777 size = i;
778 } else
779 size = ROUNDUP_TO_PAGEMAP(size)(((size) + 64 - 1) & ~(64 - 1));
780
781 pager->map = NULL((void*)0);
782 pager->size = size;
783 pager->limit = (vm_size_t)-1;
784
785#ifdef CHECKSUM
786 if (INDIRECT_PAGEMAP(size)(size > 64)) {
787 mapptr = (vm_offset_t *)
788 kalloc(INDIRECT_PAGEMAP_SIZE(size)(((((size)-1)/64) + 1) * sizeof(vm_offset_t *)));
789 for (i = INDIRECT_PAGEMAP_ENTRIES(size)((((size)-1)/64) + 1); --i >= 0;)
790 mapptr[i] = 0;
791 } else {
792 mapptr = (vm_offset_t *) kalloc(PAGEMAP_SIZE(size)((size)*sizeof(vm_offset_t)));
793 for (i = 0; i < size; i++)
794 mapptr[i] = NO_CHECKSUM;
795 }
796 pager->checksum = mapptr;
797#endif /* CHECKSUM */
798}
799
800/*
801 * Return size (in bytes) of space actually allocated to this pager.
802 * The pager is read-locked.
803 */
804
805vm_size_t
806pager_allocated(pager)
807 register dpager_t pager;
808{
809 vm_size_t size;
810 register dp_map_t map, emap;
811 vm_size_t asize;
812
813 size = pager->size; /* in pages */
814 asize = 0; /* allocated, in pages */
815 map = pager_get_direct_map(pager);
816
817 if (INDIRECT_PAGEMAP(size)(size > 64)) {
818 for (emap = &map[INDIRECT_PAGEMAP_ENTRIES(size)((((size)-1)/64) + 1)];
819 map < emap; map++) {
820
821 register dp_map_t map2, emap2;
822
823 if ((map2 = map->indirect) == 0)
824 continue;
825
826 for (emap2 = &map2[PAGEMAP_ENTRIES64];
827 map2 < emap2; map2++)
828 if ( ! no_block(*map2)((*map2).indirect == (dp_map_t)((vm_offset_t)-1)) )
829 asize++;
830
831 }
832 } else {
833 for (emap = &map[size]; map < emap; map++)
834 if ( ! no_block(*map)((*map).indirect == (dp_map_t)((vm_offset_t)-1)) )
835 asize++;
836 }
837
838 return ptoa(asize)((asize)*vm_page_size);
839}
840
841/*
842 * Find offsets (in the object) of pages actually allocated to this pager.
843 * Returns the number of allocated pages, whether or not they all fit.
844 * The pager is read-locked.
845 */
846
847unsigned int
848pager_pages(pager, pages, numpages)
849 dpager_t pager;
850 register default_pager_page_t *pages;
851 unsigned int numpages;
852{
853 vm_size_t size;
854 dp_map_t map, emap;
855 unsigned int actual;
856 vm_offset_t offset;
857
858 size = pager->size; /* in pages */
859 map = pager_get_direct_map(pager);
860 actual = 0;
861 offset = 0;
862
863 if (INDIRECT_PAGEMAP(size)(size > 64)) {
864 for (emap = &map[INDIRECT_PAGEMAP_ENTRIES(size)((((size)-1)/64) + 1)];
865 map < emap; map++) {
866
867 register dp_map_t map2, emap2;
868
869 if ((map2 = map->indirect) == 0) {
870 offset += vm_page_size * PAGEMAP_ENTRIES64;
871 continue;
872 }
873 for (emap2 = &map2[PAGEMAP_ENTRIES64];
874 map2 < emap2; map2++)
875 if ( ! no_block(*map2)((*map2).indirect == (dp_map_t)((vm_offset_t)-1)) ) {
876 if (actual++ < numpages)
877 pages++->dpp_offset = offset;
878 }
879 offset += vm_page_size;
880 }
881 } else {
882 for (emap = &map[size]; map < emap; map++)
883 if ( ! no_block(*map)((*map).indirect == (dp_map_t)((vm_offset_t)-1)) ) {
884 if (actual++ < numpages)
885 pages++->dpp_offset = offset;
886 }
887 offset += vm_page_size;
888 }
889 return actual;
890}
891
892/*
893 * Extend the map for a paging object.
894 *
895 * XXX This implementation can allocate an arbitrary large amount
896 * of wired memory when extending a big block map. Because vm-privileged
897 * threads call pager_extend, this can crash the system by exhausting
898 * system memory.
899 */
900void
901pager_extend(pager, new_size)
902 register dpager_t pager;
903 register vm_size_t new_size; /* in pages */
904{
905 register dp_map_t new_mapptr;
906 register dp_map_t old_mapptr;
907 register int i;
908 register vm_size_t old_size;
909
910 pthread_mutex_lock(&pager->lock); /* XXX lock_write */
911#if DEBUG_READER_CONFLICTS0
912 pager->writer = TRUE((boolean_t) 1);
913#endif
914 /*
915 * Double current size until we cover new size.
916 * If object is 'too big' just use new size.
917 */
918 old_size = pager->size;
919
920 if (new_size <= atop(max_doubled_size)((max_doubled_size)/vm_page_size)) {
921 /* New size cannot be less than 1 */
922 i = old_size ? old_size : 1;
923 while (i < new_size)
924 i <<= 1;
925 new_size = i;
926 } else
927 new_size = ROUNDUP_TO_PAGEMAP(new_size)(((new_size) + 64 - 1) & ~(64 - 1));
928
929 if (INDIRECT_PAGEMAP(old_size)(old_size > 64)) {
930 /*
931 * Pager already uses two levels. Allocate
932 * a larger indirect block.
933 */
934 new_mapptr = (dp_map_t)
935 kalloc(INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
936 old_mapptr = pager_get_direct_map(pager);
937 for (i = 0; i < INDIRECT_PAGEMAP_ENTRIES(old_size)((((old_size)-1)/64) + 1); i++)
938 new_mapptr[i] = old_mapptr[i];
939 for (; i < INDIRECT_PAGEMAP_ENTRIES(new_size)((((new_size)-1)/64) + 1); i++)
940 new_mapptr[i].indirect = (dp_map_t)0;
941 kfree((char *)old_mapptr, INDIRECT_PAGEMAP_SIZE(old_size)(((((old_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
942 pager->map = new_mapptr;
943 pager->size = new_size;
944#ifdef CHECKSUM
945 new_mapptr = (vm_offset_t *)
946 kalloc(INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
947 old_mapptr = pager->checksum;
948 for (i = 0; i < INDIRECT_PAGEMAP_ENTRIES(old_size)((((old_size)-1)/64) + 1); i++)
949 new_mapptr[i] = old_mapptr[i];
950 for (; i < INDIRECT_PAGEMAP_ENTRIES(new_size)((((new_size)-1)/64) + 1); i++)
951 new_mapptr[i] = 0;
952 kfree((char *)old_mapptr, INDIRECT_PAGEMAP_SIZE(old_size)(((((old_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
953 pager->checksum = new_mapptr;
954#endif /* CHECKSUM */
955#if DEBUG_READER_CONFLICTS0
956 pager->writer = FALSE((boolean_t) 0);
957#endif
958 pthread_mutex_unlock(&pager->lock);
959#if 0
960 ddprintf ("pager_extend 1 mapptr %x [3b] = %x\n", new_mapptr,
961 new_mapptr[0x3b]);
962 if (new_mapptr[0x3b].indirect > 0x10000
963 && new_mapptr[0x3b].indirect != NO_BLOCK((vm_offset_t)-1))
964 panic ("debug panic");
965#endif
966 return;
967 }
968
969 if (INDIRECT_PAGEMAP(new_size)(new_size > 64)) {
970 /*
971 * Changing from direct map to indirect map.
972 * Allocate both indirect and direct map blocks,
973 * since second-level (direct) block must be
974 * full size (PAGEMAP_SIZE(PAGEMAP_ENTRIES)).
975 */
976
977 /*
978 * Allocate new second-level map first.
979 */
980 new_mapptr = (dp_map_t) kalloc(PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
981 old_mapptr = pager_get_direct_map(pager);
982 for (i = 0; i < old_size; i++)
983 new_mapptr[i] = old_mapptr[i];
984 for (; i < PAGEMAP_ENTRIES64; i++)
985 invalidate_block(new_mapptr[i])((new_mapptr[i]).indirect = (dp_map_t)((vm_offset_t)-1));
986 kfree((char *)old_mapptr, PAGEMAP_SIZE(old_size)((old_size)*sizeof(vm_offset_t)));
987 old_mapptr = new_mapptr;
988
989#if 0
990 ddprintf ("pager_extend 2 mapptr %x [3b] = %x\n", new_mapptr,
991 new_mapptr[0x3b]);
992 if (new_mapptr[0x3b].indirect > 0x10000
993 && new_mapptr[0x3b].indirect != NO_BLOCK((vm_offset_t)-1))
994 panic ("debug panic");
995#endif
996
997 /*
998 * Now allocate indirect map.
999 */
1000 new_mapptr = (dp_map_t)
1001 kalloc(INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1002 new_mapptr[0].indirect = old_mapptr;
1003 for (i = 1; i < INDIRECT_PAGEMAP_ENTRIES(new_size)((((new_size)-1)/64) + 1); i++)
1004 new_mapptr[i].indirect = 0;
1005 pager->map = new_mapptr;
1006 pager->size = new_size;
1007#ifdef CHECKSUM
1008 /*
1009 * Allocate new second-level map first.
1010 */
1011 new_mapptr = (vm_offset_t *)kalloc(PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
1012 old_mapptr = pager->checksum;
1013 for (i = 0; i < old_size; i++)
1014 new_mapptr[i] = old_mapptr[i];
1015 for (; i < PAGEMAP_ENTRIES64; i++)
1016 new_mapptr[i] = NO_CHECKSUM;
1017 kfree((char *)old_mapptr, PAGEMAP_SIZE(old_size)((old_size)*sizeof(vm_offset_t)));
1018 old_mapptr = new_mapptr;
1019
1020 /*
1021 * Now allocate indirect map.
1022 */
1023 new_mapptr = (vm_offset_t *)
1024 kalloc(INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1025 new_mapptr[0] = (vm_offset_t) old_mapptr;
1026 for (i = 1; i < INDIRECT_PAGEMAP_ENTRIES(new_size)((((new_size)-1)/64) + 1); i++)
1027 new_mapptr[i] = 0;
1028 pager->checksum = new_mapptr;
1029#endif /* CHECKSUM */
1030#if DEBUG_READER_CONFLICTS0
1031 pager->writer = FALSE((boolean_t) 0);
1032#endif
1033 pthread_mutex_unlock(&pager->lock);
1034 return;
1035 }
1036 /*
1037 * Enlarging a direct block.
1038 */
1039 new_mapptr = (dp_map_t) kalloc(PAGEMAP_SIZE(new_size)((new_size)*sizeof(vm_offset_t)));
1040 old_mapptr = pager_get_direct_map(pager);
1041 for (i = 0; i < old_size; i++)
1042 new_mapptr[i] = old_mapptr[i];
1043 for (; i < new_size; i++)
1044 invalidate_block(new_mapptr[i])((new_mapptr[i]).indirect = (dp_map_t)((vm_offset_t)-1));
1045 kfree((char *)old_mapptr, PAGEMAP_SIZE(old_size)((old_size)*sizeof(vm_offset_t)));
1046 pager->map = new_mapptr;
1047 pager->size = new_size;
1048#ifdef CHECKSUM
1049 new_mapptr = (vm_offset_t *)
1050 kalloc(PAGEMAP_SIZE(new_size)((new_size)*sizeof(vm_offset_t)));
1051 old_mapptr = pager->checksum;
1052 for (i = 0; i < old_size; i++)
1053 new_mapptr[i] = old_mapptr[i];
1054 for (; i < new_size; i++)
1055 new_mapptr[i] = NO_CHECKSUM;
1056 kfree((char *)old_mapptr, PAGEMAP_SIZE(old_size)((old_size)*sizeof(vm_offset_t)));
1057 pager->checksum = new_mapptr;
1058#endif /* CHECKSUM */
1059#if DEBUG_READER_CONFLICTS0
1060 pager->writer = FALSE((boolean_t) 0);
1061#endif
1062 pthread_mutex_unlock(&pager->lock);
1063}
1064
1065/* This deallocates the pages necessary to truncate a direct map
1066 previously of size NEW_SIZE to the smaller size OLD_SIZE. */
1067static void
1068dealloc_direct (dp_map_t mapptr,
1069 vm_size_t old_size, vm_size_t new_size)
1070{
1071 vm_size_t i;
1072
1073 if (!mapptr)
1074 return;
1075
1076 for (i = new_size; i < old_size; ++i)
1077 {
1078 const union dp_map entry = mapptr[i];
1079 if (!no_block(entry)((entry).indirect == (dp_map_t)((vm_offset_t)-1)))
1080 {
1081 pager_dealloc_page(entry.block.p_index, entry.block.p_offset,
1082 TRUE((boolean_t) 1));
1083 invalidate_block(mapptr[i])((mapptr[i]).indirect = (dp_map_t)((vm_offset_t)-1));
1084 }
1085 }
1086}
1087
1088/* Truncate a memory object. First, any pages between the new size
1089 and the (larger) old size are deallocated. Then, the size of
1090 the pagemap may be reduced, an indirect map may be turned into
1091 a direct map.
1092
1093 The pager must be locked by the caller. */
1094static void
1095pager_truncate(dpager_t pager, vm_size_t new_size) /* in pages */
1096{
1097 dp_map_t new_mapptr;
1098 dp_map_t old_mapptr;
1099 int i;
1100 vm_size_t old_size;
1101
1102 pthread_mutex_lock(&pager->lock); /* XXX lock_write */
1103
1104 if (!pager->map)
1105 goto done;
1106
1107 old_size = pager->size;
1108
1109 if (INDIRECT_PAGEMAP(old_size)(old_size > 64))
1110 {
1111 /* First handle the entire second-levels blocks that are being freed. */
1112 for (i = INDIRECT_PAGEMAP_ENTRIES(new_size)((((new_size)-1)/64) + 1);
1113 i < INDIRECT_PAGEMAP_ENTRIES(old_size)((((old_size)-1)/64) + 1);
1114 ++i)
1115 {
1116 const dp_map_t mapptr = pager->map[i].indirect;
1117 pager->map[i].indirect = (dp_map_t)0;
1118 dealloc_direct (mapptr, PAGEMAP_ENTRIES64, 0);
1119 kfree ((char *)mapptr, PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
1120 }
1121
1122 /* Now truncate what's now the final nonempty direct block. */
1123 dealloc_direct (pager->map[(new_size - 1) / PAGEMAP_ENTRIES64].indirect,
1124 old_size & (PAGEMAP_ENTRIES64 - 1),
1125 new_size & (PAGEMAP_ENTRIES64 - 1));
1126
1127 if (INDIRECT_PAGEMAP (new_size)(new_size > 64))
1128 {
1129 const dp_map_t old_mapptr = pager->map;
1130 pager->map = (dp_map_t) kalloc (INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1131 memcpy (pager->map, old_mapptr, INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1132 kfree ((char *) old_mapptr, INDIRECT_PAGEMAP_SIZE (old_size)(((((old_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1133 }
1134 else
1135 {
1136 /* We are truncating to a size small enough that it goes to using
1137 a one-level map. We already have that map, as the first and only
1138 nonempty element in our indirect map. */
1139 const dp_map_t mapptr = pager->map[0].indirect;
1140 kfree((char *)pager->map, INDIRECT_PAGEMAP_SIZE(old_size)(((((old_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1141 pager->map = mapptr;
1142 }
1143 }
1144
1145 if (! INDIRECT_PAGEMAP(old_size)(old_size > 64))
1146 {
1147 /* First deallocate pages in the truncated region. */
1148 dealloc_direct (pager->map, old_size, new_size);
1149 /* Now reduce the size of the direct map itself. We don't bother
1150 with kalloc/kfree if it's not shrinking enough that kalloc.c
1151 would actually use less. */
1152 if (PAGEMAP_SIZE (new_size)((new_size)*sizeof(vm_offset_t)) <= PAGEMAP_SIZE (old_size)((old_size)*sizeof(vm_offset_t)) / 2)
1153 {
1154 const dp_map_t old_mapptr = pager->map;
1155 pager->map = (dp_map_t) kalloc (PAGEMAP_SIZE (new_size)((new_size)*sizeof(vm_offset_t)));
1156 memcpy (pager->map, old_mapptr, PAGEMAP_SIZE (new_size)((new_size)*sizeof(vm_offset_t)));
1157 kfree ((char *) old_mapptr, PAGEMAP_SIZE (old_size)((old_size)*sizeof(vm_offset_t)));
1158 }
1159 }
1160
1161 done:
1162 pager->size = new_size;
1163 pthread_mutex_unlock(&pager->lock);
1164
1165#ifdef CHECKSUM
1166#error write me
1167#endif /* CHECKSUM */
1168}
1169
1170
1171/*
1172 * Given an offset within a paging object, find the
1173 * corresponding block within the paging partition.
1174 * Return NO_BLOCK if none allocated.
1175 */
1176union dp_map
1177pager_read_offset(pager, offset)
1178 register dpager_t pager;
1179 vm_offset_t offset;
1180{
1181 register vm_offset_t f_page;
1182 union dp_map pager_offset;
1183
1184 f_page = atop(offset)((offset)/vm_page_size);
1185
1186#if DEBUG_READER_CONFLICTS0
1187 if (pager->readers > 0)
1188 default_pager_read_conflicts++; /* would have proceeded with
1189 read/write lock */
1190#endif
1191 pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1192#if DEBUG_READER_CONFLICTS0
1193 pager->readers++;
1194#endif
1195 if (f_page >= pager->size)
1196 {
1197 ddprintf ("%spager_read_offset pager %x: bad page %d >= size %d",
1198 my_name, pager, f_page, pager->size);
1199 pthread_mutex_unlock(&pager->lock);
1200 return (union dp_map) (union dp_map *) NO_BLOCK((vm_offset_t)-1);
1201#if 0
1202 panic("%spager_read_offset",my_name);
1203#endif
1204 }
1205
1206 invalidate_block(pager_offset)((pager_offset).indirect = (dp_map_t)((vm_offset_t)-1));
1207 if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1208 register dp_map_t mapptr;
1209
1210 if (pager->map) {
1211 mapptr = pager->map[f_page/PAGEMAP_ENTRIES64].indirect;
1212 if (mapptr)
1213 pager_offset = mapptr[f_page%PAGEMAP_ENTRIES64];
1214 }
1215 }
1216 else {
1217 if (pager->map)
1218 pager_offset = pager->map[f_page];
1219 }
1220
1221#if DEBUG_READER_CONFLICTS0
1222 pager->readers--;
1223#endif
1224 pthread_mutex_unlock(&pager->lock);
1225 return (pager_offset);
1226}
1227
1228#if USE_PRECIOUS1
1229/*
1230 * Release a single disk block.
1231 */
1232void pager_release_offset(pager, offset)
1233 register dpager_t pager;
1234 vm_offset_t offset;
1235{
1236 register union dp_map entry;
1237
1238 offset = atop(offset)((offset)/vm_page_size);
1239
1240 pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1241
1242 assert (pager->map)((pager->map) ? (void) (0) : __assert_fail ("pager->map"
, "../../mach-defpager/default_pager.c", 1242, __PRETTY_FUNCTION__
));
1243 if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1244 register dp_map_t mapptr;
1245
1246 mapptr = pager->map[offset / PAGEMAP_ENTRIES64].indirect;
1247 entry = mapptr[offset % PAGEMAP_ENTRIES64];
1248 invalidate_block(mapptr[offset % PAGEMAP_ENTRIES])((mapptr[offset % 64]).indirect = (dp_map_t)((vm_offset_t)-1)
);
1249 } else {
1250 entry = pager->map[offset];
1251 invalidate_block(pager->map[offset])((pager->map[offset]).indirect = (dp_map_t)((vm_offset_t)-
1));
1252 }
1253
1254 pthread_mutex_unlock(&pager->lock);
1255
1256 pager_dealloc_page(entry.block.p_index, entry.block.p_offset, TRUE((boolean_t) 1));
1257}
1258#endif /*USE_PRECIOUS*/
1259
1260
1261/*
1262 * Move a page from one partition to another
1263 * New partition is locked, old partition is
1264 * locked unless LOCK_OLD sez otherwise.
1265 */
1266union dp_map
1267pager_move_page(block)
1268 union dp_map block;
1269{
1270 partition_t old_part, new_part;
1271 p_index_t old_pindex, new_pindex;
1272 union dp_map ret;
1273 vm_size_t size;
1274 vm_offset_t raddr, offset, new_offset;
1275 kern_return_t rc;
1276 static char here[] = "%spager_move_page";
1277
1278 old_pindex = block.block.p_index;
1279 invalidate_block(ret)((ret).indirect = (dp_map_t)((vm_offset_t)-1));
1280
1281 /* See if we have room to put it anywhere else */
1282 new_pindex = choose_partition( ptoa(1)((1)*vm_page_size), old_pindex);
1283 if (no_partition(new_pindex)((new_pindex) == ((p_index_t)-1)))
1284 return ret;
1285
1286 /* this unlocks the new partition */
1287 new_offset = pager_alloc_page(new_pindex, FALSE((boolean_t) 0));
1288 if (new_offset == NO_BLOCK((vm_offset_t)-1))
1289 panic(here,my_name);
1290
1291 /*
1292 * Got the resources, now move the data
1293 */
1294ddprintf ("pager_move_page(%x,%d,%d)\n",block.block.p_offset,old_pindex,new_pindex);
1295 old_part = partition_of(old_pindex);
1296 offset = ptoa(block.block.p_offset)((block.block.p_offset)*vm_page_size);
1297 rc = page_read_file_direct (old_part->file,
1298 offset,
1299 vm_page_size,
1300 &raddr,
1301 &size);
1302 if (rc != 0)
1303 panic(here,my_name);
1304
1305 /* release old */
1306 pager_dealloc_page(old_pindex, block.block.p_offset, FALSE((boolean_t) 0));
1307
1308 new_part = partition_of(new_pindex);
1309 offset = ptoa(new_offset)((new_offset)*vm_page_size);
1310 rc = page_write_file_direct (new_part->file,
1311 offset,
1312 raddr,
1313 size,
1314 &size);
1315 if (rc != 0)
1316 panic(here,my_name);
1317
1318 (void) vm_deallocate( mach_task_self()((__mach_task_self_ + 0)), raddr, size);
1319
1320 ret.block.p_offset = new_offset;
1321 ret.block.p_index = new_pindex;
1322
1323 return ret;
1324}
1325
1326#ifdef CHECKSUM
1327/*
1328 * Return the checksum for a block.
1329 */
1330int
1331pager_get_checksum(pager, offset)
1332 register dpager_t pager;
1333 vm_offset_t offset;
1334{
1335 register vm_offset_t f_page;
1336 int checksum;
1337
1338 f_page = atop(offset)((offset)/vm_page_size);
1339
1340 pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1341 if (f_page >= pager->size)
1342 panic("%spager_get_checksum",my_name);
1343
1344 if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1345 register vm_offset_t *mapptr;
1346
1347 mapptr = (vm_offset_t *)pager->checksum[f_page/PAGEMAP_ENTRIES64];
1348 if (mapptr == 0)
1349 checksum = NO_CHECKSUM;
1350 else
1351 checksum = mapptr[f_page%PAGEMAP_ENTRIES64];
1352 }
1353 else {
1354 checksum = pager->checksum[f_page];
1355 }
1356
1357 pthread_mutex_unlock(&pager->lock);
1358 return (checksum);
1359}
1360
1361/*
1362 * Remember the checksum for a block.
1363 */
1364int
1365pager_put_checksum(pager, offset, checksum)
1366 register dpager_t pager;
1367 vm_offset_t offset;
1368 int checksum;
1369{
1370 register vm_offset_t f_page;
1371 static char here[] = "%spager_put_checksum";
1372
1373 f_page = atop(offset)((offset)/vm_page_size);
1374
1375 pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1376 if (f_page >= pager->size)
1377 panic(here,my_name);
1378
1379 if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1380 register vm_offset_t *mapptr;
1381
1382 mapptr = (vm_offset_t *)pager->checksum[f_page/PAGEMAP_ENTRIES64];
1383 if (mapptr == 0)
1384 panic(here,my_name);
1385
1386 mapptr[f_page%PAGEMAP_ENTRIES64] = checksum;
1387 }
1388 else {
1389 pager->checksum[f_page] = checksum;
1390 }
1391 pthread_mutex_unlock(&pager->lock);
1392}
1393
1394/*
1395 * Compute a checksum - XOR each 32-bit word.
1396 */
1397int
1398compute_checksum(addr, size)
1399 vm_offset_t addr;
1400 vm_size_t size;
1401{
1402 register int checksum = NO_CHECKSUM;
1403 register int *ptr;
1404 register int count;
1405
1406 ptr = (int *)addr;
1407 count = size / sizeof(int);
1408
1409 while (--count >= 0)
1410 checksum ^= *ptr++;
1411
1412 return (checksum);
1413}
1414#endif /* CHECKSUM */
1415
1416/*
1417 * Given an offset within a paging object, find the
1418 * corresponding block within the paging partition.
1419 * Allocate a new block if necessary.
1420 *
1421 * WARNING: paging objects apparently may be extended
1422 * without notice!
1423 */
1424union dp_map
1425pager_write_offset(pager, offset)
1426 register dpager_t pager;
1427 vm_offset_t offset;
1428{
1429 register vm_offset_t f_page;
1430 register dp_map_t mapptr;
1431 register union dp_map block;
1432
1433 invalidate_block(block)((block).indirect = (dp_map_t)((vm_offset_t)-1));
1434
1435 f_page = atop(offset)((offset)/vm_page_size);
1436
1437#if DEBUG_READER_CONFLICTS0
1438 if (pager->readers > 0)
1439 default_pager_read_conflicts++; /* would have proceeded with
1440 read/write lock */
1441#endif
1442 pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1443#if DEBUG_READER_CONFLICTS0
1444 pager->readers++;
1445#endif
1446
1447 /* Catch the case where we had no initial fit partition
1448 for this object, but one was added later on */
1449 if (no_partition(pager->cur_partition)((pager->cur_partition) == ((p_index_t)-1))) {
1450 p_index_t new_part;
1451 vm_size_t size;
1452
1453 size = (f_page > pager->size) ? f_page : pager->size;
1454 new_part = choose_partition(ptoa(size)((size)*vm_page_size), P_INDEX_INVALID((p_index_t)-1));
1455 if (no_partition(new_part)((new_part) == ((p_index_t)-1)))
1456 new_part = choose_partition(ptoa(1)((1)*vm_page_size), P_INDEX_INVALID((p_index_t)-1));
1457 if (no_partition(new_part)((new_part) == ((p_index_t)-1)))
1458 /* give up right now to avoid confusion */
1459 goto out;
1460 else
1461 pager->cur_partition = new_part;
1462 }
1463
1464 while (f_page >= pager->size) {
1465 ddprintf ("pager_write_offset: extending: %x %x\n", f_page, pager->size);
1466
1467 /*
1468 * Paging object must be extended.
1469 * Remember that offset is 0-based, but size is 1-based.
1470 */
1471#if DEBUG_READER_CONFLICTS0
1472 pager->readers--;
1473#endif
1474 pthread_mutex_unlock(&pager->lock);
1475 pager_extend(pager, f_page + 1);
1476#if DEBUG_READER_CONFLICTS0
1477 if (pager->readers > 0)
1478 default_pager_read_conflicts++; /* would have proceeded with
1479 read/write lock */
1480#endif
1481 pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1482#if DEBUG_READER_CONFLICTS0
1483 pager->readers++;
1484#endif
1485 ddprintf ("pager_write_offset: done extending: %x %x\n", f_page, pager->size);
1486 }
1487
1488 if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1489 ddprintf ("pager_write_offset: indirect\n");
1490 mapptr = pager_get_direct_map(pager);
1491 mapptr = mapptr[f_page/PAGEMAP_ENTRIES64].indirect;
1492 if (mapptr == 0) {
1493 /*
1494 * Allocate the indirect block
1495 */
1496 register int i;
1497 ddprintf ("pager_write_offset: allocating indirect\n");
1498
1499 mapptr = (dp_map_t) kalloc(PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
1500 if (mapptr == 0) {
1501 /* out of space! */
1502 no_paging_space(TRUE((boolean_t) 1));
1503 goto out;
1504 }
1505 pager->map[f_page/PAGEMAP_ENTRIES64].indirect = mapptr;
1506 for (i = 0; i < PAGEMAP_ENTRIES64; i++)
1507 invalidate_block(mapptr[i])((mapptr[i]).indirect = (dp_map_t)((vm_offset_t)-1));
1508#ifdef CHECKSUM
1509 {
1510 register vm_offset_t *cksumptr;
1511 register int j;
1512
1513 cksumptr = (vm_offset_t *)
1514 kalloc(PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
1515 if (cksumptr == 0) {
1516 /* out of space! */
1517 no_paging_space(TRUE((boolean_t) 1));
1518 goto out;
1519 }
1520 pager->checksum[f_page/PAGEMAP_ENTRIES64]
1521 = (vm_offset_t)cksumptr;
1522 for (j = 0; j < PAGEMAP_ENTRIES64; j++)
1523 cksumptr[j] = NO_CHECKSUM;
1524 }
1525#endif /* CHECKSUM */
1526 }
1527 f_page %= PAGEMAP_ENTRIES64;
1528 }
1529 else {
1530 mapptr = pager_get_direct_map(pager);
1531 }
1532
1533 block = mapptr[f_page];
1534 ddprintf ("pager_write_offset: block starts as %x[%x] %x\n", mapptr, f_page, block);
1535 if (no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1))) {
1536 vm_offset_t off;
1537
1538 /* get room now */
1539 off = pager_alloc_page(pager->cur_partition, TRUE((boolean_t) 1));
1540 if (off == NO_BLOCK((vm_offset_t)-1)) {
1541 /*
1542 * Before giving up, try all other partitions.
1543 */
1544 p_index_t new_part;
1545
1546 ddprintf ("pager_write_offset: could not allocate block\n");
1547 /* returns it locked (if any one is non-full) */
1548 new_part = choose_partition( ptoa(1)((1)*vm_page_size), pager->cur_partition);
1549 if ( ! no_partition(new_part)((new_part) == ((p_index_t)-1)) ) {
1550
1551#if debug0
1552dprintf("%s partition %x filled,", my_name, pager->cur_partition);
1553dprintf("extending object %x (size %x) to %x.\n",
1554 pager, pager->size, new_part);
1555#endif
1556
1557 /* this one tastes better */
1558 pager->cur_partition = new_part;
1559
1560 /* this unlocks the partition too */
1561 off = pager_alloc_page(pager->cur_partition, FALSE((boolean_t) 0));
1562
1563 }
1564
1565 if (off == NO_BLOCK((vm_offset_t)-1)) {
1566 /*
1567 * Oh well.
1568 */
1569 overcommitted(FALSE((boolean_t) 0), 1);
1570 goto out;
1571 }
1572 ddprintf ("pager_write_offset: decided to allocate block\n");
1573 }
1574 block.block.p_offset = off;
1575 block.block.p_index = pager->cur_partition;
1576 mapptr[f_page] = block;
1577 }
1578
1579out:
1580
1581#if DEBUG_READER_CONFLICTS0
1582 pager->readers--;
1583#endif
1584 pthread_mutex_unlock(&pager->lock);
1585 return (block);
1586}
1587
1588/*
1589 * Deallocate all of the blocks belonging to a paging object.
1590 * No locking needed because no other operations can be in progress.
1591 */
1592void
1593pager_dealloc(pager)
1594 register dpager_t pager;
1595{
1596 register int i, j;
1597 register dp_map_t mapptr;
1598 register union dp_map block;
1599
1600 if (!pager->map)
1601 return;
1602
1603 if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1604 for (i = INDIRECT_PAGEMAP_ENTRIES(pager->size)((((pager->size)-1)/64) + 1); --i >= 0; ) {
1605 mapptr = pager->map[i].indirect;
1606 if (mapptr != 0) {
1607 for (j = 0; j < PAGEMAP_ENTRIES64; j++) {
1608 block = mapptr[j];
1609 if ( ! no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)) )
1610 pager_dealloc_page(block.block.p_index,
1611 block.block.p_offset, TRUE((boolean_t) 1));
1612 }
1613 kfree((char *)mapptr, PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
1614 pager->map[i].indirect = (dp_map_t) 0;
1615 }
1616 }
1617 kfree((char *)pager->map, INDIRECT_PAGEMAP_SIZE(pager->size)(((((pager->size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1618 pager->map = (dp_map_t) 0;
1619#ifdef CHECKSUM
1620 for (i = INDIRECT_PAGEMAP_ENTRIES(pager->size)((((pager->size)-1)/64) + 1); --i >= 0; ) {
1621 mapptr = (vm_offset_t *)pager->checksum[i];
1622 if (mapptr) {
1623 kfree((char *)mapptr, PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
1624 }
1625 }
1626 kfree((char *)pager->checksum,
1627 INDIRECT_PAGEMAP_SIZE(pager->size)(((((pager->size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1628#endif /* CHECKSUM */
1629 }
1630 else {
1631 mapptr = pager->map;
1632 for (i = 0; i < pager->size; i++ ) {
1633 block = mapptr[i];
1634 if ( ! no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)) )
1635 pager_dealloc_page(block.block.p_index,
1636 block.block.p_offset, TRUE((boolean_t) 1));
1637 }
1638 kfree((char *)pager->map, PAGEMAP_SIZE(pager->size)((pager->size)*sizeof(vm_offset_t)));
1639 pager->map = (dp_map_t) 0;
1640#ifdef CHECKSUM
1641 kfree((char *)pager->checksum, PAGEMAP_SIZE(pager->size)((pager->size)*sizeof(vm_offset_t)));
1642#endif /* CHECKSUM */
1643 }
1644}
1645
1646/*
1647 * Move all the pages of a PAGER that live in a
1648 * partition PINDEX somewhere else.
1649 * Pager should be write-locked, partition too.
1650 * Returns FALSE if it could not do it, but
1651 * some pages might have been moved nonetheless.
1652 */
1653boolean_t
1654pager_realloc(pager, pindex)
1655 register dpager_t pager;
1656 p_index_t pindex;
1657{
1658 register dp_map_t map, emap;
1659 vm_size_t size;
1660 union dp_map block;
1661
1662 if (!pager->map)
1663 return TRUE((boolean_t) 1);
1664
1665 size = pager->size; /* in pages */
1666 map = pager->map;
1667
1668 if (INDIRECT_PAGEMAP(size)(size > 64)) {
1669 for (emap = &map[INDIRECT_PAGEMAP_ENTRIES(size)((((size)-1)/64) + 1)];
1670 map < emap; map++) {
1671
1672 register dp_map_t map2, emap2;
1673
1674 if ((map2 = map->indirect) == 0)
1675 continue;
1676
1677 for (emap2 = &map2[PAGEMAP_ENTRIES64];
1678 map2 < emap2; map2++)
1679 if ( map2->block.p_index == pindex) {
1680
1681 block = pager_move_page(*map2);
1682 if (!no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)))
1683 *map2 = block;
1684 else
1685 return FALSE((boolean_t) 0);
1686 }
1687
1688 }
1689 goto ok;
1690 }
1691
1692 /* A small one */
1693 for (emap = &map[size]; map < emap; map++)
1694 if (map->block.p_index == pindex) {
1695 block = pager_move_page(*map);
1696 if (!no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)))
1697 *map = block;
1698 else
1699 return FALSE((boolean_t) 0);
1700 }
1701ok:
1702 pager->cur_partition = choose_partition(0, P_INDEX_INVALID((p_index_t)-1));
1703 return TRUE((boolean_t) 1);
1704}
1705
1706/*
1707
1708 */
1709
1710/*
1711 * Read/write routines.
1712 */
1713#define PAGER_SUCCESS0 0
1714#define PAGER_ABSENT1 1
1715#define PAGER_ERROR2 2
1716
1717/*
1718 * Read data from a default pager. Addr is the address of a buffer
1719 * to fill. Out_addr returns the buffer that contains the data;
1720 * if it is different from <addr>, it must be deallocated after use.
1721 */
1722int
1723default_read(ds, addr, size, offset, out_addr, deallocate, external)
1724 register dpager_t ds;
1725 vm_offset_t addr; /* pointer to block to fill */
1726 register vm_size_t size;
1727 register vm_offset_t offset;
1728 vm_offset_t *out_addr;
1729 /* returns pointer to data */
1730 boolean_t deallocate;
1731 boolean_t external;
1732{
1733 register union dp_map block;
1734 vm_offset_t raddr;
1735 vm_size_t rsize;
1736 register int rc;
1737 boolean_t first_time;
1738 register partition_t part;
1739#ifdef CHECKSUM
1740 vm_size_t original_size = size;
1741#endif /* CHECKSUM */
1742 vm_offset_t original_offset = offset;
1743
1744 /*
1745 * Find the block in the paging partition
1746 */
1747 block = pager_read_offset(ds, offset);
1748 if ( no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)) ) {
1749 if (external) {
1750 /*
1751 * An external object is requesting unswapped data,
1752 * zero fill the page and return.
1753 */
1754 bzero((char *) addr, vm_page_size);
1755 *out_addr = addr;
1756 return (PAGER_SUCCESS0);
1757 }
1758 return (PAGER_ABSENT1);
1759 }
1760
1761 /*
1762 * Read it, trying for the entire page.
1763 */
1764 offset = ptoa(block.block.p_offset)((block.block.p_offset)*vm_page_size);
1765ddprintf ("default_read(%x,%x,%x,%d)\n",addr,size,offset,block.block.p_index);
1766 part = partition_of(block.block.p_index);
1767 first_time = TRUE((boolean_t) 1);
1768 *out_addr = addr;
1769
1770 do {
1771 rc = page_read_file_direct(part->file,
1772 offset,
1773 size,
1774 &raddr,
1775 &rsize);
1776 if (rc != 0)
1777 return (PAGER_ERROR2);
1778
1779 /*
1780 * If we got the entire page on the first read, return it.
1781 */
1782 if (first_time && rsize == size) {
1783 *out_addr = raddr;
1784 break;
1785 }
1786 /*
1787 * Otherwise, copy the data into the
1788 * buffer we were passed, and try for
1789 * the next piece.
1790 */
1791 first_time = FALSE((boolean_t) 0);
1792 bcopy((char *)raddr, (char *)addr, rsize);
1793 addr += rsize;
1794 offset += rsize;
1795 size -= rsize;
1796 } while (size != 0);
1797
1798#if USE_PRECIOUS1
1799 if (deallocate)
1800 pager_release_offset(ds, original_offset);
1801#endif /*USE_PRECIOUS*/
1802
1803#ifdef CHECKSUM
1804 {
1805 int write_checksum,
1806 read_checksum;
1807
1808 write_checksum = pager_get_checksum(ds, original_offset);
1809 read_checksum = compute_checksum(*out_addr, original_size);
1810 if (write_checksum != read_checksum) {
1811 panic(
1812 "PAGER CHECKSUM ERROR: offset 0x%x, written 0x%x, read 0x%x",
1813 original_offset, write_checksum, read_checksum);
1814 }
1815 }
1816#endif /* CHECKSUM */
1817 return (PAGER_SUCCESS0);
1818}
1819
1820int
1821default_write(ds, addr, size, offset)
1822 register dpager_t ds;
1823 register vm_offset_t addr;
1824 register vm_size_t size;
1825 register vm_offset_t offset;
1826{
1827 register union dp_map block;
1828 partition_t part;
1829 vm_size_t wsize;
1830 register int rc;
1831
1832 ddprintf ("default_write: pager offset %x\n", offset);
1833
1834 /*
1835 * Find block in paging partition
1836 */
1837 block = pager_write_offset(ds, offset);
1838 if ( no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)) )
1839 return (PAGER_ERROR2);
1840
1841#ifdef CHECKSUM
1842 /*
1843 * Save checksum
1844 */
1845 {
1846 int checksum;
1847
1848 checksum = compute_checksum(addr, size);
1849 pager_put_checksum(ds, offset, checksum);
1850 }
1851#endif /* CHECKSUM */
1852 offset = ptoa(block.block.p_offset)((block.block.p_offset)*vm_page_size);
1853ddprintf ("default_write(%x,%x,%x,%d)\n",addr,size,offset,block.block.p_index);
1854 part = partition_of(block.block.p_index);
1855
1856 /*
1857 * There are various assumptions made here,we
1858 * will not get into the next disk 'block' by
1859 * accident. It might well be non-contiguous.
1860 */
1861 do {
1862 rc = page_write_file_direct(part->file,
1863 offset,
1864 addr,
1865 size,
1866 &wsize);
1867 if (rc != 0) {
1868 dprintf("*** PAGER ERROR: default_write: ");
1869 dprintf("ds=0x%x addr=0x%x size=0x%x offset=0x%x resid=0x%x\n",
1870 ds, addr, size, offset, wsize);
1871 return (PAGER_ERROR2);
1872 }
1873 addr += wsize;
1874 offset += wsize;
1875 size -= wsize;
1876 } while (size != 0);
1877 return (PAGER_SUCCESS0);
1878}
1879
1880boolean_t
1881default_has_page(ds, offset)
1882 dpager_t ds;
1883 vm_offset_t offset;
1884{
1885 return ( ! no_block(pager_read_offset(ds, offset))((pager_read_offset(ds, offset)).indirect == (dp_map_t)((vm_offset_t
)-1)) );
1886}
1887/*
1888
1889 */
1890
1891/*
1892 * Mapping between pager port and paging object.
1893 */
1894struct dstruct {
1895 queue_chain_t links; /* Link in pager-port list */
1896
1897 pthread_mutex_t lock; /* Lock for the structure */
1898 pthread_cond_t
1899 waiting_seqno, /* someone waiting on seqno */
1900 waiting_read, /* someone waiting on readers */
1901 waiting_write, /* someone waiting on writers */
1902 waiting_refs; /* someone waiting on refs */
1903
1904 memory_object_t pager; /* Pager port */
1905 mach_port_seqno_t seqno; /* Pager port sequence number */
1906 mach_port_t pager_request; /* Request port */
1907 mach_port_urefs_t request_refs; /* Request port user-refs */
1908 mach_port_t pager_name; /* Name port */
1909 mach_port_urefs_t name_refs; /* Name port user-refs */
1910 boolean_t external; /* Is an external object? */
1911
1912 unsigned int readers; /* Reads in progress */
1913 unsigned int writers; /* Writes in progress */
1914
1915 /* This is the reply port of an outstanding
1916 default_pager_object_set_size call. */
1917 mach_port_t lock_request;
1918
1919 unsigned int errors; /* Pageout error count */
1920 struct dpager dpager; /* Actual pager */
1921};
1922typedef struct dstruct * default_pager_t;
1923#define DEFAULT_PAGER_NULL((default_pager_t)0) ((default_pager_t)0)
1924
1925#if PARALLEL1
1926#define dstruct_lock_init(ds)pthread_mutex_init(&ds->lock, ((void*)0)) pthread_mutex_init(&ds->lock, NULL((void*)0))
1927#define dstruct_lock(ds)pthread_mutex_lock(&ds->lock) pthread_mutex_lock(&ds->lock)
1928#define dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock) pthread_mutex_unlock(&ds->lock)
1929#else /* PARALLEL */
1930#define dstruct_lock_init(ds)pthread_mutex_init(&ds->lock, ((void*)0))
1931#define dstruct_lock(ds)pthread_mutex_lock(&ds->lock)
1932#define dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock)
1933#endif /* PARALLEL */
1934
1935/*
1936 * List of all pagers. A specific pager is
1937 * found directly via its port, this list is
1938 * only used for monitoring purposes by the
1939 * default_pager_object* calls
1940 */
1941struct pager_port {
1942 queue_head_t queue;
1943 pthread_mutex_t lock;
1944 int count; /* saves code */
1945 queue_head_t leak_queue;
1946} all_pagers;
1947
1948#define pager_port_list_init(){ pthread_mutex_init(&all_pagers.lock, ((void*)0)); ((&
all_pagers.queue)->next = (&all_pagers.queue)->prev
= &all_pagers.queue); ((&all_pagers.leak_queue)->
next = (&all_pagers.leak_queue)->prev = &all_pagers
.leak_queue); all_pagers.count = 0; } \
1949{ \
1950 pthread_mutex_init(&all_pagers.lock, NULL((void*)0)); \
1951 queue_init(&all_pagers.queue)((&all_pagers.queue)->next = (&all_pagers.queue)->
prev = &all_pagers.queue); \
1952 queue_init(&all_pagers.leak_queue)((&all_pagers.leak_queue)->next = (&all_pagers.leak_queue
)->prev = &all_pagers.leak_queue); \
1953 all_pagers.count = 0; \
1954}
1955
1956void pager_port_list_insert(port, ds)
1957 mach_port_t port;
1958 default_pager_t ds;
1959{
1960 pthread_mutex_lock(&all_pagers.lock);
1961 queue_enter(&all_pagers.queue, ds, default_pager_t, links){ register queue_entry_t prev; prev = (&all_pagers.queue)
->prev; if ((&all_pagers.queue) == prev) { (&all_pagers
.queue)->next = (queue_entry_t) (ds); } else { ((default_pager_t
)prev)->links.next = (queue_entry_t)(ds); } (ds)->links
.prev = prev; (ds)->links.next = &all_pagers.queue; (&
all_pagers.queue)->prev = (queue_entry_t) ds; };
1962 all_pagers.count++;
1963 pthread_mutex_unlock(&all_pagers.lock);
1964}
1965
1966/* given a data structure return a good port-name to associate it to */
1967#define pnameof(_x_)(((vm_offset_t)(_x_))+1) (((vm_offset_t)(_x_))+1)
1968/* reverse, assumes no-odd-pointers */
1969#define dnameof(_x_)(((vm_offset_t)(_x_))&~1) (((vm_offset_t)(_x_))&~1)
1970
1971/* The magic typecast */
1972#define pager_port_lookup(_port_)((! (((_port_) != ((mach_port_t) 0)) && ((_port_) != (
(mach_port_t) ~0))) || ((default_pager_t)(((vm_offset_t)(_port_
))&~1))->pager != (_port_)) ? ((default_pager_t)0) : (
default_pager_t)(((vm_offset_t)(_port_))&~1)) \
1973 ((! MACH_PORT_VALID(_port_)(((_port_) != ((mach_port_t) 0)) && ((_port_) != ((mach_port_t
) ~0))) || \
1974 ((default_pager_t)dnameof(_port_)(((vm_offset_t)(_port_))&~1))->pager != (_port_)) ? \
1975 DEFAULT_PAGER_NULL((default_pager_t)0) : (default_pager_t)dnameof(_port_)(((vm_offset_t)(_port_))&~1))
1976
1977void pager_port_list_delete(ds)
1978 default_pager_t ds;
1979{
1980 pthread_mutex_lock(&all_pagers.lock);
1981 queue_remove(&all_pagers.queue, ds, default_pager_t, links){ register queue_entry_t next, prev; next = (ds)->links.next
; prev = (ds)->links.prev; if ((&all_pagers.queue) == next
) (&all_pagers.queue)->prev = prev; else ((default_pager_t
)next)->links.prev = prev; if ((&all_pagers.queue) == prev
) (&all_pagers.queue)->next = next; else ((default_pager_t
)prev)->links.next = next; };
1982 all_pagers.count--;
1983 pthread_mutex_unlock(&all_pagers.lock);
1984}
1985
1986/*
1987 * Destroy a paging partition.
1988 * XXX this is not re-entrant XXX
1989 */
1990kern_return_t
1991destroy_paging_partition(name, pp_private)
1992 char *name;
1993 void **pp_private;
1994{
1995 register unsigned int id = part_id(name);
1996 register partition_t part;
1
Variable 'part' declared without an initial value
→
1997 boolean_t all_ok = TRUE((boolean_t) 1);
1998 default_pager_t entry;
1999 int pindex;
2000
2001 /*
2002 * Find and take partition out of list
2003 * This prevents choose_partition from
2004 * getting in the way.
2005 */
2006 pthread_mutex_lock(&all_partitions.lock);
2007 for (pindex = 0; pindex < all_partitions.n_partitions; pindex++) {
2
←
Loop condition is false. Execution continues on line 2011
→
2008 part = partition_of(pindex);
2009 if (part && (part->id == id)) break;
2010 }
2011 if (pindex == all_partitions.n_partitions) {
3
←
Taking false branch
→
2012 pthread_mutex_unlock(&all_partitions.lock);
2013 return KERN_INVALID_ARGUMENT4;
2014 }
2015 part->going_away = TRUE((boolean_t) 1);
4
←
Dereference of undefined pointer value
2016 pthread_mutex_unlock(&all_partitions.lock);
2017
2018 /*
2019 * This might take a while..
2020 */
2021all_over_again:
2022#if debug0
2023dprintf("Partition x%x (id x%x) for %s, all_ok %d\n", part, id, name, all_ok);
2024#endif
2025 all_ok = TRUE((boolean_t) 1);
2026 pthread_mutex_lock(&part->p_lock);
2027
2028 pthread_mutex_lock(&all_pagers.lock);
2029 queue_iterate(&all_pagers.queue, entry, default_pager_t, links)for ((entry) = (default_pager_t) ((&all_pagers.queue)->
next); !(((&all_pagers.queue)) == ((queue_entry_t)(entry)
)); (entry) = (default_pager_t) ((&(entry)->links)->
next)) {
2030
2031 dstruct_lock(entry)pthread_mutex_lock(&entry->lock);
2032
2033 if (pthread_mutex_trylock(&entry->dpager.lock)) {
2034
2035 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
2036 pthread_mutex_unlock(&all_pagers.lock);
2037 pthread_mutex_unlock(&part->p_lock);
2038
2039 /* yield the processor */
2040 (void) thread_switch(MACH_PORT_NULL((mach_port_t) 0),
2041 SWITCH_OPTION_NONE0, 0);
2042
2043 goto all_over_again;
2044
2045 }
2046
2047 /*
2048 * See if we can relocate all the pages of this object
2049 * currently on this partition on some other partition
2050 */
2051 all_ok = pager_realloc(&entry->dpager, pindex);
2052
2053 pthread_mutex_unlock(&entry->dpager.lock);
2054 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
2055
2056 if (!all_ok) break;
2057
2058 }
2059 pthread_mutex_unlock(&all_pagers.lock);
2060
2061 if (all_ok) {
2062 /* No need to unlock partition, there are no refs left */
2063
2064 set_partition_of(pindex, 0);
2065 *pp_private = part->file;
2066 kfree(part->bitmap, howmany(part->total_size, NB_BM)(((part->total_size) + (32) - 1)/(32)) * sizeof(bm_entry_t));
2067 kfree(part, sizeof(struct part));
2068 dprintf("%s Removed paging partition %s\n", my_name, name);
2069 return KERN_SUCCESS0;
2070 }
2071
2072 /*
2073 * Put partition back in.
2074 */
2075 part->going_away = FALSE((boolean_t) 0);
2076
2077 return KERN_FAILURE5;
2078}
2079
2080
2081/*
2082 * We use the sequence numbers on requests to regulate
2083 * our parallelism. In general, we allow multiple reads and writes
2084 * to proceed in parallel, with the exception that reads must
2085 * wait for previous writes to finish. (Because the kernel might
2086 * generate a data-request for a page on the heels of a data-write
2087 * for the same page, and we must avoid returning stale data.)
2088 * terminate requests wait for proceeding reads and writes to finish.
2089 */
2090
2091unsigned int default_pager_total = 0; /* debugging */
2092unsigned int default_pager_wait_seqno = 0; /* debugging */
2093unsigned int default_pager_wait_read = 0; /* debugging */
2094unsigned int default_pager_wait_write = 0; /* debugging */
2095unsigned int default_pager_wait_refs = 0; /* debugging */
2096
2097#if PARALLEL1
2098/*
2099 * Waits for correct sequence number. Leaves pager locked.
2100 */
2101void pager_port_lock(ds, seqno)
2102 default_pager_t ds;
2103 mach_port_seqno_t seqno;
2104{
2105 default_pager_total++;
2106 dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2107 while (ds->seqno != seqno) {
2108 default_pager_wait_seqno++;
2109 pthread_cond_wait(&ds->waiting_seqno, &ds->lock);
2110 }
2111}
2112
2113/*
2114 * Increments sequence number and unlocks pager.
2115 */
2116void pager_port_unlock(ds)
2117 default_pager_t ds;
2118{
2119 ds->seqno++;
2120 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2121 pthread_cond_broadcast(&ds->waiting_seqno);
2122}
2123
2124/*
2125 * Start a read - one more reader. Pager must be locked.
2126 */
2127void pager_port_start_read(ds)
2128 default_pager_t ds;
2129{
2130 ds->readers++;
2131}
2132
2133/*
2134 * Wait for readers. Unlocks and relocks pager if wait needed.
2135 */
2136void pager_port_wait_for_readers(ds)
2137 default_pager_t ds;
2138{
2139 while (ds->readers != 0) {
2140 default_pager_wait_read++;
2141 pthread_cond_wait(&ds->waiting_read, &ds->lock);
2142 }
2143}
2144
2145/*
2146 * Finish a read. Pager is unlocked and returns unlocked.
2147 */
2148void pager_port_finish_read(ds)
2149 default_pager_t ds;
2150{
2151 dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2152 if (--ds->readers == 0) {
2153 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2154 pthread_cond_broadcast(&ds->waiting_read);
2155 }
2156 else {
2157 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2158 }
2159}
2160
2161/*
2162 * Start a write - one more writer. Pager must be locked.
2163 */
2164void pager_port_start_write(ds)
2165 default_pager_t ds;
2166{
2167 ds->writers++;
2168}
2169
2170/*
2171 * Wait for writers. Unlocks and relocks pager if wait needed.
2172 */
2173void pager_port_wait_for_writers(ds)
2174 default_pager_t ds;
2175{
2176 while (ds->writers != 0) {
2177 default_pager_wait_write++;
2178 pthread_cond_wait(&ds->waiting_write, &ds->lock);
2179 }
2180}
2181
2182/*
2183 * Finish a write. Pager is unlocked and returns unlocked.
2184 */
2185void pager_port_finish_write(ds)
2186 default_pager_t ds;
2187{
2188 dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2189 if (--ds->writers == 0) {
2190 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2191 pthread_cond_broadcast(&ds->waiting_write);
2192 }
2193 else {
2194 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2195 }
2196}
2197
2198/*
2199 * Wait for concurrent default_pager_objects.
2200 * Unlocks and relocks pager if wait needed.
2201 */
2202void pager_port_wait_for_refs(ds)
2203 default_pager_t ds;
2204{
2205 while (ds->name_refs == 0) {
2206 default_pager_wait_refs++;
2207 pthread_cond_wait(&ds->waiting_refs, &ds->lock);
2208 }
2209}
2210
2211/*
2212 * Finished creating name refs - wake up waiters.
2213 */
2214void pager_port_finish_refs(ds)
2215 default_pager_t ds;
2216{
2217 pthread_cond_broadcast(&ds->waiting_refs);
2218}
2219
2220#else /* PARALLEL */
2221
2222#define pager_port_lock(ds,seqno)
2223#define pager_port_unlock(ds)
2224#define pager_port_start_read(ds)
2225#define pager_port_wait_for_readers(ds)
2226#define pager_port_finish_read(ds)
2227#define pager_port_start_write(ds)
2228#define pager_port_wait_for_writers(ds)
2229#define pager_port_finish_write(ds)
2230#define pager_port_wait_for_refs(ds)
2231#define pager_port_finish_refs(ds)
2232
2233#endif /* PARALLEL */
2234
2235/*
2236 * Default pager.
2237 */
2238task_t default_pager_self; /* Our task port. */
2239
2240mach_port_t default_pager_default_port; /* Port for memory_object_create. */
2241
2242/* We catch exceptions on ourself & startup using this port. */
2243mach_port_t default_pager_exception_port;
2244
2245mach_port_t default_pager_internal_set; /* Port set for internal objects. */
2246mach_port_t default_pager_external_set; /* Port set for external objects. */
2247mach_port_t default_pager_default_set; /* Port set for "default" thread. */
2248
2249typedef struct default_pager_thread {
2250 pthread_t dpt_thread; /* Server thread. */
2251 vm_offset_t dpt_buffer; /* Read buffer. */
2252 boolean_t dpt_internal; /* Do we handle internal objects? */
2253} default_pager_thread_t;
2254
2255#if PARALLEL1
2256 /* determine number of threads at run time */
2257#define DEFAULT_PAGER_INTERNAL_COUNT(0) (0)
2258
2259#else /* PARALLEL */
2260#define DEFAULT_PAGER_INTERNAL_COUNT(0) (1)
2261#endif /* PARALLEL */
2262
2263/* Memory created by default_pager_object_create should mostly be resident. */
2264#define DEFAULT_PAGER_EXTERNAL_COUNT(1) (1)
2265
2266unsigned int default_pager_internal_count = DEFAULT_PAGER_INTERNAL_COUNT(0);
2267 /* Number of "internal" threads. */
2268unsigned int default_pager_external_count = DEFAULT_PAGER_EXTERNAL_COUNT(1);
2269 /* Number of "external" threads. */
2270
2271default_pager_t pager_port_alloc(size)
2272 vm_size_t size;
2273{
2274 default_pager_t ds;
2275 p_index_t part;
2276
2277 ds = (default_pager_t) kalloc(sizeof *ds);
2278 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2279 panic("%spager_port_alloc",my_name);
2280 bzero((char *) ds, sizeof *ds);
2281
2282 dstruct_lock_init(ds)pthread_mutex_init(&ds->lock, ((void*)0));
2283
2284 /*
2285 * Get a suitable partition. If none big enough
2286 * just pick one and overcommit. If no partitions
2287 * at all.. well just fake one so that we will
2288 * kill specific objects on pageouts rather than
2289 * panicing the system now.
2290 */
2291 part = choose_partition(size, P_INDEX_INVALID((p_index_t)-1));
2292 if (no_partition(part)((part) == ((p_index_t)-1))) {
2293 overcommitted(FALSE((boolean_t) 0), atop(size)((size)/vm_page_size));
2294 part = choose_partition(0,P_INDEX_INVALID((p_index_t)-1));
2295#if debug0
2296 if (no_partition(part)((part) == ((p_index_t)-1)))
2297 dprintf("%s No paging space at all !!\n", my_name);
2298#endif
2299 }
2300 pager_alloc(&ds->dpager, part, size);
2301
2302 return ds;
2303}
2304
2305mach_port_urefs_t default_pager_max_urefs = 10000;
2306
2307/*
2308 * Check user reference count on pager_request port.
2309 * Pager must be locked.
2310 * Unlocks and re-locks pager if needs to call kernel.
2311 */
2312void pager_port_check_request(ds, pager_request)
2313 default_pager_t ds;
2314 mach_port_t pager_request;
2315{
2316 mach_port_delta_t delta;
2317 kern_return_t kr;
2318
2319 assert(ds->pager_request == pager_request)((ds->pager_request == pager_request) ? (void) (0) : __assert_fail
("ds->pager_request == pager_request", "../../mach-defpager/default_pager.c"
, 2319, __PRETTY_FUNCTION__));
2320
2321 if (++ds->request_refs > default_pager_max_urefs) {
2322 delta = 1 - ds->request_refs;
2323 ds->request_refs = 1;
2324
2325 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2326
2327 /*
2328 * Deallocate excess user references.
2329 */
2330
2331 kr = mach_port_mod_refs(default_pager_self, pager_request,
2332 MACH_PORT_RIGHT_SEND((mach_port_right_t) 0), delta);
2333 if (kr != KERN_SUCCESS0)
2334 panic("%spager_port_check_request",my_name);
2335
2336 dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2337 }
2338}
2339
2340void default_pager_add(ds, internal)
2341 default_pager_t ds;
2342 boolean_t internal;
2343{
2344 mach_port_t pager = ds->pager;
2345 mach_port_t pset;
2346 mach_port_mscount_t sync;
2347 mach_port_t previous;
2348 kern_return_t kr;
2349 static char here[] = "%sdefault_pager_add";
2350
2351 /*
2352 * The port currently has a make-send count of zero,
2353 * because either we just created the port or we just
2354 * received the port in a memory_object_create request.
2355 */
2356
2357 if (internal) {
2358 /* possibly generate an immediate no-senders notification */
2359 sync = 0;
2360 pset = default_pager_internal_set;
2361 ds->external = FALSE((boolean_t) 0);
2362 } else {
2363 /* delay notification till send right is created */
2364 sync = 1;
2365 pset = default_pager_external_set;
2366 ds->external = TRUE((boolean_t) 1);
2367 }
2368
2369 kr = mach_port_request_notification(default_pager_self, pager,
2370 MACH_NOTIFY_NO_SENDERS(0100 + 006), sync,
2371 pager, MACH_MSG_TYPE_MAKE_SEND_ONCE21,
2372 &previous);
2373 if ((kr != KERN_SUCCESS0) || (previous != MACH_PORT_NULL((mach_port_t) 0)))
2374 panic(here,my_name);
2375
2376 kr = mach_port_move_member(default_pager_self, pager, pset);
2377 if (kr != KERN_SUCCESS0)
2378 panic(here,my_name);
2379}
2380
2381/*
2382 * Routine: memory_object_create
2383 * Purpose:
2384 * Handle requests for memory objects from the
2385 * kernel.
2386 * Notes:
2387 * Because we only give out the default memory
2388 * manager port to the kernel, we don't have to
2389 * be so paranoid about the contents.
2390 */
2391kern_return_t
2392seqnos_memory_object_create(old_pager, seqno, new_pager, new_size,
2393 new_pager_request, new_pager_name, new_page_size)
2394 mach_port_t old_pager;
2395 mach_port_seqno_t seqno;
2396 mach_port_t new_pager;
2397 vm_size_t new_size;
2398 mach_port_t new_pager_request;
2399 mach_port_t new_pager_name;
2400 vm_size_t new_page_size;
2401{
2402 register default_pager_t ds;
2403 kern_return_t kr;
2404
2405 assert(old_pager == default_pager_default_port)((old_pager == default_pager_default_port) ? (void) (0) : __assert_fail
("old_pager == default_pager_default_port", "../../mach-defpager/default_pager.c"
, 2405, __PRETTY_FUNCTION__));
2406 assert(MACH_PORT_VALID(new_pager_request))(((((new_pager_request) != ((mach_port_t) 0)) && ((new_pager_request
) != ((mach_port_t) ~0)))) ? (void) (0) : __assert_fail ("(((new_pager_request) != ((mach_port_t) 0)) && ((new_pager_request) != ((mach_port_t) ~0)))"
, "../../mach-defpager/default_pager.c", 2406, __PRETTY_FUNCTION__
));
2407 assert(MACH_PORT_VALID(new_pager_name))(((((new_pager_name) != ((mach_port_t) 0)) && ((new_pager_name
) != ((mach_port_t) ~0)))) ? (void) (0) : __assert_fail ("(((new_pager_name) != ((mach_port_t) 0)) && ((new_pager_name) != ((mach_port_t) ~0)))"
, "../../mach-defpager/default_pager.c", 2407, __PRETTY_FUNCTION__
));
2408 assert(new_page_size == vm_page_size)((new_page_size == vm_page_size) ? (void) (0) : __assert_fail
("new_page_size == vm_page_size", "../../mach-defpager/default_pager.c"
, 2408, __PRETTY_FUNCTION__));
2409
2410 ds = pager_port_alloc(new_size);
2411rename_it:
2412 kr = mach_port_rename( default_pager_self,
2413 new_pager, (mach_port_t)pnameof(ds)(((vm_offset_t)(ds))+1));
2414 if (kr != KERN_SUCCESS0) {
2415 default_pager_t ds1;
2416
2417 if (kr != KERN_NAME_EXISTS13)
2418 panic("%s m_o_create", my_name);
2419 ds1 = (default_pager_t) kalloc(sizeof *ds1);
2420 *ds1 = *ds;
2421 pthread_mutex_lock(&all_pagers.lock);
2422 queue_enter(&all_pagers.leak_queue, ds, default_pager_t, links){ register queue_entry_t prev; prev = (&all_pagers.leak_queue
)->prev; if ((&all_pagers.leak_queue) == prev) { (&
all_pagers.leak_queue)->next = (queue_entry_t) (ds); } else
{ ((default_pager_t)prev)->links.next = (queue_entry_t)(ds
); } (ds)->links.prev = prev; (ds)->links.next = &all_pagers
.leak_queue; (&all_pagers.leak_queue)->prev = (queue_entry_t
) ds; };
2423 pthread_mutex_unlock(&all_pagers.lock);
2424 ds = ds1;
2425 goto rename_it;
2426 }
2427
2428 new_pager = (mach_port_t) pnameof(ds)(((vm_offset_t)(ds))+1);
2429
2430 /*
2431 * Set up associations between these ports
2432 * and this default_pager structure
2433 */
2434
2435 ds->pager = new_pager;
2436 ds->pager_request = new_pager_request;
2437 ds->request_refs = 1;
2438 ds->pager_name = new_pager_name;
2439 ds->name_refs = 1;
2440
2441 /*
2442 * After this, other threads might receive requests
2443 * for this memory object or find it in the port list.
2444 */
2445
2446 pager_port_list_insert(new_pager, ds);
2447 default_pager_add(ds, TRUE((boolean_t) 1));
2448
2449 return(KERN_SUCCESS0);
2450}
2451
2452memory_object_copy_strategy_t default_pager_copy_strategy =
2453 MEMORY_OBJECT_COPY_DELAY2;
2454
2455kern_return_t
2456seqnos_memory_object_init(pager, seqno, pager_request, pager_name,
2457 pager_page_size)
2458 mach_port_t pager;
2459 mach_port_seqno_t seqno;
2460 mach_port_t pager_request;
2461 mach_port_t pager_name;
2462 vm_size_t pager_page_size;
2463{
2464 register default_pager_t ds;
2465 kern_return_t kr;
2466 static char here[] = "%sinit";
2467
2468 assert(MACH_PORT_VALID(pager_request))(((((pager_request) != ((mach_port_t) 0)) && ((pager_request
) != ((mach_port_t) ~0)))) ? (void) (0) : __assert_fail ("(((pager_request) != ((mach_port_t) 0)) && ((pager_request) != ((mach_port_t) ~0)))"
, "../../mach-defpager/default_pager.c", 2468, __PRETTY_FUNCTION__
));
2469 assert(MACH_PORT_VALID(pager_name))(((((pager_name) != ((mach_port_t) 0)) && ((pager_name
) != ((mach_port_t) ~0)))) ? (void) (0) : __assert_fail ("(((pager_name) != ((mach_port_t) 0)) && ((pager_name) != ((mach_port_t) ~0)))"
, "../../mach-defpager/default_pager.c", 2469, __PRETTY_FUNCTION__
));
2470 assert(pager_page_size == vm_page_size)((pager_page_size == vm_page_size) ? (void) (0) : __assert_fail
("pager_page_size == vm_page_size", "../../mach-defpager/default_pager.c"
, 2470, __PRETTY_FUNCTION__));
2471
2472 ds = pager_port_lookup(pager)((! (((pager) != ((mach_port_t) 0)) && ((pager) != ((
mach_port_t) ~0))) || ((default_pager_t)(((vm_offset_t)(pager
))&~1))->pager != (pager)) ? ((default_pager_t)0) : (default_pager_t
)(((vm_offset_t)(pager))&~1));
2473 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2474 panic(here, my_name);
2475 pager_port_lock(ds, seqno);
2476
2477 if (ds->pager_request != MACH_PORT_NULL((mach_port_t) 0))
2478 panic(here, my_name);
2479
2480 ds->pager_request = pager_request;
2481 ds->request_refs = 1;
2482 ds->pager_name = pager_name;
2483 ds->name_refs = 1;
2484
2485 /*
2486 * Even if the kernel immediately terminates the object,
2487 * the pager_request port won't be destroyed until
2488 * we process the terminate request, which won't happen
2489 * until we unlock the object.
2490 */
2491
2492 kr = memory_object_ready(pager_request,
2493 FALSE((boolean_t) 0), /* Do not cache */
2494 default_pager_copy_strategy);
2495 if (kr != KERN_SUCCESS0)
2496 panic(here, my_name);
2497
2498 pager_port_unlock(ds);
2499
2500 return(KERN_SUCCESS0);
2501}
2502
2503kern_return_t
2504seqnos_memory_object_terminate(pager, seqno, pager_request, pager_name)
2505 mach_port_t pager;
2506 mach_port_seqno_t seqno;
2507 mach_port_t pager_request;
2508 mach_port_t pager_name;
2509{
2510 register default_pager_t ds;
2511 kern_return_t kr;
2512 static char here[] = "%sterminate";
2513
2514 /*
2515 * pager_request and pager_name are receive rights,
2516 * not send rights.
2517 */
2518
2519 ds = pager_port_lookup(pager)((! (((pager) != ((mach_port_t) 0)) && ((pager) != ((
mach_port_t) ~0))) || ((default_pager_t)(((vm_offset_t)(pager
))&~1))->pager != (pager)) ? ((default_pager_t)0) : (default_pager_t
)(((vm_offset_t)(pager))&~1));
2520 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2521 panic(here, my_name);
2522ddprintf ("seqnos_memory_object_terminate <%p>: pager_port_lock: <%p>[s:%d,r:%d,w:%d,l:%d], %d\n",
2523 &kr, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held, seqno);
2524 pager_port_lock(ds, seqno);
2525
2526 /*
2527 * Wait for read and write requests to terminate.
2528 */
2529
2530 pager_port_wait_for_readers(ds);
2531 pager_port_wait_for_writers(ds);
2532
2533 /*
2534 * After memory_object_terminate both memory_object_init
2535 * and a no-senders notification are possible, so we need
2536 * to clean up the request and name ports but leave
2537 * the pager port.
2538 *
2539 * A concurrent default_pager_objects might be allocating
2540 * more references for the name port. In this case,
2541 * we must first wait for it to finish.
2542 */
2543
2544 pager_port_wait_for_refs(ds);
2545
2546 if (ds->external)
2547 pager_request = ds->pager_request;
2548 ds->pager_request = MACH_PORT_NULL((mach_port_t) 0);
2549 ds->request_refs = 0;
2550 assert(ds->pager_name == pager_name)((ds->pager_name == pager_name) ? (void) (0) : __assert_fail
("ds->pager_name == pager_name", "../../mach-defpager/default_pager.c"
, 2550, __PRETTY_FUNCTION__));
2551 ds->pager_name = MACH_PORT_NULL((mach_port_t) 0);
2552 ds->name_refs = 0;
2553ddprintf ("seqnos_memory_object_terminate <%p>: pager_port_unlock: <%p>[s:%d,r:%d,w:%d,l:%d]\n",
2554 &kr, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held);
2555 pager_port_unlock(ds);
2556
2557 /*
2558 * Now we destroy our port rights.
2559 */
2560
2561 mach_port_destroy(mach_task_self()((__mach_task_self_ + 0)), pager_request);
2562 mach_port_destroy(mach_task_self()((__mach_task_self_ + 0)), pager_name);
2563
2564 return (KERN_SUCCESS0);
2565}
2566
2567void default_pager_no_senders(pager, seqno, mscount)
2568 memory_object_t pager;
2569 mach_port_seqno_t seqno;
2570 mach_port_mscount_t mscount;
2571{
2572 register default_pager_t ds;
2573 kern_return_t kr;
2574 static char here[] = "%sno_senders";
2575
2576 /*
2577 * Because we don't give out multiple send rights
2578 * for a memory object, there can't be a race
2579 * between getting a no-senders notification
2580 * and creating a new send right for the object.
2581 * Hence we don't keep track of mscount.
2582 */
2583
2584
2585 ds = pager_port_lookup(pager)((! (((pager) != ((mach_port_t) 0)) && ((pager) != ((
mach_port_t) ~0))) || ((default_pager_t)(((vm_offset_t)(pager
))&~1))->pager != (pager)) ? ((default_pager_t)0) : (default_pager_t
)(((vm_offset_t)(pager))&~1));
2586 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2587 panic(here,my_name);
2588 pager_port_lock(ds, seqno);
2589
2590 /*
2591 * We shouldn't get a no-senders notification
2592 * when the kernel has the object cached.
2593 */
2594
2595 if (ds->pager_request != MACH_PORT_NULL((mach_port_t) 0))
2596 panic(here,my_name);
2597
2598 /*
2599 * Unlock the pager (though there should be no one
2600 * waiting for it).
2601 */
2602 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2603
2604 /*
2605 * Remove the memory object port association, and then
2606 * the destroy the port itself. We must remove the object
2607 * from the port list before deallocating the pager,
2608 * because of default_pager_objects.
2609 */
2610
2611 pager_port_list_delete(ds);
2612 pager_dealloc(&ds->dpager);
2613
2614 kr = mach_port_mod_refs(default_pager_self, pager,
2615 MACH_PORT_RIGHT_RECEIVE((mach_port_right_t) 1), -1);
2616 if (kr != KERN_SUCCESS0)
2617 panic(here,my_name);
2618
2619 /*
2620 * Do this *after* deallocating the port name
2621 */
2622 kfree((char *) ds, sizeof(*ds));
2623
2624 /*
2625 * Recover memory that we might have wasted because
2626 * of name conflicts
2627 */
2628 pthread_mutex_lock(&all_pagers.lock);
2629
2630 while (!queue_empty(&all_pagers.leak_queue)(((&all_pagers.leak_queue)) == (((&all_pagers.leak_queue
)->next)))) {
2631
2632 ds = (default_pager_t) queue_first(&all_pagers.leak_queue)((&all_pagers.leak_queue)->next);
2633 queue_remove_first(&all_pagers.leak_queue, ds, default_pager_t, links){ register queue_entry_t next; (ds) = (default_pager_t) ((&
all_pagers.leak_queue)->next); next = (ds)->links.next;
if ((&all_pagers.leak_queue) == next) (&all_pagers.leak_queue
)->prev = (&all_pagers.leak_queue); else ((default_pager_t
)(next))->links.prev = (&all_pagers.leak_queue); (&
all_pagers.leak_queue)->next = next; };
2634 kfree((char *) ds, sizeof(*ds));
2635 }
2636
2637 pthread_mutex_unlock(&all_pagers.lock);
2638}
2639
2640int default_pager_pagein_count = 0;
2641int default_pager_pageout_count = 0;
2642
2643static __thread default_pager_thread_t *dpt;
2644
2645kern_return_t
2646seqnos_memory_object_data_request(pager, seqno, reply_to, offset,
2647 length, protection_required)
2648 memory_object_t pager;
2649 mach_port_seqno_t seqno;
2650 mach_port_t reply_to;
2651 vm_offset_t offset;
2652 vm_size_t length;
2653 vm_prot_t protection_required;
2654{
2655 default_pager_t ds;
2656 vm_offset_t addr;
2657 unsigned int errors;
2658 kern_return_t rc;
2659 static char here[] = "%sdata_request";
2660
2661 if (length != vm_page_size)
2662 panic(here,my_name);
2663
2664 ds = pager_port_lookup(pager)((! (((pager) != ((mach_port_t) 0)) && ((pager) != ((
mach_port_t) ~0))) || ((default_pager_t)(((vm_offset_t)(pager
))&~1))->pager != (pager)) ? ((default_pager_t)0) : (default_pager_t
)(((vm_offset_t)(pager))&~1));
2665 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2666 panic(here,my_name);
2667ddprintf ("seqnos_memory_object_data_request <%p>: pager_port_lock: <%p>[s:%d,r:%d,w:%d,l:%d], %d\n",
2668 &ds, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held, seqno);
2669 pager_port_lock(ds, seqno);
2670 pager_port_check_request(ds, reply_to);
2671 pager_port_wait_for_writers(ds);
2672 pager_port_start_read(ds);
2673
2674 /*
2675 * Get error count while pager locked.
2676 */
2677 errors = ds->errors;
2678
2679ddprintf ("seqnos_memory_object_data_request <%p>: pager_port_unlock: <%p>[s:%d,r:%d,w:%d,l:%d]\n",
2680 &ds, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held);
2681 pager_port_unlock(ds);
2682
2683 if (errors) {
2684 dprintf("%s %s\n", my_name,
2685 "dropping data_request because of previous paging errors");
2686 (void) memory_object_data_error(reply_to,
2687 offset, vm_page_size,
2688 KERN_FAILURE5);
2689 goto done;
2690 }
2691
2692 if (offset >= ds->dpager.limit)
2693 rc = PAGER_ERROR2;
2694 else
2695 rc = default_read(&ds->dpager, dpt->dpt_buffer,
2696 vm_page_size, offset,
2697 &addr, protection_required & VM_PROT_WRITE((vm_prot_t) 0x02),
2698 ds->external);
2699
2700 switch (rc) {
2701 case PAGER_SUCCESS0:
2702 if (addr != dpt->dpt_buffer) {
2703 /*
2704 * Deallocates data buffer
2705 */
2706 (void) memory_object_data_supply(
2707 reply_to, offset,
2708 addr, vm_page_size, TRUE((boolean_t) 1),
2709 VM_PROT_NONE((vm_prot_t) 0x00),
2710 FALSE((boolean_t) 0), MACH_PORT_NULL((mach_port_t) 0));
2711 } else {
2712 (void) memory_object_data_supply(
2713 reply_to, offset,
2714 addr, vm_page_size, FALSE((boolean_t) 0),
2715 VM_PROT_NONE((vm_prot_t) 0x00),
2716 FALSE((boolean_t) 0), MACH_PORT_NULL((mach_port_t) 0));
2717 }
2718 break;
2719
2720 case PAGER_ABSENT1:
2721 (void) memory_object_data_unavailable(
2722 reply_to,
2723 offset,
2724 vm_page_size);
2725 break;
2726
2727 case PAGER_ERROR2:
2728 (void) memory_object_data_error(
2729 reply_to,
2730 offset,
2731 vm_page_size,
2732 KERN_FAILURE5);
2733 break;
2734 }
2735
2736 default_pager_pagein_count++;
2737
2738 done:
2739 pager_port_finish_read(ds);
2740 return(KERN_SUCCESS0);
2741}
2742
2743/*
2744 * memory_object_data_initialize: check whether we already have each page, and
2745 * write it if we do not. The implementation is far from optimized, and
2746 * also assumes that the default_pager is single-threaded.
2747 */
2748kern_return_t
2749seqnos_memory_object_data_initialize(pager, seqno, pager_request,
2750 offset, addr, data_cnt)
2751 memory_object_t pager;
2752 mach_port_seqno_t seqno;
2753 mach_port_t pager_request;
2754 register
2755 vm_offset_t offset;
2756 register
2757 pointer_t addr;
2758 vm_size_t data_cnt;
2759{
2760 vm_offset_t amount_sent;
2761 default_pager_t ds;
2762 static char here[] = "%sdata_initialize";
2763
2764#ifdef lint
2765 pager_request++;
2766#endif /* lint */
2767
2768 ds = pager_port_lookup(pager)((! (((pager) != ((mach_port_t) 0)) && ((pager) != ((
mach_port_t) ~0))) || ((default_pager_t)(((vm_offset_t)(pager
))&~1))->pager != (pager)) ? ((default_pager_t)0) : (default_pager_t
)(((vm_offset_t)(pager))&~1));
2769 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2770 panic(here,my_name);
2771ddprintf ("seqnos_memory_object_data_initialize <%p>: pager_port_lock: <%p>[s:%d,r:%d,w:%d,l:%d], %d\n",
2772 &ds, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held, seqno);
2773 pager_port_lock(ds, seqno);
2774 pager_port_check_request(ds, pager_request);
2775 pager_port_start_write(ds);
2776ddprintf ("seqnos_memory_object_data_initialize <%p>: pager_port_unlock: <%p>[s:%d,r:%d,w:%d,l:%d]\n",
2777 &ds, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held);
2778 pager_port_unlock(ds);
2779
2780 for (amount_sent = 0;
2781 amount_sent < data_cnt;
2782 amount_sent += vm_page_size) {
2783
2784 if (!default_has_page(&ds->dpager, offset + amount_sent)) {
2785 if (default_write(&ds->dpager,
2786 addr + amount_sent,
2787 vm_page_size,
2788 offset + amount_sent)
2789 != PAGER_SUCCESS0) {
2790 dprintf("%s%s write error\n", my_name, here);
2791 dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2792 ds->errors++;
2793 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2794 }
2795 }
2796 }
2797
2798 pager_port_finish_write(ds);
2799 if (vm_deallocate(default_pager_self, addr, data_cnt) != KERN_SUCCESS0)
2800 panic(here,my_name);
2801
2802 return(KERN_SUCCESS0);
2803}
2804
2805/*
2806 * memory_object_data_write: split up the stuff coming in from
2807 * a memory_object_data_write call
2808 * into individual pages and pass them off to default_write.
2809 */
2810kern_return_t
2811seqnos_memory_object_data_write(pager, seqno, pager_request,
2812 offset, addr, data_cnt)
2813 memory_object_t pager;
2814 mach_port_seqno_t seqno;
2815 mach_port_t pager_request;
2816 register
2817 vm_offset_t offset;
2818 register
2819 pointer_t addr;
2820 vm_size_t data_cnt;
2821{
2822 register
2823 vm_size_t amount_sent;
2824 default_pager_t ds;
2825 static char here[] = "%sdata_write";
2826 int err;
2827
2828#ifdef lint
2829 pager_request++;
2830#endif /* lint */
2831
2832 if ((data_cnt % vm_page_size) != 0)
2833 panic(here,my_name);
2834
2835 ds = pager_port_lookup(pager)((! (((pager) != ((mach_port_t) 0)) && ((pager) != ((
mach_port_t) ~0))) || ((default_pager_t)(((vm_offset_t)(pager
))&~1))->pager != (pager)) ? ((default_pager_t)0) : (default_pager_t
)(((vm_offset_t)(pager))&~1));
2836 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2837 panic(here,my_name);
2838
2839 pager_port_lock(ds, seqno);
2840 pager_port_start_write(ds);
2841
2842 vm_size_t limit = ds->dpager.byte_limit;
2843 pager_port_unlock(ds);
2844 if ((limit != round_page(limit)((((vm_offset_t) (limit) + __vm_page_size - 1) / __vm_page_size
) * __vm_page_size)) && (trunc_page(limit)((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size) == offset)) {
2845 assert(trunc_page(limit) == offset)((((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size
) == offset) ? (void) (0) : __assert_fail ("((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size) == offset"
, "../../mach-defpager/default_pager.c", 2845, __PRETTY_FUNCTION__
));
2846 assert(data_cnt == vm_page_size)((data_cnt == vm_page_size) ? (void) (0) : __assert_fail ("data_cnt == vm_page_size"
, "../../mach-defpager/default_pager.c", 2846, __PRETTY_FUNCTION__
));
2847
2848 vm_offset_t tail = addr + limit - trunc_page(limit)((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size);
2849 vm_size_t tail_size = round_page(limit)((((vm_offset_t) (limit) + __vm_page_size - 1) / __vm_page_size
) * __vm_page_size) - limit;
2850 memset((void *) tail, 0, tail_size);
2851
2852 unsigned *arr = (unsigned *)addr;
2853 memory_object_data_supply(pager_request, trunc_page(limit)((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size), addr,
2854 vm_page_size, TRUE((boolean_t) 1), VM_PROT_NONE((vm_prot_t) 0x00),
2855 TRUE((boolean_t) 1), MACH_PORT_NULL((mach_port_t) 0));
2856 dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2857 ds->dpager.byte_limit = round_page(limit)((((vm_offset_t) (limit) + __vm_page_size - 1) / __vm_page_size
) * __vm_page_size);
2858 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2859 pager_port_finish_write(ds);
2860
2861 return(KERN_SUCCESS0);
2862 }
2863
2864 for (amount_sent = 0;
2865 amount_sent < data_cnt;
2866 amount_sent += vm_page_size) {
2867
2868 register int result;
2869
2870 result = default_write(&ds->dpager,
2871 addr + amount_sent,
2872 vm_page_size,
2873 offset + amount_sent);
2874 if (result != KERN_SUCCESS0) {
2875 dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2876 ds->errors++;
2877 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2878 }
2879 default_pager_pageout_count++;
2880 }
2881
2882 pager_port_finish_write(ds);
2883 err = vm_deallocate(default_pager_self, addr, data_cnt);
2884 if (err != KERN_SUCCESS0)
2885 {
2886 panic(here,my_name);
2887 }
2888
2889 return(KERN_SUCCESS0);
2890}
2891
2892/*ARGSUSED*/
2893kern_return_t
2894seqnos_memory_object_copy(old_memory_object, seqno, old_memory_control,
2895 offset, length, new_memory_object)
2896 memory_object_t old_memory_object;
2897 mach_port_seqno_t seqno;
2898 memory_object_control_t
2899 old_memory_control;
2900 vm_offset_t offset;
2901 vm_size_t length;
2902 memory_object_t new_memory_object;
2903{
2904 panic("%scopy", my_name);
2905 return KERN_FAILURE5;
2906}
2907
2908/* We get this when our memory_object_lock_request has completed
2909 after we truncated an object. */
2910kern_return_t
2911seqnos_memory_object_lock_completed (memory_object_t pager,
2912 mach_port_seqno_t seqno,
2913 mach_port_t pager_request,
2914 vm_offset_t offset,
2915 vm_size_t length)
2916{
2917 panic("%slock_completed",my_name);
2918 return KERN_FAILURE5;
2919}
2920
2921kern_return_t
2922seqnos_memory_object_data_unlock(pager, seqno, pager_request,
2923 offset, addr, data_cnt)
2924 memory_object_t pager;
2925 mach_port_seqno_t seqno;
2926 mach_port_t pager_request;
2927 vm_offset_t offset;
2928 pointer_t addr;
2929 vm_size_t data_cnt;
2930{
2931 panic("%sdata_unlock",my_name);
2932 return(KERN_FAILURE5);
2933}
2934
2935kern_return_t
2936seqnos_memory_object_supply_completed(pager, seqno, pager_request,
2937 offset, length,
2938 result, error_offset)
2939 memory_object_t pager;
2940 mach_port_seqno_t seqno;
2941 mach_port_t pager_request;
2942 vm_offset_t offset;
2943 vm_size_t length;
2944 kern_return_t result;
2945 vm_offset_t error_offset;
2946{
2947 panic("%ssupply_completed",my_name);
2948 return(KERN_FAILURE5);
2949}
2950
2951/*
2952 * memory_object_data_return: split up the stuff coming in from
2953 * a memory_object_data_write call
2954 * into individual pages and pass them off to default_write.
2955 */
2956kern_return_t
2957seqnos_memory_object_data_return(pager, seqno, pager_request,
2958 offset, addr, data_cnt,
2959 dirty, kernel_copy)
2960 memory_object_t pager;
2961 mach_port_seqno_t seqno;
2962 mach_port_t pager_request;
2963 vm_offset_t offset;
2964 pointer_t addr;
2965 vm_size_t data_cnt;
2966 boolean_t dirty;
2967 boolean_t kernel_copy;
2968{
2969
2970 return seqnos_memory_object_data_write (pager, seqno, pager_request,
2971 offset, addr, data_cnt);
2972}
2973
2974kern_return_t
2975seqnos_memory_object_change_completed(pager, seqno, may_cache, copy_strategy)
2976 memory_object_t pager;
2977 mach_port_seqno_t seqno;
2978 boolean_t may_cache;
2979 memory_object_copy_strategy_t copy_strategy;
2980{
2981 panic("%schange_completed",my_name);
2982 return(KERN_FAILURE5);
2983}
2984
2985
2986boolean_t default_pager_notify_server(in, out)
2987 mach_msg_header_t *in, *out;
2988{
2989 register mach_no_senders_notification_t *n =
2990 (mach_no_senders_notification_t *) in;
2991
2992 /*
2993 * The only send-once rights we create are for
2994 * receiving no-more-senders notifications.
2995 * Hence, if we receive a message directed to
2996 * a send-once right, we can assume it is
2997 * a genuine no-senders notification from the kernel.
2998 */
2999
3000 if ((n->not_header.msgh_bits !=
3001 MACH_MSGH_BITS(0, MACH_MSG_TYPE_PORT_SEND_ONCE)((0) | ((18) << 8))) ||
3002 (n->not_header.msgh_id != MACH_NOTIFY_NO_SENDERS(0100 + 006)))
3003 return FALSE((boolean_t) 0);
3004
3005 assert(n->not_header.msgh_size == sizeof *n)((n->not_header.msgh_size == sizeof *n) ? (void) (0) : __assert_fail
("n->not_header.msgh_size == sizeof *n", "../../mach-defpager/default_pager.c"
, 3005, __PRETTY_FUNCTION__));
3006 assert(n->not_header.msgh_remote_port == MACH_PORT_NULL)((n->not_header.msgh_remote_port == ((mach_port_t) 0)) ? (
void) (0) : __assert_fail ("n->not_header.msgh_remote_port == ((mach_port_t) 0)"
, "../../mach-defpager/default_pager.c", 3006, __PRETTY_FUNCTION__
));
3007
3008 assert(n->not_type.msgt_name == MACH_MSG_TYPE_INTEGER_32)((n->not_type.msgt_name == 2) ? (void) (0) : __assert_fail
("n->not_type.msgt_name == 2", "../../mach-defpager/default_pager.c"
, 3008, __PRETTY_FUNCTION__));
3009 assert(n->not_type.msgt_size == 32)((n->not_type.msgt_size == 32) ? (void) (0) : __assert_fail
("n->not_type.msgt_size == 32", "../../mach-defpager/default_pager.c"
, 3009, __PRETTY_FUNCTION__));
3010 assert(n->not_type.msgt_number == 1)((n->not_type.msgt_number == 1) ? (void) (0) : __assert_fail
("n->not_type.msgt_number == 1", "../../mach-defpager/default_pager.c"
, 3010, __PRETTY_FUNCTION__));
3011 assert(n->not_type.msgt_inline)((n->not_type.msgt_inline) ? (void) (0) : __assert_fail ("n->not_type.msgt_inline"
, "../../mach-defpager/default_pager.c", 3011, __PRETTY_FUNCTION__
));
3012 assert(! n->not_type.msgt_longform)((! n->not_type.msgt_longform) ? (void) (0) : __assert_fail
("! n->not_type.msgt_longform", "../../mach-defpager/default_pager.c"
, 3012, __PRETTY_FUNCTION__));
3013
3014 default_pager_no_senders(n->not_header.msgh_local_port,
3015 n->not_header.msgh_seqno, n->not_count);
3016
3017 out->msgh_remote_port = MACH_PORT_NULL((mach_port_t) 0);
3018 return TRUE((boolean_t) 1);
3019}
3020
3021extern boolean_t seqnos_memory_object_server();
3022extern boolean_t seqnos_memory_object_default_server();
3023extern boolean_t default_pager_server();
3024extern boolean_t exc_server();
3025extern boolean_t bootstrap_server();
3026extern void bootstrap_compat();
3027
3028mach_msg_size_t default_pager_msg_size_object = 128;
3029
3030boolean_t
3031default_pager_demux_object(in, out)
3032 mach_msg_header_t *in;
3033 mach_msg_header_t *out;
3034{
3035 /*
3036 * We receive memory_object_data_initialize messages in
3037 * the memory_object_default interface.
3038 */
3039
3040int rval;
3041ddprintf ("DPAGER DEMUX OBJECT <%p>: %d\n", in, in->msgh_id);
3042rval =
3043 (seqnos_memory_object_server(in, out) ||
3044 seqnos_memory_object_default_server(in, out) ||
3045 default_pager_notify_server(in, out) ||
3046 default_pager_server(in, out));
3047ddprintf ("DPAGER DEMUX OBJECT DONE <%p>: %d\n", in, in->msgh_id);
3048return rval;
3049}
3050
3051mach_msg_size_t default_pager_msg_size_default = 8 * 1024;
3052
3053boolean_t
3054default_pager_demux_default(in, out)
3055 mach_msg_header_t *in;
3056 mach_msg_header_t *out;
3057{
3058 if (in->msgh_local_port == default_pager_default_port) {
3059 /*
3060 * We receive memory_object_create messages in
3061 * the memory_object_default interface.
3062 */
3063
3064int rval;
3065ddprintf ("DPAGER DEMUX DEFAULT <%p>: %d\n", in, in->msgh_id);
3066rval =
3067 (seqnos_memory_object_default_server(in, out) ||
3068 default_pager_server(in, out));
3069ddprintf ("DPAGER DEMUX DEFAULT DONE <%p>: %d\n", in, in->msgh_id);
3070return rval;
3071 } else if (in->msgh_local_port == default_pager_exception_port) {
3072 /*
3073 * We receive exception messages for
3074 * ourself and the startup task.
3075 */
3076
3077 return exc_server(in, out);
3078 } else {
3079 panic(my_name);
3080 return FALSE((boolean_t) 0);
3081 }
3082}
3083
3084/*
3085 * We use multiple threads, for two reasons.
3086 *
3087 * First, memory objects created by default_pager_object_create
3088 * are "external", instead of "internal". This means the kernel
3089 * sends data (memory_object_data_write) to the object pageable.
3090 * To prevent deadlocks, the external and internal objects must
3091 * be managed by different threads.
3092 *
3093 * Second, the default pager uses synchronous IO operations.
3094 * Spreading requests across multiple threads should
3095 * recover some of the performance loss from synchronous IO.
3096 *
3097 * We have 3+ threads.
3098 * One receives memory_object_create and
3099 * default_pager_object_create requests.
3100 * One or more manage internal objects.
3101 * One or more manage external objects.
3102 */
3103
3104void
3105default_pager_thread_privileges()
3106{
3107 /*
3108 * Set thread privileges.
3109 */
3110 wire_thread(); /* grab a kernel stack and memory allocation
3111 privileges */
3112}
3113
3114void *
3115default_pager_default_thread(void *arg)
3116{
3117 kern_return_t kr;
3118 default_pager_thread_privileges ();
3119 for (;;) {
3120 kr = mach_msg_server(default_pager_demux_default,
3121 default_pager_msg_size_default,
3122 default_pager_default_set);
3123 panic(my_name, kr);
3124 }
3125}
3126
3127
3128
3129void *
3130default_pager_thread(void *arg)
3131{
3132 mach_port_t pset;
3133 kern_return_t kr;
3134
3135 dpt = (default_pager_thread_t *) arg;
3136
3137 /*
3138 * Threads handling external objects cannot have
3139 * privileges. Otherwise a burst of data-requests for an
3140 * external object could empty the free-page queue,
3141 * because the fault code only reserves real pages for
3142 * requests sent to internal objects.
3143 */
3144
3145 if (dpt->dpt_internal) {
3146 default_pager_thread_privileges();
3147 pset = default_pager_internal_set;
3148 } else {
3149 pset = default_pager_external_set;
3150 }
3151
3152 for (;;) {
3153 kr = mach_msg_server(default_pager_demux_object,
3154 default_pager_msg_size_object,
3155 pset);
3156 panic(my_name, kr);
3157 }
3158}
3159
3160void
3161start_default_pager_thread(internal)
3162 boolean_t internal;
3163{
3164 default_pager_thread_t *ndpt;
3165 kern_return_t kr;
3166 error_t err;
3167
3168 ndpt = (default_pager_thread_t *) kalloc(sizeof *ndpt);
3169 if (ndpt == 0)
3170 panic(my_name);
3171
3172 ndpt->dpt_internal = internal;
3173
3174 kr = vm_allocate(default_pager_self, &ndpt->dpt_buffer,
3175 vm_page_size, TRUE((boolean_t) 1));
3176 if (kr != KERN_SUCCESS0)
3177 panic(my_name);
3178 wire_memory(ndpt->dpt_buffer, vm_page_size,
3179 VM_PROT_READ((vm_prot_t) 0x01)|VM_PROT_WRITE((vm_prot_t) 0x02));
3180
3181 err = pthread_create(&ndpt->dpt_thread, NULL((void*)0), default_pager_thread,
3182 ndpt);
3183 if (!err)
3184 pthread_detach (ndpt->dpt_thread);
3185 else {
3186 errno(*__errno_location ()) = err;
3187 perror ("pthread_create");
3188 }
3189}
3190
3191void
3192default_pager_initialize(host_port)
3193 mach_port_t host_port;
3194{
3195 memory_object_t DMM;
3196 kern_return_t kr;
3197
3198 /*
3199 * This task will become the default pager.
3200 */
3201 default_pager_self = mach_task_self()((__mach_task_self_ + 0));
3202
3203 /*
3204 * Initialize the "default pager" port.
3205 */
3206 kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_RECEIVE((mach_port_right_t) 1),
3207 &default_pager_default_port);
3208 if (kr != KERN_SUCCESS0)
3209 panic(my_name);
3210
3211 DMM = default_pager_default_port;
3212 kr = vm_set_default_memory_manager(host_port, &DMM);
3213 if ((kr != KERN_SUCCESS0) || MACH_PORT_VALID(DMM)(((DMM) != ((mach_port_t) 0)) && ((DMM) != ((mach_port_t
) ~0))))
3214 panic(my_name);
3215
3216 /*
3217 * Initialize the exception port.
3218 */
3219 kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_RECEIVE((mach_port_right_t) 1),
3220 &default_pager_exception_port);
3221 if (kr != KERN_SUCCESS0)
3222 panic(my_name);
3223
3224 /*
3225 * Arrange for wiring privileges.
3226 */
3227 wire_setup(host_port);
3228
3229 /*
3230 * Find out how many CPUs we have, to determine the number
3231 * of threads to create.
3232 */
3233 if (default_pager_internal_count == 0) {
3234 host_basic_info_data_t h_info;
3235 natural_t h_info_count;
3236
3237 h_info_count = HOST_BASIC_INFO_COUNT(sizeof(host_basic_info_data_t)/sizeof(integer_t));
3238 (void) host_info(host_port, HOST_BASIC_INFO1,
3239 (host_info_t)&h_info, &h_info_count);
3240
3241 /*
3242 * Random computation to get more parallelism on
3243 * multiprocessors.
3244 */
3245 default_pager_internal_count =
3246 (h_info.avail_cpus > 32 ? 32 : h_info.avail_cpus) / 4 + 3;
3247 }
3248}
3249
3250/*
3251 * Initialize and Run the default pager
3252 */
3253void
3254default_pager()
3255{
3256 kern_return_t kr;
3257 int i;
3258
3259 default_pager_thread_privileges();
3260
3261 /*
3262 * Wire down code, data, stack
3263 */
3264 wire_all_memory();
3265
3266
3267 /*
3268 * Initialize the list of all pagers.
3269 */
3270 pager_port_list_init(){ pthread_mutex_init(&all_pagers.lock, ((void*)0)); ((&
all_pagers.queue)->next = (&all_pagers.queue)->prev
= &all_pagers.queue); ((&all_pagers.leak_queue)->
next = (&all_pagers.leak_queue)->prev = &all_pagers
.leak_queue); all_pagers.count = 0; };
3271
3272 kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_PORT_SET((mach_port_right_t) 3),
3273 &default_pager_internal_set);
3274 if (kr != KERN_SUCCESS0)
3275 panic(my_name);
3276
3277 kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_PORT_SET((mach_port_right_t) 3),
3278 &default_pager_external_set);
3279 if (kr != KERN_SUCCESS0)
3280 panic(my_name);
3281
3282 kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_PORT_SET((mach_port_right_t) 3),
3283 &default_pager_default_set);
3284 if (kr != KERN_SUCCESS0)
3285 panic(my_name);
3286
3287 kr = mach_port_move_member(default_pager_self,
3288 default_pager_default_port,
3289 default_pager_default_set);
3290 if (kr != KERN_SUCCESS0)
3291 panic(my_name);
3292
3293 kr = mach_port_move_member(default_pager_self,
3294 default_pager_exception_port,
3295 default_pager_default_set);
3296 if (kr != KERN_SUCCESS0)
3297 panic(my_name);
3298
3299 /*
3300 * Now we create the threads that will actually
3301 * manage objects.
3302 */
3303
3304 for (i = 0; i < default_pager_internal_count; i++)
3305 start_default_pager_thread(TRUE((boolean_t) 1));
3306
3307 for (i = 0; i < default_pager_external_count; i++)
3308 start_default_pager_thread(FALSE((boolean_t) 0));
3309
3310 default_pager_default_thread(0); /* Become the default_pager server */
3311#if 0
3312 cthread_fork (default_pager_default_thread, 0);
3313 /* cthread_exit (cthread_self ()); */
3314 thread_suspend (mach_thread_self ());
3315#endif
3316}
3317
3318/*
3319 * Create an external object.
3320 */
3321kern_return_t
3322S_default_pager_object_create (mach_port_t pager,
3323 mach_port_t *mem_obj,
3324 vm_size_t size)
3325{
3326 default_pager_t ds;
3327 mach_port_t port;
3328 kern_return_t result;
3329
3330 if (pager != default_pager_default_port)
3331 return KERN_INVALID_ARGUMENT4;
3332
3333 ds = pager_port_alloc(size);
3334rename_it:
3335 port = (mach_port_t) pnameof(ds)(((vm_offset_t)(ds))+1);
3336 result = mach_port_allocate_name(default_pager_self,
3337 MACH_PORT_RIGHT_RECEIVE((mach_port_right_t) 1), port);
3338 if (result != KERN_SUCCESS0) {
3339 default_pager_t ds1;
3340
3341 if (result != KERN_NAME_EXISTS13) return (result);
3342
3343 ds1 = (default_pager_t) kalloc(sizeof *ds1);
3344 *ds1 = *ds;
3345 pthread_mutex_lock(&all_pagers.lock);
3346 queue_enter(&all_pagers.leak_queue, ds, default_pager_t, links){ register queue_entry_t prev; prev = (&all_pagers.leak_queue
)->prev; if ((&all_pagers.leak_queue) == prev) { (&
all_pagers.leak_queue)->next = (queue_entry_t) (ds); } else
{ ((default_pager_t)prev)->links.next = (queue_entry_t)(ds
); } (ds)->links.prev = prev; (ds)->links.next = &all_pagers
.leak_queue; (&all_pagers.leak_queue)->prev = (queue_entry_t
) ds; };
3347 pthread_mutex_unlock(&all_pagers.lock);
3348 ds = ds1;
3349 goto rename_it;
3350 }
3351
3352 /*
3353 * Set up associations between these ports
3354 * and this default_pager structure
3355 */
3356
3357 ds->pager = port;
3358 ds->dpager.limit = size;
3359 pager_port_list_insert(port, ds);
3360 default_pager_add(ds, FALSE((boolean_t) 0));
3361
3362 *mem_obj = port;
3363 return (KERN_SUCCESS0);
3364}
3365
3366kern_return_t
3367S_default_pager_info (mach_port_t pager,
3368 default_pager_info_t *infop)
3369{
3370 vm_size_t total, free;
3371
3372 if (pager != default_pager_default_port)
3373 return KERN_INVALID_ARGUMENT4;
3374
3375 pthread_mutex_lock(&all_partitions.lock);
3376 paging_space_info(&total, &free);
3377 pthread_mutex_unlock(&all_partitions.lock);
3378
3379 infop->dpi_total_space = ptoa(total)((total)*vm_page_size);
3380 infop->dpi_free_space = ptoa(free)((free)*vm_page_size);
3381 infop->dpi_page_size = vm_page_size;
3382 return KERN_SUCCESS0;
3383}
3384
3385kern_return_t
3386S_default_pager_objects (mach_port_t pager,
3387 default_pager_object_array_t *objectsp,
3388 natural_t *ocountp,
3389 mach_port_array_t *portsp,
3390 natural_t *pcountp)
3391{
3392 vm_offset_t oaddr; /* memory for objects */
3393 vm_size_t osize; /* current size */
3394 default_pager_object_t *objects;
3395 natural_t opotential;
3396
3397 vm_offset_t paddr; /* memory for ports */
3398 vm_size_t psize; /* current size */
3399 mach_port_t *ports;
3400 natural_t ppotential;
3401
3402 unsigned int actual;
3403 unsigned int num_pagers;
3404 kern_return_t kr;
3405 default_pager_t entry;
3406
3407 if (pager != default_pager_default_port)
3408 return KERN_INVALID_ARGUMENT4;
3409
3410 /* start with the inline memory */
3411
3412 num_pagers = 0;
3413
3414 objects = *objectsp;
3415 opotential = *ocountp;
3416
3417 ports = *portsp;
3418 ppotential = *pcountp;
3419
3420 pthread_mutex_lock(&all_pagers.lock);
3421 /*
3422 * We will send no more than this many
3423 */
3424 actual = all_pagers.count;
3425 pthread_mutex_unlock(&all_pagers.lock);
3426
3427 if (opotential < actual) {
3428 vm_offset_t newaddr;
3429 vm_size_t newsize;
3430
3431 newsize = 2 * round_page(actual * sizeof *objects)((((vm_offset_t) (actual * sizeof *objects) + __vm_page_size -
1) / __vm_page_size) * __vm_page_size);
3432
3433 kr = vm_allocate(default_pager_self, &newaddr, newsize, TRUE((boolean_t) 1));
3434 if (kr != KERN_SUCCESS0)
3435 goto nomemory;
3436
3437 oaddr = newaddr;
3438 osize = newsize;
3439 opotential = osize/sizeof *objects;
3440 objects = (default_pager_object_t *) oaddr;
3441 }
3442
3443 if (ppotential < actual) {
3444 vm_offset_t newaddr;
3445 vm_size_t newsize;
3446
3447 newsize = 2 * round_page(actual * sizeof *ports)((((vm_offset_t) (actual * sizeof *ports) + __vm_page_size - 1
) / __vm_page_size) * __vm_page_size);
3448
3449 kr = vm_allocate(default_pager_self, &newaddr, newsize, TRUE((boolean_t) 1));
3450 if (kr != KERN_SUCCESS0)
3451 goto nomemory;
3452
3453 paddr = newaddr;
3454 psize = newsize;
3455 ppotential = psize/sizeof *ports;
3456 ports = (mach_port_t *) paddr;
3457 }
3458
3459 /*
3460 * Now scan the list.
3461 */
3462
3463 pthread_mutex_lock(&all_pagers.lock);
3464
3465 num_pagers = 0;
3466 queue_iterate(&all_pagers.queue, entry, default_pager_t, links)for ((entry) = (default_pager_t) ((&all_pagers.queue)->
next); !(((&all_pagers.queue)) == ((queue_entry_t)(entry)
)); (entry) = (default_pager_t) ((&(entry)->links)->
next)) {
3467
3468 mach_port_t port;
3469 vm_size_t size;
3470
3471 if ((num_pagers >= opotential) ||
3472 (num_pagers >= ppotential)) {
3473 /*
3474 * This should be rare. In any case,
3475 * we will only miss recent objects,
3476 * because they are added at the end.
3477 */
3478 break;
3479 }
3480
3481 /*
3482 * Avoid interfering with normal operations
3483 */
3484 if (pthread_mutex_trylock(&entry->dpager.lock))
3485 goto not_this_one;
3486 size = pager_allocated(&entry->dpager);
3487 pthread_mutex_unlock(&entry->dpager.lock);
3488
3489 dstruct_lock(entry)pthread_mutex_lock(&entry->lock);
3490
3491 port = entry->pager_name;
3492 if (port == MACH_PORT_NULL((mach_port_t) 0)) {
3493 /*
3494 * The object is waiting for no-senders
3495 * or memory_object_init.
3496 */
3497 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3498 goto not_this_one;
3499 }
3500
3501 /*
3502 * We need a reference for the reply message.
3503 * While we are unlocked, the bucket queue
3504 * can change and the object might be terminated.
3505 * memory_object_terminate will wait for us,
3506 * preventing deallocation of the entry.
3507 */
3508
3509 if (--entry->name_refs == 0) {
3510 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3511
3512 /* keep the list locked, wont take long */
3513
3514 kr = mach_port_mod_refs(default_pager_self,
3515 port, MACH_PORT_RIGHT_SEND((mach_port_right_t) 0),
3516 default_pager_max_urefs);
3517 if (kr != KERN_SUCCESS0)
3518 panic("%sdefault_pager_objects",my_name);
3519
3520 dstruct_lock(entry)pthread_mutex_lock(&entry->lock);
3521
3522 entry->name_refs += default_pager_max_urefs;
3523 pager_port_finish_refs(entry);
3524 }
3525 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3526
3527 /* the arrays are wired, so no deadlock worries */
3528
3529 objects[num_pagers].dpo_object = (vm_offset_t) entry;
3530 objects[num_pagers].dpo_size = size;
3531 ports [num_pagers++] = port;
3532 continue;
3533not_this_one:
3534 /*
3535 * Do not return garbage
3536 */
3537 objects[num_pagers].dpo_object = (vm_offset_t) 0;
3538 objects[num_pagers].dpo_size = 0;
3539 ports [num_pagers++] = MACH_PORT_NULL((mach_port_t) 0);
3540
3541 }
3542
3543 pthread_mutex_unlock(&all_pagers.lock);
3544
3545 /*
3546 * Deallocate and clear unused memory.
3547 * (Returned memory will automagically become pageable.)
3548 */
3549
3550 if (objects == *objectsp) {
3551 /*
3552 * Our returned information fit inline.
3553 * Nothing to deallocate.
3554 */
3555
3556 *ocountp = num_pagers;
3557 } else if (actual == 0) {
3558 (void) vm_deallocate(default_pager_self, oaddr, osize);
3559
3560 /* return zero items inline */
3561 *ocountp = 0;
3562 } else {
3563 vm_offset_t used;
3564
3565 used = round_page(actual * sizeof *objects)((((vm_offset_t) (actual * sizeof *objects) + __vm_page_size -
1) / __vm_page_size) * __vm_page_size);
3566
3567 if (used != osize)
3568 (void) vm_deallocate(default_pager_self,
3569 oaddr + used, osize - used);
3570
3571 *objectsp = objects;
3572 *ocountp = num_pagers;
3573 }
3574
3575 if (ports == *portsp) {
3576 /*
3577 * Our returned information fit inline.
3578 * Nothing to deallocate.
3579 */
3580
3581 *pcountp = num_pagers;
3582 } else if (actual == 0) {
3583 (void) vm_deallocate(default_pager_self, paddr, psize);
3584
3585 /* return zero items inline */
3586 *pcountp = 0;
3587 } else {
3588 vm_offset_t used;
3589
3590 used = round_page(actual * sizeof *ports)((((vm_offset_t) (actual * sizeof *ports) + __vm_page_size - 1
) / __vm_page_size) * __vm_page_size);
3591
3592 if (used != psize)
3593 (void) vm_deallocate(default_pager_self,
3594 paddr + used, psize - used);
3595
3596 *portsp = ports;
3597 *pcountp = num_pagers;
3598 }
3599
3600 return KERN_SUCCESS0;
3601
3602 nomemory:
3603
3604 {
3605 register int i;
3606 for (i = 0; i < num_pagers; i++)
3607 (void) mach_port_deallocate(default_pager_self, ports[i]);
3608 }
3609
3610 if (objects != *objectsp)
3611 (void) vm_deallocate(default_pager_self, oaddr, osize);
3612
3613 if (ports != *portsp)
3614 (void) vm_deallocate(default_pager_self, paddr, psize);
3615
3616 return KERN_RESOURCE_SHORTAGE6;
3617}
3618
3619
3620kern_return_t
3621S_default_pager_object_pages (mach_port_t pager,
3622 mach_port_t object,
3623 default_pager_page_array_t *pagesp,
3624 natural_t *countp)
3625{
3626 vm_offset_t addr; /* memory for page offsets */
3627 vm_size_t size; /* current memory size */
3628 default_pager_page_t *pages;
3629 natural_t potential, actual;
3630 kern_return_t kr;
3631
3632 if (pager != default_pager_default_port)
3633 return KERN_INVALID_ARGUMENT4;
3634
3635 /* we start with the inline space */
3636
3637 pages = *pagesp;
3638 potential = *countp;
3639
3640 for (;;) {
3641 default_pager_t entry;
3642
3643 pthread_mutex_lock(&all_pagers.lock);
3644 queue_iterate(&all_pagers.queue, entry, default_pager_t, links)for ((entry) = (default_pager_t) ((&all_pagers.queue)->
next); !(((&all_pagers.queue)) == ((queue_entry_t)(entry)
)); (entry) = (default_pager_t) ((&(entry)->links)->
next)) {
3645 dstruct_lock(entry)pthread_mutex_lock(&entry->lock);
3646 if (entry->pager_name == object) {
3647 pthread_mutex_unlock(&all_pagers.lock);
3648 goto found_object;
3649 }
3650 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3651 }
3652 pthread_mutex_unlock(&all_pagers.lock);
3653
3654 /* did not find the object */
3655
3656 if (pages != *pagesp)
3657 (void) vm_deallocate(default_pager_self, addr, size);
3658 return KERN_INVALID_ARGUMENT4;
3659
3660 found_object:
3661
3662 if (pthread_mutex_trylock(&entry->dpager.lock)) {
3663 /* oh well bad luck */
3664
3665 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3666
3667 /* yield the processor */
3668 (void) thread_switch(MACH_PORT_NULL((mach_port_t) 0),
3669 SWITCH_OPTION_NONE0, 0);
3670 continue;
3671 }
3672
3673 actual = pager_pages(&entry->dpager, pages, potential);
3674 pthread_mutex_unlock(&entry->dpager.lock);
3675 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3676
3677 if (actual <= potential)
3678 break;
3679
3680 /* allocate more memory */
3681
3682 if (pages != *pagesp)
3683 (void) vm_deallocate(default_pager_self, addr, size);
3684 size = round_page(actual * sizeof *pages)((((vm_offset_t) (actual * sizeof *pages) + __vm_page_size - 1
) / __vm_page_size) * __vm_page_size);
3685 kr = vm_allocate(default_pager_self, &addr, size, TRUE((boolean_t) 1));
3686 if (kr != KERN_SUCCESS0)
3687 return kr;
3688 pages = (default_pager_page_t *) addr;
3689 potential = size/sizeof *pages;
3690 }
3691
3692 /*
3693 * Deallocate and clear unused memory.
3694 * (Returned memory will automagically become pageable.)
3695 */
3696
3697 if (pages == *pagesp) {
3698 /*
3699 * Our returned information fit inline.
3700 * Nothing to deallocate.
3701 */
3702
3703 *countp = actual;
3704 } else if (actual == 0) {
3705 (void) vm_deallocate(default_pager_self, addr, size);
3706
3707 /* return zero items inline */
3708 *countp = 0;
3709 } else {
3710 vm_offset_t used;
3711
3712 used = round_page(actual * sizeof *pages)((((vm_offset_t) (actual * sizeof *pages) + __vm_page_size - 1
) / __vm_page_size) * __vm_page_size);
3713
3714 if (used != size)
3715 (void) vm_deallocate(default_pager_self,
3716 addr + used, size - used);
3717
3718 *pagesp = pages;
3719 *countp = actual;
3720 }
3721 return KERN_SUCCESS0;
3722}
3723
3724
3725kern_return_t
3726S_default_pager_object_set_size (mach_port_t pager,
3727 mach_port_seqno_t seqno,
3728 vm_size_t limit)
3729{
3730 kern_return_t kr;
3731 default_pager_t ds;
3732
3733 ds = pager_port_lookup(pager)((! (((pager) != ((mach_port_t) 0)) && ((pager) != ((
mach_port_t) ~0))) || ((default_pager_t)(((vm_offset_t)(pager
))&~1))->pager != (pager)) ? ((default_pager_t)0) : (default_pager_t
)(((vm_offset_t)(pager))&~1));
3734 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
3735 return KERN_INVALID_ARGUMENT4;
3736
3737 pager_port_lock(ds, seqno);
3738 pager_port_wait_for_readers(ds);
3739 pager_port_wait_for_writers(ds);
3740
3741 vm_size_t rounded_limit = round_page (limit)((((vm_offset_t) (limit) + __vm_page_size - 1) / __vm_page_size
) * __vm_page_size);
3742 vm_size_t trunc_limit = trunc_page (limit)((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size);
3743
3744
3745 if (ds->dpager.limit < rounded_limit)
3746 {
3747 /* The limit has not been exceeded heretofore. Just change it. */
3748 ds->dpager.limit = rounded_limit;
3749
3750 /* Byte limit is used for truncation of file, which aren't rounded to
3751 page boundary. But by enlarging of file we are free to increase this value*/
3752 ds->dpager.byte_limit = rounded_limit;
3753 kr = memory_object_lock_request(ds->pager_request, 0,
3754 rounded_limit,
3755 MEMORY_OBJECT_RETURN_NONE0, FALSE((boolean_t) 0),
3756 VM_PROT_NONE((vm_prot_t) 0x00), MACH_PORT_NULL((mach_port_t) 0));
3757 if (kr != KERN_SUCCESS0)
3758 panic ("memory_object_lock_request: %d", kr);
3759 }
3760 else
3761 {
3762 if (ds->dpager.limit != rounded_limit)
3763 {
3764 kr = memory_object_lock_request(ds->pager_request, rounded_limit,
3765 ds->dpager.limit - rounded_limit,
3766 MEMORY_OBJECT_RETURN_NONE0, TRUE((boolean_t) 1),
3767 VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04)), MACH_PORT_NULL((mach_port_t) 0));
3768 if (kr != KERN_SUCCESS0)
3769 panic ("memory_object_lock_request: %d", kr);
3770
3771 ds->dpager.limit = rounded_limit;
3772 }
3773
3774 /* Deallocate the old backing store pages and shrink the page map. */
3775 if (ds->dpager.size > ds->dpager.limit / vm_page_size)
3776 pager_truncate (&ds->dpager, ds->dpager.limit / vm_page_size);
3777
3778 /* If memory object size isn't page aligned, fill the tail
3779 of last page with zeroes */
3780 if ((limit != rounded_limit) && (ds->dpager.limit > limit))
3781 {
3782 /* Clean part of last page which isn't part of file.
3783 For file sizes that aren't multiple of vm_page_size */
3784 ds->dpager.byte_limit = limit;
3785 kr = memory_object_lock_request(ds->pager_request, trunc_limit,
3786 vm_page_size,
3787 MEMORY_OBJECT_RETURN_ALL2, TRUE((boolean_t) 1),
3788 VM_PROT_NONE((vm_prot_t) 0x00), MACH_PORT_NULL((mach_port_t) 0));
3789 }
3790 }
3791
3792 pager_port_unlock(ds);
3793
3794 return kr;
3795}
3796
3797/*
3798 * Add/remove extra paging space
3799 */
3800
3801extern mach_port_t bootstrap_master_device_port;
3802extern mach_port_t bootstrap_master_host_port;
3803
3804kern_return_t
3805S_default_pager_paging_file (pager, mdport, file_name, add)
3806 mach_port_t pager;
3807 mach_port_t mdport;
3808 default_pager_filename_t file_name;
3809 boolean_t add;
3810{
3811 kern_return_t kr;
3812
3813 if (pager != default_pager_default_port)
3814 return KERN_INVALID_ARGUMENT4;
3815
3816#if 0
3817dprintf("bmd %x md %x\n", bootstrap_master_device_port, mdport);
3818#endif
3819 if (add) {
3820 kr = add_paging_file(bootstrap_master_device_port,
3821 file_name, 0);
3822 } else {
3823 kr = remove_paging_file(file_name);
3824 }
3825
3826 /* XXXX more code needed */
3827 if (mdport != bootstrap_master_device_port)
3828 mach_port_deallocate( mach_task_self()((__mach_task_self_ + 0)), mdport);
3829
3830 return kr;
3831}
3832
3833kern_return_t
3834default_pager_register_fileserver(pager, fileserver)
3835 mach_port_t pager;
3836 mach_port_t fileserver;
3837{
3838 if (pager != default_pager_default_port)
3839 return KERN_INVALID_ARGUMENT4;
3840#if notyet
3841 mach_port_deallocate(mach_task_self()((__mach_task_self_ + 0)), fileserver);
3842 if (0) dp_helper_paging_space(0,0,0);/*just linkit*/
3843#endif
3844 return KERN_SUCCESS0;
3845}
3846
3847/*
3848 * When things do not quite workout...
3849 */
3850void no_paging_space(out_of_memory)
3851 boolean_t out_of_memory;
3852{
3853 static char here[] = "%s *** NOT ENOUGH PAGING SPACE ***";
3854
3855 if (out_of_memory)
3856 dprintf("*** OUT OF MEMORY *** ");
3857 panic(here, my_name);
3858}
3859
3860void overcommitted(got_more_space, space)
3861 boolean_t got_more_space;
3862 vm_size_t space; /* in pages */
3863{
3864 vm_size_t pages_free, pages_total;
3865
3866 static boolean_t user_warned = FALSE((boolean_t) 0);
3867 static vm_size_t pages_shortage = 0;
3868
3869 paging_space_info(&pages_total, &pages_free);
3870
3871 /*
3872 * If user added more space, see if it is enough
3873 */
3874 if (got_more_space) {
3875 pages_free -= pages_shortage;
3876 if (pages_free > 0) {
3877 pages_shortage = 0;
3878 if (user_warned)
3879 dprintf("%s paging space ok now.\n", my_name);
3880 } else
3881 pages_shortage = pages_free;
3882 user_warned = FALSE((boolean_t) 0);
3883 return;
3884 }
3885 /*
3886 * We ran out of gas, let user know.
3887 */
3888 pages_free -= space;
3889 pages_shortage = (pages_free > 0) ? 0 : -pages_free;
3890 if (!user_warned && pages_shortage) {
3891 user_warned = TRUE((boolean_t) 1);
3892 dprintf("%s paging space over-committed.\n", my_name);
3893 }
3894#if debug0
3895 user_warned = FALSE((boolean_t) 0);
3896 dprintf("%s paging space over-committed [+%d (%d) pages].\n",
3897 my_name, space, pages_shortage);
3898#endif
3899}
3900
3901void paging_space_info(totp, freep)
3902 vm_size_t *totp, *freep;
3903{
3904 register vm_size_t total, free;
3905 register partition_t part;
3906 register int i;
3907
3908 total = free = 0;
3909 for (i = 0; i < all_partitions.n_partitions; i++) {
3910
3911 if ((part = partition_of(i)) == 0) continue;
3912
3913 /* no need to lock: by the time this data
3914 gets back to any remote requestor it
3915 will be obsolete anyways */
3916 total += part->total_size;
3917 free += part->free;
3918#if debug0
3919 dprintf("Partition %d: x%x total, x%x free\n",
3920 i, part->total_size, part->free);
3921#endif
3922 }
3923 *totp = total;
3924 *freep = free;
3925}
3926
3927/*
3928 * Catch exceptions.
3929 */
3930
3931kern_return_t
3932catch_exception_raise(exception_port, thread, task, exception, code, subcode)
3933 mach_port_t exception_port;
3934 mach_port_t thread, task;
3935 int exception, code, subcode;
3936{
3937 ddprintf ("(default_pager)catch_exception_raise(%d,%d,%d)\n",
3938 exception, code, subcode);
3939 panic(my_name);
3940
3941 /* mach_msg_server will deallocate thread/task for us */
3942
3943 return KERN_FAILURE5;
3944}