Bug Summary

File:obj-scan-build/mach-defpager/../../mach-defpager/default_pager.c
Location:line 2440, column 3
Description:Value stored to 'ds' is never read

Annotated Source Code

1/*
2 * Mach Operating System
3 * Copyright (c) 1993-1989 Carnegie Mellon University
4 * All Rights Reserved.
5 *
6 * Permission to use, copy, modify and distribute this software and its
7 * documentation is hereby granted, provided that both the copyright
8 * notice and this permission notice appear in all copies of the
9 * software, derivative works or modified versions, and any portions
10 * thereof, and that both notices appear in supporting documentation.
11 *
12 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
13 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
14 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
15 *
16 * Carnegie Mellon requests users of this software to return to
17 *
18 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
19 * School of Computer Science
20 * Carnegie Mellon University
21 * Pittsburgh PA 15213-3890
22 *
23 * any improvements or extensions that they make and grant Carnegie Mellon
24 * the rights to redistribute these changes.
25 */
26/*
27 * Default pager. Pages to paging partition.
28 *
29 * MUST BE ABLE TO ALLOCATE WIRED-DOWN MEMORY!!!
30 */
31
32#include <mach.h>
33#include <mach/message.h>
34#include <mach/notify.h>
35#include <mach/mig_errors.h>
36#include <mach/thread_switch.h>
37#include <mach/task_info.h>
38#include <mach/default_pager_types.h>
39
40#include <pthread.h>
41#include <stddef.h>
42
43#include <device/device_types.h>
44#include <device/device.h>
45
46#include <hurd/ihash.h>
47
48#include "queue.h"
49#include "wiring.h"
50#include "kalloc.h"
51#include "default_pager.h"
52
53#include <assert.h>
54#include <errno(*__errno_location ()).h>
55#include <stdio.h>
56#include <string.h>
57#include <stdarg.h>
58
59#include <file_io.h>
60
61#include "memory_object_S.h"
62#include "memory_object_default_S.h"
63#include "default_pager_S.h"
64#include "exc_S.h"
65
66#include "priv.h"
67
68#define debug0 0
69
70static char my_name[] = "(default pager):";
71
72static void __attribute__ ((format (printf, 1, 2), unused))
73synchronized_printf (const char *fmt, ...)
74{
75 static pthread_mutex_t printf_lock = PTHREAD_MUTEX_INITIALIZER{ ((__pthread_spinlock_t) 0), ((__pthread_spinlock_t) 0), 0, 0
, 0, 0, 0, 0 };
76 va_list ap;
77
78 va_start (ap, fmt)__builtin_va_start(ap, fmt);
79 pthread_mutex_lock (&printf_lock);
80
81 vprintf (fmt, ap);
82 fflush (stdoutstdout);
83
84 pthread_mutex_unlock (&printf_lock);
85 va_end (ap)__builtin_va_end(ap);
86}
87
88#if 0
89#define dprintf(f, x...) synchronized_printf (f, ##x)
90#else
91#define dprintf(f, x...)
92#endif
93
94#if 0
95#define ddprintf(f, x...) synchronized_printf (f, ##x)
96#else
97#define ddprintf(f, x...)
98#endif
99
100/*
101 * parallel vs serial switch
102 */
103#define PARALLEL1 1
104
105#if 0
106#define CHECKSUM 1
107#endif
108
109#define USE_PRECIOUS1 1
110
111#define ptoa(p)((p)*vm_page_size) ((p)*vm_page_size)
112#define atop(a)((a)/vm_page_size) ((a)/vm_page_size)
113
114partition_t partition_of(x)
115 int x;
116{
117 if (x >= all_partitions.n_partitions || x < 0)
118 panic("partition_of x%x", x);
119 return all_partitions.partition_list[x];
120}
121
122void set_partition_of(x, p)
123 int x;
124 partition_t p;
125{
126 if (x >= all_partitions.n_partitions || x < 0)
127 panic("set_partition_of x%x", x);
128 all_partitions.partition_list[x] = p;
129}
130
131/*
132 * Simple mapping from (file)NAME to id
133 * Saves space, filenames can be long.
134 */
135unsigned int
136part_id(const char *name)
137{
138 unsigned int id, xorid;
139 size_t len;
140
141 len = strlen(name);
142 id = xorid = 0;
143 while (len--) {
144 xorid ^= *name;
145 id += *name++;
146 }
147 return (id << 8) | xorid;
148}
149
150void
151partition_init()
152{
153 pthread_mutex_init(&all_partitions.lock, NULL((void*)0));
154 all_partitions.n_partitions = 0;
155}
156
157static partition_t
158new_partition (const char *name, struct file_direct *fdp,
159 int check_linux_signature)
160{
161 partition_t part;
162 vm_size_t size, bmsize;
163 vm_offset_t raddr;
164 mach_msg_type_number_t rsize;
165 int rc;
166 unsigned int id = part_id(name);
167
168 pthread_mutex_lock(&all_partitions.lock);
169 {
170 unsigned int i;
171 for (i = 0; i < all_partitions.n_partitions; i++)
172 {
173 part = partition_of(i);
174 if (part && part->id == id)
175 {
176 pthread_mutex_unlock(&all_partitions.lock);
177 return 0;
178 }
179 }
180 }
181 pthread_mutex_unlock(&all_partitions.lock);
182
183 size = atop(fdp->fd_size * fdp->fd_bsize)((fdp->fd_size * fdp->fd_bsize)/vm_page_size);
184 bmsize = howmany(size, NB_BM)(((size) + (32) - 1)/(32)) * sizeof(bm_entry_t);
185
186 part = (partition_t) kalloc(sizeof(struct part));
187 pthread_mutex_init(&part->p_lock, NULL((void*)0));
188 part->total_size = size;
189 part->free = size;
190 part->id = id;
191 part->bitmap = (bm_entry_t *)kalloc(bmsize);
192 part->going_away= FALSE((boolean_t) 0);
193 part->file = fdp;
194
195 bzero((char *)part->bitmap, bmsize);
196
197 if (check_linux_signature < 0)
198 {
199 if (check_linux_signature != -3)
200 printf("(default pager): "
201 "Paging to raw partition %s (%uk paging space)\n",
202 name, part->total_size * (vm_page_size / 1024));
203 return part;
204 }
205
206#define LINUX_PAGE_SIZE4096 4096 /* size of pages in Linux swap partitions */
207 rc = page_read_file_direct(part->file,
208 0, LINUX_PAGE_SIZE4096,
209 &raddr,
210 &rsize);
211 if (rc)
212 panic("(default pager): cannot read first page of %s! rc=%#x\n",
213 name, rc);
214 while (rsize < LINUX_PAGE_SIZE4096)
215 {
216 /* Filesystem block size is smaller than page size,
217 so we must do several reads to get the whole page. */
218 vm_address_t baddr;
219 vm_size_t bsize;
220 rc = page_read_file_direct(part->file,
221 rsize, LINUX_PAGE_SIZE4096-rsize,
222 &baddr,
223 &bsize);
224 if (rc)
225 panic("(default pager): "
226 "cannot read first page of %s! rc=%#x at %#x\n",
227 name, rc, rsize);
228
229 memcpy ((char *) raddr + rsize, (void *) baddr, bsize);
230 rsize += bsize;
231 vm_deallocate (mach_task_self ()((__mach_task_self_ + 0)), baddr, bsize);
232 }
233
234 if (!memcmp("SWAP-SPACE", (char *) raddr + LINUX_PAGE_SIZE4096-10, 10))
235 {
236 /* The partition's first page has a Linux swap signature.
237 This means the beginning of the page contains a bitmap
238 of good pages, and all others are bad. */
239 unsigned int i, j, bad, max;
240 int waste;
241
242 printf("(default pager): Found Linux 2.0 swap signature in %s\n",
243 name);
244
245 /* The first page, and the pages corresponding to the bits
246 occupied by the signature in the final 10 bytes of the page,
247 are always unavailable ("bad"). */
248 *(u_int32_t *)raddr &= ~(u_int32_t) 1;
249 memset((char *) raddr + LINUX_PAGE_SIZE4096-10, 0, 10);
250
251 max = LINUX_PAGE_SIZE4096 / sizeof(u_int32_t);
252 if (max > (part->total_size + 31) / 32)
253 max = (part->total_size + 31) / 32;
254
255 bad = 0;
256 for (i = 0; i < max; ++i)
257 {
258 u_int32_t bm = ((u_int32_t *) raddr)[i];
259 if (bm == ~(u_int32_t) 0)
260 continue;
261 /* There are some zero bits in this word. */
262 for (j = 0; j < 32; ++j)
263 if ((bm & (1 << j)) == 0)
264 {
265 unsigned int p = i*32 + j;
266 if (p >= part->total_size)
267 break;
268 ++bad;
269 part->bitmap[p / NB_BM32] |= 1 << (p % NB_BM32);
270 }
271 }
272 part->free -= bad;
273
274 --bad; /* Don't complain about first page. */
275 waste = part->total_size - (8 * (LINUX_PAGE_SIZE4096-10));
276 if (waste > 0)
277 {
278 /* The wasted pages were already marked "bad". */
279 bad -= waste;
280 if (bad > 0)
281 printf("\
282(default pager): Paging to %s, %dk swap-space (%dk bad, %dk wasted at end)\n",
283 name,
284 part->free * (LINUX_PAGE_SIZE4096 / 1024),
285 bad * (LINUX_PAGE_SIZE4096 / 1024),
286 waste * (LINUX_PAGE_SIZE4096 / 1024));
287 else
288 printf("\
289(default pager): Paging to %s, %dk swap-space (%dk wasted at end)\n",
290 name,
291 part->free * (LINUX_PAGE_SIZE4096 / 1024),
292 waste * (LINUX_PAGE_SIZE4096 / 1024));
293 }
294 else if (bad > 0)
295 printf("\
296(default pager): Paging to %s, %dk swap-space (excludes %dk marked bad)\n",
297 name,
298 part->free * (LINUX_PAGE_SIZE4096 / 1024),
299 bad * (LINUX_PAGE_SIZE4096 / 1024));
300 else
301 printf("\
302(default pager): Paging to %s, %dk swap-space\n",
303 name,
304 part->free * (LINUX_PAGE_SIZE4096 / 1024));
305 }
306 else if (!memcmp("SWAPSPACE2",
307 (char *) raddr + LINUX_PAGE_SIZE4096-10, 10))
308 {
309 struct
310 {
311 u_int8_t bootbits[1024];
312 u_int32_t version;
313 u_int32_t last_page;
314 u_int32_t nr_badpages;
315 u_int32_t padding[125];
316 u_int32_t badpages[1];
317 } *hdr = (void *) raddr;
318
319 printf("\
320(default pager): Found Linux 2.2 swap signature (v%u) in %s...",
321 hdr->version, name);
322
323 part->bitmap[0] |= 1; /* first page unusable */
324 part->free--;
325
326 switch (hdr->version)
327 {
328 default:
329 if (check_linux_signature)
330 {
331 printf ("version %u unknown! SKIPPING %s!\n",
332 hdr->version,
333 name);
334 vm_deallocate(mach_task_self()((__mach_task_self_ + 0)), raddr, rsize);
335 kfree(part->bitmap, bmsize);
336 kfree(part, sizeof *part);
337 return 0;
338 }
339 else
340 printf ("version %u unknown! IGNORING SIGNATURE PAGE!"
341 " %dk swap-space\n",
342 hdr->version,
343 part->free * (LINUX_PAGE_SIZE4096 / 1024));
344 break;
345
346 case 1:
347 {
348 unsigned int waste, i;
349 if (hdr->last_page > part->total_size)
350 {
351 printf ("signature says %uk, partition has only %uk! ",
352 hdr->last_page * (LINUX_PAGE_SIZE4096 / 1024),
353 part->total_size * (LINUX_PAGE_SIZE4096 / 1024));
354 waste = 0;
355 }
356 else
357 {
358 waste = part->total_size - hdr->last_page;
359 part->total_size = hdr->last_page;
360 part->free = part->total_size - 1;
361 }
362 for (i = 0; i < hdr->nr_badpages; ++i)
363 {
364 const u_int32_t bad = hdr->badpages[i];
365 part->bitmap[bad / NB_BM32] |= 1 << (bad % NB_BM32);
366 part->free--;
367 }
368 printf ("%uk swap-space",
369 part->free * (LINUX_PAGE_SIZE4096 / 1024));
370 if (hdr->nr_badpages != 0)
371 printf (" (excludes %uk marked bad)",
372 hdr->nr_badpages * (LINUX_PAGE_SIZE4096 / 1024));
373 if (waste != 0)
374 printf (" (excludes %uk at end of partition)",
375 waste * (LINUX_PAGE_SIZE4096 / 1024));
376 printf ("\n");
377 }
378 }
379 }
380 else if (check_linux_signature)
381 {
382 printf ("(default pager): "
383 "Cannot find Linux swap signature page! "
384 "SKIPPING %s (%uk partition)!",
385 name, part->total_size * (vm_page_size / 1024));
386 kfree(part->bitmap, bmsize);
387 kfree(part, sizeof *part);
388 part = 0;
389 }
390 else
391 printf("(default pager): "
392 "Paging to raw partition %s (%uk paging space)\n",
393 name, part->total_size * (vm_page_size / 1024));
394
395 vm_deallocate(mach_task_self()((__mach_task_self_ + 0)), raddr, rsize);
396
397 return part;
398}
399
400/*
401 * Create a partition descriptor,
402 * add it to the list of all such.
403 * size is in BYTES.
404 */
405void
406create_paging_partition(const char *name,
407 struct file_direct *fdp, int isa_file,
408 int linux_signature)
409{
410 partition_t part;
411
412 part = new_partition (name, fdp, linux_signature);
413 if (!part)
414 return;
415
416 pthread_mutex_lock(&all_partitions.lock);
417 {
418 int i;
419
420 for (i = 0; i < all_partitions.n_partitions; i++)
421 if (partition_of(i) == 0) break;
422
423 if (i == all_partitions.n_partitions) {
424 partition_t *new_list, *old_list;
425 int n;
426
427 n = i ? (i<<1) : 2;
428 new_list = (partition_t *)
429 kalloc( n * sizeof(partition_t) );
430 if (new_list == 0) no_paging_space(TRUE((boolean_t) 1));
431 bzero(new_list, n*sizeof(partition_t));
432 if (i) {
433 old_list = all_partitions.partition_list;
434 bcopy(old_list, new_list, i*sizeof(partition_t));
435 }
436 all_partitions.partition_list = new_list;
437 all_partitions.n_partitions = n;
438 if (i) kfree(old_list, i*sizeof(partition_t));
439 }
440 set_partition_of(i, part);
441 }
442 pthread_mutex_unlock(&all_partitions.lock);
443
444#if 0
445 dprintf("%s Added paging %s %s\n", my_name,
446 (isa_file) ? "file" : "device", name);
447#endif
448 overcommitted(TRUE((boolean_t) 1), part->free);
449}
450
451/*
452 * Choose the most appropriate default partition
453 * for an object of SIZE bytes.
454 * Return the partition locked, unless
455 * the object has no CUR_PARTition.
456 */
457p_index_t
458choose_partition(size, cur_part)
459 unsigned int size;
460 p_index_t cur_part;
461{
462 partition_t part;
463 boolean_t found = FALSE((boolean_t) 0);
464 int i;
465
466 pthread_mutex_lock(&all_partitions.lock);
467 for (i = 0; i < all_partitions.n_partitions; i++) {
468
469 /* the undesirable one ? */
470 if (i == cur_part)
471 continue;
472
473ddprintf ("choose_partition(%x,%d,%d)\n",size,cur_part,i);
474 /* one that was removed ? */
475 if ((part = partition_of(i)) == 0)
476 continue;
477
478 /* one that is being removed ? */
479 if (part->going_away)
480 continue;
481
482 /* is it big enough ? */
483 pthread_mutex_lock(&part->p_lock);
484 if (ptoa(part->free)((part->free)*vm_page_size) >= size) {
485 if (cur_part != P_INDEX_INVALID((p_index_t)-1)) {
486 pthread_mutex_unlock(&all_partitions.lock);
487 return (p_index_t)i;
488 } else
489 found = TRUE((boolean_t) 1);
490 }
491 pthread_mutex_unlock(&part->p_lock);
492
493 if (found) break;
494 }
495 pthread_mutex_unlock(&all_partitions.lock);
496 return (found) ? (p_index_t)i : P_INDEX_INVALID((p_index_t)-1);
497}
498
499/*
500 * Allocate a page in a paging partition
501 * The partition is returned unlocked.
502 */
503vm_offset_t
504pager_alloc_page(pindex, lock_it)
505 p_index_t pindex;
506 boolean_t lock_it;
507{
508 int bm_e;
509 int bit;
510 int limit;
511 bm_entry_t *bm;
512 partition_t part;
513 static char here[] = "%spager_alloc_page";
514
515 if (no_partition(pindex)((pindex) == ((p_index_t)-1)))
516 return (NO_BLOCK((vm_offset_t)-1));
517ddprintf ("pager_alloc_page(%d,%d)\n",pindex,lock_it);
518 part = partition_of(pindex);
519
520 /* unlikely, but possible deadlock against destroy_partition */
521 if (!part || part->going_away)
522 return (NO_BLOCK((vm_offset_t)-1));
523
524 if (lock_it)
525 pthread_mutex_lock(&part->p_lock);
526
527 if (part->free == 0) {
528 /* out of paging space */
529 pthread_mutex_unlock(&part->p_lock);
530 return (NO_BLOCK((vm_offset_t)-1));
531 }
532
533 limit = howmany(part->total_size, NB_BM)(((part->total_size) + (32) - 1)/(32));
534 bm = part->bitmap;
535 for (bm_e = 0; bm_e < limit; bm_e++, bm++)
536 if (*bm != BM_MASK0xffffffff)
537 break;
538
539 if (bm_e == limit)
540 panic(here,my_name);
541
542 /*
543 * Find and set the proper bit
544 */
545 {
546 bm_entry_t b = *bm;
547
548 for (bit = 0; bit < NB_BM32; bit++)
549 if ((b & (1<<bit)) == 0)
550 break;
551 if (bit == NB_BM32)
552 panic(here,my_name);
553
554 *bm = b | (1<<bit);
555 part->free--;
556
557 }
558
559 pthread_mutex_unlock(&part->p_lock);
560
561 return (bm_e*NB_BM32+bit);
562}
563
564/*
565 * Deallocate a page in a paging partition
566 */
567void
568pager_dealloc_page(pindex, page, lock_it)
569 p_index_t pindex;
570 vm_offset_t page;
571 boolean_t lock_it;
572{
573 partition_t part;
574 int bit, bm_e;
575
576 /* be paranoid */
577 if (no_partition(pindex)((pindex) == ((p_index_t)-1)))
578 panic("%sdealloc_page",my_name);
579ddprintf ("pager_dealloc_page(%d,%x,%d)\n",pindex,page,lock_it);
580 part = partition_of(pindex);
581
582 if (page >= part->total_size)
583 panic("%sdealloc_page",my_name);
584
585 bm_e = page / NB_BM32;
586 bit = page % NB_BM32;
587
588 if (lock_it)
589 pthread_mutex_lock(&part->p_lock);
590
591 part->bitmap[bm_e] &= ~(1<<bit);
592 part->free++;
593
594 if (lock_it)
595 pthread_mutex_unlock(&part->p_lock);
596}
597
598/*
599 * Object sizes are rounded up to the next power of 2,
600 * unless they are bigger than a given maximum size.
601 */
602vm_size_t max_doubled_size = 4 * 1024 * 1024; /* 4 meg */
603
604/*
605 * Return first level map for pager.
606 * If there is no such map, than allocate it.
607 */
608dp_map_t pager_get_direct_map(pager)
609 dpager_t pager;
610{
611 dp_map_t mapptr, emapptr;
612 vm_size_t size = pager->size;
613
614 if (pager->map)
615 return pager->map;
616 /*
617 * Allocate and initialize the block map
618 */
619 {
620 vm_size_t alloc_size;
621 dp_map_t init_value;
622
623 if (INDIRECT_PAGEMAP(size)(size > 64)) {
624 alloc_size = INDIRECT_PAGEMAP_SIZE(size)(((((size)-1)/64) + 1) * sizeof(vm_offset_t *));
625 init_value = (dp_map_t)0;
626 } else {
627 alloc_size = PAGEMAP_SIZE(size)((size)*sizeof(vm_offset_t));
628 init_value = (dp_map_t)NO_BLOCK((vm_offset_t)-1);
629 }
630
631 mapptr = (dp_map_t) kalloc(alloc_size);
632 for (emapptr = &mapptr[(alloc_size-1) / sizeof(vm_offset_t)];
633 emapptr >= mapptr;
634 emapptr--)
635 emapptr->indirect = init_value;
636 }
637 pager->map = mapptr;
638 return mapptr;
639}
640
641/*
642 * Attach a new paging object to a paging partition
643 */
644void
645pager_alloc(pager, part, size)
646 dpager_t pager;
647 p_index_t part;
648 vm_size_t size; /* in BYTES */
649{
650 int i;
651#ifdef CHECKSUM
652 dp_map_t mapptr, emapptr;
653#endif
654
655 pthread_mutex_init(&pager->lock, NULL((void*)0));
656#if DEBUG_READER_CONFLICTS0
657 pager->readers = 0;
658 pager->writer = FALSE((boolean_t) 0);
659#endif
660 pager->cur_partition = part;
661
662 /*
663 * Convert byte size to number of pages, then increase to the nearest
664 * power of 2.
665 */
666 size = atop(size)((size)/vm_page_size);
667 if (size <= atop(max_doubled_size)((max_doubled_size)/vm_page_size)) {
668 i = 1;
669 while (i < size)
670 i <<= 1;
671 size = i;
672 } else
673 size = ROUNDUP_TO_PAGEMAP(size)(((size) + 64 - 1) & ~(64 - 1));
674
675 pager->map = NULL((void*)0);
676 pager->size = size;
677 pager->limit = (vm_size_t)-1;
678
679#ifdef CHECKSUM
680 if (INDIRECT_PAGEMAP(size)(size > 64)) {
681 mapptr = (vm_offset_t *)
682 kalloc(INDIRECT_PAGEMAP_SIZE(size)(((((size)-1)/64) + 1) * sizeof(vm_offset_t *)));
683 for (i = INDIRECT_PAGEMAP_ENTRIES(size)((((size)-1)/64) + 1); --i >= 0;)
684 mapptr[i] = 0;
685 } else {
686 mapptr = (vm_offset_t *) kalloc(PAGEMAP_SIZE(size)((size)*sizeof(vm_offset_t)));
687 for (i = 0; i < size; i++)
688 mapptr[i] = NO_CHECKSUM;
689 }
690 pager->checksum = mapptr;
691#endif /* CHECKSUM */
692}
693
694/*
695 * Return size (in bytes) of space actually allocated to this pager.
696 * The pager is read-locked.
697 */
698
699vm_size_t
700pager_allocated(pager)
701 dpager_t pager;
702{
703 vm_size_t size;
704 dp_map_t map, emap;
705 vm_size_t asize;
706
707 size = pager->size; /* in pages */
708 asize = 0; /* allocated, in pages */
709 map = pager_get_direct_map(pager);
710
711 if (INDIRECT_PAGEMAP(size)(size > 64)) {
712 for (emap = &map[INDIRECT_PAGEMAP_ENTRIES(size)((((size)-1)/64) + 1)];
713 map < emap; map++) {
714
715 dp_map_t map2, emap2;
716
717 if ((map2 = map->indirect) == 0)
718 continue;
719
720 for (emap2 = &map2[PAGEMAP_ENTRIES64];
721 map2 < emap2; map2++)
722 if ( ! no_block(*map2)((*map2).indirect == (dp_map_t)((vm_offset_t)-1)) )
723 asize++;
724
725 }
726 } else {
727 for (emap = &map[size]; map < emap; map++)
728 if ( ! no_block(*map)((*map).indirect == (dp_map_t)((vm_offset_t)-1)) )
729 asize++;
730 }
731
732 return ptoa(asize)((asize)*vm_page_size);
733}
734
735/*
736 * Find offsets (in the object) of pages actually allocated to this pager.
737 * Returns the number of allocated pages, whether or not they all fit.
738 * The pager is read-locked.
739 */
740
741unsigned int
742pager_pages(pager, pages, numpages)
743 dpager_t pager;
744 default_pager_page_t *pages;
745 unsigned int numpages;
746{
747 vm_size_t size;
748 dp_map_t map, emap;
749 unsigned int actual;
750 vm_offset_t offset;
751
752 size = pager->size; /* in pages */
753 map = pager_get_direct_map(pager);
754 actual = 0;
755 offset = 0;
756
757 if (INDIRECT_PAGEMAP(size)(size > 64)) {
758 for (emap = &map[INDIRECT_PAGEMAP_ENTRIES(size)((((size)-1)/64) + 1)];
759 map < emap; map++) {
760
761 dp_map_t map2, emap2;
762
763 if ((map2 = map->indirect) == 0) {
764 offset += vm_page_size * PAGEMAP_ENTRIES64;
765 continue;
766 }
767 for (emap2 = &map2[PAGEMAP_ENTRIES64];
768 map2 < emap2; map2++)
769 if ( ! no_block(*map2)((*map2).indirect == (dp_map_t)((vm_offset_t)-1)) ) {
770 if (actual++ < numpages)
771 pages++->dpp_offset = offset;
772 }
773 offset += vm_page_size;
774 }
775 } else {
776 for (emap = &map[size]; map < emap; map++) {
777 if ( ! no_block(*map)((*map).indirect == (dp_map_t)((vm_offset_t)-1)) ) {
778 if (actual++ < numpages)
779 pages++->dpp_offset = offset;
780 }
781 offset += vm_page_size;
782 }
783 }
784 return actual;
785}
786
787/*
788 * Extend the map for a paging object.
789 *
790 * XXX This implementation can allocate an arbitrary large amount
791 * of wired memory when extending a big block map. Because vm-privileged
792 * threads call pager_extend, this can crash the system by exhausting
793 * system memory.
794 */
795void
796pager_extend(pager, new_size)
797 dpager_t pager;
798 vm_size_t new_size; /* in pages */
799{
800 dp_map_t new_mapptr;
801 dp_map_t old_mapptr;
802 int i;
803 vm_size_t old_size;
804
805 pthread_mutex_lock(&pager->lock); /* XXX lock_write */
806#if DEBUG_READER_CONFLICTS0
807 pager->writer = TRUE((boolean_t) 1);
808#endif
809 /*
810 * Double current size until we cover new size.
811 * If object is 'too big' just use new size.
812 */
813 old_size = pager->size;
814
815 if (new_size <= atop(max_doubled_size)((max_doubled_size)/vm_page_size)) {
816 /* New size cannot be less than 1 */
817 i = old_size ? old_size : 1;
818 while (i < new_size)
819 i <<= 1;
820 new_size = i;
821 } else
822 new_size = ROUNDUP_TO_PAGEMAP(new_size)(((new_size) + 64 - 1) & ~(64 - 1));
823
824 if (INDIRECT_PAGEMAP(old_size)(old_size > 64)) {
825 /*
826 * Pager already uses two levels. Allocate
827 * a larger indirect block.
828 */
829 new_mapptr = (dp_map_t)
830 kalloc(INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
831 old_mapptr = pager_get_direct_map(pager);
832 for (i = 0; i < INDIRECT_PAGEMAP_ENTRIES(old_size)((((old_size)-1)/64) + 1); i++)
833 new_mapptr[i] = old_mapptr[i];
834 for (; i < INDIRECT_PAGEMAP_ENTRIES(new_size)((((new_size)-1)/64) + 1); i++)
835 new_mapptr[i].indirect = (dp_map_t)0;
836 kfree((char *)old_mapptr, INDIRECT_PAGEMAP_SIZE(old_size)(((((old_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
837 pager->map = new_mapptr;
838 pager->size = new_size;
839#ifdef CHECKSUM
840 new_mapptr = (vm_offset_t *)
841 kalloc(INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
842 old_mapptr = pager->checksum;
843 for (i = 0; i < INDIRECT_PAGEMAP_ENTRIES(old_size)((((old_size)-1)/64) + 1); i++)
844 new_mapptr[i] = old_mapptr[i];
845 for (; i < INDIRECT_PAGEMAP_ENTRIES(new_size)((((new_size)-1)/64) + 1); i++)
846 new_mapptr[i] = 0;
847 kfree((char *)old_mapptr, INDIRECT_PAGEMAP_SIZE(old_size)(((((old_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
848 pager->checksum = new_mapptr;
849#endif /* CHECKSUM */
850#if DEBUG_READER_CONFLICTS0
851 pager->writer = FALSE((boolean_t) 0);
852#endif
853 pthread_mutex_unlock(&pager->lock);
854#if 0
855 ddprintf ("pager_extend 1 mapptr %x [3b] = %x\n", new_mapptr,
856 new_mapptr[0x3b]);
857 if (new_mapptr[0x3b].indirect > 0x10000
858 && new_mapptr[0x3b].indirect != NO_BLOCK((vm_offset_t)-1))
859 panic ("debug panic");
860#endif
861 return;
862 }
863
864 if (INDIRECT_PAGEMAP(new_size)(new_size > 64)) {
865 /*
866 * Changing from direct map to indirect map.
867 * Allocate both indirect and direct map blocks,
868 * since second-level (direct) block must be
869 * full size (PAGEMAP_SIZE(PAGEMAP_ENTRIES)).
870 */
871
872 /*
873 * Allocate new second-level map first.
874 */
875 new_mapptr = (dp_map_t) kalloc(PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
876 old_mapptr = pager_get_direct_map(pager);
877 for (i = 0; i < old_size; i++)
878 new_mapptr[i] = old_mapptr[i];
879 for (; i < PAGEMAP_ENTRIES64; i++)
880 invalidate_block(new_mapptr[i])((new_mapptr[i]).indirect = (dp_map_t)((vm_offset_t)-1));
881 kfree((char *)old_mapptr, PAGEMAP_SIZE(old_size)((old_size)*sizeof(vm_offset_t)));
882 old_mapptr = new_mapptr;
883
884#if 0
885 ddprintf ("pager_extend 2 mapptr %x [3b] = %x\n", new_mapptr,
886 new_mapptr[0x3b]);
887 if (new_mapptr[0x3b].indirect > 0x10000
888 && new_mapptr[0x3b].indirect != NO_BLOCK((vm_offset_t)-1))
889 panic ("debug panic");
890#endif
891
892 /*
893 * Now allocate indirect map.
894 */
895 new_mapptr = (dp_map_t)
896 kalloc(INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
897 new_mapptr[0].indirect = old_mapptr;
898 for (i = 1; i < INDIRECT_PAGEMAP_ENTRIES(new_size)((((new_size)-1)/64) + 1); i++)
899 new_mapptr[i].indirect = 0;
900 pager->map = new_mapptr;
901 pager->size = new_size;
902#ifdef CHECKSUM
903 /*
904 * Allocate new second-level map first.
905 */
906 new_mapptr = (vm_offset_t *)kalloc(PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
907 old_mapptr = pager->checksum;
908 for (i = 0; i < old_size; i++)
909 new_mapptr[i] = old_mapptr[i];
910 for (; i < PAGEMAP_ENTRIES64; i++)
911 new_mapptr[i] = NO_CHECKSUM;
912 kfree((char *)old_mapptr, PAGEMAP_SIZE(old_size)((old_size)*sizeof(vm_offset_t)));
913 old_mapptr = new_mapptr;
914
915 /*
916 * Now allocate indirect map.
917 */
918 new_mapptr = (vm_offset_t *)
919 kalloc(INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
920 new_mapptr[0] = (vm_offset_t) old_mapptr;
921 for (i = 1; i < INDIRECT_PAGEMAP_ENTRIES(new_size)((((new_size)-1)/64) + 1); i++)
922 new_mapptr[i] = 0;
923 pager->checksum = new_mapptr;
924#endif /* CHECKSUM */
925#if DEBUG_READER_CONFLICTS0
926 pager->writer = FALSE((boolean_t) 0);
927#endif
928 pthread_mutex_unlock(&pager->lock);
929 return;
930 }
931 /*
932 * Enlarging a direct block.
933 */
934 new_mapptr = (dp_map_t) kalloc(PAGEMAP_SIZE(new_size)((new_size)*sizeof(vm_offset_t)));
935 old_mapptr = pager_get_direct_map(pager);
936 for (i = 0; i < old_size; i++)
937 new_mapptr[i] = old_mapptr[i];
938 for (; i < new_size; i++)
939 invalidate_block(new_mapptr[i])((new_mapptr[i]).indirect = (dp_map_t)((vm_offset_t)-1));
940 kfree((char *)old_mapptr, PAGEMAP_SIZE(old_size)((old_size)*sizeof(vm_offset_t)));
941 pager->map = new_mapptr;
942 pager->size = new_size;
943#ifdef CHECKSUM
944 new_mapptr = (vm_offset_t *)
945 kalloc(PAGEMAP_SIZE(new_size)((new_size)*sizeof(vm_offset_t)));
946 old_mapptr = pager->checksum;
947 for (i = 0; i < old_size; i++)
948 new_mapptr[i] = old_mapptr[i];
949 for (; i < new_size; i++)
950 new_mapptr[i] = NO_CHECKSUM;
951 kfree((char *)old_mapptr, PAGEMAP_SIZE(old_size)((old_size)*sizeof(vm_offset_t)));
952 pager->checksum = new_mapptr;
953#endif /* CHECKSUM */
954#if DEBUG_READER_CONFLICTS0
955 pager->writer = FALSE((boolean_t) 0);
956#endif
957 pthread_mutex_unlock(&pager->lock);
958}
959
960/* This deallocates the pages necessary to truncate a direct map
961 previously of size NEW_SIZE to the smaller size OLD_SIZE. */
962static void
963dealloc_direct (dp_map_t mapptr,
964 vm_size_t old_size, vm_size_t new_size)
965{
966 vm_size_t i;
967
968 if (!mapptr)
969 return;
970
971 for (i = new_size; i < old_size; ++i)
972 {
973 const union dp_map entry = mapptr[i];
974 if (!no_block(entry)((entry).indirect == (dp_map_t)((vm_offset_t)-1)))
975 {
976 pager_dealloc_page(entry.block.p_index, entry.block.p_offset,
977 TRUE((boolean_t) 1));
978 invalidate_block(mapptr[i])((mapptr[i]).indirect = (dp_map_t)((vm_offset_t)-1));
979 }
980 }
981}
982
983/* Truncate a memory object. First, any pages between the new size
984 and the (larger) old size are deallocated. Then, the size of
985 the pagemap may be reduced, an indirect map may be turned into
986 a direct map.
987
988 The pager must be locked by the caller. */
989static void
990pager_truncate(dpager_t pager, vm_size_t new_size) /* in pages */
991{
992 int i;
993 vm_size_t old_size;
994
995 pthread_mutex_lock(&pager->lock); /* XXX lock_write */
996
997 if (!pager->map)
998 goto done;
999
1000 old_size = pager->size;
1001
1002 if (INDIRECT_PAGEMAP(old_size)(old_size > 64))
1003 {
1004 /* First handle the entire second-levels blocks that are being freed. */
1005 for (i = INDIRECT_PAGEMAP_ENTRIES(new_size)((((new_size)-1)/64) + 1);
1006 i < INDIRECT_PAGEMAP_ENTRIES(old_size)((((old_size)-1)/64) + 1);
1007 ++i)
1008 {
1009 const dp_map_t mapptr = pager->map[i].indirect;
1010 pager->map[i].indirect = (dp_map_t)0;
1011 dealloc_direct (mapptr, PAGEMAP_ENTRIES64, 0);
1012 kfree ((char *)mapptr, PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
1013 }
1014
1015 /* Now truncate what's now the final nonempty direct block. */
1016 dealloc_direct (pager->map[(new_size - 1) / PAGEMAP_ENTRIES64].indirect,
1017 old_size & (PAGEMAP_ENTRIES64 - 1),
1018 new_size & (PAGEMAP_ENTRIES64 - 1));
1019
1020 if (INDIRECT_PAGEMAP (new_size)(new_size > 64))
1021 {
1022 const dp_map_t old_mapptr = pager->map;
1023 pager->map = (dp_map_t) kalloc (INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1024 memcpy (pager->map, old_mapptr, INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1025 kfree ((char *) old_mapptr, INDIRECT_PAGEMAP_SIZE (old_size)(((((old_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1026 }
1027 else
1028 {
1029 /* We are truncating to a size small enough that it goes to using
1030 a one-level map. We already have that map, as the first and only
1031 nonempty element in our indirect map. */
1032 const dp_map_t mapptr = pager->map[0].indirect;
1033 kfree((char *)pager->map, INDIRECT_PAGEMAP_SIZE(old_size)(((((old_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1034 pager->map = mapptr;
1035 }
1036 }
1037
1038 if (! INDIRECT_PAGEMAP(old_size)(old_size > 64))
1039 {
1040 /* First deallocate pages in the truncated region. */
1041 dealloc_direct (pager->map, old_size, new_size);
1042 /* Now reduce the size of the direct map itself. We don't bother
1043 with kalloc/kfree if it's not shrinking enough that kalloc.c
1044 would actually use less. */
1045 if (PAGEMAP_SIZE (new_size)((new_size)*sizeof(vm_offset_t)) <= PAGEMAP_SIZE (old_size)((old_size)*sizeof(vm_offset_t)) / 2)
1046 {
1047 const dp_map_t old_mapptr = pager->map;
1048 pager->map = (dp_map_t) kalloc (PAGEMAP_SIZE (new_size)((new_size)*sizeof(vm_offset_t)));
1049 memcpy (pager->map, old_mapptr, PAGEMAP_SIZE (new_size)((new_size)*sizeof(vm_offset_t)));
1050 kfree ((char *) old_mapptr, PAGEMAP_SIZE (old_size)((old_size)*sizeof(vm_offset_t)));
1051 }
1052 }
1053
1054 done:
1055 pager->size = new_size;
1056 pthread_mutex_unlock(&pager->lock);
1057
1058#ifdef CHECKSUM
1059#error write me
1060#endif /* CHECKSUM */
1061}
1062
1063
1064/*
1065 * Given an offset within a paging object, find the
1066 * corresponding block within the paging partition.
1067 * Return NO_BLOCK if none allocated.
1068 */
1069union dp_map
1070pager_read_offset(pager, offset)
1071 dpager_t pager;
1072 vm_offset_t offset;
1073{
1074 vm_offset_t f_page;
1075 union dp_map pager_offset;
1076
1077 f_page = atop(offset)((offset)/vm_page_size);
1078
1079#if DEBUG_READER_CONFLICTS0
1080 if (pager->readers > 0)
1081 default_pager_read_conflicts++; /* would have proceeded with
1082 read/write lock */
1083#endif
1084 pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1085#if DEBUG_READER_CONFLICTS0
1086 pager->readers++;
1087#endif
1088 if (f_page >= pager->size)
1089 {
1090 ddprintf ("%spager_read_offset pager %x: bad page %d >= size %d",
1091 my_name, pager, f_page, pager->size);
1092 pthread_mutex_unlock(&pager->lock);
1093 return (union dp_map) (union dp_map *) NO_BLOCK((vm_offset_t)-1);
1094#if 0
1095 panic("%spager_read_offset",my_name);
1096#endif
1097 }
1098
1099 invalidate_block(pager_offset)((pager_offset).indirect = (dp_map_t)((vm_offset_t)-1));
1100 if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1101 dp_map_t mapptr;
1102
1103 if (pager->map) {
1104 mapptr = pager->map[f_page/PAGEMAP_ENTRIES64].indirect;
1105 if (mapptr)
1106 pager_offset = mapptr[f_page%PAGEMAP_ENTRIES64];
1107 }
1108 }
1109 else {
1110 if (pager->map)
1111 pager_offset = pager->map[f_page];
1112 }
1113
1114#if DEBUG_READER_CONFLICTS0
1115 pager->readers--;
1116#endif
1117 pthread_mutex_unlock(&pager->lock);
1118 return (pager_offset);
1119}
1120
1121#if USE_PRECIOUS1
1122/*
1123 * Release a single disk block.
1124 */
1125void pager_release_offset(pager, offset)
1126 dpager_t pager;
1127 vm_offset_t offset;
1128{
1129 union dp_map entry;
1130
1131 offset = atop(offset)((offset)/vm_page_size);
1132
1133 pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1134
1135 assert (pager->map)((pager->map) ? (void) (0) : __assert_fail ("pager->map"
, "../../mach-defpager/default_pager.c", 1135, __PRETTY_FUNCTION__
));
1136 if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1137 dp_map_t mapptr;
1138
1139 mapptr = pager->map[offset / PAGEMAP_ENTRIES64].indirect;
1140 entry = mapptr[offset % PAGEMAP_ENTRIES64];
1141 invalidate_block(mapptr[offset % PAGEMAP_ENTRIES])((mapptr[offset % 64]).indirect = (dp_map_t)((vm_offset_t)-1)
);
1142 } else {
1143 entry = pager->map[offset];
1144 invalidate_block(pager->map[offset])((pager->map[offset]).indirect = (dp_map_t)((vm_offset_t)-
1));
1145 }
1146
1147 pthread_mutex_unlock(&pager->lock);
1148
1149 pager_dealloc_page(entry.block.p_index, entry.block.p_offset, TRUE((boolean_t) 1));
1150}
1151#endif /*USE_PRECIOUS*/
1152
1153
1154/*
1155 * Move a page from one partition to another
1156 * New partition is locked, old partition is
1157 * locked unless LOCK_OLD sez otherwise.
1158 */
1159union dp_map
1160pager_move_page(block)
1161 union dp_map block;
1162{
1163 partition_t old_part, new_part;
1164 p_index_t old_pindex, new_pindex;
1165 union dp_map ret;
1166 vm_size_t size;
1167 vm_offset_t raddr, offset, new_offset;
1168 kern_return_t rc;
1169 static char here[] = "%spager_move_page";
1170
1171 old_pindex = block.block.p_index;
1172 invalidate_block(ret)((ret).indirect = (dp_map_t)((vm_offset_t)-1));
1173
1174 /* See if we have room to put it anywhere else */
1175 new_pindex = choose_partition( ptoa(1)((1)*vm_page_size), old_pindex);
1176 if (no_partition(new_pindex)((new_pindex) == ((p_index_t)-1)))
1177 return ret;
1178
1179 /* this unlocks the new partition */
1180 new_offset = pager_alloc_page(new_pindex, FALSE((boolean_t) 0));
1181 if (new_offset == NO_BLOCK((vm_offset_t)-1))
1182 panic(here,my_name);
1183
1184 /*
1185 * Got the resources, now move the data
1186 */
1187ddprintf ("pager_move_page(%x,%d,%d)\n",block.block.p_offset,old_pindex,new_pindex);
1188 old_part = partition_of(old_pindex);
1189 offset = ptoa(block.block.p_offset)((block.block.p_offset)*vm_page_size);
1190 rc = page_read_file_direct (old_part->file,
1191 offset,
1192 vm_page_size,
1193 &raddr,
1194 &size);
1195 if (rc != 0)
1196 panic(here,my_name);
1197
1198 /* release old */
1199 pager_dealloc_page(old_pindex, block.block.p_offset, FALSE((boolean_t) 0));
1200
1201 new_part = partition_of(new_pindex);
1202 offset = ptoa(new_offset)((new_offset)*vm_page_size);
1203 rc = page_write_file_direct (new_part->file,
1204 offset,
1205 raddr,
1206 size,
1207 &size);
1208 if (rc != 0)
1209 panic(here,my_name);
1210
1211 (void) vm_deallocate( mach_task_self()((__mach_task_self_ + 0)), raddr, size);
1212
1213 ret.block.p_offset = new_offset;
1214 ret.block.p_index = new_pindex;
1215
1216 return ret;
1217}
1218
1219#ifdef CHECKSUM
1220/*
1221 * Return the checksum for a block.
1222 */
1223int
1224pager_get_checksum(pager, offset)
1225 dpager_t pager;
1226 vm_offset_t offset;
1227{
1228 vm_offset_t f_page;
1229 int checksum;
1230
1231 f_page = atop(offset)((offset)/vm_page_size);
1232
1233 pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1234 if (f_page >= pager->size)
1235 panic("%spager_get_checksum",my_name);
1236
1237 if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1238 vm_offset_t *mapptr;
1239
1240 mapptr = (vm_offset_t *)pager->checksum[f_page/PAGEMAP_ENTRIES64];
1241 if (mapptr == 0)
1242 checksum = NO_CHECKSUM;
1243 else
1244 checksum = mapptr[f_page%PAGEMAP_ENTRIES64];
1245 }
1246 else {
1247 checksum = pager->checksum[f_page];
1248 }
1249
1250 pthread_mutex_unlock(&pager->lock);
1251 return (checksum);
1252}
1253
1254/*
1255 * Remember the checksum for a block.
1256 */
1257int
1258pager_put_checksum(pager, offset, checksum)
1259 dpager_t pager;
1260 vm_offset_t offset;
1261 int checksum;
1262{
1263 vm_offset_t f_page;
1264 static char here[] = "%spager_put_checksum";
1265
1266 f_page = atop(offset)((offset)/vm_page_size);
1267
1268 pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1269 if (f_page >= pager->size)
1270 panic(here,my_name);
1271
1272 if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1273 vm_offset_t *mapptr;
1274
1275 mapptr = (vm_offset_t *)pager->checksum[f_page/PAGEMAP_ENTRIES64];
1276 if (mapptr == 0)
1277 panic(here,my_name);
1278
1279 mapptr[f_page%PAGEMAP_ENTRIES64] = checksum;
1280 }
1281 else {
1282 pager->checksum[f_page] = checksum;
1283 }
1284 pthread_mutex_unlock(&pager->lock);
1285}
1286
1287/*
1288 * Compute a checksum - XOR each 32-bit word.
1289 */
1290int
1291compute_checksum(addr, size)
1292 vm_offset_t addr;
1293 vm_size_t size;
1294{
1295 int checksum = NO_CHECKSUM;
1296 int *ptr;
1297 int count;
1298
1299 ptr = (int *)addr;
1300 count = size / sizeof(int);
1301
1302 while (--count >= 0)
1303 checksum ^= *ptr++;
1304
1305 return (checksum);
1306}
1307#endif /* CHECKSUM */
1308
1309/*
1310 * Given an offset within a paging object, find the
1311 * corresponding block within the paging partition.
1312 * Allocate a new block if necessary.
1313 *
1314 * WARNING: paging objects apparently may be extended
1315 * without notice!
1316 */
1317union dp_map
1318pager_write_offset(pager, offset)
1319 dpager_t pager;
1320 vm_offset_t offset;
1321{
1322 vm_offset_t f_page;
1323 dp_map_t mapptr;
1324 union dp_map block;
1325
1326 invalidate_block(block)((block).indirect = (dp_map_t)((vm_offset_t)-1));
1327
1328 f_page = atop(offset)((offset)/vm_page_size);
1329
1330#if DEBUG_READER_CONFLICTS0
1331 if (pager->readers > 0)
1332 default_pager_read_conflicts++; /* would have proceeded with
1333 read/write lock */
1334#endif
1335 pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1336#if DEBUG_READER_CONFLICTS0
1337 pager->readers++;
1338#endif
1339
1340 /* Catch the case where we had no initial fit partition
1341 for this object, but one was added later on */
1342 if (no_partition(pager->cur_partition)((pager->cur_partition) == ((p_index_t)-1))) {
1343 p_index_t new_part;
1344 vm_size_t size;
1345
1346 size = (f_page > pager->size) ? f_page : pager->size;
1347 new_part = choose_partition(ptoa(size)((size)*vm_page_size), P_INDEX_INVALID((p_index_t)-1));
1348 if (no_partition(new_part)((new_part) == ((p_index_t)-1)))
1349 new_part = choose_partition(ptoa(1)((1)*vm_page_size), P_INDEX_INVALID((p_index_t)-1));
1350 if (no_partition(new_part)((new_part) == ((p_index_t)-1)))
1351 /* give up right now to avoid confusion */
1352 goto out;
1353 else
1354 pager->cur_partition = new_part;
1355 }
1356
1357 while (f_page >= pager->size) {
1358 ddprintf ("pager_write_offset: extending: %x %x\n", f_page, pager->size);
1359
1360 /*
1361 * Paging object must be extended.
1362 * Remember that offset is 0-based, but size is 1-based.
1363 */
1364#if DEBUG_READER_CONFLICTS0
1365 pager->readers--;
1366#endif
1367 pthread_mutex_unlock(&pager->lock);
1368 pager_extend(pager, f_page + 1);
1369#if DEBUG_READER_CONFLICTS0
1370 if (pager->readers > 0)
1371 default_pager_read_conflicts++; /* would have proceeded with
1372 read/write lock */
1373#endif
1374 pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1375#if DEBUG_READER_CONFLICTS0
1376 pager->readers++;
1377#endif
1378 ddprintf ("pager_write_offset: done extending: %x %x\n", f_page, pager->size);
1379 }
1380
1381 if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1382 ddprintf ("pager_write_offset: indirect\n");
1383 mapptr = pager_get_direct_map(pager);
1384 mapptr = mapptr[f_page/PAGEMAP_ENTRIES64].indirect;
1385 if (mapptr == 0) {
1386 /*
1387 * Allocate the indirect block
1388 */
1389 int i;
1390 ddprintf ("pager_write_offset: allocating indirect\n");
1391
1392 mapptr = (dp_map_t) kalloc(PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
1393 if (mapptr == 0) {
1394 /* out of space! */
1395 no_paging_space(TRUE((boolean_t) 1));
1396 goto out;
1397 }
1398 pager->map[f_page/PAGEMAP_ENTRIES64].indirect = mapptr;
1399 for (i = 0; i < PAGEMAP_ENTRIES64; i++)
1400 invalidate_block(mapptr[i])((mapptr[i]).indirect = (dp_map_t)((vm_offset_t)-1));
1401#ifdef CHECKSUM
1402 {
1403 vm_offset_t *cksumptr;
1404 int j;
1405
1406 cksumptr = (vm_offset_t *)
1407 kalloc(PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
1408 if (cksumptr == 0) {
1409 /* out of space! */
1410 no_paging_space(TRUE((boolean_t) 1));
1411 goto out;
1412 }
1413 pager->checksum[f_page/PAGEMAP_ENTRIES64]
1414 = (vm_offset_t)cksumptr;
1415 for (j = 0; j < PAGEMAP_ENTRIES64; j++)
1416 cksumptr[j] = NO_CHECKSUM;
1417 }
1418#endif /* CHECKSUM */
1419 }
1420 f_page %= PAGEMAP_ENTRIES64;
1421 }
1422 else {
1423 mapptr = pager_get_direct_map(pager);
1424 }
1425
1426 block = mapptr[f_page];
1427 ddprintf ("pager_write_offset: block starts as %x[%x] %x\n", mapptr, f_page, block);
1428 if (no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1))) {
1429 vm_offset_t off;
1430
1431 /* get room now */
1432 off = pager_alloc_page(pager->cur_partition, TRUE((boolean_t) 1));
1433 if (off == NO_BLOCK((vm_offset_t)-1)) {
1434 /*
1435 * Before giving up, try all other partitions.
1436 */
1437 p_index_t new_part;
1438
1439 ddprintf ("pager_write_offset: could not allocate block\n");
1440 /* returns it locked (if any one is non-full) */
1441 new_part = choose_partition( ptoa(1)((1)*vm_page_size), pager->cur_partition);
1442 if ( ! no_partition(new_part)((new_part) == ((p_index_t)-1)) ) {
1443
1444#if debug0
1445dprintf("%s partition %x filled,", my_name, pager->cur_partition);
1446dprintf("extending object %x (size %x) to %x.\n",
1447 pager, pager->size, new_part);
1448#endif
1449
1450 /* this one tastes better */
1451 pager->cur_partition = new_part;
1452
1453 /* this unlocks the partition too */
1454 off = pager_alloc_page(pager->cur_partition, FALSE((boolean_t) 0));
1455
1456 }
1457
1458 if (off == NO_BLOCK((vm_offset_t)-1)) {
1459 /*
1460 * Oh well.
1461 */
1462 overcommitted(FALSE((boolean_t) 0), 1);
1463 goto out;
1464 }
1465 ddprintf ("pager_write_offset: decided to allocate block\n");
1466 }
1467 block.block.p_offset = off;
1468 block.block.p_index = pager->cur_partition;
1469 mapptr[f_page] = block;
1470 }
1471
1472out:
1473
1474#if DEBUG_READER_CONFLICTS0
1475 pager->readers--;
1476#endif
1477 pthread_mutex_unlock(&pager->lock);
1478 return (block);
1479}
1480
1481/*
1482 * Deallocate all of the blocks belonging to a paging object.
1483 * No locking needed because no other operations can be in progress.
1484 */
1485void
1486pager_dealloc(pager)
1487 dpager_t pager;
1488{
1489 int i, j;
1490 dp_map_t mapptr;
1491 union dp_map block;
1492
1493 if (!pager->map)
1494 return;
1495
1496 if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1497 for (i = INDIRECT_PAGEMAP_ENTRIES(pager->size)((((pager->size)-1)/64) + 1); --i >= 0; ) {
1498 mapptr = pager->map[i].indirect;
1499 if (mapptr != 0) {
1500 for (j = 0; j < PAGEMAP_ENTRIES64; j++) {
1501 block = mapptr[j];
1502 if ( ! no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)) )
1503 pager_dealloc_page(block.block.p_index,
1504 block.block.p_offset, TRUE((boolean_t) 1));
1505 }
1506 kfree((char *)mapptr, PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
1507 pager->map[i].indirect = (dp_map_t) 0;
1508 }
1509 }
1510 kfree((char *)pager->map, INDIRECT_PAGEMAP_SIZE(pager->size)(((((pager->size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1511 pager->map = (dp_map_t) 0;
1512#ifdef CHECKSUM
1513 for (i = INDIRECT_PAGEMAP_ENTRIES(pager->size)((((pager->size)-1)/64) + 1); --i >= 0; ) {
1514 mapptr = (vm_offset_t *)pager->checksum[i];
1515 if (mapptr) {
1516 kfree((char *)mapptr, PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
1517 }
1518 }
1519 kfree((char *)pager->checksum,
1520 INDIRECT_PAGEMAP_SIZE(pager->size)(((((pager->size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1521#endif /* CHECKSUM */
1522 }
1523 else {
1524 mapptr = pager->map;
1525 for (i = 0; i < pager->size; i++ ) {
1526 block = mapptr[i];
1527 if ( ! no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)) )
1528 pager_dealloc_page(block.block.p_index,
1529 block.block.p_offset, TRUE((boolean_t) 1));
1530 }
1531 kfree((char *)pager->map, PAGEMAP_SIZE(pager->size)((pager->size)*sizeof(vm_offset_t)));
1532 pager->map = (dp_map_t) 0;
1533#ifdef CHECKSUM
1534 kfree((char *)pager->checksum, PAGEMAP_SIZE(pager->size)((pager->size)*sizeof(vm_offset_t)));
1535#endif /* CHECKSUM */
1536 }
1537}
1538
1539/*
1540 * Move all the pages of a PAGER that live in a
1541 * partition PINDEX somewhere else.
1542 * Pager should be write-locked, partition too.
1543 * Returns FALSE if it could not do it, but
1544 * some pages might have been moved nonetheless.
1545 */
1546boolean_t
1547pager_realloc(pager, pindex)
1548 dpager_t pager;
1549 p_index_t pindex;
1550{
1551 dp_map_t map, emap;
1552 vm_size_t size;
1553 union dp_map block;
1554
1555 if (!pager->map)
1556 return TRUE((boolean_t) 1);
1557
1558 size = pager->size; /* in pages */
1559 map = pager->map;
1560
1561 if (INDIRECT_PAGEMAP(size)(size > 64)) {
1562 for (emap = &map[INDIRECT_PAGEMAP_ENTRIES(size)((((size)-1)/64) + 1)];
1563 map < emap; map++) {
1564
1565 dp_map_t map2, emap2;
1566
1567 if ((map2 = map->indirect) == 0)
1568 continue;
1569
1570 for (emap2 = &map2[PAGEMAP_ENTRIES64];
1571 map2 < emap2; map2++)
1572 if ( map2->block.p_index == pindex) {
1573
1574 block = pager_move_page(*map2);
1575 if (!no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)))
1576 *map2 = block;
1577 else
1578 return FALSE((boolean_t) 0);
1579 }
1580
1581 }
1582 goto ok;
1583 }
1584
1585 /* A small one */
1586 for (emap = &map[size]; map < emap; map++)
1587 if (map->block.p_index == pindex) {
1588 block = pager_move_page(*map);
1589 if (!no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)))
1590 *map = block;
1591 else
1592 return FALSE((boolean_t) 0);
1593 }
1594ok:
1595 pager->cur_partition = choose_partition(0, P_INDEX_INVALID((p_index_t)-1));
1596 return TRUE((boolean_t) 1);
1597}
1598
1599/*
1600 * Read/write routines.
1601 */
1602#define PAGER_SUCCESS0 0
1603#define PAGER_ABSENT1 1
1604#define PAGER_ERROR2 2
1605
1606/*
1607 * Read data from a default pager. Addr is the address of a buffer
1608 * to fill. Out_addr returns the buffer that contains the data;
1609 * if it is different from <addr>, it must be deallocated after use.
1610 */
1611int
1612default_read(ds, addr, size, offset, out_addr, deallocate, external)
1613 dpager_t ds;
1614 vm_offset_t addr; /* pointer to block to fill */
1615 vm_size_t size;
1616 vm_offset_t offset;
1617 vm_offset_t *out_addr;
1618 /* returns pointer to data */
1619 boolean_t deallocate;
1620 boolean_t external;
1621{
1622 union dp_map block;
1623 vm_offset_t raddr;
1624 vm_size_t rsize;
1625 int rc;
1626 boolean_t first_time;
1627 partition_t part;
1628#ifdef CHECKSUM
1629 vm_size_t original_size = size;
1630#endif /* CHECKSUM */
1631 vm_offset_t original_offset = offset;
1632
1633 /*
1634 * Find the block in the paging partition
1635 */
1636 block = pager_read_offset(ds, offset);
1637 if ( no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)) ) {
1638 if (external) {
1639 /*
1640 * An external object is requesting unswapped data,
1641 * zero fill the page and return.
1642 */
1643 bzero((char *) addr, vm_page_size);
1644 *out_addr = addr;
1645 return (PAGER_SUCCESS0);
1646 }
1647 return (PAGER_ABSENT1);
1648 }
1649
1650 /*
1651 * Read it, trying for the entire page.
1652 */
1653 offset = ptoa(block.block.p_offset)((block.block.p_offset)*vm_page_size);
1654ddprintf ("default_read(%x,%x,%x,%d)\n",addr,size,offset,block.block.p_index);
1655 part = partition_of(block.block.p_index);
1656 first_time = TRUE((boolean_t) 1);
1657 *out_addr = addr;
1658
1659 do {
1660 rc = page_read_file_direct(part->file,
1661 offset,
1662 size,
1663 &raddr,
1664 &rsize);
1665 if (rc != 0)
1666 return (PAGER_ERROR2);
1667
1668 /*
1669 * If we got the entire page on the first read, return it.
1670 */
1671 if (first_time && rsize == size) {
1672 *out_addr = raddr;
1673 break;
1674 }
1675 /*
1676 * Otherwise, copy the data into the
1677 * buffer we were passed, and try for
1678 * the next piece.
1679 */
1680 first_time = FALSE((boolean_t) 0);
1681 bcopy((char *)raddr, (char *)addr, rsize);
1682 addr += rsize;
1683 offset += rsize;
1684 size -= rsize;
1685 } while (size != 0);
1686
1687#if USE_PRECIOUS1
1688 if (deallocate)
1689 pager_release_offset(ds, original_offset);
1690#endif /*USE_PRECIOUS*/
1691
1692#ifdef CHECKSUM
1693 {
1694 int write_checksum,
1695 read_checksum;
1696
1697 write_checksum = pager_get_checksum(ds, original_offset);
1698 read_checksum = compute_checksum(*out_addr, original_size);
1699 if (write_checksum != read_checksum) {
1700 panic(
1701 "PAGER CHECKSUM ERROR: offset 0x%x, written 0x%x, read 0x%x",
1702 original_offset, write_checksum, read_checksum);
1703 }
1704 }
1705#endif /* CHECKSUM */
1706 return (PAGER_SUCCESS0);
1707}
1708
1709int
1710default_write(ds, addr, size, offset)
1711 dpager_t ds;
1712 vm_offset_t addr;
1713 vm_size_t size;
1714 vm_offset_t offset;
1715{
1716 union dp_map block;
1717 partition_t part;
1718 vm_size_t wsize;
1719 int rc;
1720
1721 ddprintf ("default_write: pager offset %x\n", offset);
1722
1723 /*
1724 * Find block in paging partition
1725 */
1726 block = pager_write_offset(ds, offset);
1727 if ( no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)) )
1728 return (PAGER_ERROR2);
1729
1730#ifdef CHECKSUM
1731 /*
1732 * Save checksum
1733 */
1734 {
1735 int checksum;
1736
1737 checksum = compute_checksum(addr, size);
1738 pager_put_checksum(ds, offset, checksum);
1739 }
1740#endif /* CHECKSUM */
1741 offset = ptoa(block.block.p_offset)((block.block.p_offset)*vm_page_size);
1742ddprintf ("default_write(%x,%x,%x,%d)\n",addr,size,offset,block.block.p_index);
1743 part = partition_of(block.block.p_index);
1744
1745 /*
1746 * There are various assumptions made here,we
1747 * will not get into the next disk 'block' by
1748 * accident. It might well be non-contiguous.
1749 */
1750 do {
1751 rc = page_write_file_direct(part->file,
1752 offset,
1753 addr,
1754 size,
1755 &wsize);
1756 if (rc != 0) {
1757 dprintf("*** PAGER ERROR: default_write: ");
1758 dprintf("ds=0x%x addr=0x%x size=0x%x offset=0x%x resid=0x%x\n",
1759 ds, addr, size, offset, wsize);
1760 return (PAGER_ERROR2);
1761 }
1762 addr += wsize;
1763 offset += wsize;
1764 size -= wsize;
1765 } while (size != 0);
1766 return (PAGER_SUCCESS0);
1767}
1768
1769boolean_t
1770default_has_page(ds, offset)
1771 dpager_t ds;
1772 vm_offset_t offset;
1773{
1774 return ( ! no_block(pager_read_offset(ds, offset))((pager_read_offset(ds, offset)).indirect == (dp_map_t)((vm_offset_t
)-1)) );
1775}
1776
1777#if PARALLEL1
1778#define dstruct_lock_init(ds)pthread_mutex_init(&ds->lock, ((void*)0)) pthread_mutex_init(&ds->lock, NULL((void*)0))
1779#define dstruct_lock(ds)pthread_mutex_lock(&ds->lock) pthread_mutex_lock(&ds->lock)
1780#define dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock) pthread_mutex_unlock(&ds->lock)
1781#else /* PARALLEL */
1782#define dstruct_lock_init(ds)pthread_mutex_init(&ds->lock, ((void*)0))
1783#define dstruct_lock(ds)pthread_mutex_lock(&ds->lock)
1784#define dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock)
1785#endif /* PARALLEL */
1786
1787struct pager_port all_pagers;
1788
1789#define pager_port_list_init(){ pthread_mutex_init(&all_pagers.lock, ((void*)0)); hurd_ihash_init
(&all_pagers.htable, __builtin_offsetof(struct dstruct, htable_locp
)); ((&all_pagers.leak_queue)->next = (&all_pagers
.leak_queue)->prev = &all_pagers.leak_queue); } \
1790{ \
1791 pthread_mutex_init(&all_pagers.lock, NULL((void*)0)); \
1792 hurd_ihash_init (&all_pagers.htable, \
1793 offsetof (struct dstruct, htable_locp)__builtin_offsetof(struct dstruct, htable_locp)); \
1794 queue_init(&all_pagers.leak_queue)((&all_pagers.leak_queue)->next = (&all_pagers.leak_queue
)->prev = &all_pagers.leak_queue); \
1795}
1796
1797void pager_port_list_insert(port, ds)
1798 mach_port_t port;
1799 default_pager_t ds;
1800{
1801 pthread_mutex_lock(&all_pagers.lock);
1802 hurd_ihash_add (&all_pagers.htable,
1803 (hurd_ihash_key_t) port,
1804 (hurd_ihash_value_t) ds);
1805 pthread_mutex_unlock(&all_pagers.lock);
1806}
1807
1808void pager_port_list_delete(ds)
1809 default_pager_t ds;
1810{
1811 pthread_mutex_lock(&all_pagers.lock);
1812 hurd_ihash_locp_remove (&all_pagers.htable,
1813 ds->htable_locp);
1814 pthread_mutex_unlock(&all_pagers.lock);
1815}
1816
1817/*
1818 * Destroy a paging partition.
1819 * XXX this is not re-entrant XXX
1820 */
1821kern_return_t
1822destroy_paging_partition(name, pp_private)
1823 char *name;
1824 void **pp_private;
1825{
1826 unsigned int id = part_id(name);
1827 partition_t part = NULL((void*)0);
1828 boolean_t all_ok = TRUE((boolean_t) 1);
1829 default_pager_t entry;
1830 int pindex;
1831
1832 /*
1833 * Find and take partition out of list
1834 * This prevents choose_partition from
1835 * getting in the way.
1836 */
1837 pthread_mutex_lock(&all_partitions.lock);
1838 for (pindex = 0; pindex < all_partitions.n_partitions; pindex++) {
1839 part = partition_of(pindex);
1840 if (part && (part->id == id)) break;
1841 }
1842 if (! part) {
1843 pthread_mutex_unlock(&all_partitions.lock);
1844 return KERN_INVALID_ARGUMENT4;
1845 }
1846 part->going_away = TRUE((boolean_t) 1);
1847 pthread_mutex_unlock(&all_partitions.lock);
1848
1849 /*
1850 * This might take a while..
1851 */
1852all_over_again:
1853#if debug0
1854dprintf("Partition x%x (id x%x) for %s, all_ok %d\n", part, id, name, all_ok);
1855#endif
1856 all_ok = TRUE((boolean_t) 1);
1857 pthread_mutex_lock(&part->p_lock);
1858
1859 pthread_mutex_lock(&all_pagers.lock);
1860 HURD_IHASH_ITERATE (&all_pagers.htable, val)for (hurd_ihash_value_t val, *_hurd_ihash_valuep = (&all_pagers
.htable)->size ? &(&all_pagers.htable)->items[0
].value : 0; (&all_pagers.htable)->size && ((_hurd_ihash_item_t
) _hurd_ihash_valuep) - &(&all_pagers.htable)->items
[0] < (&all_pagers.htable)->size && (val = *
_hurd_ihash_valuep, 1); _hurd_ihash_valuep = (hurd_ihash_value_t
*) (((_hurd_ihash_item_t) _hurd_ihash_valuep) + 1)) if (val !=
((hurd_ihash_value_t) 0) && val != ((hurd_ihash_value_t
) -1)) {
1861 entry = (default_pager_t) val;
1862
1863 dstruct_lock(entry)pthread_mutex_lock(&entry->lock);
1864
1865 if (pthread_mutex_trylock(&entry->dpager.lock)) {
1866
1867 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
1868 pthread_mutex_unlock(&all_pagers.lock);
1869 pthread_mutex_unlock(&part->p_lock);
1870
1871 /* yield the processor */
1872 (void) thread_switch(MACH_PORT_NULL((mach_port_t) 0),
1873 SWITCH_OPTION_NONE0, 0);
1874
1875 goto all_over_again;
1876
1877 }
1878
1879 /*
1880 * See if we can relocate all the pages of this object
1881 * currently on this partition on some other partition
1882 */
1883 all_ok = pager_realloc(&entry->dpager, pindex);
1884
1885 pthread_mutex_unlock(&entry->dpager.lock);
1886 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
1887
1888 if (!all_ok) break;
1889
1890 }
1891 pthread_mutex_unlock(&all_pagers.lock);
1892
1893 if (all_ok) {
1894 /* No need to unlock partition, there are no refs left */
1895
1896 set_partition_of(pindex, 0);
1897 *pp_private = part->file;
1898 kfree(part->bitmap, howmany(part->total_size, NB_BM)(((part->total_size) + (32) - 1)/(32)) * sizeof(bm_entry_t));
1899 kfree(part, sizeof(struct part));
1900 dprintf("%s Removed paging partition %s\n", my_name, name);
1901 return KERN_SUCCESS0;
1902 }
1903
1904 /*
1905 * Put partition back in.
1906 */
1907 part->going_away = FALSE((boolean_t) 0);
1908
1909 return KERN_FAILURE5;
1910}
1911
1912
1913/*
1914 * We use the sequence numbers on requests to regulate
1915 * our parallelism. In general, we allow multiple reads and writes
1916 * to proceed in parallel, with the exception that reads must
1917 * wait for previous writes to finish. (Because the kernel might
1918 * generate a data-request for a page on the heels of a data-write
1919 * for the same page, and we must avoid returning stale data.)
1920 * terminate requests wait for proceeding reads and writes to finish.
1921 */
1922
1923unsigned int default_pager_total = 0; /* debugging */
1924unsigned int default_pager_wait_seqno = 0; /* debugging */
1925unsigned int default_pager_wait_read = 0; /* debugging */
1926unsigned int default_pager_wait_write = 0; /* debugging */
1927unsigned int default_pager_wait_refs = 0; /* debugging */
1928
1929#if PARALLEL1
1930/*
1931 * Waits for correct sequence number. Leaves pager locked.
1932 */
1933void pager_port_lock(ds, seqno)
1934 default_pager_t ds;
1935 mach_port_seqno_t seqno;
1936{
1937 default_pager_total++;
1938 dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
1939 while (ds->seqno != seqno) {
1940 default_pager_wait_seqno++;
1941 pthread_cond_wait(&ds->waiting_seqno, &ds->lock);
1942 }
1943}
1944
1945/*
1946 * Increments sequence number and unlocks pager.
1947 */
1948void pager_port_unlock(ds)
1949 default_pager_t ds;
1950{
1951 ds->seqno++;
1952 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
1953 pthread_cond_broadcast(&ds->waiting_seqno);
1954}
1955
1956/*
1957 * Start a read - one more reader. Pager must be locked.
1958 */
1959void pager_port_start_read(ds)
1960 default_pager_t ds;
1961{
1962 ds->readers++;
1963}
1964
1965/*
1966 * Wait for readers. Unlocks and relocks pager if wait needed.
1967 */
1968void pager_port_wait_for_readers(ds)
1969 default_pager_t ds;
1970{
1971 while (ds->readers != 0) {
1972 default_pager_wait_read++;
1973 pthread_cond_wait(&ds->waiting_read, &ds->lock);
1974 }
1975}
1976
1977/*
1978 * Finish a read. Pager is unlocked and returns unlocked.
1979 */
1980void pager_port_finish_read(ds)
1981 default_pager_t ds;
1982{
1983 dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
1984 if (--ds->readers == 0) {
1985 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
1986 pthread_cond_broadcast(&ds->waiting_read);
1987 }
1988 else {
1989 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
1990 }
1991}
1992
1993/*
1994 * Start a write - one more writer. Pager must be locked.
1995 */
1996void pager_port_start_write(ds)
1997 default_pager_t ds;
1998{
1999 ds->writers++;
2000}
2001
2002/*
2003 * Wait for writers. Unlocks and relocks pager if wait needed.
2004 */
2005void pager_port_wait_for_writers(ds)
2006 default_pager_t ds;
2007{
2008 while (ds->writers != 0) {
2009 default_pager_wait_write++;
2010 pthread_cond_wait(&ds->waiting_write, &ds->lock);
2011 }
2012}
2013
2014/*
2015 * Finish a write. Pager is unlocked and returns unlocked.
2016 */
2017void pager_port_finish_write(ds)
2018 default_pager_t ds;
2019{
2020 dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2021 if (--ds->writers == 0) {
2022 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2023 pthread_cond_broadcast(&ds->waiting_write);
2024 }
2025 else {
2026 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2027 }
2028}
2029
2030/*
2031 * Wait for concurrent default_pager_objects.
2032 * Unlocks and relocks pager if wait needed.
2033 */
2034void pager_port_wait_for_refs(ds)
2035 default_pager_t ds;
2036{
2037 while (ds->name_refs == 0) {
2038 default_pager_wait_refs++;
2039 pthread_cond_wait(&ds->waiting_refs, &ds->lock);
2040 }
2041}
2042
2043/*
2044 * Finished creating name refs - wake up waiters.
2045 */
2046void pager_port_finish_refs(ds)
2047 default_pager_t ds;
2048{
2049 pthread_cond_broadcast(&ds->waiting_refs);
2050}
2051
2052#else /* PARALLEL */
2053
2054#define pager_port_lock(ds,seqno)
2055#define pager_port_unlock(ds)
2056#define pager_port_start_read(ds)
2057#define pager_port_wait_for_readers(ds)
2058#define pager_port_finish_read(ds)
2059#define pager_port_start_write(ds)
2060#define pager_port_wait_for_writers(ds)
2061#define pager_port_finish_write(ds)
2062#define pager_port_wait_for_refs(ds)
2063#define pager_port_finish_refs(ds)
2064
2065#endif /* PARALLEL */
2066
2067/*
2068 * Default pager.
2069 */
2070task_t default_pager_self; /* Our task port. */
2071
2072mach_port_t default_pager_default_port; /* Port for memory_object_create. */
2073
2074/* We catch exceptions on ourself & startup using this port. */
2075mach_port_t default_pager_exception_port;
2076
2077mach_port_t default_pager_internal_set; /* Port set for internal objects. */
2078mach_port_t default_pager_external_set; /* Port set for external objects. */
2079mach_port_t default_pager_default_set; /* Port set for "default" thread. */
2080
2081typedef struct default_pager_thread {
2082 pthread_t dpt_thread; /* Server thread. */
2083 vm_offset_t dpt_buffer; /* Read buffer. */
2084 boolean_t dpt_internal; /* Do we handle internal objects? */
2085} default_pager_thread_t;
2086
2087#if PARALLEL1
2088 /* determine number of threads at run time */
2089#define DEFAULT_PAGER_INTERNAL_COUNT(0) (0)
2090
2091#else /* PARALLEL */
2092#define DEFAULT_PAGER_INTERNAL_COUNT(0) (1)
2093#endif /* PARALLEL */
2094
2095/* Memory created by default_pager_object_create should mostly be resident. */
2096#define DEFAULT_PAGER_EXTERNAL_COUNT(1) (1)
2097
2098unsigned int default_pager_internal_count = DEFAULT_PAGER_INTERNAL_COUNT(0);
2099 /* Number of "internal" threads. */
2100unsigned int default_pager_external_count = DEFAULT_PAGER_EXTERNAL_COUNT(1);
2101 /* Number of "external" threads. */
2102
2103default_pager_t pager_port_alloc(size)
2104 vm_size_t size;
2105{
2106 default_pager_t ds;
2107 p_index_t part;
2108
2109 ds = (default_pager_t) kalloc(sizeof *ds);
2110 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2111 panic("%spager_port_alloc",my_name);
2112 bzero((char *) ds, sizeof *ds);
2113
2114 dstruct_lock_init(ds)pthread_mutex_init(&ds->lock, ((void*)0));
2115
2116 /*
2117 * Get a suitable partition. If none big enough
2118 * just pick one and overcommit. If no partitions
2119 * at all.. well just fake one so that we will
2120 * kill specific objects on pageouts rather than
2121 * panicing the system now.
2122 */
2123 part = choose_partition(size, P_INDEX_INVALID((p_index_t)-1));
2124 if (no_partition(part)((part) == ((p_index_t)-1))) {
2125 overcommitted(FALSE((boolean_t) 0), atop(size)((size)/vm_page_size));
2126 part = choose_partition(0,P_INDEX_INVALID((p_index_t)-1));
2127#if debug0
2128 if (no_partition(part)((part) == ((p_index_t)-1)))
2129 dprintf("%s No paging space at all !!\n", my_name);
2130#endif
2131 }
2132 pager_alloc(&ds->dpager, part, size);
2133
2134 return ds;
2135}
2136
2137mach_port_urefs_t default_pager_max_urefs = 10000;
2138
2139/*
2140 * Check user reference count on pager_request port.
2141 * Pager must be locked.
2142 * Unlocks and re-locks pager if needs to call kernel.
2143 */
2144void pager_port_check_request(ds, pager_request)
2145 default_pager_t ds;
2146 mach_port_t pager_request;
2147{
2148 mach_port_delta_t delta;
2149 kern_return_t kr;
2150
2151 assert(ds->pager_request == pager_request)((ds->pager_request == pager_request) ? (void) (0) : __assert_fail
("ds->pager_request == pager_request", "../../mach-defpager/default_pager.c"
, 2151, __PRETTY_FUNCTION__));
2152
2153 if (++ds->request_refs > default_pager_max_urefs) {
2154 delta = 1 - ds->request_refs;
2155 ds->request_refs = 1;
2156
2157 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2158
2159 /*
2160 * Deallocate excess user references.
2161 */
2162
2163 kr = mach_port_mod_refs(default_pager_self, pager_request,
2164 MACH_PORT_RIGHT_SEND((mach_port_right_t) 0), delta);
2165 if (kr != KERN_SUCCESS0)
2166 panic("%spager_port_check_request",my_name);
2167
2168 dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2169 }
2170}
2171
2172void default_pager_add(ds, internal)
2173 default_pager_t ds;
2174 boolean_t internal;
2175{
2176 mach_port_t pager = ds->pager;
2177 mach_port_t pset;
2178 mach_port_mscount_t sync;
2179 mach_port_t previous;
2180 kern_return_t kr;
2181 static char here[] = "%sdefault_pager_add";
2182
2183 /*
2184 * The port currently has a make-send count of zero,
2185 * because either we just created the port or we just
2186 * received the port in a memory_object_create request.
2187 */
2188
2189 if (internal) {
2190 /* possibly generate an immediate no-senders notification */
2191 sync = 0;
2192 pset = default_pager_internal_set;
2193 ds->external = FALSE((boolean_t) 0);
2194 } else {
2195 /* delay notification till send right is created */
2196 sync = 1;
2197 pset = default_pager_external_set;
2198 ds->external = TRUE((boolean_t) 1);
2199 }
2200
2201 kr = mach_port_request_notification(default_pager_self, pager,
2202 MACH_NOTIFY_NO_SENDERS(0100 + 006), sync,
2203 pager, MACH_MSG_TYPE_MAKE_SEND_ONCE21,
2204 &previous);
2205 if ((kr != KERN_SUCCESS0) || (previous != MACH_PORT_NULL((mach_port_t) 0)))
2206 panic(here,my_name);
2207
2208 kr = mach_port_move_member(default_pager_self, pager, pset);
2209 if (kr != KERN_SUCCESS0)
2210 panic(here,my_name);
2211}
2212
2213/*
2214 * Routine: memory_object_create
2215 * Purpose:
2216 * Handle requests for memory objects from the
2217 * kernel.
2218 * Notes:
2219 * Because we only give out the default memory
2220 * manager port to the kernel, we don't have to
2221 * be so paranoid about the contents.
2222 */
2223kern_return_t
2224seqnos_memory_object_create(old_pager, seqno, new_pager, new_size,
2225 new_pager_request, new_pager_name, new_page_size)
2226 mach_port_t old_pager;
2227 mach_port_seqno_t seqno;
2228 mach_port_t new_pager;
2229 vm_size_t new_size;
2230 mach_port_t new_pager_request;
2231 mach_port_t new_pager_name;
2232 vm_size_t new_page_size;
2233{
2234 default_pager_t ds;
2235
2236 assert(old_pager == default_pager_default_port)((old_pager == default_pager_default_port) ? (void) (0) : __assert_fail
("old_pager == default_pager_default_port", "../../mach-defpager/default_pager.c"
, 2236, __PRETTY_FUNCTION__));
2237 assert(MACH_PORT_VALID(new_pager_request))(((((new_pager_request) != ((mach_port_t) 0)) && ((new_pager_request
) != ((mach_port_t) ~0)))) ? (void) (0) : __assert_fail ("(((new_pager_request) != ((mach_port_t) 0)) && ((new_pager_request) != ((mach_port_t) ~0)))"
, "../../mach-defpager/default_pager.c", 2237, __PRETTY_FUNCTION__
));
2238 assert(MACH_PORT_VALID(new_pager_name))(((((new_pager_name) != ((mach_port_t) 0)) && ((new_pager_name
) != ((mach_port_t) ~0)))) ? (void) (0) : __assert_fail ("(((new_pager_name) != ((mach_port_t) 0)) && ((new_pager_name) != ((mach_port_t) ~0)))"
, "../../mach-defpager/default_pager.c", 2238, __PRETTY_FUNCTION__
));
2239 assert(new_page_size == vm_page_size)((new_page_size == vm_page_size) ? (void) (0) : __assert_fail
("new_page_size == vm_page_size", "../../mach-defpager/default_pager.c"
, 2239, __PRETTY_FUNCTION__));
2240
2241 ds = pager_port_alloc(new_size);
2242
2243 /*
2244 * Set up associations between these ports
2245 * and this default_pager structure
2246 */
2247
2248 ds->pager = new_pager;
2249 ds->pager_request = new_pager_request;
2250 ds->request_refs = 1;
2251 ds->pager_name = new_pager_name;
2252 ds->name_refs = 1;
2253
2254 /*
2255 * After this, other threads might receive requests
2256 * for this memory object or find it in the port list.
2257 */
2258
2259 pager_port_list_insert(new_pager, ds);
2260 default_pager_add(ds, TRUE((boolean_t) 1));
2261
2262 return(KERN_SUCCESS0);
2263}
2264
2265memory_object_copy_strategy_t default_pager_copy_strategy =
2266 MEMORY_OBJECT_COPY_DELAY2;
2267
2268kern_return_t
2269seqnos_memory_object_init(ds, seqno, pager_request, pager_name,
2270 pager_page_size)
2271 default_pager_t ds;
2272 mach_port_seqno_t seqno;
2273 mach_port_t pager_request;
2274 mach_port_t pager_name;
2275 vm_size_t pager_page_size;
2276{
2277 kern_return_t kr;
2278 static char here[] = "%sinit";
2279
2280 assert(MACH_PORT_VALID(pager_request))(((((pager_request) != ((mach_port_t) 0)) && ((pager_request
) != ((mach_port_t) ~0)))) ? (void) (0) : __assert_fail ("(((pager_request) != ((mach_port_t) 0)) && ((pager_request) != ((mach_port_t) ~0)))"
, "../../mach-defpager/default_pager.c", 2280, __PRETTY_FUNCTION__
));
2281 assert(MACH_PORT_VALID(pager_name))(((((pager_name) != ((mach_port_t) 0)) && ((pager_name
) != ((mach_port_t) ~0)))) ? (void) (0) : __assert_fail ("(((pager_name) != ((mach_port_t) 0)) && ((pager_name) != ((mach_port_t) ~0)))"
, "../../mach-defpager/default_pager.c", 2281, __PRETTY_FUNCTION__
));
2282 assert(pager_page_size == vm_page_size)((pager_page_size == vm_page_size) ? (void) (0) : __assert_fail
("pager_page_size == vm_page_size", "../../mach-defpager/default_pager.c"
, 2282, __PRETTY_FUNCTION__));
2283
2284 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2285 panic(here, my_name);
2286 pager_port_lock(ds, seqno);
2287
2288 if (ds->pager_request != MACH_PORT_NULL((mach_port_t) 0))
2289 panic(here, my_name);
2290
2291 ds->pager_request = pager_request;
2292 ds->request_refs = 1;
2293 ds->pager_name = pager_name;
2294 ds->name_refs = 1;
2295
2296 /*
2297 * Even if the kernel immediately terminates the object,
2298 * the pager_request port won't be destroyed until
2299 * we process the terminate request, which won't happen
2300 * until we unlock the object.
2301 */
2302
2303 kr = memory_object_ready(pager_request,
2304 FALSE((boolean_t) 0), /* Do not cache */
2305 default_pager_copy_strategy);
2306 if (kr != KERN_SUCCESS0)
2307 panic(here, my_name);
2308
2309 pager_port_unlock(ds);
2310
2311 return(KERN_SUCCESS0);
2312}
2313
2314kern_return_t
2315seqnos_memory_object_terminate(ds, seqno, pager_request, pager_name)
2316 default_pager_t ds;
2317 mach_port_seqno_t seqno;
2318 mach_port_t pager_request;
2319 mach_port_t pager_name;
2320{
2321 static char here[] = "%sterminate";
2322
2323 /*
2324 * pager_request and pager_name are receive rights,
2325 * not send rights.
2326 */
2327
2328 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2329 panic(here, my_name);
2330ddprintf ("seqnos_memory_object_terminate <%p>: pager_port_lock: <%p>[s:%d,r:%d,w:%d,l:%d], %d\n",
2331 &ds, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held, seqno);
2332 pager_port_lock(ds, seqno);
2333
2334 /*
2335 * Wait for read and write requests to terminate.
2336 */
2337
2338 pager_port_wait_for_readers(ds);
2339 pager_port_wait_for_writers(ds);
2340
2341 /*
2342 * After memory_object_terminate both memory_object_init
2343 * and a no-senders notification are possible, so we need
2344 * to clean up the request and name ports but leave
2345 * the pager port.
2346 *
2347 * A concurrent default_pager_objects might be allocating
2348 * more references for the name port. In this case,
2349 * we must first wait for it to finish.
2350 */
2351
2352 pager_port_wait_for_refs(ds);
2353
2354 if (ds->external)
2355 pager_request = ds->pager_request;
2356 ds->pager_request = MACH_PORT_NULL((mach_port_t) 0);
2357 ds->request_refs = 0;
2358 assert(ds->pager_name == pager_name)((ds->pager_name == pager_name) ? (void) (0) : __assert_fail
("ds->pager_name == pager_name", "../../mach-defpager/default_pager.c"
, 2358, __PRETTY_FUNCTION__));
2359 ds->pager_name = MACH_PORT_NULL((mach_port_t) 0);
2360 ds->name_refs = 0;
2361ddprintf ("seqnos_memory_object_terminate <%p>: pager_port_unlock: <%p>[s:%d,r:%d,w:%d,l:%d]\n",
2362 &ds, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held);
2363 pager_port_unlock(ds);
2364
2365 /*
2366 * Now we destroy our port rights.
2367 */
2368
2369 mach_port_destroy(mach_task_self()((__mach_task_self_ + 0)), pager_request);
2370 mach_port_destroy(mach_task_self()((__mach_task_self_ + 0)), pager_name);
2371
2372 return (KERN_SUCCESS0);
2373}
2374
2375void default_pager_no_senders(pager, seqno, mscount)
2376 memory_object_t pager;
2377 mach_port_seqno_t seqno;
2378 mach_port_mscount_t mscount;
2379{
2380 default_pager_t ds;
2381 kern_return_t kr;
2382 static char here[] = "%sno_senders";
2383
2384 /*
2385 * Because we don't give out multiple send rights
2386 * for a memory object, there can't be a race
2387 * between getting a no-senders notification
2388 * and creating a new send right for the object.
2389 * Hence we don't keep track of mscount.
2390 */
2391
2392
2393 ds = begin_using_default_pager(pager);
2394 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2395 panic(here,my_name);
2396 pager_port_lock(ds, seqno);
2397
2398 /*
2399 * We shouldn't get a no-senders notification
2400 * when the kernel has the object cached.
2401 */
2402
2403 if (ds->pager_request != MACH_PORT_NULL((mach_port_t) 0))
2404 panic(here,my_name);
2405
2406 /*
2407 * Unlock the pager (though there should be no one
2408 * waiting for it).
2409 */
2410 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2411
2412 /*
2413 * Remove the memory object port association, and then
2414 * the destroy the port itself. We must remove the object
2415 * from the port list before deallocating the pager,
2416 * because of default_pager_objects.
2417 */
2418
2419 pager_port_list_delete(ds);
2420 pager_dealloc(&ds->dpager);
2421
2422 kr = mach_port_mod_refs(default_pager_self, pager,
2423 MACH_PORT_RIGHT_RECEIVE((mach_port_right_t) 1), -1);
2424 if (kr != KERN_SUCCESS0)
2425 panic(here,my_name);
2426
2427 /*
2428 * Do this *after* deallocating the port name
2429 */
2430 kfree((char *) ds, sizeof(*ds));
2431
2432 /*
2433 * Recover memory that we might have wasted because
2434 * of name conflicts
2435 */
2436 pthread_mutex_lock(&all_pagers.lock);
2437
2438 while (!queue_empty(&all_pagers.leak_queue)(((&all_pagers.leak_queue)) == (((&all_pagers.leak_queue
)->next)))) {
2439
2440 ds = (default_pager_t) queue_first(&all_pagers.leak_queue)((&all_pagers.leak_queue)->next);
Value stored to 'ds' is never read
2441 queue_remove_first(&all_pagers.leak_queue, ds, default_pager_t, links){ queue_entry_t next; (ds) = (default_pager_t) ((&all_pagers
.leak_queue)->next); next = (ds)->links.next; if ((&
all_pagers.leak_queue) == next) (&all_pagers.leak_queue)->
prev = (&all_pagers.leak_queue); else ((default_pager_t)(
next))->links.prev = (&all_pagers.leak_queue); (&all_pagers
.leak_queue)->next = next; };
2442 kfree((char *) ds, sizeof(*ds));
2443 }
2444
2445 pthread_mutex_unlock(&all_pagers.lock);
2446}
2447
2448int default_pager_pagein_count = 0;
2449int default_pager_pageout_count = 0;
2450
2451static __thread default_pager_thread_t *dpt;
2452
2453kern_return_t
2454seqnos_memory_object_data_request(ds, seqno, reply_to, offset,
2455 length, protection_required)
2456 default_pager_t ds;
2457 mach_port_seqno_t seqno;
2458 mach_port_t reply_to;
2459 vm_offset_t offset;
2460 vm_size_t length;
2461 vm_prot_t protection_required;
2462{
2463 vm_offset_t addr;
2464 unsigned int errors;
2465 kern_return_t rc;
2466 static char here[] = "%sdata_request";
2467
2468 if (length != vm_page_size)
2469 panic(here,my_name);
2470
2471 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2472 panic(here,my_name);
2473ddprintf ("seqnos_memory_object_data_request <%p>: pager_port_lock: <%p>[s:%d,r:%d,w:%d,l:%d], %d\n",
2474 &ds, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held, seqno);
2475 pager_port_lock(ds, seqno);
2476 pager_port_check_request(ds, reply_to);
2477 pager_port_wait_for_writers(ds);
2478 pager_port_start_read(ds);
2479
2480 /*
2481 * Get error count while pager locked.
2482 */
2483 errors = ds->errors;
2484
2485ddprintf ("seqnos_memory_object_data_request <%p>: pager_port_unlock: <%p>[s:%d,r:%d,w:%d,l:%d]\n",
2486 &ds, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held);
2487 pager_port_unlock(ds);
2488
2489 if (errors) {
2490 dprintf("%s %s\n", my_name,
2491 "dropping data_request because of previous paging errors");
2492 (void) memory_object_data_error(reply_to,
2493 offset, vm_page_size,
2494 KERN_FAILURE5);
2495 goto done;
2496 }
2497
2498 if (offset >= ds->dpager.limit)
2499 rc = PAGER_ERROR2;
2500 else
2501 rc = default_read(&ds->dpager, dpt->dpt_buffer,
2502 vm_page_size, offset,
2503 &addr, protection_required & VM_PROT_WRITE((vm_prot_t) 0x02),
2504 ds->external);
2505
2506 switch (rc) {
2507 case PAGER_SUCCESS0:
2508 if (addr != dpt->dpt_buffer) {
2509 /*
2510 * Deallocates data buffer
2511 */
2512 (void) memory_object_data_supply(
2513 reply_to, offset,
2514 addr, vm_page_size, TRUE((boolean_t) 1),
2515 VM_PROT_NONE((vm_prot_t) 0x00),
2516 FALSE((boolean_t) 0), MACH_PORT_NULL((mach_port_t) 0));
2517 } else {
2518 (void) memory_object_data_supply(
2519 reply_to, offset,
2520 addr, vm_page_size, FALSE((boolean_t) 0),
2521 VM_PROT_NONE((vm_prot_t) 0x00),
2522 FALSE((boolean_t) 0), MACH_PORT_NULL((mach_port_t) 0));
2523 }
2524 break;
2525
2526 case PAGER_ABSENT1:
2527 (void) memory_object_data_unavailable(
2528 reply_to,
2529 offset,
2530 vm_page_size);
2531 break;
2532
2533 case PAGER_ERROR2:
2534 (void) memory_object_data_error(
2535 reply_to,
2536 offset,
2537 vm_page_size,
2538 KERN_FAILURE5);
2539 break;
2540 }
2541
2542 default_pager_pagein_count++;
2543
2544 done:
2545 pager_port_finish_read(ds);
2546 return(KERN_SUCCESS0);
2547}
2548
2549/*
2550 * memory_object_data_initialize: check whether we already have each page, and
2551 * write it if we do not. The implementation is far from optimized, and
2552 * also assumes that the default_pager is single-threaded.
2553 */
2554kern_return_t
2555seqnos_memory_object_data_initialize(ds, seqno, pager_request,
2556 offset, addr, data_cnt)
2557 default_pager_t ds;
2558 mach_port_seqno_t seqno;
2559 mach_port_t pager_request;
2560 register
2561 vm_offset_t offset;
2562 register
2563 pointer_t addr;
2564 vm_size_t data_cnt;
2565{
2566 vm_offset_t amount_sent;
2567 static char here[] = "%sdata_initialize";
2568
2569#ifdef lint
2570 pager_request++;
2571#endif /* lint */
2572
2573 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2574 panic(here,my_name);
2575ddprintf ("seqnos_memory_object_data_initialize <%p>: pager_port_lock: <%p>[s:%d,r:%d,w:%d,l:%d], %d\n",
2576 &ds, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held, seqno);
2577 pager_port_lock(ds, seqno);
2578 pager_port_check_request(ds, pager_request);
2579 pager_port_start_write(ds);
2580ddprintf ("seqnos_memory_object_data_initialize <%p>: pager_port_unlock: <%p>[s:%d,r:%d,w:%d,l:%d]\n",
2581 &ds, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held);
2582 pager_port_unlock(ds);
2583
2584 for (amount_sent = 0;
2585 amount_sent < data_cnt;
2586 amount_sent += vm_page_size) {
2587
2588 if (!default_has_page(&ds->dpager, offset + amount_sent)) {
2589 if (default_write(&ds->dpager,
2590 addr + amount_sent,
2591 vm_page_size,
2592 offset + amount_sent)
2593 != PAGER_SUCCESS0) {
2594 dprintf("%s%s write error\n", my_name, here);
2595 dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2596 ds->errors++;
2597 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2598 }
2599 }
2600 }
2601
2602 pager_port_finish_write(ds);
2603 if (vm_deallocate(default_pager_self, addr, data_cnt) != KERN_SUCCESS0)
2604 panic(here,my_name);
2605
2606 return(KERN_SUCCESS0);
2607}
2608
2609/*
2610 * memory_object_data_write: split up the stuff coming in from
2611 * a memory_object_data_write call
2612 * into individual pages and pass them off to default_write.
2613 */
2614kern_return_t
2615seqnos_memory_object_data_write(ds, seqno, pager_request,
2616 offset, addr, data_cnt)
2617 default_pager_t ds;
2618 mach_port_seqno_t seqno;
2619 mach_port_t pager_request;
2620 register
2621 vm_offset_t offset;
2622 register
2623 pointer_t addr;
2624 vm_size_t data_cnt;
2625{
2626 register
2627 vm_size_t amount_sent;
2628 static char here[] = "%sdata_write";
2629 int err;
2630
2631#ifdef lint
2632 pager_request++;
2633#endif /* lint */
2634
2635 if ((data_cnt % vm_page_size) != 0)
2636 panic(here,my_name);
2637
2638 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2639 panic(here,my_name);
2640
2641 pager_port_lock(ds, seqno);
2642 pager_port_start_write(ds);
2643
2644 vm_size_t limit = ds->dpager.byte_limit;
2645 pager_port_unlock(ds);
2646 if ((limit != round_page(limit)((((vm_offset_t) (limit) + __vm_page_size - 1) / __vm_page_size
) * __vm_page_size)) && (trunc_page(limit)((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size) == offset)) {
2647 assert(trunc_page(limit) == offset)((((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size
) == offset) ? (void) (0) : __assert_fail ("((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size) == offset"
, "../../mach-defpager/default_pager.c", 2647, __PRETTY_FUNCTION__
));
2648 assert(data_cnt == vm_page_size)((data_cnt == vm_page_size) ? (void) (0) : __assert_fail ("data_cnt == vm_page_size"
, "../../mach-defpager/default_pager.c", 2648, __PRETTY_FUNCTION__
));
2649
2650 vm_offset_t tail = addr + limit - trunc_page(limit)((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size);
2651 vm_size_t tail_size = round_page(limit)((((vm_offset_t) (limit) + __vm_page_size - 1) / __vm_page_size
) * __vm_page_size) - limit;
2652 memset((void *) tail, 0, tail_size);
2653
2654 memory_object_data_supply(pager_request, trunc_page(limit)((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size), addr,
2655 vm_page_size, TRUE((boolean_t) 1), VM_PROT_NONE((vm_prot_t) 0x00),
2656 TRUE((boolean_t) 1), MACH_PORT_NULL((mach_port_t) 0));
2657 dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2658 ds->dpager.byte_limit = round_page(limit)((((vm_offset_t) (limit) + __vm_page_size - 1) / __vm_page_size
) * __vm_page_size);
2659 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2660 pager_port_finish_write(ds);
2661
2662 return(KERN_SUCCESS0);
2663 }
2664
2665 for (amount_sent = 0;
2666 amount_sent < data_cnt;
2667 amount_sent += vm_page_size) {
2668
2669 int result;
2670
2671 result = default_write(&ds->dpager,
2672 addr + amount_sent,
2673 vm_page_size,
2674 offset + amount_sent);
2675 if (result != KERN_SUCCESS0) {
2676 dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2677 ds->errors++;
2678 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2679 }
2680 default_pager_pageout_count++;
2681 }
2682
2683 pager_port_finish_write(ds);
2684 err = vm_deallocate(default_pager_self, addr, data_cnt);
2685 if (err != KERN_SUCCESS0)
2686 {
2687 panic(here,my_name);
2688 }
2689
2690 return(KERN_SUCCESS0);
2691}
2692
2693/*ARGSUSED*/
2694kern_return_t
2695seqnos_memory_object_copy(old_memory_object, seqno, old_memory_control,
2696 offset, length, new_memory_object)
2697 default_pager_t old_memory_object;
2698 mach_port_seqno_t seqno;
2699 memory_object_control_t
2700 old_memory_control;
2701 vm_offset_t offset;
2702 vm_size_t length;
2703 memory_object_t new_memory_object;
2704{
2705 panic("%scopy", my_name);
2706 return KERN_FAILURE5;
2707}
2708
2709/* We get this when our memory_object_lock_request has completed
2710 after we truncated an object. */
2711kern_return_t
2712seqnos_memory_object_lock_completed (default_pager_t ds,
2713 mach_port_seqno_t seqno,
2714 mach_port_t pager_request,
2715 vm_offset_t offset,
2716 vm_size_t length)
2717{
2718 panic("%slock_completed",my_name);
2719 return KERN_FAILURE5;
2720}
2721
2722kern_return_t
2723seqnos_memory_object_data_unlock(pager, seqno, pager_request,
2724 offset, length, protection_required)
2725 default_pager_t pager;
2726 mach_port_seqno_t seqno;
2727 mach_port_t pager_request;
2728 vm_offset_t offset;
2729 vm_size_t length;
2730 vm_prot_t protection_required;
2731{
2732 panic("%sdata_unlock",my_name);
2733 return(KERN_FAILURE5);
2734}
2735
2736kern_return_t
2737seqnos_memory_object_supply_completed(ds, seqno, pager_request,
2738 offset, length,
2739 result, error_offset)
2740 default_pager_t ds;
2741 mach_port_seqno_t seqno;
2742 mach_port_t pager_request;
2743 vm_offset_t offset;
2744 vm_size_t length;
2745 kern_return_t result;
2746 vm_offset_t error_offset;
2747{
2748 panic("%ssupply_completed",my_name);
2749 return(KERN_FAILURE5);
2750}
2751
2752/*
2753 * memory_object_data_return: split up the stuff coming in from
2754 * a memory_object_data_write call
2755 * into individual pages and pass them off to default_write.
2756 */
2757kern_return_t
2758seqnos_memory_object_data_return(ds, seqno, pager_request,
2759 offset, addr, data_cnt,
2760 dirty, kernel_copy)
2761 default_pager_t ds;
2762 mach_port_seqno_t seqno;
2763 mach_port_t pager_request;
2764 vm_offset_t offset;
2765 pointer_t addr;
2766 vm_size_t data_cnt;
2767 boolean_t dirty;
2768 boolean_t kernel_copy;
2769{
2770
2771 return seqnos_memory_object_data_write (ds, seqno, pager_request,
2772 offset, addr, data_cnt);
2773}
2774
2775kern_return_t
2776seqnos_memory_object_change_completed(ds, seqno, may_cache, copy_strategy)
2777 default_pager_t ds;
2778 mach_port_seqno_t seqno;
2779 boolean_t may_cache;
2780 memory_object_copy_strategy_t copy_strategy;
2781{
2782 panic("%schange_completed",my_name);
2783 return(KERN_FAILURE5);
2784}
2785
2786
2787boolean_t default_pager_notify_server(in, out)
2788 mach_msg_header_t *in, *out;
2789{
2790 mach_no_senders_notification_t *n =
2791 (mach_no_senders_notification_t *) in;
2792
2793 /*
2794 * The only send-once rights we create are for
2795 * receiving no-more-senders notifications.
2796 * Hence, if we receive a message directed to
2797 * a send-once right, we can assume it is
2798 * a genuine no-senders notification from the kernel.
2799 */
2800
2801 if ((n->not_header.msgh_bits !=
2802 MACH_MSGH_BITS(0, MACH_MSG_TYPE_PORT_SEND_ONCE)((0) | ((18) << 8))) ||
2803 (n->not_header.msgh_id != MACH_NOTIFY_NO_SENDERS(0100 + 006)))
2804 return FALSE((boolean_t) 0);
2805
2806 assert(n->not_header.msgh_size == sizeof *n)((n->not_header.msgh_size == sizeof *n) ? (void) (0) : __assert_fail
("n->not_header.msgh_size == sizeof *n", "../../mach-defpager/default_pager.c"
, 2806, __PRETTY_FUNCTION__));
2807 assert(n->not_header.msgh_remote_port == MACH_PORT_NULL)((n->not_header.msgh_remote_port == ((mach_port_t) 0)) ? (
void) (0) : __assert_fail ("n->not_header.msgh_remote_port == ((mach_port_t) 0)"
, "../../mach-defpager/default_pager.c", 2807, __PRETTY_FUNCTION__
));
2808
2809 assert(n->not_type.msgt_name == MACH_MSG_TYPE_INTEGER_32)((n->not_type.msgt_name == 2) ? (void) (0) : __assert_fail
("n->not_type.msgt_name == 2", "../../mach-defpager/default_pager.c"
, 2809, __PRETTY_FUNCTION__));
2810 assert(n->not_type.msgt_size == 32)((n->not_type.msgt_size == 32) ? (void) (0) : __assert_fail
("n->not_type.msgt_size == 32", "../../mach-defpager/default_pager.c"
, 2810, __PRETTY_FUNCTION__));
2811 assert(n->not_type.msgt_number == 1)((n->not_type.msgt_number == 1) ? (void) (0) : __assert_fail
("n->not_type.msgt_number == 1", "../../mach-defpager/default_pager.c"
, 2811, __PRETTY_FUNCTION__));
2812 assert(n->not_type.msgt_inline)((n->not_type.msgt_inline) ? (void) (0) : __assert_fail ("n->not_type.msgt_inline"
, "../../mach-defpager/default_pager.c", 2812, __PRETTY_FUNCTION__
));
2813 assert(! n->not_type.msgt_longform)((! n->not_type.msgt_longform) ? (void) (0) : __assert_fail
("! n->not_type.msgt_longform", "../../mach-defpager/default_pager.c"
, 2813, __PRETTY_FUNCTION__));
2814
2815 default_pager_no_senders(n->not_header.msgh_local_port,
2816 n->not_header.msgh_seqno, n->not_count);
2817
2818 out->msgh_remote_port = MACH_PORT_NULL((mach_port_t) 0);
2819 return TRUE((boolean_t) 1);
2820}
2821
2822extern boolean_t seqnos_memory_object_default_server();
2823extern boolean_t default_pager_server();
2824extern boolean_t exc_server();
2825extern boolean_t bootstrap_server();
2826extern void bootstrap_compat();
2827
2828mach_msg_size_t default_pager_msg_size_object = 128;
2829
2830/* Fill in default response. */
2831static void
2832mig_reply_setup (
2833 const mach_msg_header_t *in,
2834 mach_msg_header_t *out)
2835{
2836 static const mach_msg_type_t RetCodeType = {
2837 /* msgt_name = */ MACH_MSG_TYPE_INTEGER_322,
2838 /* msgt_size = */ 32,
2839 /* msgt_number = */ 1,
2840 /* msgt_inline = */ TRUE((boolean_t) 1),
2841 /* msgt_longform = */ FALSE((boolean_t) 0),
2842 /* msgt_deallocate = */ FALSE((boolean_t) 0),
2843 /* msgt_unused = */ 0
2844 };
2845
2846#define InP (in)
2847#define OutP ((mig_reply_header_t *) out)
2848 OutP->Head.msgh_bits =
2849 MACH_MSGH_BITS(MACH_MSGH_BITS_REMOTE(InP->msgh_bits), 0)((((InP->msgh_bits) & 0x000000ff)) | ((0) << 8));
2850 OutP->Head.msgh_size = sizeof *OutP;
2851 OutP->Head.msgh_remote_port = InP->msgh_remote_port;
2852 OutP->Head.msgh_local_port = MACH_PORT_NULL((mach_port_t) 0);
2853 OutP->Head.msgh_seqno = 0;
2854 OutP->Head.msgh_id = InP->msgh_id + 100;
2855 OutP->RetCodeType = RetCodeType;
2856 OutP->RetCode = MIG_BAD_ID-303;
2857#undef InP
2858#undef OutP
2859}
2860
2861boolean_t
2862default_pager_demux_object(in, out)
2863 mach_msg_header_t *in;
2864 mach_msg_header_t *out;
2865{
2866 /*
2867 * We receive memory_object_data_initialize messages in
2868 * the memory_object_default interface.
2869 */
2870
2871 int rval = FALSE((boolean_t) 0);
2872 ddprintf ("DPAGER DEMUX OBJECT <%p>: %d\n", in, in->msgh_id);
2873 mig_reply_setup (in, out);
2874
2875 mig_routine_t routine;
2876 if ((routine = seqnos_memory_object_server_routine (in)) ||
2877 (routine = seqnos_memory_object_default_server_routine (in)) ||
2878 (routine = NULL((void*)0), default_pager_notify_server (in, out)) ||
2879 (routine = default_pager_server_routine (in)))
2880 {
2881 if (routine)
2882 (*routine) (in, out);
2883 rval = TRUE((boolean_t) 1);
2884 }
2885
2886 ddprintf ("DPAGER DEMUX OBJECT DONE <%p>: %d\n", in, in->msgh_id);
2887 return rval;
2888}
2889
2890mach_msg_size_t default_pager_msg_size_default = 8 * 1024;
2891
2892boolean_t
2893default_pager_demux_default(in, out)
2894 mach_msg_header_t *in;
2895 mach_msg_header_t *out;
2896{
2897 if (in->msgh_local_port == default_pager_default_port) {
2898 /*
2899 * We receive memory_object_create messages in
2900 * the memory_object_default interface.
2901 */
2902
2903int rval;
2904ddprintf ("DPAGER DEMUX DEFAULT <%p>: %d\n", in, in->msgh_id);
2905rval =
2906 (seqnos_memory_object_default_server(in, out) ||
2907 default_pager_server(in, out));
2908ddprintf ("DPAGER DEMUX DEFAULT DONE <%p>: %d\n", in, in->msgh_id);
2909return rval;
2910 } else if (in->msgh_local_port == default_pager_exception_port) {
2911 /*
2912 * We receive exception messages for
2913 * ourself and the startup task.
2914 */
2915
2916 return exc_server(in, out);
2917 } else {
2918 panic(my_name);
2919 return FALSE((boolean_t) 0);
2920 }
2921}
2922
2923/*
2924 * We use multiple threads, for two reasons.
2925 *
2926 * First, memory objects created by default_pager_object_create
2927 * are "external", instead of "internal". This means the kernel
2928 * sends data (memory_object_data_write) to the object pageable.
2929 * To prevent deadlocks, the external and internal objects must
2930 * be managed by different threads.
2931 *
2932 * Second, the default pager uses synchronous IO operations.
2933 * Spreading requests across multiple threads should
2934 * recover some of the performance loss from synchronous IO.
2935 *
2936 * We have 3+ threads.
2937 * One receives memory_object_create and
2938 * default_pager_object_create requests.
2939 * One or more manage internal objects.
2940 * One or more manage external objects.
2941 */
2942
2943void
2944default_pager_thread_privileges()
2945{
2946 /*
2947 * Set thread privileges.
2948 */
2949 wire_thread(); /* grab a kernel stack and memory allocation
2950 privileges */
2951}
2952
2953void *
2954default_pager_default_thread(void *arg)
2955{
2956 kern_return_t kr;
2957 default_pager_thread_privileges ();
2958 for (;;) {
2959 kr = mach_msg_server(default_pager_demux_default,
2960 default_pager_msg_size_default,
2961 default_pager_default_set);
2962 panic(my_name, kr);
2963 }
2964}
2965
2966
2967
2968void *
2969default_pager_thread(void *arg)
2970{
2971 mach_port_t pset;
2972 kern_return_t kr;
2973
2974 dpt = (default_pager_thread_t *) arg;
2975
2976 /*
2977 * Threads handling external objects cannot have
2978 * privileges. Otherwise a burst of data-requests for an
2979 * external object could empty the free-page queue,
2980 * because the fault code only reserves real pages for
2981 * requests sent to internal objects.
2982 */
2983
2984 if (dpt->dpt_internal) {
2985 default_pager_thread_privileges();
2986 pset = default_pager_internal_set;
2987 } else {
2988 pset = default_pager_external_set;
2989 }
2990
2991 for (;;) {
2992 kr = mach_msg_server(default_pager_demux_object,
2993 default_pager_msg_size_object,
2994 pset);
2995 panic(my_name, kr);
2996 }
2997}
2998
2999void
3000start_default_pager_thread(internal)
3001 boolean_t internal;
3002{
3003 default_pager_thread_t *ndpt;
3004 kern_return_t kr;
3005 error_t err;
3006
3007 ndpt = (default_pager_thread_t *) kalloc(sizeof *ndpt);
3008 if (ndpt == 0)
3009 panic(my_name);
3010
3011 ndpt->dpt_internal = internal;
3012
3013 kr = vm_allocate(default_pager_self, &ndpt->dpt_buffer,
3014 vm_page_size, TRUE((boolean_t) 1));
3015 if (kr != KERN_SUCCESS0)
3016 panic(my_name);
3017 wire_memory(ndpt->dpt_buffer, vm_page_size,
3018 VM_PROT_READ((vm_prot_t) 0x01)|VM_PROT_WRITE((vm_prot_t) 0x02));
3019
3020 err = pthread_create(&ndpt->dpt_thread, NULL((void*)0), default_pager_thread,
3021 ndpt);
3022 if (!err)
3023 pthread_detach (ndpt->dpt_thread);
3024 else {
3025 errno(*__errno_location ()) = err;
3026 perror ("pthread_create");
3027 }
3028}
3029
3030void
3031default_pager_initialize(host_port)
3032 mach_port_t host_port;
3033{
3034 memory_object_t DMM;
3035 kern_return_t kr;
3036
3037 /*
3038 * This task will become the default pager.
3039 */
3040 default_pager_self = mach_task_self()((__mach_task_self_ + 0));
3041
3042 /*
3043 * Initialize the "default pager" port.
3044 */
3045 kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_RECEIVE((mach_port_right_t) 1),
3046 &default_pager_default_port);
3047 if (kr != KERN_SUCCESS0)
3048 panic(my_name);
3049
3050 DMM = default_pager_default_port;
3051 kr = vm_set_default_memory_manager(host_port, &DMM);
3052 if ((kr != KERN_SUCCESS0) || MACH_PORT_VALID(DMM)(((DMM) != ((mach_port_t) 0)) && ((DMM) != ((mach_port_t
) ~0))))
3053 panic(my_name);
3054
3055 /*
3056 * Initialize the exception port.
3057 */
3058 kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_RECEIVE((mach_port_right_t) 1),
3059 &default_pager_exception_port);
3060 if (kr != KERN_SUCCESS0)
3061 panic(my_name);
3062
3063 /*
3064 * Arrange for wiring privileges.
3065 */
3066 wire_setup(host_port);
3067
3068 /*
3069 * Find out how many CPUs we have, to determine the number
3070 * of threads to create.
3071 */
3072 if (default_pager_internal_count == 0) {
3073 host_basic_info_data_t h_info;
3074 natural_t h_info_count;
3075
3076 h_info_count = HOST_BASIC_INFO_COUNT(sizeof(host_basic_info_data_t)/sizeof(integer_t));
3077 (void) host_info(host_port, HOST_BASIC_INFO1,
3078 (host_info_t)&h_info, &h_info_count);
3079
3080 /*
3081 * Random computation to get more parallelism on
3082 * multiprocessors.
3083 */
3084 default_pager_internal_count =
3085 (h_info.avail_cpus > 32 ? 32 : h_info.avail_cpus) / 4 + 3;
3086 }
3087}
3088
3089/*
3090 * Initialize and Run the default pager
3091 */
3092void
3093default_pager()
3094{
3095 kern_return_t kr;
3096 int i;
3097
3098 default_pager_thread_privileges();
3099
3100 /*
3101 * Wire down code, data, stack
3102 */
3103 wire_all_memory();
3104
3105
3106 /*
3107 * Initialize the list of all pagers.
3108 */
3109 pager_port_list_init(){ pthread_mutex_init(&all_pagers.lock, ((void*)0)); hurd_ihash_init
(&all_pagers.htable, __builtin_offsetof(struct dstruct, htable_locp
)); ((&all_pagers.leak_queue)->next = (&all_pagers
.leak_queue)->prev = &all_pagers.leak_queue); };
3110
3111 kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_PORT_SET((mach_port_right_t) 3),
3112 &default_pager_internal_set);
3113 if (kr != KERN_SUCCESS0)
3114 panic(my_name);
3115
3116 kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_PORT_SET((mach_port_right_t) 3),
3117 &default_pager_external_set);
3118 if (kr != KERN_SUCCESS0)
3119 panic(my_name);
3120
3121 kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_PORT_SET((mach_port_right_t) 3),
3122 &default_pager_default_set);
3123 if (kr != KERN_SUCCESS0)
3124 panic(my_name);
3125
3126 kr = mach_port_move_member(default_pager_self,
3127 default_pager_default_port,
3128 default_pager_default_set);
3129 if (kr != KERN_SUCCESS0)
3130 panic(my_name);
3131
3132 kr = mach_port_move_member(default_pager_self,
3133 default_pager_exception_port,
3134 default_pager_default_set);
3135 if (kr != KERN_SUCCESS0)
3136 panic(my_name);
3137
3138 /*
3139 * Now we create the threads that will actually
3140 * manage objects.
3141 */
3142
3143 for (i = 0; i < default_pager_internal_count; i++)
3144 start_default_pager_thread(TRUE((boolean_t) 1));
3145
3146 for (i = 0; i < default_pager_external_count; i++)
3147 start_default_pager_thread(FALSE((boolean_t) 0));
3148
3149 default_pager_default_thread(0); /* Become the default_pager server */
3150#if 0
3151 cthread_fork (default_pager_default_thread, 0);
3152 /* cthread_exit (cthread_self ()); */
3153 thread_suspend (mach_thread_self ());
3154#endif
3155}
3156
3157/*
3158 * Create an external object.
3159 */
3160kern_return_t
3161S_default_pager_object_create (mach_port_t pager,
3162 mach_port_t *mem_obj,
3163 vm_size_t size)
3164{
3165 default_pager_t ds;
3166 mach_port_t port;
3167 kern_return_t result;
3168
3169 if (pager != default_pager_default_port)
3170 return KERN_INVALID_ARGUMENT4;
3171
3172 ds = pager_port_alloc(size);
3173 result = mach_port_allocate (default_pager_self,
3174 MACH_PORT_RIGHT_RECEIVE((mach_port_right_t) 1),
3175 &port);
3176 if (result != KERN_SUCCESS0)
3177 {
3178 kfree ((char *) ds, sizeof *ds);
3179 return result;
3180 }
3181
3182 /*
3183 * Set up associations between these ports
3184 * and this default_pager structure
3185 */
3186
3187 ds->pager = port;
3188 ds->dpager.limit = size;
3189 pager_port_list_insert(port, ds);
3190 default_pager_add(ds, FALSE((boolean_t) 0));
3191
3192 *mem_obj = port;
3193 return (KERN_SUCCESS0);
3194}
3195
3196kern_return_t
3197S_default_pager_info (mach_port_t pager,
3198 default_pager_info_t *infop)
3199{
3200 vm_size_t total, free;
3201
3202 if (pager != default_pager_default_port)
3203 return KERN_INVALID_ARGUMENT4;
3204
3205 pthread_mutex_lock(&all_partitions.lock);
3206 paging_space_info(&total, &free);
3207 pthread_mutex_unlock(&all_partitions.lock);
3208
3209 infop->dpi_total_space = ptoa(total)((total)*vm_page_size);
3210 infop->dpi_free_space = ptoa(free)((free)*vm_page_size);
3211 infop->dpi_page_size = vm_page_size;
3212 return KERN_SUCCESS0;
3213}
3214
3215kern_return_t
3216S_default_pager_objects (mach_port_t pager,
3217 default_pager_object_array_t *objectsp,
3218 natural_t *ocountp,
3219 mach_port_array_t *portsp,
3220 natural_t *pcountp)
3221{
3222 vm_offset_t oaddr = 0; /* memory for objects */
3223 vm_size_t osize = 0; /* current size */
3224 default_pager_object_t *objects;
3225 natural_t opotential;
3226
3227 vm_offset_t paddr = 0; /* memory for ports */
3228 vm_size_t psize = 0; /* current size */
3229 mach_port_t *ports;
3230 natural_t ppotential;
3231
3232 unsigned int actual;
3233 unsigned int num_pagers;
3234 kern_return_t kr;
3235 default_pager_t entry;
3236
3237 if (pager != default_pager_default_port)
3238 return KERN_INVALID_ARGUMENT4;
3239
3240 /* start with the inline memory */
3241
3242 num_pagers = 0;
3243
3244 objects = *objectsp;
3245 opotential = *ocountp;
3246
3247 ports = *portsp;
3248 ppotential = *pcountp;
3249
3250 pthread_mutex_lock(&all_pagers.lock);
3251 /*
3252 * We will send no more than this many
3253 */
3254 actual = all_pagers.htable.nr_items;
3255 pthread_mutex_unlock(&all_pagers.lock);
3256
3257 if (opotential < actual) {
3258 vm_offset_t newaddr;
3259 vm_size_t newsize;
3260
3261 newsize = 2 * round_page(actual * sizeof *objects)((((vm_offset_t) (actual * sizeof *objects) + __vm_page_size -
1) / __vm_page_size) * __vm_page_size);
3262
3263 kr = vm_allocate(default_pager_self, &newaddr, newsize, TRUE((boolean_t) 1));
3264 if (kr != KERN_SUCCESS0)
3265 goto nomemory;
3266
3267 oaddr = newaddr;
3268 osize = newsize;
3269 opotential = osize/sizeof *objects;
3270 objects = (default_pager_object_t *) oaddr;
3271 }
3272
3273 if (ppotential < actual) {
3274 vm_offset_t newaddr;
3275 vm_size_t newsize;
3276
3277 newsize = 2 * round_page(actual * sizeof *ports)((((vm_offset_t) (actual * sizeof *ports) + __vm_page_size - 1
) / __vm_page_size) * __vm_page_size);
3278
3279 kr = vm_allocate(default_pager_self, &newaddr, newsize, TRUE((boolean_t) 1));
3280 if (kr != KERN_SUCCESS0)
3281 goto nomemory;
3282
3283 paddr = newaddr;
3284 psize = newsize;
3285 ppotential = psize/sizeof *ports;
3286 ports = (mach_port_t *) paddr;
3287 }
3288
3289 /*
3290 * Now scan the list.
3291 */
3292
3293 pthread_mutex_lock(&all_pagers.lock);
3294
3295 num_pagers = 0;
3296 HURD_IHASH_ITERATE (&all_pagers.htable, val)for (hurd_ihash_value_t val, *_hurd_ihash_valuep = (&all_pagers
.htable)->size ? &(&all_pagers.htable)->items[0
].value : 0; (&all_pagers.htable)->size && ((_hurd_ihash_item_t
) _hurd_ihash_valuep) - &(&all_pagers.htable)->items
[0] < (&all_pagers.htable)->size && (val = *
_hurd_ihash_valuep, 1); _hurd_ihash_valuep = (hurd_ihash_value_t
*) (((_hurd_ihash_item_t) _hurd_ihash_valuep) + 1)) if (val !=
((hurd_ihash_value_t) 0) && val != ((hurd_ihash_value_t
) -1)) {
3297 entry = (default_pager_t) val;
3298
3299 mach_port_t port;
3300 vm_size_t size;
3301
3302 if ((num_pagers >= opotential) ||
3303 (num_pagers >= ppotential)) {
3304 /*
3305 * This should be rare. In any case,
3306 * we will only miss recent objects,
3307 * because they are added at the end.
3308 */
3309 break;
3310 }
3311
3312 /*
3313 * Avoid interfering with normal operations
3314 */
3315 if (pthread_mutex_trylock(&entry->dpager.lock))
3316 goto not_this_one;
3317 size = pager_allocated(&entry->dpager);
3318 pthread_mutex_unlock(&entry->dpager.lock);
3319
3320 dstruct_lock(entry)pthread_mutex_lock(&entry->lock);
3321
3322 port = entry->pager_name;
3323 if (port == MACH_PORT_NULL((mach_port_t) 0)) {
3324 /*
3325 * The object is waiting for no-senders
3326 * or memory_object_init.
3327 */
3328 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3329 goto not_this_one;
3330 }
3331
3332 /*
3333 * We need a reference for the reply message.
3334 * While we are unlocked, the bucket queue
3335 * can change and the object might be terminated.
3336 * memory_object_terminate will wait for us,
3337 * preventing deallocation of the entry.
3338 */
3339
3340 if (--entry->name_refs == 0) {
3341 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3342
3343 /* keep the list locked, wont take long */
3344
3345 kr = mach_port_mod_refs(default_pager_self,
3346 port, MACH_PORT_RIGHT_SEND((mach_port_right_t) 0),
3347 default_pager_max_urefs);
3348 if (kr != KERN_SUCCESS0)
3349 panic("%sdefault_pager_objects",my_name);
3350
3351 dstruct_lock(entry)pthread_mutex_lock(&entry->lock);
3352
3353 entry->name_refs += default_pager_max_urefs;
3354 pager_port_finish_refs(entry);
3355 }
3356 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3357
3358 /* the arrays are wired, so no deadlock worries */
3359
3360 objects[num_pagers].dpo_object = (vm_offset_t) entry;
3361 objects[num_pagers].dpo_size = size;
3362 ports [num_pagers++] = port;
3363 continue;
3364not_this_one:
3365 /*
3366 * Do not return garbage
3367 */
3368 objects[num_pagers].dpo_object = (vm_offset_t) 0;
3369 objects[num_pagers].dpo_size = 0;
3370 ports [num_pagers++] = MACH_PORT_NULL((mach_port_t) 0);
3371
3372 }
3373
3374 pthread_mutex_unlock(&all_pagers.lock);
3375
3376 /*
3377 * Deallocate and clear unused memory.
3378 * (Returned memory will automagically become pageable.)
3379 */
3380
3381 if (objects == *objectsp) {
3382 /*
3383 * Our returned information fit inline.
3384 * Nothing to deallocate.
3385 */
3386
3387 *ocountp = num_pagers;
3388 } else if (actual == 0) {
3389 (void) vm_deallocate(default_pager_self, oaddr, osize);
3390
3391 /* return zero items inline */
3392 *ocountp = 0;
3393 } else {
3394 vm_offset_t used;
3395
3396 used = round_page(actual * sizeof *objects)((((vm_offset_t) (actual * sizeof *objects) + __vm_page_size -
1) / __vm_page_size) * __vm_page_size);
3397
3398 if (used != osize)
3399 (void) vm_deallocate(default_pager_self,
3400 oaddr + used, osize - used);
3401
3402 *objectsp = objects;
3403 *ocountp = num_pagers;
3404 }
3405
3406 if (ports == *portsp) {
3407 /*
3408 * Our returned information fit inline.
3409 * Nothing to deallocate.
3410 */
3411
3412 *pcountp = num_pagers;
3413 } else if (actual == 0) {
3414 (void) vm_deallocate(default_pager_self, paddr, psize);
3415
3416 /* return zero items inline */
3417 *pcountp = 0;
3418 } else {
3419 vm_offset_t used;
3420
3421 used = round_page(actual * sizeof *ports)((((vm_offset_t) (actual * sizeof *ports) + __vm_page_size - 1
) / __vm_page_size) * __vm_page_size);
3422
3423 if (used != psize)
3424 (void) vm_deallocate(default_pager_self,
3425 paddr + used, psize - used);
3426
3427 *portsp = ports;
3428 *pcountp = num_pagers;
3429 }
3430
3431 return KERN_SUCCESS0;
3432
3433 nomemory:
3434
3435 {
3436 int i;
3437 for (i = 0; i < num_pagers; i++)
3438 (void) mach_port_deallocate(default_pager_self, ports[i]);
3439 }
3440
3441 if (objects != *objectsp)
3442 (void) vm_deallocate(default_pager_self, oaddr, osize);
3443
3444 if (ports != *portsp)
3445 (void) vm_deallocate(default_pager_self, paddr, psize);
3446
3447 return KERN_RESOURCE_SHORTAGE6;
3448}
3449
3450
3451kern_return_t
3452S_default_pager_object_pages (mach_port_t pager,
3453 mach_port_t object,
3454 default_pager_page_array_t *pagesp,
3455 natural_t *countp)
3456{
3457 vm_offset_t addr = 0; /* memory for page offsets */
3458 vm_size_t size = 0; /* current memory size */
3459 default_pager_page_t *pages;
3460 natural_t potential, actual;
3461 kern_return_t kr;
3462
3463 if (pager != default_pager_default_port)
3464 return KERN_INVALID_ARGUMENT4;
3465
3466 /* we start with the inline space */
3467
3468 pages = *pagesp;
3469 potential = *countp;
3470
3471 for (;;) {
3472 default_pager_t entry;
3473
3474 pthread_mutex_lock(&all_pagers.lock);
3475 HURD_IHASH_ITERATE (&all_pagers.htable, val)for (hurd_ihash_value_t val, *_hurd_ihash_valuep = (&all_pagers
.htable)->size ? &(&all_pagers.htable)->items[0
].value : 0; (&all_pagers.htable)->size && ((_hurd_ihash_item_t
) _hurd_ihash_valuep) - &(&all_pagers.htable)->items
[0] < (&all_pagers.htable)->size && (val = *
_hurd_ihash_valuep, 1); _hurd_ihash_valuep = (hurd_ihash_value_t
*) (((_hurd_ihash_item_t) _hurd_ihash_valuep) + 1)) if (val !=
((hurd_ihash_value_t) 0) && val != ((hurd_ihash_value_t
) -1)) {
3476 entry = (default_pager_t) val;
3477 dstruct_lock(entry)pthread_mutex_lock(&entry->lock);
3478 if (entry->pager_name == object) {
3479 pthread_mutex_unlock(&all_pagers.lock);
3480 goto found_object;
3481 }
3482 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3483 }
3484 pthread_mutex_unlock(&all_pagers.lock);
3485
3486 /* did not find the object */
3487
3488 if (pages != *pagesp)
3489 (void) vm_deallocate(default_pager_self, addr, size);
3490 return KERN_INVALID_ARGUMENT4;
3491
3492 found_object:
3493
3494 if (pthread_mutex_trylock(&entry->dpager.lock)) {
3495 /* oh well bad luck */
3496
3497 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3498
3499 /* yield the processor */
3500 (void) thread_switch(MACH_PORT_NULL((mach_port_t) 0),
3501 SWITCH_OPTION_NONE0, 0);
3502 continue;
3503 }
3504
3505 actual = pager_pages(&entry->dpager, pages, potential);
3506 pthread_mutex_unlock(&entry->dpager.lock);
3507 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3508
3509 if (actual <= potential)
3510 break;
3511
3512 /* allocate more memory */
3513
3514 if (pages != *pagesp)
3515 (void) vm_deallocate(default_pager_self, addr, size);
3516 size = round_page(actual * sizeof *pages)((((vm_offset_t) (actual * sizeof *pages) + __vm_page_size - 1
) / __vm_page_size) * __vm_page_size);
3517 kr = vm_allocate(default_pager_self, &addr, size, TRUE((boolean_t) 1));
3518 if (kr != KERN_SUCCESS0)
3519 return kr;
3520 pages = (default_pager_page_t *) addr;
3521 potential = size/sizeof *pages;
3522 }
3523
3524 /*
3525 * Deallocate and clear unused memory.
3526 * (Returned memory will automagically become pageable.)
3527 */
3528
3529 if (pages == *pagesp) {
3530 /*
3531 * Our returned information fit inline.
3532 * Nothing to deallocate.
3533 */
3534
3535 *countp = actual;
3536 } else if (actual == 0) {
3537 (void) vm_deallocate(default_pager_self, addr, size);
3538
3539 /* return zero items inline */
3540 *countp = 0;
3541 } else {
3542 vm_offset_t used;
3543
3544 used = round_page(actual * sizeof *pages)((((vm_offset_t) (actual * sizeof *pages) + __vm_page_size - 1
) / __vm_page_size) * __vm_page_size);
3545
3546 if (used != size)
3547 (void) vm_deallocate(default_pager_self,
3548 addr + used, size - used);
3549
3550 *pagesp = pages;
3551 *countp = actual;
3552 }
3553 return KERN_SUCCESS0;
3554}
3555
3556
3557kern_return_t
3558S_default_pager_object_set_size (default_pager_t ds,
3559 mach_port_seqno_t seqno,
3560 vm_size_t limit)
3561{
3562 kern_return_t kr = KERN_SUCCESS0;
3563
3564 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
3565 return KERN_INVALID_ARGUMENT4;
3566
3567 pager_port_lock(ds, seqno);
3568 pager_port_wait_for_readers(ds);
3569 pager_port_wait_for_writers(ds);
3570
3571 vm_size_t rounded_limit = round_page (limit)((((vm_offset_t) (limit) + __vm_page_size - 1) / __vm_page_size
) * __vm_page_size);
3572 vm_size_t trunc_limit = trunc_page (limit)((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size);
3573
3574
3575 if (ds->dpager.limit < rounded_limit)
3576 {
3577 /* The limit has not been exceeded heretofore. Just change it. */
3578 ds->dpager.limit = rounded_limit;
3579
3580 /* Byte limit is used for truncation of file, which aren't rounded to
3581 page boundary. But by enlarging of file we are free to increase this value*/
3582 ds->dpager.byte_limit = rounded_limit;
3583 kr = memory_object_lock_request(ds->pager_request, 0,
3584 rounded_limit,
3585 MEMORY_OBJECT_RETURN_NONE0, FALSE((boolean_t) 0),
3586 VM_PROT_NONE((vm_prot_t) 0x00), MACH_PORT_NULL((mach_port_t) 0));
3587 if (kr != KERN_SUCCESS0)
3588 panic ("memory_object_lock_request: %d", kr);
3589 }
3590 else
3591 {
3592 if (ds->dpager.limit != rounded_limit)
3593 {
3594 kr = memory_object_lock_request(ds->pager_request, rounded_limit,
3595 ds->dpager.limit - rounded_limit,
3596 MEMORY_OBJECT_RETURN_NONE0, TRUE((boolean_t) 1),
3597 VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04)), MACH_PORT_NULL((mach_port_t) 0));
3598 if (kr != KERN_SUCCESS0)
3599 panic ("memory_object_lock_request: %d", kr);
3600
3601 ds->dpager.limit = rounded_limit;
3602 }
3603
3604 /* Deallocate the old backing store pages and shrink the page map. */
3605 if (ds->dpager.size > ds->dpager.limit / vm_page_size)
3606 pager_truncate (&ds->dpager, ds->dpager.limit / vm_page_size);
3607
3608 /* If memory object size isn't page aligned, fill the tail
3609 of last page with zeroes */
3610 if ((limit != rounded_limit) && (ds->dpager.limit > limit))
3611 {
3612 /* Clean part of last page which isn't part of file.
3613 For file sizes that aren't multiple of vm_page_size */
3614 ds->dpager.byte_limit = limit;
3615 kr = memory_object_lock_request(ds->pager_request, trunc_limit,
3616 vm_page_size,
3617 MEMORY_OBJECT_RETURN_ALL2, TRUE((boolean_t) 1),
3618 VM_PROT_NONE((vm_prot_t) 0x00), MACH_PORT_NULL((mach_port_t) 0));
3619 }
3620 }
3621
3622 pager_port_unlock(ds);
3623
3624 return kr;
3625}
3626
3627/*
3628 * Add/remove extra paging space
3629 */
3630
3631extern mach_port_t bootstrap_master_device_port;
3632extern mach_port_t bootstrap_master_host_port;
3633
3634kern_return_t
3635S_default_pager_paging_file (pager, mdport, file_name, add)
3636 mach_port_t pager;
3637 mach_port_t mdport;
3638 default_pager_filename_t file_name;
3639 boolean_t add;
3640{
3641 kern_return_t kr;
3642
3643 if (pager != default_pager_default_port)
3644 return KERN_INVALID_ARGUMENT4;
3645
3646#if 0
3647dprintf("bmd %x md %x\n", bootstrap_master_device_port, mdport);
3648#endif
3649 if (add) {
3650 kr = add_paging_file(bootstrap_master_device_port,
3651 file_name, 0);
3652 } else {
3653 kr = remove_paging_file(file_name);
3654 }
3655
3656 /* XXXX more code needed */
3657 if (mdport != bootstrap_master_device_port)
3658 mach_port_deallocate( mach_task_self()((__mach_task_self_ + 0)), mdport);
3659
3660 return kr;
3661}
3662
3663kern_return_t
3664default_pager_register_fileserver(pager, fileserver)
3665 mach_port_t pager;
3666 mach_port_t fileserver;
3667{
3668 if (pager != default_pager_default_port)
3669 return KERN_INVALID_ARGUMENT4;
3670#if notyet
3671 mach_port_deallocate(mach_task_self()((__mach_task_self_ + 0)), fileserver);
3672 if (0) dp_helper_paging_space(0,0,0);/*just linkit*/
3673#endif
3674 return KERN_SUCCESS0;
3675}
3676
3677/*
3678 * When things do not quite workout...
3679 */
3680void no_paging_space(out_of_memory)
3681 boolean_t out_of_memory;
3682{
3683 static char here[] = "%s *** NOT ENOUGH PAGING SPACE ***";
3684
3685 if (out_of_memory)
3686 dprintf("*** OUT OF MEMORY *** ");
3687 panic(here, my_name);
3688}
3689
3690void overcommitted(got_more_space, space)
3691 boolean_t got_more_space;
3692 vm_size_t space; /* in pages */
3693{
3694 vm_size_t pages_free, pages_total;
3695
3696 static boolean_t user_warned = FALSE((boolean_t) 0);
3697 static vm_size_t pages_shortage = 0;
3698
3699 paging_space_info(&pages_total, &pages_free);
3700
3701 /*
3702 * If user added more space, see if it is enough
3703 */
3704 if (got_more_space) {
3705 pages_free -= pages_shortage;
3706 if (pages_free > 0) {
3707 pages_shortage = 0;
3708 if (user_warned)
3709 dprintf("%s paging space ok now.\n", my_name);
3710 } else
3711 pages_shortage = pages_free;
3712 user_warned = FALSE((boolean_t) 0);
3713 return;
3714 }
3715 /*
3716 * We ran out of gas, let user know.
3717 */
3718 pages_free -= space;
3719 pages_shortage = (pages_free > 0) ? 0 : -pages_free;
3720 if (!user_warned && pages_shortage) {
3721 user_warned = TRUE((boolean_t) 1);
3722 dprintf("%s paging space over-committed.\n", my_name);
3723 }
3724#if debug0
3725 user_warned = FALSE((boolean_t) 0);
3726 dprintf("%s paging space over-committed [+%d (%d) pages].\n",
3727 my_name, space, pages_shortage);
3728#endif
3729}
3730
3731void paging_space_info(totp, freep)
3732 vm_size_t *totp, *freep;
3733{
3734 vm_size_t total, free;
3735 partition_t part;
3736 int i;
3737
3738 total = free = 0;
3739 for (i = 0; i < all_partitions.n_partitions; i++) {
3740
3741 if ((part = partition_of(i)) == 0) continue;
3742
3743 /* no need to lock: by the time this data
3744 gets back to any remote requestor it
3745 will be obsolete anyways */
3746 total += part->total_size;
3747 free += part->free;
3748#if debug0
3749 dprintf("Partition %d: x%x total, x%x free\n",
3750 i, part->total_size, part->free);
3751#endif
3752 }
3753 *totp = total;
3754 *freep = free;
3755}
3756
3757/*
3758 * Catch exceptions.
3759 */
3760
3761kern_return_t
3762catch_exception_raise(exception_port, thread, task, exception, code, subcode)
3763 mach_port_t exception_port;
3764 mach_port_t thread, task;
3765 int exception, code, subcode;
3766{
3767 ddprintf ("(default_pager)catch_exception_raise(%d,%d,%d)\n",
3768 exception, code, subcode);
3769 panic(my_name);
3770
3771 /* mach_msg_server will deallocate thread/task for us */
3772
3773 return KERN_FAILURE5;
3774}