Bug Summary

File:obj-scan-build/mach-defpager/../../mach-defpager/default_pager.c
Location:line 2449, column 3
Description:Value stored to 'ds' is never read

Annotated Source Code

1/*
2 * Mach Operating System
3 * Copyright (c) 1993-1989 Carnegie Mellon University
4 * All Rights Reserved.
5 *
6 * Permission to use, copy, modify and distribute this software and its
7 * documentation is hereby granted, provided that both the copyright
8 * notice and this permission notice appear in all copies of the
9 * software, derivative works or modified versions, and any portions
10 * thereof, and that both notices appear in supporting documentation.
11 *
12 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
13 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
14 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
15 *
16 * Carnegie Mellon requests users of this software to return to
17 *
18 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
19 * School of Computer Science
20 * Carnegie Mellon University
21 * Pittsburgh PA 15213-3890
22 *
23 * any improvements or extensions that they make and grant Carnegie Mellon
24 * the rights to redistribute these changes.
25 */
26/*
27 * Default pager. Pages to paging partition.
28 *
29 * MUST BE ABLE TO ALLOCATE WIRED-DOWN MEMORY!!!
30 */
31
32#include <mach.h>
33#include <mach/message.h>
34#include <mach/notify.h>
35#include <mach/mig_errors.h>
36#include <mach/thread_switch.h>
37#include <mach/task_info.h>
38#include <mach/default_pager_types.h>
39
40#include <pthread.h>
41#include <stddef.h>
42
43#include <device/device_types.h>
44#include <device/device.h>
45
46#include <hurd/ihash.h>
47
48#include "queue.h"
49#include "wiring.h"
50#include "kalloc.h"
51#include "default_pager.h"
52
53#include <assert.h>
54#include <errno(*__errno_location ()).h>
55#include <stdio.h>
56#include <string.h>
57#include <stdarg.h>
58
59#include <file_io.h>
60
61#include "memory_object_S.h"
62#include "memory_object_default_S.h"
63#include "default_pager_S.h"
64#include "exc_S.h"
65
66#include "priv.h"
67
68#define debug0 0
69
70static char my_name[] = "(default pager):";
71
72static void __attribute__ ((format (printf, 1, 2), unused))
73synchronized_printf (const char *fmt, ...)
74{
75 static pthread_mutex_t printf_lock = PTHREAD_MUTEX_INITIALIZER{ ((__pthread_spinlock_t) 0), ((__pthread_spinlock_t) 0), 0, 0
, 0, 0, 0, 0 };
76 va_list ap;
77
78 va_start (ap, fmt)__builtin_va_start(ap, fmt);
79 pthread_mutex_lock (&printf_lock);
80
81 vprintf (fmt, ap);
82 fflush (stdoutstdout);
83
84 pthread_mutex_unlock (&printf_lock);
85 va_end (ap)__builtin_va_end(ap);
86}
87
88#if 0
89#define dprintf(f, x...)(void) 0 synchronized_printf (f, ##x)
90#else
91#define dprintf(f, x...)(void) 0 (void) 0
92#endif
93
94#if 0
95#define ddprintf(f, x...)(void) 0 synchronized_printf (f, ##x)
96#else
97#define ddprintf(f, x...)(void) 0 (void) 0
98#endif
99
100/*
101 * parallel vs serial switch
102 */
103#define PARALLEL1 1
104
105#if 0
106#define CHECKSUM 1
107#endif
108
109#define USE_PRECIOUS1 1
110
111#define ptoa(p)((p)*vm_page_size) ((p)*vm_page_size)
112#define atop(a)((a)/vm_page_size) ((a)/vm_page_size)
113
114partition_t partition_of(x)
115 int x;
116{
117 if (x >= all_partitions.n_partitions || x < 0)
118 panic("partition_of x%x", x);
119 return all_partitions.partition_list[x];
120}
121
122void set_partition_of(x, p)
123 int x;
124 partition_t p;
125{
126 if (x >= all_partitions.n_partitions || x < 0)
127 panic("set_partition_of x%x", x);
128 all_partitions.partition_list[x] = p;
129}
130
131/*
132 * Simple mapping from (file)NAME to id
133 * Saves space, filenames can be long.
134 */
135unsigned int
136part_id(const char *name)
137{
138 unsigned int id, xorid;
139 size_t len;
140
141 len = strlen(name);
142 id = xorid = 0;
143 while (len--) {
144 xorid ^= *name;
145 id += *name++;
146 }
147 return (id << 8) | xorid;
148}
149
150void
151partition_init()
152{
153 pthread_mutex_init(&all_partitions.lock, NULL((void*)0));
154 all_partitions.n_partitions = 0;
155}
156
157static partition_t
158new_partition (const char *name, struct file_direct *fdp,
159 int check_linux_signature)
160{
161 partition_t part;
162 vm_size_t size, bmsize;
163 vm_offset_t raddr;
164 mach_msg_type_number_t rsize;
165 int rc;
166 unsigned int id = part_id(name);
167
168 pthread_mutex_lock(&all_partitions.lock);
169 {
170 unsigned int i;
171 for (i = 0; i < all_partitions.n_partitions; i++)
172 {
173 part = partition_of(i);
174 if (part && part->id == id)
175 {
176 pthread_mutex_unlock(&all_partitions.lock);
177 return 0;
178 }
179 }
180 }
181 pthread_mutex_unlock(&all_partitions.lock);
182
183 size = atop(fdp->fd_size * fdp->fd_bsize)((fdp->fd_size * fdp->fd_bsize)/vm_page_size);
184 bmsize = howmany(size, NB_BM)(((size) + (32) - 1)/(32)) * sizeof(bm_entry_t);
185
186 part = (partition_t) kalloc(sizeof(struct part));
187 pthread_mutex_init(&part->p_lock, NULL((void*)0));
188 part->total_size = size;
189 part->free = size;
190 part->id = id;
191 part->bitmap = (bm_entry_t *)kalloc(bmsize);
192 part->going_away= FALSE((boolean_t) 0);
193 part->file = fdp;
194
195 bzero((char *)part->bitmap, bmsize);
196
197 if (check_linux_signature < 0)
198 {
199 if (check_linux_signature != -3)
200 printf("(default pager): "
201 "Paging to raw partition %s (%uk paging space)\n",
202 name, part->total_size * (vm_page_size / 1024));
203 return part;
204 }
205
206#define LINUX_PAGE_SIZE4096 4096 /* size of pages in Linux swap partitions */
207 rc = page_read_file_direct(part->file,
208 0, LINUX_PAGE_SIZE4096,
209 &raddr,
210 &rsize);
211 if (rc)
212 panic("(default pager): cannot read first page of %s! rc=%#x\n",
213 name, rc);
214 while (rsize < LINUX_PAGE_SIZE4096)
215 {
216 /* Filesystem block size is smaller than page size,
217 so we must do several reads to get the whole page. */
218 vm_address_t baddr;
219 vm_size_t bsize;
220 rc = page_read_file_direct(part->file,
221 rsize, LINUX_PAGE_SIZE4096-rsize,
222 &baddr,
223 &bsize);
224 if (rc)
225 panic("(default pager): "
226 "cannot read first page of %s! rc=%#x at %#x\n",
227 name, rc, rsize);
228
229 memcpy ((char *) raddr + rsize, (void *) baddr, bsize);
230 rsize += bsize;
231 vm_deallocate (mach_task_self ()((__mach_task_self_ + 0)), baddr, bsize);
232 }
233
234 if (!memcmp("SWAP-SPACE", (char *) raddr + LINUX_PAGE_SIZE4096-10, 10))
235 {
236 /* The partition's first page has a Linux swap signature.
237 This means the beginning of the page contains a bitmap
238 of good pages, and all others are bad. */
239 unsigned int i, j, bad, max;
240 int waste;
241
242 printf("(default pager): Found Linux 2.0 swap signature in %s\n",
243 name);
244
245 /* The first page, and the pages corresponding to the bits
246 occupied by the signature in the final 10 bytes of the page,
247 are always unavailable ("bad"). */
248 *(u_int32_t *)raddr &= ~(u_int32_t) 1;
249 memset((char *) raddr + LINUX_PAGE_SIZE4096-10, 0, 10);
250
251 max = LINUX_PAGE_SIZE4096 / sizeof(u_int32_t);
252 if (max > (part->total_size + 31) / 32)
253 max = (part->total_size + 31) / 32;
254
255 bad = 0;
256 for (i = 0; i < max; ++i)
257 {
258 u_int32_t bm = ((u_int32_t *) raddr)[i];
259 if (bm == ~(u_int32_t) 0)
260 continue;
261 /* There are some zero bits in this word. */
262 for (j = 0; j < 32; ++j)
263 if ((bm & (1 << j)) == 0)
264 {
265 unsigned int p = i*32 + j;
266 if (p >= part->total_size)
267 break;
268 ++bad;
269 part->bitmap[p / NB_BM32] |= 1 << (p % NB_BM32);
270 }
271 }
272 part->free -= bad;
273
274 --bad; /* Don't complain about first page. */
275 waste = part->total_size - (8 * (LINUX_PAGE_SIZE4096-10));
276 if (waste > 0)
277 {
278 /* The wasted pages were already marked "bad". */
279 bad -= waste;
280 if (bad > 0)
281 printf("\
282(default pager): Paging to %s, %dk swap-space (%dk bad, %dk wasted at end)\n",
283 name,
284 part->free * (LINUX_PAGE_SIZE4096 / 1024),
285 bad * (LINUX_PAGE_SIZE4096 / 1024),
286 waste * (LINUX_PAGE_SIZE4096 / 1024));
287 else
288 printf("\
289(default pager): Paging to %s, %dk swap-space (%dk wasted at end)\n",
290 name,
291 part->free * (LINUX_PAGE_SIZE4096 / 1024),
292 waste * (LINUX_PAGE_SIZE4096 / 1024));
293 }
294 else if (bad > 0)
295 printf("\
296(default pager): Paging to %s, %dk swap-space (excludes %dk marked bad)\n",
297 name,
298 part->free * (LINUX_PAGE_SIZE4096 / 1024),
299 bad * (LINUX_PAGE_SIZE4096 / 1024));
300 else
301 printf("\
302(default pager): Paging to %s, %dk swap-space\n",
303 name,
304 part->free * (LINUX_PAGE_SIZE4096 / 1024));
305 }
306 else if (!memcmp("SWAPSPACE2",
307 (char *) raddr + LINUX_PAGE_SIZE4096-10, 10))
308 {
309 struct
310 {
311 u_int8_t bootbits[1024];
312 u_int32_t version;
313 u_int32_t last_page;
314 u_int32_t nr_badpages;
315 u_int32_t padding[125];
316 u_int32_t badpages[1];
317 } *hdr = (void *) raddr;
318
319 printf("\
320(default pager): Found Linux 2.2 swap signature (v%u) in %s...",
321 hdr->version, name);
322
323 part->bitmap[0] |= 1; /* first page unusable */
324 part->free--;
325
326 switch (hdr->version)
327 {
328 default:
329 if (check_linux_signature)
330 {
331 printf ("version %u unknown! SKIPPING %s!\n",
332 hdr->version,
333 name);
334 vm_deallocate(mach_task_self()((__mach_task_self_ + 0)), raddr, rsize);
335 kfree(part->bitmap, bmsize);
336 kfree(part, sizeof *part);
337 return 0;
338 }
339 else
340 printf ("version %u unknown! IGNORING SIGNATURE PAGE!"
341 " %dk swap-space\n",
342 hdr->version,
343 part->free * (LINUX_PAGE_SIZE4096 / 1024));
344 break;
345
346 case 1:
347 {
348 unsigned int waste, i;
349 if (hdr->last_page > part->total_size)
350 {
351 printf ("signature says %uk, partition has only %uk! ",
352 hdr->last_page * (LINUX_PAGE_SIZE4096 / 1024),
353 part->total_size * (LINUX_PAGE_SIZE4096 / 1024));
354 waste = 0;
355 }
356 else
357 {
358 waste = part->total_size - hdr->last_page;
359 part->total_size = hdr->last_page;
360 part->free = part->total_size - 1;
361 }
362 for (i = 0; i < hdr->nr_badpages; ++i)
363 {
364 const u_int32_t bad = hdr->badpages[i];
365 part->bitmap[bad / NB_BM32] |= 1 << (bad % NB_BM32);
366 part->free--;
367 }
368 printf ("%uk swap-space",
369 part->free * (LINUX_PAGE_SIZE4096 / 1024));
370 if (hdr->nr_badpages != 0)
371 printf (" (excludes %uk marked bad)",
372 hdr->nr_badpages * (LINUX_PAGE_SIZE4096 / 1024));
373 if (waste != 0)
374 printf (" (excludes %uk at end of partition)",
375 waste * (LINUX_PAGE_SIZE4096 / 1024));
376 printf ("\n");
377 }
378 }
379 }
380 else if (check_linux_signature)
381 {
382 printf ("(default pager): "
383 "Cannot find Linux swap signature page! "
384 "SKIPPING %s (%uk partition)!",
385 name, part->total_size * (vm_page_size / 1024));
386 kfree(part->bitmap, bmsize);
387 kfree(part, sizeof *part);
388 part = 0;
389 }
390 else
391 printf("(default pager): "
392 "Paging to raw partition %s (%uk paging space)\n",
393 name, part->total_size * (vm_page_size / 1024));
394
395 vm_deallocate(mach_task_self()((__mach_task_self_ + 0)), raddr, rsize);
396
397 return part;
398}
399
400/*
401 * Create a partition descriptor,
402 * add it to the list of all such.
403 * size is in BYTES.
404 */
405void
406create_paging_partition(const char *name,
407 struct file_direct *fdp, int isa_file,
408 int linux_signature)
409{
410 partition_t part;
411
412 part = new_partition (name, fdp, linux_signature);
413 if (!part)
414 return;
415
416 pthread_mutex_lock(&all_partitions.lock);
417 {
418 int i;
419
420 for (i = 0; i < all_partitions.n_partitions; i++)
421 if (partition_of(i) == 0) break;
422
423 if (i == all_partitions.n_partitions) {
424 partition_t *new_list, *old_list;
425 int n;
426
427 n = i ? (i<<1) : 2;
428 new_list = (partition_t *)
429 kalloc( n * sizeof(partition_t) );
430 if (new_list == 0) no_paging_space(TRUE((boolean_t) 1));
431 bzero(new_list, n*sizeof(partition_t));
432 if (i) {
433 old_list = all_partitions.partition_list;
434 bcopy(old_list, new_list, i*sizeof(partition_t));
435 }
436 all_partitions.partition_list = new_list;
437 all_partitions.n_partitions = n;
438 if (i) kfree(old_list, i*sizeof(partition_t));
439 }
440 set_partition_of(i, part);
441 }
442 pthread_mutex_unlock(&all_partitions.lock);
443
444#if 0
445 dprintf("%s Added paging %s %s\n", my_name,(void) 0
446 (isa_file) ? "file" : "device", name)(void) 0;
447#endif
448 overcommitted(TRUE((boolean_t) 1), part->free);
449}
450
451/*
452 * Choose the most appropriate default partition
453 * for an object of SIZE bytes.
454 * Return the partition locked, unless
455 * the object has no CUR_PARTition.
456 */
457p_index_t
458choose_partition(size, cur_part)
459 unsigned int size;
460 p_index_t cur_part;
461{
462 partition_t part;
463 boolean_t found = FALSE((boolean_t) 0);
464 int i;
465
466 pthread_mutex_lock(&all_partitions.lock);
467 for (i = 0; i < all_partitions.n_partitions; i++) {
468
469 /* the undesirable one ? */
470 if (i == cur_part)
471 continue;
472
473ddprintf ("choose_partition(%x,%d,%d)\n",size,cur_part,i)(void) 0;
474 /* one that was removed ? */
475 if ((part = partition_of(i)) == 0)
476 continue;
477
478 /* one that is being removed ? */
479 if (part->going_away)
480 continue;
481
482 /* is it big enough ? */
483 pthread_mutex_lock(&part->p_lock);
484 if (ptoa(part->free)((part->free)*vm_page_size) >= size) {
485 if (cur_part != P_INDEX_INVALID((p_index_t)-1)) {
486 pthread_mutex_unlock(&all_partitions.lock);
487 return (p_index_t)i;
488 } else
489 found = TRUE((boolean_t) 1);
490 }
491 pthread_mutex_unlock(&part->p_lock);
492
493 if (found) break;
494 }
495 pthread_mutex_unlock(&all_partitions.lock);
496 return (found) ? (p_index_t)i : P_INDEX_INVALID((p_index_t)-1);
497}
498
499/*
500 * Allocate a page in a paging partition
501 * The partition is returned unlocked.
502 */
503vm_offset_t
504pager_alloc_page(pindex, lock_it)
505 p_index_t pindex;
506 boolean_t lock_it;
507{
508 int bm_e;
509 int bit;
510 int limit;
511 bm_entry_t *bm;
512 partition_t part;
513 static char here[] = "%spager_alloc_page";
514
515 if (no_partition(pindex)((pindex) == ((p_index_t)-1)))
516 return (NO_BLOCK((vm_offset_t)-1));
517ddprintf ("pager_alloc_page(%d,%d)\n",pindex,lock_it)(void) 0;
518 part = partition_of(pindex);
519
520 /* unlikely, but possible deadlock against destroy_partition */
521 if (!part || part->going_away)
522 return (NO_BLOCK((vm_offset_t)-1));
523
524 if (lock_it)
525 pthread_mutex_lock(&part->p_lock);
526
527 if (part->free == 0) {
528 /* out of paging space */
529 pthread_mutex_unlock(&part->p_lock);
530 return (NO_BLOCK((vm_offset_t)-1));
531 }
532
533 limit = howmany(part->total_size, NB_BM)(((part->total_size) + (32) - 1)/(32));
534 bm = part->bitmap;
535 for (bm_e = 0; bm_e < limit; bm_e++, bm++)
536 if (*bm != BM_MASK0xffffffff)
537 break;
538
539 if (bm_e == limit)
540 panic(here,my_name);
541
542 /*
543 * Find and set the proper bit
544 */
545 {
546 bm_entry_t b = *bm;
547
548 for (bit = 0; bit < NB_BM32; bit++)
549 if ((b & (1<<bit)) == 0)
550 break;
551 if (bit == NB_BM32)
552 panic(here,my_name);
553
554 *bm = b | (1<<bit);
555 part->free--;
556
557 }
558
559 pthread_mutex_unlock(&part->p_lock);
560
561 return (bm_e*NB_BM32+bit);
562}
563
564/*
565 * Deallocate a page in a paging partition
566 */
567void
568pager_dealloc_page(pindex, page, lock_it)
569 p_index_t pindex;
570 vm_offset_t page;
571 boolean_t lock_it;
572{
573 partition_t part;
574 int bit, bm_e;
575
576 /* be paranoid */
577 if (no_partition(pindex)((pindex) == ((p_index_t)-1)))
578 panic("%sdealloc_page",my_name);
579ddprintf ("pager_dealloc_page(%d,%x,%d)\n",pindex,page,lock_it)(void) 0;
580 part = partition_of(pindex);
581
582 if (page >= part->total_size)
583 panic("%sdealloc_page",my_name);
584
585 bm_e = page / NB_BM32;
586 bit = page % NB_BM32;
587
588 if (lock_it)
589 pthread_mutex_lock(&part->p_lock);
590
591 part->bitmap[bm_e] &= ~(1<<bit);
592 part->free++;
593
594 if (lock_it)
595 pthread_mutex_unlock(&part->p_lock);
596}
597
598/*
599 * Object sizes are rounded up to the next power of 2,
600 * unless they are bigger than a given maximum size.
601 */
602vm_size_t max_doubled_size = 4 * 1024 * 1024; /* 4 meg */
603
604/*
605 * Return first level map for pager.
606 * If there is no such map, than allocate it.
607 */
608dp_map_t pager_get_direct_map(pager)
609 dpager_t pager;
610{
611 dp_map_t mapptr, emapptr;
612 vm_size_t size = pager->size;
613
614 if (pager->map)
615 return pager->map;
616 /*
617 * Allocate and initialize the block map
618 */
619 {
620 vm_size_t alloc_size;
621 dp_map_t init_value;
622
623 if (INDIRECT_PAGEMAP(size)(size > 64)) {
624 alloc_size = INDIRECT_PAGEMAP_SIZE(size)(((((size)-1)/64) + 1) * sizeof(vm_offset_t *));
625 init_value = (dp_map_t)0;
626 } else {
627 alloc_size = PAGEMAP_SIZE(size)((size)*sizeof(vm_offset_t));
628 init_value = (dp_map_t)NO_BLOCK((vm_offset_t)-1);
629 }
630
631 mapptr = (dp_map_t) kalloc(alloc_size);
632 for (emapptr = &mapptr[(alloc_size-1) / sizeof(vm_offset_t)];
633 emapptr >= mapptr;
634 emapptr--)
635 emapptr->indirect = init_value;
636 }
637 pager->map = mapptr;
638 return mapptr;
639}
640
641/*
642 * Attach a new paging object to a paging partition
643 */
644void
645pager_alloc(pager, part, size)
646 dpager_t pager;
647 p_index_t part;
648 vm_size_t size; /* in BYTES */
649{
650 int i;
651#ifdef CHECKSUM
652 dp_map_t mapptr, emapptr;
653#endif
654
655 pthread_mutex_init(&pager->lock, NULL((void*)0));
656#if DEBUG_READER_CONFLICTS0
657 pager->readers = 0;
658 pager->writer = FALSE((boolean_t) 0);
659#endif
660 pager->cur_partition = part;
661
662 /*
663 * Convert byte size to number of pages, then increase to the nearest
664 * power of 2.
665 */
666 size = atop(size)((size)/vm_page_size);
667 if (size <= atop(max_doubled_size)((max_doubled_size)/vm_page_size)) {
668 i = 1;
669 while (i < size)
670 i <<= 1;
671 size = i;
672 } else
673 size = ROUNDUP_TO_PAGEMAP(size)(((size) + 64 - 1) & ~(64 - 1));
674
675 pager->map = NULL((void*)0);
676 pager->size = size;
677 pager->limit = (vm_size_t)-1;
678
679#ifdef CHECKSUM
680 if (INDIRECT_PAGEMAP(size)(size > 64)) {
681 mapptr = (vm_offset_t *)
682 kalloc(INDIRECT_PAGEMAP_SIZE(size)(((((size)-1)/64) + 1) * sizeof(vm_offset_t *)));
683 for (i = INDIRECT_PAGEMAP_ENTRIES(size)((((size)-1)/64) + 1); --i >= 0;)
684 mapptr[i] = 0;
685 } else {
686 mapptr = (vm_offset_t *) kalloc(PAGEMAP_SIZE(size)((size)*sizeof(vm_offset_t)));
687 for (i = 0; i < size; i++)
688 mapptr[i] = NO_CHECKSUM;
689 }
690 pager->checksum = mapptr;
691#endif /* CHECKSUM */
692}
693
694/*
695 * Return size (in bytes) of space actually allocated to this pager.
696 * The pager is read-locked.
697 */
698
699vm_size_t
700pager_allocated(pager)
701 dpager_t pager;
702{
703 vm_size_t size;
704 dp_map_t map, emap;
705 vm_size_t asize;
706
707 size = pager->size; /* in pages */
708 asize = 0; /* allocated, in pages */
709 map = pager_get_direct_map(pager);
710
711 if (INDIRECT_PAGEMAP(size)(size > 64)) {
712 for (emap = &map[INDIRECT_PAGEMAP_ENTRIES(size)((((size)-1)/64) + 1)];
713 map < emap; map++) {
714
715 dp_map_t map2, emap2;
716
717 if ((map2 = map->indirect) == 0)
718 continue;
719
720 for (emap2 = &map2[PAGEMAP_ENTRIES64];
721 map2 < emap2; map2++)
722 if ( ! no_block(*map2)((*map2).indirect == (dp_map_t)((vm_offset_t)-1)) )
723 asize++;
724
725 }
726 } else {
727 for (emap = &map[size]; map < emap; map++)
728 if ( ! no_block(*map)((*map).indirect == (dp_map_t)((vm_offset_t)-1)) )
729 asize++;
730 }
731
732 return ptoa(asize)((asize)*vm_page_size);
733}
734
735/*
736 * Find offsets (in the object) of pages actually allocated to this pager.
737 * Returns the number of allocated pages, whether or not they all fit.
738 * The pager is read-locked.
739 */
740
741unsigned int
742pager_pages(pager, pages, numpages)
743 dpager_t pager;
744 default_pager_page_t *pages;
745 unsigned int numpages;
746{
747 vm_size_t size;
748 dp_map_t map, emap;
749 unsigned int actual;
750 vm_offset_t offset;
751
752 size = pager->size; /* in pages */
753 map = pager_get_direct_map(pager);
754 actual = 0;
755 offset = 0;
756
757 if (INDIRECT_PAGEMAP(size)(size > 64)) {
758 for (emap = &map[INDIRECT_PAGEMAP_ENTRIES(size)((((size)-1)/64) + 1)];
759 map < emap; map++) {
760
761 dp_map_t map2, emap2;
762
763 if ((map2 = map->indirect) == 0) {
764 offset += vm_page_size * PAGEMAP_ENTRIES64;
765 continue;
766 }
767 for (emap2 = &map2[PAGEMAP_ENTRIES64];
768 map2 < emap2; map2++)
769 if ( ! no_block(*map2)((*map2).indirect == (dp_map_t)((vm_offset_t)-1)) ) {
770 if (actual++ < numpages)
771 pages++->dpp_offset = offset;
772 }
773 offset += vm_page_size;
774 }
775 } else {
776 for (emap = &map[size]; map < emap; map++) {
777 if ( ! no_block(*map)((*map).indirect == (dp_map_t)((vm_offset_t)-1)) ) {
778 if (actual++ < numpages)
779 pages++->dpp_offset = offset;
780 }
781 offset += vm_page_size;
782 }
783 }
784 return actual;
785}
786
787/*
788 * Extend the map for a paging object.
789 *
790 * XXX This implementation can allocate an arbitrary large amount
791 * of wired memory when extending a big block map. Because vm-privileged
792 * threads call pager_extend, this can crash the system by exhausting
793 * system memory.
794 */
795void
796pager_extend(pager, new_size)
797 dpager_t pager;
798 vm_size_t new_size; /* in pages */
799{
800 dp_map_t new_mapptr;
801 dp_map_t old_mapptr;
802 int i;
803 vm_size_t old_size;
804
805 pthread_mutex_lock(&pager->lock); /* XXX lock_write */
806#if DEBUG_READER_CONFLICTS0
807 pager->writer = TRUE((boolean_t) 1);
808#endif
809 /*
810 * Double current size until we cover new size.
811 * If object is 'too big' just use new size.
812 */
813 old_size = pager->size;
814
815 if (new_size <= atop(max_doubled_size)((max_doubled_size)/vm_page_size)) {
816 /* New size cannot be less than 1 */
817 i = old_size ? old_size : 1;
818 while (i < new_size)
819 i <<= 1;
820 new_size = i;
821 } else
822 new_size = ROUNDUP_TO_PAGEMAP(new_size)(((new_size) + 64 - 1) & ~(64 - 1));
823
824 if (INDIRECT_PAGEMAP(old_size)(old_size > 64)) {
825 /*
826 * Pager already uses two levels. Allocate
827 * a larger indirect block.
828 */
829 new_mapptr = (dp_map_t)
830 kalloc(INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
831 old_mapptr = pager_get_direct_map(pager);
832 for (i = 0; i < INDIRECT_PAGEMAP_ENTRIES(old_size)((((old_size)-1)/64) + 1); i++)
833 new_mapptr[i] = old_mapptr[i];
834 for (; i < INDIRECT_PAGEMAP_ENTRIES(new_size)((((new_size)-1)/64) + 1); i++)
835 new_mapptr[i].indirect = (dp_map_t)0;
836 kfree((char *)old_mapptr, INDIRECT_PAGEMAP_SIZE(old_size)(((((old_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
837 pager->map = new_mapptr;
838 pager->size = new_size;
839#ifdef CHECKSUM
840 new_mapptr = (vm_offset_t *)
841 kalloc(INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
842 old_mapptr = pager->checksum;
843 for (i = 0; i < INDIRECT_PAGEMAP_ENTRIES(old_size)((((old_size)-1)/64) + 1); i++)
844 new_mapptr[i] = old_mapptr[i];
845 for (; i < INDIRECT_PAGEMAP_ENTRIES(new_size)((((new_size)-1)/64) + 1); i++)
846 new_mapptr[i] = 0;
847 kfree((char *)old_mapptr, INDIRECT_PAGEMAP_SIZE(old_size)(((((old_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
848 pager->checksum = new_mapptr;
849#endif /* CHECKSUM */
850#if DEBUG_READER_CONFLICTS0
851 pager->writer = FALSE((boolean_t) 0);
852#endif
853 pthread_mutex_unlock(&pager->lock);
854#if 0
855 ddprintf ("pager_extend 1 mapptr %x [3b] = %x\n", new_mapptr,(void) 0
856 new_mapptr[0x3b])(void) 0;
857 if (new_mapptr[0x3b].indirect > 0x10000
858 && new_mapptr[0x3b].indirect != NO_BLOCK((vm_offset_t)-1))
859 panic ("debug panic");
860#endif
861 return;
862 }
863
864 if (INDIRECT_PAGEMAP(new_size)(new_size > 64)) {
865 /*
866 * Changing from direct map to indirect map.
867 * Allocate both indirect and direct map blocks,
868 * since second-level (direct) block must be
869 * full size (PAGEMAP_SIZE(PAGEMAP_ENTRIES)).
870 */
871
872 /*
873 * Allocate new second-level map first.
874 */
875 new_mapptr = (dp_map_t) kalloc(PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
876 old_mapptr = pager_get_direct_map(pager);
877 for (i = 0; i < old_size; i++)
878 new_mapptr[i] = old_mapptr[i];
879 for (; i < PAGEMAP_ENTRIES64; i++)
880 invalidate_block(new_mapptr[i])((new_mapptr[i]).indirect = (dp_map_t)((vm_offset_t)-1));
881 kfree((char *)old_mapptr, PAGEMAP_SIZE(old_size)((old_size)*sizeof(vm_offset_t)));
882 old_mapptr = new_mapptr;
883
884#if 0
885 ddprintf ("pager_extend 2 mapptr %x [3b] = %x\n", new_mapptr,(void) 0
886 new_mapptr[0x3b])(void) 0;
887 if (new_mapptr[0x3b].indirect > 0x10000
888 && new_mapptr[0x3b].indirect != NO_BLOCK((vm_offset_t)-1))
889 panic ("debug panic");
890#endif
891
892 /*
893 * Now allocate indirect map.
894 */
895 new_mapptr = (dp_map_t)
896 kalloc(INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
897 new_mapptr[0].indirect = old_mapptr;
898 for (i = 1; i < INDIRECT_PAGEMAP_ENTRIES(new_size)((((new_size)-1)/64) + 1); i++)
899 new_mapptr[i].indirect = 0;
900 pager->map = new_mapptr;
901 pager->size = new_size;
902#ifdef CHECKSUM
903 /*
904 * Allocate new second-level map first.
905 */
906 new_mapptr = (vm_offset_t *)kalloc(PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
907 old_mapptr = pager->checksum;
908 for (i = 0; i < old_size; i++)
909 new_mapptr[i] = old_mapptr[i];
910 for (; i < PAGEMAP_ENTRIES64; i++)
911 new_mapptr[i] = NO_CHECKSUM;
912 kfree((char *)old_mapptr, PAGEMAP_SIZE(old_size)((old_size)*sizeof(vm_offset_t)));
913 old_mapptr = new_mapptr;
914
915 /*
916 * Now allocate indirect map.
917 */
918 new_mapptr = (vm_offset_t *)
919 kalloc(INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
920 new_mapptr[0] = (vm_offset_t) old_mapptr;
921 for (i = 1; i < INDIRECT_PAGEMAP_ENTRIES(new_size)((((new_size)-1)/64) + 1); i++)
922 new_mapptr[i] = 0;
923 pager->checksum = new_mapptr;
924#endif /* CHECKSUM */
925#if DEBUG_READER_CONFLICTS0
926 pager->writer = FALSE((boolean_t) 0);
927#endif
928 pthread_mutex_unlock(&pager->lock);
929 return;
930 }
931 /*
932 * Enlarging a direct block.
933 */
934 new_mapptr = (dp_map_t) kalloc(PAGEMAP_SIZE(new_size)((new_size)*sizeof(vm_offset_t)));
935 old_mapptr = pager_get_direct_map(pager);
936 for (i = 0; i < old_size; i++)
937 new_mapptr[i] = old_mapptr[i];
938 for (; i < new_size; i++)
939 invalidate_block(new_mapptr[i])((new_mapptr[i]).indirect = (dp_map_t)((vm_offset_t)-1));
940 kfree((char *)old_mapptr, PAGEMAP_SIZE(old_size)((old_size)*sizeof(vm_offset_t)));
941 pager->map = new_mapptr;
942 pager->size = new_size;
943#ifdef CHECKSUM
944 new_mapptr = (vm_offset_t *)
945 kalloc(PAGEMAP_SIZE(new_size)((new_size)*sizeof(vm_offset_t)));
946 old_mapptr = pager->checksum;
947 for (i = 0; i < old_size; i++)
948 new_mapptr[i] = old_mapptr[i];
949 for (; i < new_size; i++)
950 new_mapptr[i] = NO_CHECKSUM;
951 kfree((char *)old_mapptr, PAGEMAP_SIZE(old_size)((old_size)*sizeof(vm_offset_t)));
952 pager->checksum = new_mapptr;
953#endif /* CHECKSUM */
954#if DEBUG_READER_CONFLICTS0
955 pager->writer = FALSE((boolean_t) 0);
956#endif
957 pthread_mutex_unlock(&pager->lock);
958}
959
960/* This deallocates the pages necessary to truncate a direct map
961 previously of size NEW_SIZE to the smaller size OLD_SIZE. */
962static void
963dealloc_direct (dp_map_t mapptr,
964 vm_size_t old_size, vm_size_t new_size)
965{
966 vm_size_t i;
967
968 if (!mapptr)
969 return;
970
971 for (i = new_size; i < old_size; ++i)
972 {
973 const union dp_map entry = mapptr[i];
974 if (!no_block(entry)((entry).indirect == (dp_map_t)((vm_offset_t)-1)))
975 {
976 pager_dealloc_page(entry.block.p_index, entry.block.p_offset,
977 TRUE((boolean_t) 1));
978 invalidate_block(mapptr[i])((mapptr[i]).indirect = (dp_map_t)((vm_offset_t)-1));
979 }
980 }
981}
982
983/* Truncate a memory object. First, any pages between the new size
984 and the (larger) old size are deallocated. Then, the size of
985 the pagemap may be reduced, an indirect map may be turned into
986 a direct map.
987
988 The pager must be locked by the caller. */
989static void
990pager_truncate(dpager_t pager, vm_size_t new_size) /* in pages */
991{
992 int i;
993 vm_size_t old_size;
994
995 pthread_mutex_lock(&pager->lock); /* XXX lock_write */
996
997 if (!pager->map)
998 goto done;
999
1000 old_size = pager->size;
1001
1002 if (INDIRECT_PAGEMAP(old_size)(old_size > 64))
1003 {
1004 /* First handle the entire second-levels blocks that are being freed. */
1005 for (i = INDIRECT_PAGEMAP_ENTRIES(new_size)((((new_size)-1)/64) + 1);
1006 i < INDIRECT_PAGEMAP_ENTRIES(old_size)((((old_size)-1)/64) + 1);
1007 ++i)
1008 {
1009 const dp_map_t mapptr = pager->map[i].indirect;
1010 pager->map[i].indirect = (dp_map_t)0;
1011 dealloc_direct (mapptr, PAGEMAP_ENTRIES64, 0);
1012 kfree ((char *)mapptr, PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
1013 }
1014
1015 /* Now truncate what's now the final nonempty direct block. */
1016 dealloc_direct (pager->map[(new_size - 1) / PAGEMAP_ENTRIES64].indirect,
1017 old_size & (PAGEMAP_ENTRIES64 - 1),
1018 new_size & (PAGEMAP_ENTRIES64 - 1));
1019
1020 if (INDIRECT_PAGEMAP (new_size)(new_size > 64))
1021 {
1022 const dp_map_t old_mapptr = pager->map;
1023 pager->map = (dp_map_t) kalloc (INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1024 memcpy (pager->map, old_mapptr, INDIRECT_PAGEMAP_SIZE(new_size)(((((new_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1025 kfree ((char *) old_mapptr, INDIRECT_PAGEMAP_SIZE (old_size)(((((old_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1026 }
1027 else
1028 {
1029 /* We are truncating to a size small enough that it goes to using
1030 a one-level map. We already have that map, as the first and only
1031 nonempty element in our indirect map. */
1032 const dp_map_t mapptr = pager->map[0].indirect;
1033 kfree((char *)pager->map, INDIRECT_PAGEMAP_SIZE(old_size)(((((old_size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1034 pager->map = mapptr;
1035 }
1036 }
1037
1038 if (! INDIRECT_PAGEMAP(old_size)(old_size > 64))
1039 {
1040 /* First deallocate pages in the truncated region. */
1041 dealloc_direct (pager->map, old_size, new_size);
1042 /* Now reduce the size of the direct map itself. We don't bother
1043 with kalloc/kfree if it's not shrinking enough that kalloc.c
1044 would actually use less. */
1045 if (PAGEMAP_SIZE (new_size)((new_size)*sizeof(vm_offset_t)) <= PAGEMAP_SIZE (old_size)((old_size)*sizeof(vm_offset_t)) / 2)
1046 {
1047 const dp_map_t old_mapptr = pager->map;
1048 pager->map = (dp_map_t) kalloc (PAGEMAP_SIZE (new_size)((new_size)*sizeof(vm_offset_t)));
1049 memcpy (pager->map, old_mapptr, PAGEMAP_SIZE (new_size)((new_size)*sizeof(vm_offset_t)));
1050 kfree ((char *) old_mapptr, PAGEMAP_SIZE (old_size)((old_size)*sizeof(vm_offset_t)));
1051 }
1052 }
1053
1054 done:
1055 pager->size = new_size;
1056 pthread_mutex_unlock(&pager->lock);
1057
1058#ifdef CHECKSUM
1059#error write me
1060#endif /* CHECKSUM */
1061}
1062
1063
1064/*
1065 * Given an offset within a paging object, find the
1066 * corresponding block within the paging partition.
1067 * Return NO_BLOCK if none allocated.
1068 */
1069union dp_map
1070pager_read_offset(pager, offset)
1071 dpager_t pager;
1072 vm_offset_t offset;
1073{
1074 vm_offset_t f_page;
1075 union dp_map pager_offset;
1076
1077 f_page = atop(offset)((offset)/vm_page_size);
1078
1079#if DEBUG_READER_CONFLICTS0
1080 if (pager->readers > 0)
1081 default_pager_read_conflicts++; /* would have proceeded with
1082 read/write lock */
1083#endif
1084 pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1085#if DEBUG_READER_CONFLICTS0
1086 pager->readers++;
1087#endif
1088 if (f_page >= pager->size)
1089 {
1090 ddprintf ("%spager_read_offset pager %x: bad page %d >= size %d",(void) 0
1091 my_name, pager, f_page, pager->size)(void) 0;
1092 pthread_mutex_unlock(&pager->lock);
1093 return (union dp_map) (union dp_map *) NO_BLOCK((vm_offset_t)-1);
1094#if 0
1095 panic("%spager_read_offset",my_name);
1096#endif
1097 }
1098
1099 invalidate_block(pager_offset)((pager_offset).indirect = (dp_map_t)((vm_offset_t)-1));
1100 if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1101 dp_map_t mapptr;
1102
1103 if (pager->map) {
1104 mapptr = pager->map[f_page/PAGEMAP_ENTRIES64].indirect;
1105 if (mapptr)
1106 pager_offset = mapptr[f_page%PAGEMAP_ENTRIES64];
1107 }
1108 }
1109 else {
1110 if (pager->map)
1111 pager_offset = pager->map[f_page];
1112 }
1113
1114#if DEBUG_READER_CONFLICTS0
1115 pager->readers--;
1116#endif
1117 pthread_mutex_unlock(&pager->lock);
1118 return (pager_offset);
1119}
1120
1121#if USE_PRECIOUS1
1122/*
1123 * Release a single disk block.
1124 */
1125void pager_release_offset(pager, offset)
1126 dpager_t pager;
1127 vm_offset_t offset;
1128{
1129 union dp_map entry;
1130
1131 offset = atop(offset)((offset)/vm_page_size);
1132
1133 pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1134
1135 assert (pager->map)((pager->map) ? (void) (0) : __assert_fail ("pager->map"
, "../../mach-defpager/default_pager.c", 1135, __PRETTY_FUNCTION__
));
1136 if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1137 dp_map_t mapptr;
1138
1139 mapptr = pager->map[offset / PAGEMAP_ENTRIES64].indirect;
1140 entry = mapptr[offset % PAGEMAP_ENTRIES64];
1141 invalidate_block(mapptr[offset % PAGEMAP_ENTRIES])((mapptr[offset % 64]).indirect = (dp_map_t)((vm_offset_t)-1)
);
1142 } else {
1143 entry = pager->map[offset];
1144 invalidate_block(pager->map[offset])((pager->map[offset]).indirect = (dp_map_t)((vm_offset_t)-
1));
1145 }
1146
1147 pthread_mutex_unlock(&pager->lock);
1148
1149 pager_dealloc_page(entry.block.p_index, entry.block.p_offset, TRUE((boolean_t) 1));
1150}
1151#endif /*USE_PRECIOUS*/
1152
1153
1154/*
1155 * Move a page from one partition to another
1156 * New partition is locked, old partition is
1157 * locked unless LOCK_OLD sez otherwise.
1158 */
1159union dp_map
1160pager_move_page(block)
1161 union dp_map block;
1162{
1163 partition_t old_part, new_part;
1164 p_index_t old_pindex, new_pindex;
1165 union dp_map ret;
1166 vm_size_t size;
1167 vm_offset_t raddr, offset, new_offset;
1168 kern_return_t rc;
1169 static char here[] = "%spager_move_page";
1170
1171 old_pindex = block.block.p_index;
1172 invalidate_block(ret)((ret).indirect = (dp_map_t)((vm_offset_t)-1));
1173
1174 /* See if we have room to put it anywhere else */
1175 new_pindex = choose_partition( ptoa(1)((1)*vm_page_size), old_pindex);
1176 if (no_partition(new_pindex)((new_pindex) == ((p_index_t)-1)))
1177 return ret;
1178
1179 /* this unlocks the new partition */
1180 new_offset = pager_alloc_page(new_pindex, FALSE((boolean_t) 0));
1181 if (new_offset == NO_BLOCK((vm_offset_t)-1))
1182 panic(here,my_name);
1183
1184 /*
1185 * Got the resources, now move the data
1186 */
1187ddprintf ("pager_move_page(%x,%d,%d)\n",block.block.p_offset,old_pindex,new_pindex)(void) 0;
1188 old_part = partition_of(old_pindex);
1189 offset = ptoa(block.block.p_offset)((block.block.p_offset)*vm_page_size);
1190 rc = page_read_file_direct (old_part->file,
1191 offset,
1192 vm_page_size,
1193 &raddr,
1194 &size);
1195 if (rc != 0)
1196 panic(here,my_name);
1197
1198 /* release old */
1199 pager_dealloc_page(old_pindex, block.block.p_offset, FALSE((boolean_t) 0));
1200
1201 new_part = partition_of(new_pindex);
1202 offset = ptoa(new_offset)((new_offset)*vm_page_size);
1203 rc = page_write_file_direct (new_part->file,
1204 offset,
1205 raddr,
1206 size,
1207 &size);
1208 if (rc != 0)
1209 panic(here,my_name);
1210
1211 (void) vm_deallocate( mach_task_self()((__mach_task_self_ + 0)), raddr, size);
1212
1213 ret.block.p_offset = new_offset;
1214 ret.block.p_index = new_pindex;
1215
1216 return ret;
1217}
1218
1219#ifdef CHECKSUM
1220/*
1221 * Return the checksum for a block.
1222 */
1223int
1224pager_get_checksum(pager, offset)
1225 dpager_t pager;
1226 vm_offset_t offset;
1227{
1228 vm_offset_t f_page;
1229 int checksum;
1230
1231 f_page = atop(offset)((offset)/vm_page_size);
1232
1233 pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1234 if (f_page >= pager->size)
1235 panic("%spager_get_checksum",my_name);
1236
1237 if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1238 vm_offset_t *mapptr;
1239
1240 mapptr = (vm_offset_t *)pager->checksum[f_page/PAGEMAP_ENTRIES64];
1241 if (mapptr == 0)
1242 checksum = NO_CHECKSUM;
1243 else
1244 checksum = mapptr[f_page%PAGEMAP_ENTRIES64];
1245 }
1246 else {
1247 checksum = pager->checksum[f_page];
1248 }
1249
1250 pthread_mutex_unlock(&pager->lock);
1251 return (checksum);
1252}
1253
1254/*
1255 * Remember the checksum for a block.
1256 */
1257int
1258pager_put_checksum(pager, offset, checksum)
1259 dpager_t pager;
1260 vm_offset_t offset;
1261 int checksum;
1262{
1263 vm_offset_t f_page;
1264 static char here[] = "%spager_put_checksum";
1265
1266 f_page = atop(offset)((offset)/vm_page_size);
1267
1268 pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1269 if (f_page >= pager->size)
1270 panic(here,my_name);
1271
1272 if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1273 vm_offset_t *mapptr;
1274
1275 mapptr = (vm_offset_t *)pager->checksum[f_page/PAGEMAP_ENTRIES64];
1276 if (mapptr == 0)
1277 panic(here,my_name);
1278
1279 mapptr[f_page%PAGEMAP_ENTRIES64] = checksum;
1280 }
1281 else {
1282 pager->checksum[f_page] = checksum;
1283 }
1284 pthread_mutex_unlock(&pager->lock);
1285}
1286
1287/*
1288 * Compute a checksum - XOR each 32-bit word.
1289 */
1290int
1291compute_checksum(addr, size)
1292 vm_offset_t addr;
1293 vm_size_t size;
1294{
1295 int checksum = NO_CHECKSUM;
1296 int *ptr;
1297 int count;
1298
1299 ptr = (int *)addr;
1300 count = size / sizeof(int);
1301
1302 while (--count >= 0)
1303 checksum ^= *ptr++;
1304
1305 return (checksum);
1306}
1307#endif /* CHECKSUM */
1308
1309/*
1310 * Given an offset within a paging object, find the
1311 * corresponding block within the paging partition.
1312 * Allocate a new block if necessary.
1313 *
1314 * WARNING: paging objects apparently may be extended
1315 * without notice!
1316 */
1317union dp_map
1318pager_write_offset(pager, offset)
1319 dpager_t pager;
1320 vm_offset_t offset;
1321{
1322 vm_offset_t f_page;
1323 dp_map_t mapptr;
1324 union dp_map block;
1325
1326 invalidate_block(block)((block).indirect = (dp_map_t)((vm_offset_t)-1));
1327
1328 f_page = atop(offset)((offset)/vm_page_size);
1329
1330#if DEBUG_READER_CONFLICTS0
1331 if (pager->readers > 0)
1332 default_pager_read_conflicts++; /* would have proceeded with
1333 read/write lock */
1334#endif
1335 pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1336#if DEBUG_READER_CONFLICTS0
1337 pager->readers++;
1338#endif
1339
1340 /* Catch the case where we had no initial fit partition
1341 for this object, but one was added later on */
1342 if (no_partition(pager->cur_partition)((pager->cur_partition) == ((p_index_t)-1))) {
1343 p_index_t new_part;
1344 vm_size_t size;
1345
1346 size = (f_page > pager->size) ? f_page : pager->size;
1347 new_part = choose_partition(ptoa(size)((size)*vm_page_size), P_INDEX_INVALID((p_index_t)-1));
1348 if (no_partition(new_part)((new_part) == ((p_index_t)-1)))
1349 new_part = choose_partition(ptoa(1)((1)*vm_page_size), P_INDEX_INVALID((p_index_t)-1));
1350 if (no_partition(new_part)((new_part) == ((p_index_t)-1)))
1351 /* give up right now to avoid confusion */
1352 goto out;
1353 else
1354 pager->cur_partition = new_part;
1355 }
1356
1357 while (f_page >= pager->size) {
1358 ddprintf ("pager_write_offset: extending: %x %x\n", f_page, pager->size)(void) 0;
1359
1360 /*
1361 * Paging object must be extended.
1362 * Remember that offset is 0-based, but size is 1-based.
1363 */
1364#if DEBUG_READER_CONFLICTS0
1365 pager->readers--;
1366#endif
1367 pthread_mutex_unlock(&pager->lock);
1368 pager_extend(pager, f_page + 1);
1369#if DEBUG_READER_CONFLICTS0
1370 if (pager->readers > 0)
1371 default_pager_read_conflicts++; /* would have proceeded with
1372 read/write lock */
1373#endif
1374 pthread_mutex_lock(&pager->lock); /* XXX lock_read */
1375#if DEBUG_READER_CONFLICTS0
1376 pager->readers++;
1377#endif
1378 ddprintf ("pager_write_offset: done extending: %x %x\n", f_page, pager->size)(void) 0;
1379 }
1380
1381 if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1382 ddprintf ("pager_write_offset: indirect\n")(void) 0;
1383 mapptr = pager_get_direct_map(pager);
1384 mapptr = mapptr[f_page/PAGEMAP_ENTRIES64].indirect;
1385 if (mapptr == 0) {
1386 /*
1387 * Allocate the indirect block
1388 */
1389 int i;
1390 ddprintf ("pager_write_offset: allocating indirect\n")(void) 0;
1391
1392 mapptr = (dp_map_t) kalloc(PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
1393 if (mapptr == 0) {
1394 /* out of space! */
1395 no_paging_space(TRUE((boolean_t) 1));
1396 goto out;
1397 }
1398 pager->map[f_page/PAGEMAP_ENTRIES64].indirect = mapptr;
1399 for (i = 0; i < PAGEMAP_ENTRIES64; i++)
1400 invalidate_block(mapptr[i])((mapptr[i]).indirect = (dp_map_t)((vm_offset_t)-1));
1401#ifdef CHECKSUM
1402 {
1403 vm_offset_t *cksumptr;
1404 int j;
1405
1406 cksumptr = (vm_offset_t *)
1407 kalloc(PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
1408 if (cksumptr == 0) {
1409 /* out of space! */
1410 no_paging_space(TRUE((boolean_t) 1));
1411 goto out;
1412 }
1413 pager->checksum[f_page/PAGEMAP_ENTRIES64]
1414 = (vm_offset_t)cksumptr;
1415 for (j = 0; j < PAGEMAP_ENTRIES64; j++)
1416 cksumptr[j] = NO_CHECKSUM;
1417 }
1418#endif /* CHECKSUM */
1419 }
1420 f_page %= PAGEMAP_ENTRIES64;
1421 }
1422 else {
1423 mapptr = pager_get_direct_map(pager);
1424 }
1425
1426 block = mapptr[f_page];
1427 ddprintf ("pager_write_offset: block starts as %x[%x] %x\n", mapptr, f_page, block)(void) 0;
1428 if (no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1))) {
1429 vm_offset_t off;
1430
1431 /* get room now */
1432 off = pager_alloc_page(pager->cur_partition, TRUE((boolean_t) 1));
1433 if (off == NO_BLOCK((vm_offset_t)-1)) {
1434 /*
1435 * Before giving up, try all other partitions.
1436 */
1437 p_index_t new_part;
1438
1439 ddprintf ("pager_write_offset: could not allocate block\n")(void) 0;
1440 /* returns it locked (if any one is non-full) */
1441 new_part = choose_partition( ptoa(1)((1)*vm_page_size), pager->cur_partition);
1442 if ( ! no_partition(new_part)((new_part) == ((p_index_t)-1)) ) {
1443
1444#if debug0
1445dprintf("%s partition %x filled,", my_name, pager->cur_partition)(void) 0;
1446dprintf("extending object %x (size %x) to %x.\n",(void) 0
1447 pager, pager->size, new_part)(void) 0;
1448#endif
1449
1450 /* this one tastes better */
1451 pager->cur_partition = new_part;
1452
1453 /* this unlocks the partition too */
1454 off = pager_alloc_page(pager->cur_partition, FALSE((boolean_t) 0));
1455
1456 }
1457
1458 if (off == NO_BLOCK((vm_offset_t)-1)) {
1459 /*
1460 * Oh well.
1461 */
1462 overcommitted(FALSE((boolean_t) 0), 1);
1463 goto out;
1464 }
1465 ddprintf ("pager_write_offset: decided to allocate block\n")(void) 0;
1466 }
1467 block.block.p_offset = off;
1468 block.block.p_index = pager->cur_partition;
1469 mapptr[f_page] = block;
1470 }
1471
1472out:
1473
1474#if DEBUG_READER_CONFLICTS0
1475 pager->readers--;
1476#endif
1477 pthread_mutex_unlock(&pager->lock);
1478 return (block);
1479}
1480
1481/*
1482 * Deallocate all of the blocks belonging to a paging object.
1483 * No locking needed because no other operations can be in progress.
1484 */
1485void
1486pager_dealloc(pager)
1487 dpager_t pager;
1488{
1489 int i, j;
1490 dp_map_t mapptr;
1491 union dp_map block;
1492
1493 if (!pager->map)
1494 return;
1495
1496 if (INDIRECT_PAGEMAP(pager->size)(pager->size > 64)) {
1497 for (i = INDIRECT_PAGEMAP_ENTRIES(pager->size)((((pager->size)-1)/64) + 1); --i >= 0; ) {
1498 mapptr = pager->map[i].indirect;
1499 if (mapptr != 0) {
1500 for (j = 0; j < PAGEMAP_ENTRIES64; j++) {
1501 block = mapptr[j];
1502 if ( ! no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)) )
1503 pager_dealloc_page(block.block.p_index,
1504 block.block.p_offset, TRUE((boolean_t) 1));
1505 }
1506 kfree((char *)mapptr, PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
1507 pager->map[i].indirect = (dp_map_t) 0;
1508 }
1509 }
1510 kfree((char *)pager->map, INDIRECT_PAGEMAP_SIZE(pager->size)(((((pager->size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1511 pager->map = (dp_map_t) 0;
1512#ifdef CHECKSUM
1513 for (i = INDIRECT_PAGEMAP_ENTRIES(pager->size)((((pager->size)-1)/64) + 1); --i >= 0; ) {
1514 mapptr = (vm_offset_t *)pager->checksum[i];
1515 if (mapptr) {
1516 kfree((char *)mapptr, PAGEMAP_SIZE(PAGEMAP_ENTRIES)((64)*sizeof(vm_offset_t)));
1517 }
1518 }
1519 kfree((char *)pager->checksum,
1520 INDIRECT_PAGEMAP_SIZE(pager->size)(((((pager->size)-1)/64) + 1) * sizeof(vm_offset_t *)));
1521#endif /* CHECKSUM */
1522 }
1523 else {
1524 mapptr = pager->map;
1525 for (i = 0; i < pager->size; i++ ) {
1526 block = mapptr[i];
1527 if ( ! no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)) )
1528 pager_dealloc_page(block.block.p_index,
1529 block.block.p_offset, TRUE((boolean_t) 1));
1530 }
1531 kfree((char *)pager->map, PAGEMAP_SIZE(pager->size)((pager->size)*sizeof(vm_offset_t)));
1532 pager->map = (dp_map_t) 0;
1533#ifdef CHECKSUM
1534 kfree((char *)pager->checksum, PAGEMAP_SIZE(pager->size)((pager->size)*sizeof(vm_offset_t)));
1535#endif /* CHECKSUM */
1536 }
1537}
1538
1539/*
1540 * Move all the pages of a PAGER that live in a
1541 * partition PINDEX somewhere else.
1542 * Pager should be write-locked, partition too.
1543 * Returns FALSE if it could not do it, but
1544 * some pages might have been moved nonetheless.
1545 */
1546boolean_t
1547pager_realloc(pager, pindex)
1548 dpager_t pager;
1549 p_index_t pindex;
1550{
1551 dp_map_t map, emap;
1552 vm_size_t size;
1553 union dp_map block;
1554
1555 if (!pager->map)
1556 return TRUE((boolean_t) 1);
1557
1558 size = pager->size; /* in pages */
1559 map = pager->map;
1560
1561 if (INDIRECT_PAGEMAP(size)(size > 64)) {
1562 for (emap = &map[INDIRECT_PAGEMAP_ENTRIES(size)((((size)-1)/64) + 1)];
1563 map < emap; map++) {
1564
1565 dp_map_t map2, emap2;
1566
1567 if ((map2 = map->indirect) == 0)
1568 continue;
1569
1570 for (emap2 = &map2[PAGEMAP_ENTRIES64];
1571 map2 < emap2; map2++)
1572 if ( map2->block.p_index == pindex) {
1573
1574 block = pager_move_page(*map2);
1575 if (!no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)))
1576 *map2 = block;
1577 else
1578 return FALSE((boolean_t) 0);
1579 }
1580
1581 }
1582 goto ok;
1583 }
1584
1585 /* A small one */
1586 for (emap = &map[size]; map < emap; map++)
1587 if (map->block.p_index == pindex) {
1588 block = pager_move_page(*map);
1589 if (!no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)))
1590 *map = block;
1591 else
1592 return FALSE((boolean_t) 0);
1593 }
1594ok:
1595 pager->cur_partition = choose_partition(0, P_INDEX_INVALID((p_index_t)-1));
1596 return TRUE((boolean_t) 1);
1597}
1598
1599/*
1600 * Read/write routines.
1601 */
1602#define PAGER_SUCCESS0 0
1603#define PAGER_ABSENT1 1
1604#define PAGER_ERROR2 2
1605
1606/*
1607 * Read data from a default pager. Addr is the address of a buffer
1608 * to fill. Out_addr returns the buffer that contains the data;
1609 * if it is different from <addr>, it must be deallocated after use.
1610 */
1611int
1612default_read(ds, addr, size, offset, out_addr, deallocate, external)
1613 dpager_t ds;
1614 vm_offset_t addr; /* pointer to block to fill */
1615 vm_size_t size;
1616 vm_offset_t offset;
1617 vm_offset_t *out_addr;
1618 /* returns pointer to data */
1619 boolean_t deallocate;
1620 boolean_t external;
1621{
1622 union dp_map block;
1623 vm_offset_t raddr;
1624 vm_size_t rsize;
1625 int rc;
1626 boolean_t first_time;
1627 partition_t part;
1628#ifdef CHECKSUM
1629 vm_size_t original_size = size;
1630#endif /* CHECKSUM */
1631 vm_offset_t original_offset = offset;
1632
1633 /*
1634 * Find the block in the paging partition
1635 */
1636 block = pager_read_offset(ds, offset);
1637 if ( no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)) ) {
1638 if (external) {
1639 /*
1640 * An external object is requesting unswapped data,
1641 * zero fill the page and return.
1642 */
1643 bzero((char *) addr, vm_page_size);
1644 *out_addr = addr;
1645 return (PAGER_SUCCESS0);
1646 }
1647 return (PAGER_ABSENT1);
1648 }
1649
1650 /*
1651 * Read it, trying for the entire page.
1652 */
1653 offset = ptoa(block.block.p_offset)((block.block.p_offset)*vm_page_size);
1654ddprintf ("default_read(%x,%x,%x,%d)\n",addr,size,offset,block.block.p_index)(void) 0;
1655 part = partition_of(block.block.p_index);
1656 first_time = TRUE((boolean_t) 1);
1657 *out_addr = addr;
1658
1659 do {
1660 rc = page_read_file_direct(part->file,
1661 offset,
1662 size,
1663 &raddr,
1664 &rsize);
1665 if (rc != 0)
1666 return (PAGER_ERROR2);
1667
1668 /*
1669 * If we got the entire page on the first read, return it.
1670 */
1671 if (first_time && rsize == size) {
1672 *out_addr = raddr;
1673 break;
1674 }
1675 /*
1676 * Otherwise, copy the data into the
1677 * buffer we were passed, and try for
1678 * the next piece.
1679 */
1680 first_time = FALSE((boolean_t) 0);
1681 bcopy((char *)raddr, (char *)addr, rsize);
1682 addr += rsize;
1683 offset += rsize;
1684 size -= rsize;
1685 } while (size != 0);
1686
1687#if USE_PRECIOUS1
1688 if (deallocate)
1689 pager_release_offset(ds, original_offset);
1690#endif /*USE_PRECIOUS*/
1691
1692#ifdef CHECKSUM
1693 {
1694 int write_checksum,
1695 read_checksum;
1696
1697 write_checksum = pager_get_checksum(ds, original_offset);
1698 read_checksum = compute_checksum(*out_addr, original_size);
1699 if (write_checksum != read_checksum) {
1700 panic(
1701 "PAGER CHECKSUM ERROR: offset 0x%x, written 0x%x, read 0x%x",
1702 original_offset, write_checksum, read_checksum);
1703 }
1704 }
1705#endif /* CHECKSUM */
1706 return (PAGER_SUCCESS0);
1707}
1708
1709int
1710default_write(ds, addr, size, offset)
1711 dpager_t ds;
1712 vm_offset_t addr;
1713 vm_size_t size;
1714 vm_offset_t offset;
1715{
1716 union dp_map block;
1717 partition_t part;
1718 vm_size_t wsize;
1719 int rc;
1720
1721 ddprintf ("default_write: pager offset %x\n", offset)(void) 0;
1722
1723 /*
1724 * Find block in paging partition
1725 */
1726 block = pager_write_offset(ds, offset);
1727 if ( no_block(block)((block).indirect == (dp_map_t)((vm_offset_t)-1)) )
1728 return (PAGER_ERROR2);
1729
1730#ifdef CHECKSUM
1731 /*
1732 * Save checksum
1733 */
1734 {
1735 int checksum;
1736
1737 checksum = compute_checksum(addr, size);
1738 pager_put_checksum(ds, offset, checksum);
1739 }
1740#endif /* CHECKSUM */
1741 offset = ptoa(block.block.p_offset)((block.block.p_offset)*vm_page_size);
1742ddprintf ("default_write(%x,%x,%x,%d)\n",addr,size,offset,block.block.p_index)(void) 0;
1743 part = partition_of(block.block.p_index);
1744
1745 /*
1746 * There are various assumptions made here,we
1747 * will not get into the next disk 'block' by
1748 * accident. It might well be non-contiguous.
1749 */
1750 do {
1751 rc = page_write_file_direct(part->file,
1752 offset,
1753 addr,
1754 size,
1755 &wsize);
1756 if (rc != 0) {
1757 dprintf("*** PAGER ERROR: default_write: ")(void) 0;
1758 dprintf("ds=0x%x addr=0x%x size=0x%x offset=0x%x resid=0x%x\n",(void) 0
1759 ds, addr, size, offset, wsize)(void) 0;
1760 return (PAGER_ERROR2);
1761 }
1762 addr += wsize;
1763 offset += wsize;
1764 size -= wsize;
1765 } while (size != 0);
1766 return (PAGER_SUCCESS0);
1767}
1768
1769boolean_t
1770default_has_page(ds, offset)
1771 dpager_t ds;
1772 vm_offset_t offset;
1773{
1774 return ( ! no_block(pager_read_offset(ds, offset))((pager_read_offset(ds, offset)).indirect == (dp_map_t)((vm_offset_t
)-1)) );
1775}
1776
1777#if PARALLEL1
1778#define dstruct_lock_init(ds)pthread_mutex_init(&ds->lock, ((void*)0)) pthread_mutex_init(&ds->lock, NULL((void*)0))
1779#define dstruct_lock(ds)pthread_mutex_lock(&ds->lock) pthread_mutex_lock(&ds->lock)
1780#define dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock) pthread_mutex_unlock(&ds->lock)
1781#else /* PARALLEL */
1782#define dstruct_lock_init(ds)pthread_mutex_init(&ds->lock, ((void*)0))
1783#define dstruct_lock(ds)pthread_mutex_lock(&ds->lock)
1784#define dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock)
1785#endif /* PARALLEL */
1786
1787struct pager_port all_pagers;
1788
1789#define pager_port_list_init(){ pthread_mutex_init(&all_pagers.lock, ((void*)0)); hurd_ihash_init
(&all_pagers.htable, __builtin_offsetof(struct dstruct, htable_locp
)); ((&all_pagers.leak_queue)->next = (&all_pagers
.leak_queue)->prev = &all_pagers.leak_queue); } \
1790{ \
1791 pthread_mutex_init(&all_pagers.lock, NULL((void*)0)); \
1792 hurd_ihash_init (&all_pagers.htable, \
1793 offsetof (struct dstruct, htable_locp)__builtin_offsetof(struct dstruct, htable_locp)); \
1794 queue_init(&all_pagers.leak_queue)((&all_pagers.leak_queue)->next = (&all_pagers.leak_queue
)->prev = &all_pagers.leak_queue); \
1795}
1796
1797void pager_port_list_insert(port, ds)
1798 mach_port_t port;
1799 default_pager_t ds;
1800{
1801 pthread_mutex_lock(&all_pagers.lock);
1802 hurd_ihash_add (&all_pagers.htable,
1803 (hurd_ihash_key_t) port,
1804 (hurd_ihash_value_t) ds);
1805 pthread_mutex_unlock(&all_pagers.lock);
1806
1807 /* Try to set a protected payload. This is an optimization,
1808 if it fails we degrade gracefully. */
1809 mach_port_set_protected_payload (mach_task_self ()((__mach_task_self_ + 0)),
1810 port,
1811 (unsigned long) ds);
1812}
1813
1814void pager_port_list_delete(ds)
1815 default_pager_t ds;
1816{
1817 pthread_mutex_lock(&all_pagers.lock);
1818 hurd_ihash_locp_remove (&all_pagers.htable,
1819 ds->htable_locp);
1820 pthread_mutex_unlock(&all_pagers.lock);
1821
1822 mach_port_clear_protected_payload (mach_task_self ()((__mach_task_self_ + 0)),
1823 ds->pager);
1824}
1825
1826/*
1827 * Destroy a paging partition.
1828 * XXX this is not re-entrant XXX
1829 */
1830kern_return_t
1831destroy_paging_partition(name, pp_private)
1832 char *name;
1833 void **pp_private;
1834{
1835 unsigned int id = part_id(name);
1836 partition_t part = NULL((void*)0);
1837 boolean_t all_ok = TRUE((boolean_t) 1);
1838 default_pager_t entry;
1839 int pindex;
1840
1841 /*
1842 * Find and take partition out of list
1843 * This prevents choose_partition from
1844 * getting in the way.
1845 */
1846 pthread_mutex_lock(&all_partitions.lock);
1847 for (pindex = 0; pindex < all_partitions.n_partitions; pindex++) {
1848 part = partition_of(pindex);
1849 if (part && (part->id == id)) break;
1850 }
1851 if (! part) {
1852 pthread_mutex_unlock(&all_partitions.lock);
1853 return KERN_INVALID_ARGUMENT4;
1854 }
1855 part->going_away = TRUE((boolean_t) 1);
1856 pthread_mutex_unlock(&all_partitions.lock);
1857
1858 /*
1859 * This might take a while..
1860 */
1861all_over_again:
1862#if debug0
1863dprintf("Partition x%x (id x%x) for %s, all_ok %d\n", part, id, name, all_ok)(void) 0;
1864#endif
1865 all_ok = TRUE((boolean_t) 1);
1866 pthread_mutex_lock(&part->p_lock);
1867
1868 pthread_mutex_lock(&all_pagers.lock);
1869 HURD_IHASH_ITERATE (&all_pagers.htable, val)for (hurd_ihash_value_t val, *_hurd_ihash_valuep = (&all_pagers
.htable)->size ? &(&all_pagers.htable)->items[0
].value : 0; (&all_pagers.htable)->size && (size_t
) ((_hurd_ihash_item_t) _hurd_ihash_valuep - &(&all_pagers
.htable)->items[0]) < (&all_pagers.htable)->size
&& (val = *_hurd_ihash_valuep, 1); _hurd_ihash_valuep
= (hurd_ihash_value_t *) (((_hurd_ihash_item_t) _hurd_ihash_valuep
) + 1)) if (val != ((hurd_ihash_value_t) 0) && val !=
((hurd_ihash_value_t) -1)) {
1870 entry = (default_pager_t) val;
1871
1872 dstruct_lock(entry)pthread_mutex_lock(&entry->lock);
1873
1874 if (pthread_mutex_trylock(&entry->dpager.lock)) {
1875
1876 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
1877 pthread_mutex_unlock(&all_pagers.lock);
1878 pthread_mutex_unlock(&part->p_lock);
1879
1880 /* yield the processor */
1881 (void) thread_switch(MACH_PORT_NULL((mach_port_t) 0),
1882 SWITCH_OPTION_NONE0, 0);
1883
1884 goto all_over_again;
1885
1886 }
1887
1888 /*
1889 * See if we can relocate all the pages of this object
1890 * currently on this partition on some other partition
1891 */
1892 all_ok = pager_realloc(&entry->dpager, pindex);
1893
1894 pthread_mutex_unlock(&entry->dpager.lock);
1895 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
1896
1897 if (!all_ok) break;
1898
1899 }
1900 pthread_mutex_unlock(&all_pagers.lock);
1901
1902 if (all_ok) {
1903 /* No need to unlock partition, there are no refs left */
1904
1905 set_partition_of(pindex, 0);
1906 *pp_private = part->file;
1907 kfree(part->bitmap, howmany(part->total_size, NB_BM)(((part->total_size) + (32) - 1)/(32)) * sizeof(bm_entry_t));
1908 kfree(part, sizeof(struct part));
1909 dprintf("%s Removed paging partition %s\n", my_name, name)(void) 0;
1910 return KERN_SUCCESS0;
1911 }
1912
1913 /*
1914 * Put partition back in.
1915 */
1916 part->going_away = FALSE((boolean_t) 0);
1917
1918 return KERN_FAILURE5;
1919}
1920
1921
1922/*
1923 * We use the sequence numbers on requests to regulate
1924 * our parallelism. In general, we allow multiple reads and writes
1925 * to proceed in parallel, with the exception that reads must
1926 * wait for previous writes to finish. (Because the kernel might
1927 * generate a data-request for a page on the heels of a data-write
1928 * for the same page, and we must avoid returning stale data.)
1929 * terminate requests wait for proceeding reads and writes to finish.
1930 */
1931
1932unsigned int default_pager_total = 0; /* debugging */
1933unsigned int default_pager_wait_seqno = 0; /* debugging */
1934unsigned int default_pager_wait_read = 0; /* debugging */
1935unsigned int default_pager_wait_write = 0; /* debugging */
1936unsigned int default_pager_wait_refs = 0; /* debugging */
1937
1938#if PARALLEL1
1939/*
1940 * Waits for correct sequence number. Leaves pager locked.
1941 */
1942void pager_port_lock(ds, seqno)
1943 default_pager_t ds;
1944 mach_port_seqno_t seqno;
1945{
1946 default_pager_total++;
1947 dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
1948 while (ds->seqno != seqno) {
1949 default_pager_wait_seqno++;
1950 pthread_cond_wait(&ds->waiting_seqno, &ds->lock);
1951 }
1952}
1953
1954/*
1955 * Increments sequence number and unlocks pager.
1956 */
1957void pager_port_unlock(ds)
1958 default_pager_t ds;
1959{
1960 ds->seqno++;
1961 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
1962 pthread_cond_broadcast(&ds->waiting_seqno);
1963}
1964
1965/*
1966 * Start a read - one more reader. Pager must be locked.
1967 */
1968void pager_port_start_read(ds)
1969 default_pager_t ds;
1970{
1971 ds->readers++;
1972}
1973
1974/*
1975 * Wait for readers. Unlocks and relocks pager if wait needed.
1976 */
1977void pager_port_wait_for_readers(ds)
1978 default_pager_t ds;
1979{
1980 while (ds->readers != 0) {
1981 default_pager_wait_read++;
1982 pthread_cond_wait(&ds->waiting_read, &ds->lock);
1983 }
1984}
1985
1986/*
1987 * Finish a read. Pager is unlocked and returns unlocked.
1988 */
1989void pager_port_finish_read(ds)
1990 default_pager_t ds;
1991{
1992 dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
1993 if (--ds->readers == 0) {
1994 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
1995 pthread_cond_broadcast(&ds->waiting_read);
1996 }
1997 else {
1998 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
1999 }
2000}
2001
2002/*
2003 * Start a write - one more writer. Pager must be locked.
2004 */
2005void pager_port_start_write(ds)
2006 default_pager_t ds;
2007{
2008 ds->writers++;
2009}
2010
2011/*
2012 * Wait for writers. Unlocks and relocks pager if wait needed.
2013 */
2014void pager_port_wait_for_writers(ds)
2015 default_pager_t ds;
2016{
2017 while (ds->writers != 0) {
2018 default_pager_wait_write++;
2019 pthread_cond_wait(&ds->waiting_write, &ds->lock);
2020 }
2021}
2022
2023/*
2024 * Finish a write. Pager is unlocked and returns unlocked.
2025 */
2026void pager_port_finish_write(ds)
2027 default_pager_t ds;
2028{
2029 dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2030 if (--ds->writers == 0) {
2031 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2032 pthread_cond_broadcast(&ds->waiting_write);
2033 }
2034 else {
2035 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2036 }
2037}
2038
2039/*
2040 * Wait for concurrent default_pager_objects.
2041 * Unlocks and relocks pager if wait needed.
2042 */
2043void pager_port_wait_for_refs(ds)
2044 default_pager_t ds;
2045{
2046 while (ds->name_refs == 0) {
2047 default_pager_wait_refs++;
2048 pthread_cond_wait(&ds->waiting_refs, &ds->lock);
2049 }
2050}
2051
2052/*
2053 * Finished creating name refs - wake up waiters.
2054 */
2055void pager_port_finish_refs(ds)
2056 default_pager_t ds;
2057{
2058 pthread_cond_broadcast(&ds->waiting_refs);
2059}
2060
2061#else /* PARALLEL */
2062
2063#define pager_port_lock(ds,seqno)
2064#define pager_port_unlock(ds)
2065#define pager_port_start_read(ds)
2066#define pager_port_wait_for_readers(ds)
2067#define pager_port_finish_read(ds)
2068#define pager_port_start_write(ds)
2069#define pager_port_wait_for_writers(ds)
2070#define pager_port_finish_write(ds)
2071#define pager_port_wait_for_refs(ds)
2072#define pager_port_finish_refs(ds)
2073
2074#endif /* PARALLEL */
2075
2076/*
2077 * Default pager.
2078 */
2079task_t default_pager_self; /* Our task port. */
2080
2081mach_port_t default_pager_default_port; /* Port for memory_object_create. */
2082
2083/* We catch exceptions on ourself & startup using this port. */
2084mach_port_t default_pager_exception_port;
2085
2086mach_port_t default_pager_internal_set; /* Port set for internal objects. */
2087mach_port_t default_pager_external_set; /* Port set for external objects. */
2088mach_port_t default_pager_default_set; /* Port set for "default" thread. */
2089
2090typedef struct default_pager_thread {
2091 pthread_t dpt_thread; /* Server thread. */
2092 vm_offset_t dpt_buffer; /* Read buffer. */
2093 boolean_t dpt_internal; /* Do we handle internal objects? */
2094} default_pager_thread_t;
2095
2096#if PARALLEL1
2097 /* determine number of threads at run time */
2098#define DEFAULT_PAGER_INTERNAL_COUNT(0) (0)
2099
2100#else /* PARALLEL */
2101#define DEFAULT_PAGER_INTERNAL_COUNT(0) (1)
2102#endif /* PARALLEL */
2103
2104/* Memory created by default_pager_object_create should mostly be resident. */
2105#define DEFAULT_PAGER_EXTERNAL_COUNT(1) (1)
2106
2107unsigned int default_pager_internal_count = DEFAULT_PAGER_INTERNAL_COUNT(0);
2108 /* Number of "internal" threads. */
2109unsigned int default_pager_external_count = DEFAULT_PAGER_EXTERNAL_COUNT(1);
2110 /* Number of "external" threads. */
2111
2112default_pager_t pager_port_alloc(size)
2113 vm_size_t size;
2114{
2115 default_pager_t ds;
2116 p_index_t part;
2117
2118 ds = (default_pager_t) kalloc(sizeof *ds);
2119 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2120 panic("%spager_port_alloc",my_name);
2121 bzero((char *) ds, sizeof *ds);
2122
2123 dstruct_lock_init(ds)pthread_mutex_init(&ds->lock, ((void*)0));
2124
2125 /*
2126 * Get a suitable partition. If none big enough
2127 * just pick one and overcommit. If no partitions
2128 * at all.. well just fake one so that we will
2129 * kill specific objects on pageouts rather than
2130 * panicing the system now.
2131 */
2132 part = choose_partition(size, P_INDEX_INVALID((p_index_t)-1));
2133 if (no_partition(part)((part) == ((p_index_t)-1))) {
2134 overcommitted(FALSE((boolean_t) 0), atop(size)((size)/vm_page_size));
2135 part = choose_partition(0,P_INDEX_INVALID((p_index_t)-1));
2136#if debug0
2137 if (no_partition(part)((part) == ((p_index_t)-1)))
2138 dprintf("%s No paging space at all !!\n", my_name)(void) 0;
2139#endif
2140 }
2141 pager_alloc(&ds->dpager, part, size);
2142
2143 return ds;
2144}
2145
2146mach_port_urefs_t default_pager_max_urefs = 10000;
2147
2148/*
2149 * Check user reference count on pager_request port.
2150 * Pager must be locked.
2151 * Unlocks and re-locks pager if needs to call kernel.
2152 */
2153void pager_port_check_request(ds, pager_request)
2154 default_pager_t ds;
2155 mach_port_t pager_request;
2156{
2157 mach_port_delta_t delta;
2158 kern_return_t kr;
2159
2160 assert(ds->pager_request == pager_request)((ds->pager_request == pager_request) ? (void) (0) : __assert_fail
("ds->pager_request == pager_request", "../../mach-defpager/default_pager.c"
, 2160, __PRETTY_FUNCTION__));
2161
2162 if (++ds->request_refs > default_pager_max_urefs) {
2163 delta = 1 - ds->request_refs;
2164 ds->request_refs = 1;
2165
2166 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2167
2168 /*
2169 * Deallocate excess user references.
2170 */
2171
2172 kr = mach_port_mod_refs(default_pager_self, pager_request,
2173 MACH_PORT_RIGHT_SEND((mach_port_right_t) 0), delta);
2174 if (kr != KERN_SUCCESS0)
2175 panic("%spager_port_check_request",my_name);
2176
2177 dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2178 }
2179}
2180
2181void default_pager_add(ds, internal)
2182 default_pager_t ds;
2183 boolean_t internal;
2184{
2185 mach_port_t pager = ds->pager;
2186 mach_port_t pset;
2187 mach_port_mscount_t sync;
2188 mach_port_t previous;
2189 kern_return_t kr;
2190 static char here[] = "%sdefault_pager_add";
2191
2192 /*
2193 * The port currently has a make-send count of zero,
2194 * because either we just created the port or we just
2195 * received the port in a memory_object_create request.
2196 */
2197
2198 if (internal) {
2199 /* possibly generate an immediate no-senders notification */
2200 sync = 0;
2201 pset = default_pager_internal_set;
2202 ds->external = FALSE((boolean_t) 0);
2203 } else {
2204 /* delay notification till send right is created */
2205 sync = 1;
2206 pset = default_pager_external_set;
2207 ds->external = TRUE((boolean_t) 1);
2208 }
2209
2210 kr = mach_port_request_notification(default_pager_self, pager,
2211 MACH_NOTIFY_NO_SENDERS(0100 + 006), sync,
2212 pager, MACH_MSG_TYPE_MAKE_SEND_ONCE21,
2213 &previous);
2214 if ((kr != KERN_SUCCESS0) || (previous != MACH_PORT_NULL((mach_port_t) 0)))
2215 panic(here,my_name);
2216
2217 kr = mach_port_move_member(default_pager_self, pager, pset);
2218 if (kr != KERN_SUCCESS0)
2219 panic(here,my_name);
2220}
2221
2222/*
2223 * Routine: memory_object_create
2224 * Purpose:
2225 * Handle requests for memory objects from the
2226 * kernel.
2227 * Notes:
2228 * Because we only give out the default memory
2229 * manager port to the kernel, we don't have to
2230 * be so paranoid about the contents.
2231 */
2232kern_return_t
2233seqnos_memory_object_create(old_pager, seqno, new_pager, new_size,
2234 new_pager_request, new_pager_name, new_page_size)
2235 mach_port_t old_pager;
2236 mach_port_seqno_t seqno;
2237 mach_port_t new_pager;
2238 vm_size_t new_size;
2239 mach_port_t new_pager_request;
2240 mach_port_t new_pager_name;
2241 vm_size_t new_page_size;
2242{
2243 default_pager_t ds;
2244
2245 assert(old_pager == default_pager_default_port)((old_pager == default_pager_default_port) ? (void) (0) : __assert_fail
("old_pager == default_pager_default_port", "../../mach-defpager/default_pager.c"
, 2245, __PRETTY_FUNCTION__));
2246 assert(MACH_PORT_VALID(new_pager_request))(((((new_pager_request) != ((mach_port_t) 0)) && ((new_pager_request
) != ((mach_port_t) ~0)))) ? (void) (0) : __assert_fail ("(((new_pager_request) != ((mach_port_t) 0)) && ((new_pager_request) != ((mach_port_t) ~0)))"
, "../../mach-defpager/default_pager.c", 2246, __PRETTY_FUNCTION__
));
2247 assert(MACH_PORT_VALID(new_pager_name))(((((new_pager_name) != ((mach_port_t) 0)) && ((new_pager_name
) != ((mach_port_t) ~0)))) ? (void) (0) : __assert_fail ("(((new_pager_name) != ((mach_port_t) 0)) && ((new_pager_name) != ((mach_port_t) ~0)))"
, "../../mach-defpager/default_pager.c", 2247, __PRETTY_FUNCTION__
));
2248 assert(new_page_size == vm_page_size)((new_page_size == vm_page_size) ? (void) (0) : __assert_fail
("new_page_size == vm_page_size", "../../mach-defpager/default_pager.c"
, 2248, __PRETTY_FUNCTION__));
2249
2250 ds = pager_port_alloc(new_size);
2251
2252 /*
2253 * Set up associations between these ports
2254 * and this default_pager structure
2255 */
2256
2257 ds->pager = new_pager;
2258 ds->pager_request = new_pager_request;
2259 ds->request_refs = 1;
2260 ds->pager_name = new_pager_name;
2261 ds->name_refs = 1;
2262
2263 /*
2264 * After this, other threads might receive requests
2265 * for this memory object or find it in the port list.
2266 */
2267
2268 pager_port_list_insert(new_pager, ds);
2269 default_pager_add(ds, TRUE((boolean_t) 1));
2270
2271 return(KERN_SUCCESS0);
2272}
2273
2274memory_object_copy_strategy_t default_pager_copy_strategy =
2275 MEMORY_OBJECT_COPY_DELAY2;
2276
2277kern_return_t
2278seqnos_memory_object_init(ds, seqno, pager_request, pager_name,
2279 pager_page_size)
2280 default_pager_t ds;
2281 mach_port_seqno_t seqno;
2282 mach_port_t pager_request;
2283 mach_port_t pager_name;
2284 vm_size_t pager_page_size;
2285{
2286 kern_return_t kr;
2287 static char here[] = "%sinit";
2288
2289 assert(MACH_PORT_VALID(pager_request))(((((pager_request) != ((mach_port_t) 0)) && ((pager_request
) != ((mach_port_t) ~0)))) ? (void) (0) : __assert_fail ("(((pager_request) != ((mach_port_t) 0)) && ((pager_request) != ((mach_port_t) ~0)))"
, "../../mach-defpager/default_pager.c", 2289, __PRETTY_FUNCTION__
));
2290 assert(MACH_PORT_VALID(pager_name))(((((pager_name) != ((mach_port_t) 0)) && ((pager_name
) != ((mach_port_t) ~0)))) ? (void) (0) : __assert_fail ("(((pager_name) != ((mach_port_t) 0)) && ((pager_name) != ((mach_port_t) ~0)))"
, "../../mach-defpager/default_pager.c", 2290, __PRETTY_FUNCTION__
));
2291 assert(pager_page_size == vm_page_size)((pager_page_size == vm_page_size) ? (void) (0) : __assert_fail
("pager_page_size == vm_page_size", "../../mach-defpager/default_pager.c"
, 2291, __PRETTY_FUNCTION__));
2292
2293 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2294 panic(here, my_name);
2295 pager_port_lock(ds, seqno);
2296
2297 if (ds->pager_request != MACH_PORT_NULL((mach_port_t) 0))
2298 panic(here, my_name);
2299
2300 ds->pager_request = pager_request;
2301 ds->request_refs = 1;
2302 ds->pager_name = pager_name;
2303 ds->name_refs = 1;
2304
2305 /*
2306 * Even if the kernel immediately terminates the object,
2307 * the pager_request port won't be destroyed until
2308 * we process the terminate request, which won't happen
2309 * until we unlock the object.
2310 */
2311
2312 kr = memory_object_ready(pager_request,
2313 FALSE((boolean_t) 0), /* Do not cache */
2314 default_pager_copy_strategy);
2315 if (kr != KERN_SUCCESS0)
2316 panic(here, my_name);
2317
2318 pager_port_unlock(ds);
2319
2320 return(KERN_SUCCESS0);
2321}
2322
2323kern_return_t
2324seqnos_memory_object_terminate(ds, seqno, pager_request, pager_name)
2325 default_pager_t ds;
2326 mach_port_seqno_t seqno;
2327 mach_port_t pager_request;
2328 mach_port_t pager_name;
2329{
2330 static char here[] = "%sterminate";
2331
2332 /*
2333 * pager_request and pager_name are receive rights,
2334 * not send rights.
2335 */
2336
2337 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2338 panic(here, my_name);
2339ddprintf ("seqnos_memory_object_terminate <%p>: pager_port_lock: <%p>[s:%d,r:%d,w:%d,l:%d], %d\n",(void) 0
2340 &ds, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held, seqno)(void) 0;
2341 pager_port_lock(ds, seqno);
2342
2343 /*
2344 * Wait for read and write requests to terminate.
2345 */
2346
2347 pager_port_wait_for_readers(ds);
2348 pager_port_wait_for_writers(ds);
2349
2350 /*
2351 * After memory_object_terminate both memory_object_init
2352 * and a no-senders notification are possible, so we need
2353 * to clean up the request and name ports but leave
2354 * the pager port.
2355 *
2356 * A concurrent default_pager_objects might be allocating
2357 * more references for the name port. In this case,
2358 * we must first wait for it to finish.
2359 */
2360
2361 pager_port_wait_for_refs(ds);
2362
2363 if (ds->external)
2364 pager_request = ds->pager_request;
2365 ds->pager_request = MACH_PORT_NULL((mach_port_t) 0);
2366 ds->request_refs = 0;
2367 assert(ds->pager_name == pager_name)((ds->pager_name == pager_name) ? (void) (0) : __assert_fail
("ds->pager_name == pager_name", "../../mach-defpager/default_pager.c"
, 2367, __PRETTY_FUNCTION__));
2368 ds->pager_name = MACH_PORT_NULL((mach_port_t) 0);
2369 ds->name_refs = 0;
2370ddprintf ("seqnos_memory_object_terminate <%p>: pager_port_unlock: <%p>[s:%d,r:%d,w:%d,l:%d]\n",(void) 0
2371 &ds, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held)(void) 0;
2372 pager_port_unlock(ds);
2373
2374 /*
2375 * Now we destroy our port rights.
2376 */
2377
2378 mach_port_destroy(mach_task_self()((__mach_task_self_ + 0)), pager_request);
2379 mach_port_destroy(mach_task_self()((__mach_task_self_ + 0)), pager_name);
2380
2381 return (KERN_SUCCESS0);
2382}
2383
2384void default_pager_no_senders(pager, seqno, mscount)
2385 memory_object_t pager;
2386 mach_port_seqno_t seqno;
2387 mach_port_mscount_t mscount;
2388{
2389 default_pager_t ds;
2390 kern_return_t kr;
2391 static char here[] = "%sno_senders";
2392
2393 /*
2394 * Because we don't give out multiple send rights
2395 * for a memory object, there can't be a race
2396 * between getting a no-senders notification
2397 * and creating a new send right for the object.
2398 * Hence we don't keep track of mscount.
2399 */
2400
2401
2402 ds = begin_using_default_pager(pager);
2403 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2404 panic(here,my_name);
2405 pager_port_lock(ds, seqno);
2406
2407 /*
2408 * We shouldn't get a no-senders notification
2409 * when the kernel has the object cached.
2410 */
2411
2412 if (ds->pager_request != MACH_PORT_NULL((mach_port_t) 0))
2413 panic(here,my_name);
2414
2415 /*
2416 * Unlock the pager (though there should be no one
2417 * waiting for it).
2418 */
2419 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2420
2421 /*
2422 * Remove the memory object port association, and then
2423 * the destroy the port itself. We must remove the object
2424 * from the port list before deallocating the pager,
2425 * because of default_pager_objects.
2426 */
2427
2428 pager_port_list_delete(ds);
2429 pager_dealloc(&ds->dpager);
2430
2431 kr = mach_port_mod_refs(default_pager_self, pager,
2432 MACH_PORT_RIGHT_RECEIVE((mach_port_right_t) 1), -1);
2433 if (kr != KERN_SUCCESS0)
2434 panic(here,my_name);
2435
2436 /*
2437 * Do this *after* deallocating the port name
2438 */
2439 kfree((char *) ds, sizeof(*ds));
2440
2441 /*
2442 * Recover memory that we might have wasted because
2443 * of name conflicts
2444 */
2445 pthread_mutex_lock(&all_pagers.lock);
2446
2447 while (!queue_empty(&all_pagers.leak_queue)(((&all_pagers.leak_queue)) == (((&all_pagers.leak_queue
)->next)))) {
2448
2449 ds = (default_pager_t) queue_first(&all_pagers.leak_queue)((&all_pagers.leak_queue)->next);
Value stored to 'ds' is never read
2450 queue_remove_first(&all_pagers.leak_queue, ds, default_pager_t, links){ queue_entry_t next; (ds) = (default_pager_t) ((&all_pagers
.leak_queue)->next); next = (ds)->links.next; if ((&
all_pagers.leak_queue) == next) (&all_pagers.leak_queue)->
prev = (&all_pagers.leak_queue); else ((default_pager_t)(
next))->links.prev = (&all_pagers.leak_queue); (&all_pagers
.leak_queue)->next = next; };
2451 kfree((char *) ds, sizeof(*ds));
2452 }
2453
2454 pthread_mutex_unlock(&all_pagers.lock);
2455}
2456
2457int default_pager_pagein_count = 0;
2458int default_pager_pageout_count = 0;
2459
2460static __thread default_pager_thread_t *dpt;
2461
2462kern_return_t
2463seqnos_memory_object_data_request(ds, seqno, reply_to, offset,
2464 length, protection_required)
2465 default_pager_t ds;
2466 mach_port_seqno_t seqno;
2467 mach_port_t reply_to;
2468 vm_offset_t offset;
2469 vm_size_t length;
2470 vm_prot_t protection_required;
2471{
2472 vm_offset_t addr;
2473 unsigned int errors;
2474 kern_return_t rc;
2475 static char here[] = "%sdata_request";
2476
2477 if (length != vm_page_size)
2478 panic(here,my_name);
2479
2480 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2481 panic(here,my_name);
2482ddprintf ("seqnos_memory_object_data_request <%p>: pager_port_lock: <%p>[s:%d,r:%d,w:%d,l:%d], %d\n",(void) 0
2483 &ds, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held, seqno)(void) 0;
2484 pager_port_lock(ds, seqno);
2485 pager_port_check_request(ds, reply_to);
2486 pager_port_wait_for_writers(ds);
2487 pager_port_start_read(ds);
2488
2489 /*
2490 * Get error count while pager locked.
2491 */
2492 errors = ds->errors;
2493
2494ddprintf ("seqnos_memory_object_data_request <%p>: pager_port_unlock: <%p>[s:%d,r:%d,w:%d,l:%d]\n",(void) 0
2495 &ds, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held)(void) 0;
2496 pager_port_unlock(ds);
2497
2498 if (errors) {
2499 dprintf("%s %s\n", my_name,(void) 0
2500 "dropping data_request because of previous paging errors")(void) 0;
2501 (void) memory_object_data_error(reply_to,
2502 offset, vm_page_size,
2503 KERN_FAILURE5);
2504 goto done;
2505 }
2506
2507 if (offset >= ds->dpager.limit)
2508 rc = PAGER_ERROR2;
2509 else
2510 rc = default_read(&ds->dpager, dpt->dpt_buffer,
2511 vm_page_size, offset,
2512 &addr, protection_required & VM_PROT_WRITE((vm_prot_t) 0x02),
2513 ds->external);
2514
2515 switch (rc) {
2516 case PAGER_SUCCESS0:
2517 if (addr != dpt->dpt_buffer) {
2518 /*
2519 * Deallocates data buffer
2520 */
2521 (void) memory_object_data_supply(
2522 reply_to, offset,
2523 addr, vm_page_size, TRUE((boolean_t) 1),
2524 VM_PROT_NONE((vm_prot_t) 0x00),
2525 FALSE((boolean_t) 0), MACH_PORT_NULL((mach_port_t) 0));
2526 } else {
2527 (void) memory_object_data_supply(
2528 reply_to, offset,
2529 addr, vm_page_size, FALSE((boolean_t) 0),
2530 VM_PROT_NONE((vm_prot_t) 0x00),
2531 FALSE((boolean_t) 0), MACH_PORT_NULL((mach_port_t) 0));
2532 }
2533 break;
2534
2535 case PAGER_ABSENT1:
2536 (void) memory_object_data_unavailable(
2537 reply_to,
2538 offset,
2539 vm_page_size);
2540 break;
2541
2542 case PAGER_ERROR2:
2543 (void) memory_object_data_error(
2544 reply_to,
2545 offset,
2546 vm_page_size,
2547 KERN_FAILURE5);
2548 break;
2549 }
2550
2551 default_pager_pagein_count++;
2552
2553 done:
2554 pager_port_finish_read(ds);
2555 return(KERN_SUCCESS0);
2556}
2557
2558/*
2559 * memory_object_data_initialize: check whether we already have each page, and
2560 * write it if we do not. The implementation is far from optimized, and
2561 * also assumes that the default_pager is single-threaded.
2562 */
2563kern_return_t
2564seqnos_memory_object_data_initialize(ds, seqno, pager_request,
2565 offset, addr, data_cnt)
2566 default_pager_t ds;
2567 mach_port_seqno_t seqno;
2568 mach_port_t pager_request;
2569 register
2570 vm_offset_t offset;
2571 register
2572 pointer_t addr;
2573 vm_size_t data_cnt;
2574{
2575 vm_offset_t amount_sent;
2576 static char here[] = "%sdata_initialize";
2577
2578#ifdef lint
2579 pager_request++;
2580#endif /* lint */
2581
2582 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2583 panic(here,my_name);
2584ddprintf ("seqnos_memory_object_data_initialize <%p>: pager_port_lock: <%p>[s:%d,r:%d,w:%d,l:%d], %d\n",(void) 0
2585 &ds, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held, seqno)(void) 0;
2586 pager_port_lock(ds, seqno);
2587 pager_port_check_request(ds, pager_request);
2588 pager_port_start_write(ds);
2589ddprintf ("seqnos_memory_object_data_initialize <%p>: pager_port_unlock: <%p>[s:%d,r:%d,w:%d,l:%d]\n",(void) 0
2590 &ds, ds, ds->seqno, ds->readers, ds->writers, ds->lock.held)(void) 0;
2591 pager_port_unlock(ds);
2592
2593 for (amount_sent = 0;
2594 amount_sent < data_cnt;
2595 amount_sent += vm_page_size) {
2596
2597 if (!default_has_page(&ds->dpager, offset + amount_sent)) {
2598 if (default_write(&ds->dpager,
2599 addr + amount_sent,
2600 vm_page_size,
2601 offset + amount_sent)
2602 != PAGER_SUCCESS0) {
2603 dprintf("%s%s write error\n", my_name, here)(void) 0;
2604 dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2605 ds->errors++;
2606 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2607 }
2608 }
2609 }
2610
2611 pager_port_finish_write(ds);
2612 if (vm_deallocate(default_pager_self, addr, data_cnt) != KERN_SUCCESS0)
2613 panic(here,my_name);
2614
2615 return(KERN_SUCCESS0);
2616}
2617
2618/*
2619 * memory_object_data_write: split up the stuff coming in from
2620 * a memory_object_data_write call
2621 * into individual pages and pass them off to default_write.
2622 */
2623kern_return_t
2624seqnos_memory_object_data_write(ds, seqno, pager_request,
2625 offset, addr, data_cnt)
2626 default_pager_t ds;
2627 mach_port_seqno_t seqno;
2628 mach_port_t pager_request;
2629 register
2630 vm_offset_t offset;
2631 register
2632 pointer_t addr;
2633 vm_size_t data_cnt;
2634{
2635 register
2636 vm_size_t amount_sent;
2637 static char here[] = "%sdata_write";
2638 int err;
2639
2640#ifdef lint
2641 pager_request++;
2642#endif /* lint */
2643
2644 if ((data_cnt % vm_page_size) != 0)
2645 panic(here,my_name);
2646
2647 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
2648 panic(here,my_name);
2649
2650 pager_port_lock(ds, seqno);
2651 pager_port_start_write(ds);
2652
2653 vm_size_t limit = ds->dpager.byte_limit;
2654 pager_port_unlock(ds);
2655 if ((limit != round_page(limit)((((vm_offset_t) (limit) + __vm_page_size - 1) / __vm_page_size
) * __vm_page_size)) && (trunc_page(limit)((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size) == offset)) {
2656 assert(trunc_page(limit) == offset)((((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size
) == offset) ? (void) (0) : __assert_fail ("((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size) == offset"
, "../../mach-defpager/default_pager.c", 2656, __PRETTY_FUNCTION__
));
2657 assert(data_cnt == vm_page_size)((data_cnt == vm_page_size) ? (void) (0) : __assert_fail ("data_cnt == vm_page_size"
, "../../mach-defpager/default_pager.c", 2657, __PRETTY_FUNCTION__
));
2658
2659 vm_offset_t tail = addr + limit - trunc_page(limit)((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size);
2660 vm_size_t tail_size = round_page(limit)((((vm_offset_t) (limit) + __vm_page_size - 1) / __vm_page_size
) * __vm_page_size) - limit;
2661 memset((void *) tail, 0, tail_size);
2662
2663 memory_object_data_supply(pager_request, trunc_page(limit)((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size), addr,
2664 vm_page_size, TRUE((boolean_t) 1), VM_PROT_NONE((vm_prot_t) 0x00),
2665 TRUE((boolean_t) 1), MACH_PORT_NULL((mach_port_t) 0));
2666 dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2667 ds->dpager.byte_limit = round_page(limit)((((vm_offset_t) (limit) + __vm_page_size - 1) / __vm_page_size
) * __vm_page_size);
2668 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2669 pager_port_finish_write(ds);
2670
2671 return(KERN_SUCCESS0);
2672 }
2673
2674 for (amount_sent = 0;
2675 amount_sent < data_cnt;
2676 amount_sent += vm_page_size) {
2677
2678 int result;
2679
2680 result = default_write(&ds->dpager,
2681 addr + amount_sent,
2682 vm_page_size,
2683 offset + amount_sent);
2684 if (result != KERN_SUCCESS0) {
2685 dstruct_lock(ds)pthread_mutex_lock(&ds->lock);
2686 ds->errors++;
2687 dstruct_unlock(ds)pthread_mutex_unlock(&ds->lock);
2688 }
2689 default_pager_pageout_count++;
2690 }
2691
2692 pager_port_finish_write(ds);
2693 err = vm_deallocate(default_pager_self, addr, data_cnt);
2694 if (err != KERN_SUCCESS0)
2695 {
2696 panic(here,my_name);
2697 }
2698
2699 return(KERN_SUCCESS0);
2700}
2701
2702/*ARGSUSED*/
2703kern_return_t
2704seqnos_memory_object_copy(old_memory_object, seqno, old_memory_control,
2705 offset, length, new_memory_object)
2706 default_pager_t old_memory_object;
2707 mach_port_seqno_t seqno;
2708 memory_object_control_t
2709 old_memory_control;
2710 vm_offset_t offset;
2711 vm_size_t length;
2712 memory_object_t new_memory_object;
2713{
2714 panic("%scopy", my_name);
2715 return KERN_FAILURE5;
2716}
2717
2718/* We get this when our memory_object_lock_request has completed
2719 after we truncated an object. */
2720kern_return_t
2721seqnos_memory_object_lock_completed (default_pager_t ds,
2722 mach_port_seqno_t seqno,
2723 mach_port_t pager_request,
2724 vm_offset_t offset,
2725 vm_size_t length)
2726{
2727 panic("%slock_completed",my_name);
2728 return KERN_FAILURE5;
2729}
2730
2731kern_return_t
2732seqnos_memory_object_data_unlock(pager, seqno, pager_request,
2733 offset, length, protection_required)
2734 default_pager_t pager;
2735 mach_port_seqno_t seqno;
2736 mach_port_t pager_request;
2737 vm_offset_t offset;
2738 vm_size_t length;
2739 vm_prot_t protection_required;
2740{
2741 panic("%sdata_unlock",my_name);
2742 return(KERN_FAILURE5);
2743}
2744
2745kern_return_t
2746seqnos_memory_object_supply_completed(ds, seqno, pager_request,
2747 offset, length,
2748 result, error_offset)
2749 default_pager_t ds;
2750 mach_port_seqno_t seqno;
2751 mach_port_t pager_request;
2752 vm_offset_t offset;
2753 vm_size_t length;
2754 kern_return_t result;
2755 vm_offset_t error_offset;
2756{
2757 panic("%ssupply_completed",my_name);
2758 return(KERN_FAILURE5);
2759}
2760
2761/*
2762 * memory_object_data_return: split up the stuff coming in from
2763 * a memory_object_data_write call
2764 * into individual pages and pass them off to default_write.
2765 */
2766kern_return_t
2767seqnos_memory_object_data_return(ds, seqno, pager_request,
2768 offset, addr, data_cnt,
2769 dirty, kernel_copy)
2770 default_pager_t ds;
2771 mach_port_seqno_t seqno;
2772 mach_port_t pager_request;
2773 vm_offset_t offset;
2774 pointer_t addr;
2775 vm_size_t data_cnt;
2776 boolean_t dirty;
2777 boolean_t kernel_copy;
2778{
2779
2780 return seqnos_memory_object_data_write (ds, seqno, pager_request,
2781 offset, addr, data_cnt);
2782}
2783
2784kern_return_t
2785seqnos_memory_object_change_completed(ds, seqno, may_cache, copy_strategy)
2786 default_pager_t ds;
2787 mach_port_seqno_t seqno;
2788 boolean_t may_cache;
2789 memory_object_copy_strategy_t copy_strategy;
2790{
2791 panic("%schange_completed",my_name);
2792 return(KERN_FAILURE5);
2793}
2794
2795
2796boolean_t default_pager_notify_server(in, out)
2797 mach_msg_header_t *in, *out;
2798{
2799 mach_no_senders_notification_t *n =
2800 (mach_no_senders_notification_t *) in;
2801
2802 /*
2803 * The only send-once rights we create are for
2804 * receiving no-more-senders notifications.
2805 * Hence, if we receive a message directed to
2806 * a send-once right, we can assume it is
2807 * a genuine no-senders notification from the kernel.
2808 */
2809
2810 if ((n->not_header.msgh_bits !=
2811 MACH_MSGH_BITS(0, MACH_MSG_TYPE_PORT_SEND_ONCE)((0) | ((18) << 8))) ||
2812 (n->not_header.msgh_id != MACH_NOTIFY_NO_SENDERS(0100 + 006)))
2813 return FALSE((boolean_t) 0);
2814
2815 assert(n->not_header.msgh_size == sizeof *n)((n->not_header.msgh_size == sizeof *n) ? (void) (0) : __assert_fail
("n->not_header.msgh_size == sizeof *n", "../../mach-defpager/default_pager.c"
, 2815, __PRETTY_FUNCTION__));
2816 assert(n->not_header.msgh_remote_port == MACH_PORT_NULL)((n->not_header.msgh_remote_port == ((mach_port_t) 0)) ? (
void) (0) : __assert_fail ("n->not_header.msgh_remote_port == ((mach_port_t) 0)"
, "../../mach-defpager/default_pager.c", 2816, __PRETTY_FUNCTION__
));
2817
2818 assert(n->not_type.msgt_name == MACH_MSG_TYPE_INTEGER_32)((n->not_type.msgt_name == 2) ? (void) (0) : __assert_fail
("n->not_type.msgt_name == 2", "../../mach-defpager/default_pager.c"
, 2818, __PRETTY_FUNCTION__));
2819 assert(n->not_type.msgt_size == 32)((n->not_type.msgt_size == 32) ? (void) (0) : __assert_fail
("n->not_type.msgt_size == 32", "../../mach-defpager/default_pager.c"
, 2819, __PRETTY_FUNCTION__));
2820 assert(n->not_type.msgt_number == 1)((n->not_type.msgt_number == 1) ? (void) (0) : __assert_fail
("n->not_type.msgt_number == 1", "../../mach-defpager/default_pager.c"
, 2820, __PRETTY_FUNCTION__));
2821 assert(n->not_type.msgt_inline)((n->not_type.msgt_inline) ? (void) (0) : __assert_fail ("n->not_type.msgt_inline"
, "../../mach-defpager/default_pager.c", 2821, __PRETTY_FUNCTION__
));
2822 assert(! n->not_type.msgt_longform)((! n->not_type.msgt_longform) ? (void) (0) : __assert_fail
("! n->not_type.msgt_longform", "../../mach-defpager/default_pager.c"
, 2822, __PRETTY_FUNCTION__));
2823
2824 default_pager_no_senders(n->not_header.msgh_local_port,
2825 n->not_header.msgh_seqno, n->not_count);
2826
2827 out->msgh_remote_port = MACH_PORT_NULL((mach_port_t) 0);
2828 return TRUE((boolean_t) 1);
2829}
2830
2831extern boolean_t seqnos_memory_object_default_server();
2832extern boolean_t default_pager_server();
2833extern boolean_t exc_server();
2834extern boolean_t bootstrap_server();
2835extern void bootstrap_compat();
2836
2837mach_msg_size_t default_pager_msg_size_object = 128;
2838
2839/* Fill in default response. */
2840static void
2841mig_reply_setup (
2842 const mach_msg_header_t *in,
2843 mach_msg_header_t *out)
2844{
2845 static const mach_msg_type_t RetCodeType = {
2846 /* msgt_name = */ MACH_MSG_TYPE_INTEGER_322,
2847 /* msgt_size = */ 32,
2848 /* msgt_number = */ 1,
2849 /* msgt_inline = */ TRUE((boolean_t) 1),
2850 /* msgt_longform = */ FALSE((boolean_t) 0),
2851 /* msgt_deallocate = */ FALSE((boolean_t) 0),
2852 /* msgt_unused = */ 0
2853 };
2854
2855#define InP (in)
2856#define OutP ((mig_reply_header_t *) out)
2857 OutP->Head.msgh_bits =
2858 MACH_MSGH_BITS(MACH_MSGH_BITS_REMOTE(InP->msgh_bits), 0)((((InP->msgh_bits) & 0x000000ff)) | ((0) << 8));
2859 OutP->Head.msgh_size = sizeof *OutP;
2860 OutP->Head.msgh_remote_port = InP->msgh_remote_port;
2861 OutP->Head.msgh_local_port = MACH_PORT_NULL((mach_port_t) 0);
2862 OutP->Head.msgh_seqno = 0;
2863 OutP->Head.msgh_id = InP->msgh_id + 100;
2864 OutP->RetCodeType = RetCodeType;
2865 OutP->RetCode = MIG_BAD_ID-303;
2866#undef InP
2867#undef OutP
2868}
2869
2870boolean_t
2871default_pager_demux_object(in, out)
2872 mach_msg_header_t *in;
2873 mach_msg_header_t *out;
2874{
2875 /*
2876 * We receive memory_object_data_initialize messages in
2877 * the memory_object_default interface.
2878 */
2879
2880 int rval = FALSE((boolean_t) 0);
2881 ddprintf ("DPAGER DEMUX OBJECT <%p>: %d\n", in, in->msgh_id)(void) 0;
2882 mig_reply_setup (in, out);
2883
2884 mig_routine_t routine;
2885 if ((routine = seqnos_memory_object_server_routine (in)) ||
2886 (routine = seqnos_memory_object_default_server_routine (in)) ||
2887 (routine = NULL((void*)0), default_pager_notify_server (in, out)) ||
2888 (routine = default_pager_server_routine (in)))
2889 {
2890 if (routine)
2891 (*routine) (in, out);
2892 rval = TRUE((boolean_t) 1);
2893 }
2894
2895 ddprintf ("DPAGER DEMUX OBJECT DONE <%p>: %d\n", in, in->msgh_id)(void) 0;
2896 return rval;
2897}
2898
2899mach_msg_size_t default_pager_msg_size_default = 8 * 1024;
2900
2901boolean_t
2902default_pager_demux_default(in, out)
2903 mach_msg_header_t *in;
2904 mach_msg_header_t *out;
2905{
2906 if (in->msgh_local_port == default_pager_default_port) {
2907 /*
2908 * We receive memory_object_create messages in
2909 * the memory_object_default interface.
2910 */
2911
2912int rval;
2913ddprintf ("DPAGER DEMUX DEFAULT <%p>: %d\n", in, in->msgh_id)(void) 0;
2914rval =
2915 (seqnos_memory_object_default_server(in, out) ||
2916 default_pager_server(in, out));
2917ddprintf ("DPAGER DEMUX DEFAULT DONE <%p>: %d\n", in, in->msgh_id)(void) 0;
2918return rval;
2919 } else if (in->msgh_local_port == default_pager_exception_port) {
2920 /*
2921 * We receive exception messages for
2922 * ourself and the startup task.
2923 */
2924
2925 return exc_server(in, out);
2926 } else {
2927 panic(my_name);
2928 return FALSE((boolean_t) 0);
2929 }
2930}
2931
2932/*
2933 * We use multiple threads, for two reasons.
2934 *
2935 * First, memory objects created by default_pager_object_create
2936 * are "external", instead of "internal". This means the kernel
2937 * sends data (memory_object_data_write) to the object pageable.
2938 * To prevent deadlocks, the external and internal objects must
2939 * be managed by different threads.
2940 *
2941 * Second, the default pager uses synchronous IO operations.
2942 * Spreading requests across multiple threads should
2943 * recover some of the performance loss from synchronous IO.
2944 *
2945 * We have 3+ threads.
2946 * One receives memory_object_create and
2947 * default_pager_object_create requests.
2948 * One or more manage internal objects.
2949 * One or more manage external objects.
2950 */
2951
2952void
2953default_pager_thread_privileges()
2954{
2955 /*
2956 * Set thread privileges.
2957 */
2958 wire_thread(); /* grab a kernel stack and memory allocation
2959 privileges */
2960}
2961
2962void *
2963default_pager_default_thread(void *arg)
2964{
2965 kern_return_t kr;
2966 default_pager_thread_privileges ();
2967 for (;;) {
2968 kr = mach_msg_server(default_pager_demux_default,
2969 default_pager_msg_size_default,
2970 default_pager_default_set);
2971 panic(my_name, kr);
2972 }
2973}
2974
2975
2976
2977void *
2978default_pager_thread(void *arg)
2979{
2980 mach_port_t pset;
2981 kern_return_t kr;
2982
2983 dpt = (default_pager_thread_t *) arg;
2984
2985 /*
2986 * Threads handling external objects cannot have
2987 * privileges. Otherwise a burst of data-requests for an
2988 * external object could empty the free-page queue,
2989 * because the fault code only reserves real pages for
2990 * requests sent to internal objects.
2991 */
2992
2993 if (dpt->dpt_internal) {
2994 default_pager_thread_privileges();
2995 pset = default_pager_internal_set;
2996 } else {
2997 pset = default_pager_external_set;
2998 }
2999
3000 for (;;) {
3001 kr = mach_msg_server(default_pager_demux_object,
3002 default_pager_msg_size_object,
3003 pset);
3004 panic(my_name, kr);
3005 }
3006}
3007
3008void
3009start_default_pager_thread(internal)
3010 boolean_t internal;
3011{
3012 default_pager_thread_t *ndpt;
3013 kern_return_t kr;
3014 error_t err;
3015
3016 ndpt = (default_pager_thread_t *) kalloc(sizeof *ndpt);
3017 if (ndpt == 0)
3018 panic(my_name);
3019
3020 ndpt->dpt_internal = internal;
3021
3022 kr = vm_allocate(default_pager_self, &ndpt->dpt_buffer,
3023 vm_page_size, TRUE((boolean_t) 1));
3024 if (kr != KERN_SUCCESS0)
3025 panic(my_name);
3026 wire_memory(ndpt->dpt_buffer, vm_page_size,
3027 VM_PROT_READ((vm_prot_t) 0x01)|VM_PROT_WRITE((vm_prot_t) 0x02));
3028
3029 err = pthread_create(&ndpt->dpt_thread, NULL((void*)0), default_pager_thread,
3030 ndpt);
3031 if (!err)
3032 pthread_detach (ndpt->dpt_thread);
3033 else {
3034 errno(*__errno_location ()) = err;
3035 perror ("pthread_create");
3036 }
3037}
3038
3039void
3040default_pager_initialize(host_port)
3041 mach_port_t host_port;
3042{
3043 memory_object_t DMM;
3044 kern_return_t kr;
3045
3046 /*
3047 * This task will become the default pager.
3048 */
3049 default_pager_self = mach_task_self()((__mach_task_self_ + 0));
3050
3051 /*
3052 * Initialize the "default pager" port.
3053 */
3054 kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_RECEIVE((mach_port_right_t) 1),
3055 &default_pager_default_port);
3056 if (kr != KERN_SUCCESS0)
3057 panic(my_name);
3058
3059 DMM = default_pager_default_port;
3060 kr = vm_set_default_memory_manager(host_port, &DMM);
3061 if ((kr != KERN_SUCCESS0) || MACH_PORT_VALID(DMM)(((DMM) != ((mach_port_t) 0)) && ((DMM) != ((mach_port_t
) ~0))))
3062 panic(my_name);
3063
3064 /*
3065 * Initialize the exception port.
3066 */
3067 kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_RECEIVE((mach_port_right_t) 1),
3068 &default_pager_exception_port);
3069 if (kr != KERN_SUCCESS0)
3070 panic(my_name);
3071
3072 /*
3073 * Arrange for wiring privileges.
3074 */
3075 wire_setup(host_port);
3076
3077 /*
3078 * Find out how many CPUs we have, to determine the number
3079 * of threads to create.
3080 */
3081 if (default_pager_internal_count == 0) {
3082 host_basic_info_data_t h_info;
3083 natural_t h_info_count;
3084
3085 h_info_count = HOST_BASIC_INFO_COUNT(sizeof(host_basic_info_data_t)/sizeof(integer_t));
3086 (void) host_info(host_port, HOST_BASIC_INFO1,
3087 (host_info_t)&h_info, &h_info_count);
3088
3089 /*
3090 * Random computation to get more parallelism on
3091 * multiprocessors.
3092 */
3093 default_pager_internal_count =
3094 (h_info.avail_cpus > 32 ? 32 : h_info.avail_cpus) / 4 + 3;
3095 }
3096}
3097
3098/*
3099 * Initialize and Run the default pager
3100 */
3101void
3102default_pager()
3103{
3104 kern_return_t kr;
3105 int i;
3106
3107 default_pager_thread_privileges();
3108
3109 /*
3110 * Wire down code, data, stack
3111 */
3112 wire_all_memory();
3113
3114
3115 /*
3116 * Initialize the list of all pagers.
3117 */
3118 pager_port_list_init(){ pthread_mutex_init(&all_pagers.lock, ((void*)0)); hurd_ihash_init
(&all_pagers.htable, __builtin_offsetof(struct dstruct, htable_locp
)); ((&all_pagers.leak_queue)->next = (&all_pagers
.leak_queue)->prev = &all_pagers.leak_queue); };
3119
3120 kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_PORT_SET((mach_port_right_t) 3),
3121 &default_pager_internal_set);
3122 if (kr != KERN_SUCCESS0)
3123 panic(my_name);
3124
3125 kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_PORT_SET((mach_port_right_t) 3),
3126 &default_pager_external_set);
3127 if (kr != KERN_SUCCESS0)
3128 panic(my_name);
3129
3130 kr = mach_port_allocate(default_pager_self, MACH_PORT_RIGHT_PORT_SET((mach_port_right_t) 3),
3131 &default_pager_default_set);
3132 if (kr != KERN_SUCCESS0)
3133 panic(my_name);
3134
3135 kr = mach_port_move_member(default_pager_self,
3136 default_pager_default_port,
3137 default_pager_default_set);
3138 if (kr != KERN_SUCCESS0)
3139 panic(my_name);
3140
3141 kr = mach_port_move_member(default_pager_self,
3142 default_pager_exception_port,
3143 default_pager_default_set);
3144 if (kr != KERN_SUCCESS0)
3145 panic(my_name);
3146
3147 /*
3148 * Now we create the threads that will actually
3149 * manage objects.
3150 */
3151
3152 for (i = 0; i < default_pager_internal_count; i++)
3153 start_default_pager_thread(TRUE((boolean_t) 1));
3154
3155 for (i = 0; i < default_pager_external_count; i++)
3156 start_default_pager_thread(FALSE((boolean_t) 0));
3157
3158 default_pager_default_thread(0); /* Become the default_pager server */
3159#if 0
3160 cthread_fork (default_pager_default_thread, 0);
3161 /* cthread_exit (cthread_self ()); */
3162 thread_suspend (mach_thread_self ());
3163#endif
3164}
3165
3166/*
3167 * Create an external object.
3168 */
3169kern_return_t
3170S_default_pager_object_create (mach_port_t pager,
3171 mach_port_t *mem_obj,
3172 mach_msg_type_name_t *mem_obj_type,
3173 vm_size_t size)
3174{
3175 default_pager_t ds;
3176 mach_port_t port;
3177 kern_return_t result;
3178
3179 if (pager != default_pager_default_port)
3180 return KERN_INVALID_ARGUMENT4;
3181
3182 ds = pager_port_alloc(size);
3183 result = mach_port_allocate (default_pager_self,
3184 MACH_PORT_RIGHT_RECEIVE((mach_port_right_t) 1),
3185 &port);
3186 if (result != KERN_SUCCESS0)
3187 {
3188 kfree ((char *) ds, sizeof *ds);
3189 return result;
3190 }
3191
3192 /*
3193 * Set up associations between these ports
3194 * and this default_pager structure
3195 */
3196
3197 ds->pager = port;
3198 ds->dpager.limit = size;
3199 pager_port_list_insert(port, ds);
3200 default_pager_add(ds, FALSE((boolean_t) 0));
3201
3202 *mem_obj = port;
3203 *mem_obj_type = MACH_MSG_TYPE_MAKE_SEND20;
3204 return (KERN_SUCCESS0);
3205}
3206
3207kern_return_t
3208S_default_pager_info (mach_port_t pager,
3209 default_pager_info_t *infop)
3210{
3211 vm_size_t total, free;
3212
3213 if (pager != default_pager_default_port)
3214 return KERN_INVALID_ARGUMENT4;
3215
3216 pthread_mutex_lock(&all_partitions.lock);
3217 paging_space_info(&total, &free);
3218 pthread_mutex_unlock(&all_partitions.lock);
3219
3220 infop->dpi_total_space = ptoa(total)((total)*vm_page_size);
3221 infop->dpi_free_space = ptoa(free)((free)*vm_page_size);
3222 infop->dpi_page_size = vm_page_size;
3223 return KERN_SUCCESS0;
3224}
3225
3226kern_return_t
3227S_default_pager_objects (mach_port_t pager,
3228 default_pager_object_array_t *objectsp,
3229 natural_t *ocountp,
3230 mach_port_array_t *portsp,
3231 natural_t *pcountp)
3232{
3233 vm_offset_t oaddr = 0; /* memory for objects */
3234 vm_size_t osize = 0; /* current size */
3235 default_pager_object_t *objects;
3236 natural_t opotential;
3237
3238 vm_offset_t paddr = 0; /* memory for ports */
3239 vm_size_t psize = 0; /* current size */
3240 mach_port_t *ports;
3241 natural_t ppotential;
3242
3243 unsigned int actual;
3244 unsigned int num_pagers;
3245 kern_return_t kr;
3246 default_pager_t entry;
3247
3248 if (pager != default_pager_default_port)
3249 return KERN_INVALID_ARGUMENT4;
3250
3251 /* start with the inline memory */
3252
3253 num_pagers = 0;
3254
3255 objects = *objectsp;
3256 opotential = *ocountp;
3257
3258 ports = *portsp;
3259 ppotential = *pcountp;
3260
3261 pthread_mutex_lock(&all_pagers.lock);
3262 /*
3263 * We will send no more than this many
3264 */
3265 actual = all_pagers.htable.nr_items;
3266 pthread_mutex_unlock(&all_pagers.lock);
3267
3268 if (opotential < actual) {
3269 vm_offset_t newaddr;
3270 vm_size_t newsize;
3271
3272 newsize = 2 * round_page(actual * sizeof *objects)((((vm_offset_t) (actual * sizeof *objects) + __vm_page_size -
1) / __vm_page_size) * __vm_page_size);
3273
3274 kr = vm_allocate(default_pager_self, &newaddr, newsize, TRUE((boolean_t) 1));
3275 if (kr != KERN_SUCCESS0)
3276 goto nomemory;
3277
3278 oaddr = newaddr;
3279 osize = newsize;
3280 opotential = osize/sizeof *objects;
3281 objects = (default_pager_object_t *) oaddr;
3282 }
3283
3284 if (ppotential < actual) {
3285 vm_offset_t newaddr;
3286 vm_size_t newsize;
3287
3288 newsize = 2 * round_page(actual * sizeof *ports)((((vm_offset_t) (actual * sizeof *ports) + __vm_page_size - 1
) / __vm_page_size) * __vm_page_size);
3289
3290 kr = vm_allocate(default_pager_self, &newaddr, newsize, TRUE((boolean_t) 1));
3291 if (kr != KERN_SUCCESS0)
3292 goto nomemory;
3293
3294 paddr = newaddr;
3295 psize = newsize;
3296 ppotential = psize/sizeof *ports;
3297 ports = (mach_port_t *) paddr;
3298 }
3299
3300 /*
3301 * Now scan the list.
3302 */
3303
3304 pthread_mutex_lock(&all_pagers.lock);
3305
3306 num_pagers = 0;
3307 HURD_IHASH_ITERATE (&all_pagers.htable, val)for (hurd_ihash_value_t val, *_hurd_ihash_valuep = (&all_pagers
.htable)->size ? &(&all_pagers.htable)->items[0
].value : 0; (&all_pagers.htable)->size && (size_t
) ((_hurd_ihash_item_t) _hurd_ihash_valuep - &(&all_pagers
.htable)->items[0]) < (&all_pagers.htable)->size
&& (val = *_hurd_ihash_valuep, 1); _hurd_ihash_valuep
= (hurd_ihash_value_t *) (((_hurd_ihash_item_t) _hurd_ihash_valuep
) + 1)) if (val != ((hurd_ihash_value_t) 0) && val !=
((hurd_ihash_value_t) -1)) {
3308 entry = (default_pager_t) val;
3309
3310 mach_port_t port;
3311 vm_size_t size;
3312
3313 if ((num_pagers >= opotential) ||
3314 (num_pagers >= ppotential)) {
3315 /*
3316 * This should be rare. In any case,
3317 * we will only miss recent objects,
3318 * because they are added at the end.
3319 */
3320 break;
3321 }
3322
3323 /*
3324 * Avoid interfering with normal operations
3325 */
3326 if (pthread_mutex_trylock(&entry->dpager.lock))
3327 goto not_this_one;
3328 size = pager_allocated(&entry->dpager);
3329 pthread_mutex_unlock(&entry->dpager.lock);
3330
3331 dstruct_lock(entry)pthread_mutex_lock(&entry->lock);
3332
3333 port = entry->pager_name;
3334 if (port == MACH_PORT_NULL((mach_port_t) 0)) {
3335 /*
3336 * The object is waiting for no-senders
3337 * or memory_object_init.
3338 */
3339 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3340 goto not_this_one;
3341 }
3342
3343 /*
3344 * We need a reference for the reply message.
3345 * While we are unlocked, the bucket queue
3346 * can change and the object might be terminated.
3347 * memory_object_terminate will wait for us,
3348 * preventing deallocation of the entry.
3349 */
3350
3351 if (--entry->name_refs == 0) {
3352 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3353
3354 /* keep the list locked, wont take long */
3355
3356 kr = mach_port_mod_refs(default_pager_self,
3357 port, MACH_PORT_RIGHT_SEND((mach_port_right_t) 0),
3358 default_pager_max_urefs);
3359 if (kr != KERN_SUCCESS0)
3360 panic("%sdefault_pager_objects",my_name);
3361
3362 dstruct_lock(entry)pthread_mutex_lock(&entry->lock);
3363
3364 entry->name_refs += default_pager_max_urefs;
3365 pager_port_finish_refs(entry);
3366 }
3367 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3368
3369 /* the arrays are wired, so no deadlock worries */
3370
3371 objects[num_pagers].dpo_object = (vm_offset_t) entry;
3372 objects[num_pagers].dpo_size = size;
3373 ports [num_pagers++] = port;
3374 continue;
3375not_this_one:
3376 /*
3377 * Do not return garbage
3378 */
3379 objects[num_pagers].dpo_object = (vm_offset_t) 0;
3380 objects[num_pagers].dpo_size = 0;
3381 ports [num_pagers++] = MACH_PORT_NULL((mach_port_t) 0);
3382
3383 }
3384
3385 pthread_mutex_unlock(&all_pagers.lock);
3386
3387 /*
3388 * Deallocate and clear unused memory.
3389 * (Returned memory will automagically become pageable.)
3390 */
3391
3392 if (objects == *objectsp) {
3393 /*
3394 * Our returned information fit inline.
3395 * Nothing to deallocate.
3396 */
3397
3398 *ocountp = num_pagers;
3399 } else if (actual == 0) {
3400 (void) vm_deallocate(default_pager_self, oaddr, osize);
3401
3402 /* return zero items inline */
3403 *ocountp = 0;
3404 } else {
3405 vm_offset_t used;
3406
3407 used = round_page(actual * sizeof *objects)((((vm_offset_t) (actual * sizeof *objects) + __vm_page_size -
1) / __vm_page_size) * __vm_page_size);
3408
3409 if (used != osize)
3410 (void) vm_deallocate(default_pager_self,
3411 oaddr + used, osize - used);
3412
3413 *objectsp = objects;
3414 *ocountp = num_pagers;
3415 }
3416
3417 if (ports == *portsp) {
3418 /*
3419 * Our returned information fit inline.
3420 * Nothing to deallocate.
3421 */
3422
3423 *pcountp = num_pagers;
3424 } else if (actual == 0) {
3425 (void) vm_deallocate(default_pager_self, paddr, psize);
3426
3427 /* return zero items inline */
3428 *pcountp = 0;
3429 } else {
3430 vm_offset_t used;
3431
3432 used = round_page(actual * sizeof *ports)((((vm_offset_t) (actual * sizeof *ports) + __vm_page_size - 1
) / __vm_page_size) * __vm_page_size);
3433
3434 if (used != psize)
3435 (void) vm_deallocate(default_pager_self,
3436 paddr + used, psize - used);
3437
3438 *portsp = ports;
3439 *pcountp = num_pagers;
3440 }
3441
3442 return KERN_SUCCESS0;
3443
3444 nomemory:
3445
3446 {
3447 int i;
3448 for (i = 0; i < num_pagers; i++)
3449 (void) mach_port_deallocate(default_pager_self, ports[i]);
3450 }
3451
3452 if (objects != *objectsp)
3453 (void) vm_deallocate(default_pager_self, oaddr, osize);
3454
3455 if (ports != *portsp)
3456 (void) vm_deallocate(default_pager_self, paddr, psize);
3457
3458 return KERN_RESOURCE_SHORTAGE6;
3459}
3460
3461
3462kern_return_t
3463S_default_pager_object_pages (mach_port_t pager,
3464 mach_port_t object,
3465 default_pager_page_array_t *pagesp,
3466 natural_t *countp)
3467{
3468 vm_offset_t addr = 0; /* memory for page offsets */
3469 vm_size_t size = 0; /* current memory size */
3470 default_pager_page_t *pages;
3471 natural_t potential, actual;
3472 kern_return_t kr;
3473
3474 if (pager != default_pager_default_port)
3475 return KERN_INVALID_ARGUMENT4;
3476
3477 /* we start with the inline space */
3478
3479 pages = *pagesp;
3480 potential = *countp;
3481
3482 for (;;) {
3483 default_pager_t entry;
3484
3485 pthread_mutex_lock(&all_pagers.lock);
3486 HURD_IHASH_ITERATE (&all_pagers.htable, val)for (hurd_ihash_value_t val, *_hurd_ihash_valuep = (&all_pagers
.htable)->size ? &(&all_pagers.htable)->items[0
].value : 0; (&all_pagers.htable)->size && (size_t
) ((_hurd_ihash_item_t) _hurd_ihash_valuep - &(&all_pagers
.htable)->items[0]) < (&all_pagers.htable)->size
&& (val = *_hurd_ihash_valuep, 1); _hurd_ihash_valuep
= (hurd_ihash_value_t *) (((_hurd_ihash_item_t) _hurd_ihash_valuep
) + 1)) if (val != ((hurd_ihash_value_t) 0) && val !=
((hurd_ihash_value_t) -1)) {
3487 entry = (default_pager_t) val;
3488 dstruct_lock(entry)pthread_mutex_lock(&entry->lock);
3489 if (entry->pager_name == object) {
3490 pthread_mutex_unlock(&all_pagers.lock);
3491 goto found_object;
3492 }
3493 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3494 }
3495 pthread_mutex_unlock(&all_pagers.lock);
3496
3497 /* did not find the object */
3498
3499 if (pages != *pagesp)
3500 (void) vm_deallocate(default_pager_self, addr, size);
3501 return KERN_INVALID_ARGUMENT4;
3502
3503 found_object:
3504
3505 if (pthread_mutex_trylock(&entry->dpager.lock)) {
3506 /* oh well bad luck */
3507
3508 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3509
3510 /* yield the processor */
3511 (void) thread_switch(MACH_PORT_NULL((mach_port_t) 0),
3512 SWITCH_OPTION_NONE0, 0);
3513 continue;
3514 }
3515
3516 actual = pager_pages(&entry->dpager, pages, potential);
3517 pthread_mutex_unlock(&entry->dpager.lock);
3518 dstruct_unlock(entry)pthread_mutex_unlock(&entry->lock);
3519
3520 if (actual <= potential)
3521 break;
3522
3523 /* allocate more memory */
3524
3525 if (pages != *pagesp)
3526 (void) vm_deallocate(default_pager_self, addr, size);
3527 size = round_page(actual * sizeof *pages)((((vm_offset_t) (actual * sizeof *pages) + __vm_page_size - 1
) / __vm_page_size) * __vm_page_size);
3528 kr = vm_allocate(default_pager_self, &addr, size, TRUE((boolean_t) 1));
3529 if (kr != KERN_SUCCESS0)
3530 return kr;
3531 pages = (default_pager_page_t *) addr;
3532 potential = size/sizeof *pages;
3533 }
3534
3535 /*
3536 * Deallocate and clear unused memory.
3537 * (Returned memory will automagically become pageable.)
3538 */
3539
3540 if (pages == *pagesp) {
3541 /*
3542 * Our returned information fit inline.
3543 * Nothing to deallocate.
3544 */
3545
3546 *countp = actual;
3547 } else if (actual == 0) {
3548 (void) vm_deallocate(default_pager_self, addr, size);
3549
3550 /* return zero items inline */
3551 *countp = 0;
3552 } else {
3553 vm_offset_t used;
3554
3555 used = round_page(actual * sizeof *pages)((((vm_offset_t) (actual * sizeof *pages) + __vm_page_size - 1
) / __vm_page_size) * __vm_page_size);
3556
3557 if (used != size)
3558 (void) vm_deallocate(default_pager_self,
3559 addr + used, size - used);
3560
3561 *pagesp = pages;
3562 *countp = actual;
3563 }
3564 return KERN_SUCCESS0;
3565}
3566
3567
3568kern_return_t
3569S_default_pager_object_set_size (default_pager_t ds,
3570 mach_port_seqno_t seqno,
3571 vm_size_t limit)
3572{
3573 kern_return_t kr = KERN_SUCCESS0;
3574
3575 if (ds == DEFAULT_PAGER_NULL((default_pager_t)0))
3576 return KERN_INVALID_ARGUMENT4;
3577
3578 pager_port_lock(ds, seqno);
3579 pager_port_wait_for_readers(ds);
3580 pager_port_wait_for_writers(ds);
3581
3582 vm_size_t rounded_limit = round_page (limit)((((vm_offset_t) (limit) + __vm_page_size - 1) / __vm_page_size
) * __vm_page_size);
3583 vm_size_t trunc_limit = trunc_page (limit)((((vm_offset_t) (limit)) / __vm_page_size) * __vm_page_size);
3584
3585
3586 if (ds->dpager.limit < rounded_limit)
3587 {
3588 /* The limit has not been exceeded heretofore. Just change it. */
3589 ds->dpager.limit = rounded_limit;
3590
3591 /* Byte limit is used for truncation of file, which aren't rounded to
3592 page boundary. But by enlarging of file we are free to increase this value*/
3593 ds->dpager.byte_limit = rounded_limit;
3594 kr = memory_object_lock_request(ds->pager_request, 0,
3595 rounded_limit,
3596 MEMORY_OBJECT_RETURN_NONE0, FALSE((boolean_t) 0),
3597 VM_PROT_NONE((vm_prot_t) 0x00), MACH_PORT_NULL((mach_port_t) 0));
3598 if (kr != KERN_SUCCESS0)
3599 panic ("memory_object_lock_request: %d", kr);
3600 }
3601 else
3602 {
3603 if (ds->dpager.limit != rounded_limit)
3604 {
3605 kr = memory_object_lock_request(ds->pager_request, rounded_limit,
3606 ds->dpager.limit - rounded_limit,
3607 MEMORY_OBJECT_RETURN_NONE0, TRUE((boolean_t) 1),
3608 VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04)), MACH_PORT_NULL((mach_port_t) 0));
3609 if (kr != KERN_SUCCESS0)
3610 panic ("memory_object_lock_request: %d", kr);
3611
3612 ds->dpager.limit = rounded_limit;
3613 }
3614
3615 /* Deallocate the old backing store pages and shrink the page map. */
3616 if (ds->dpager.size > ds->dpager.limit / vm_page_size)
3617 pager_truncate (&ds->dpager, ds->dpager.limit / vm_page_size);
3618
3619 /* If memory object size isn't page aligned, fill the tail
3620 of last page with zeroes */
3621 if ((limit != rounded_limit) && (ds->dpager.limit > limit))
3622 {
3623 /* Clean part of last page which isn't part of file.
3624 For file sizes that aren't multiple of vm_page_size */
3625 ds->dpager.byte_limit = limit;
3626 kr = memory_object_lock_request(ds->pager_request, trunc_limit,
3627 vm_page_size,
3628 MEMORY_OBJECT_RETURN_ALL2, TRUE((boolean_t) 1),
3629 VM_PROT_NONE((vm_prot_t) 0x00), MACH_PORT_NULL((mach_port_t) 0));
3630 }
3631 }
3632
3633 pager_port_unlock(ds);
3634
3635 return kr;
3636}
3637
3638/*
3639 * Add/remove extra paging space
3640 */
3641
3642extern mach_port_t bootstrap_master_device_port;
3643extern mach_port_t bootstrap_master_host_port;
3644
3645kern_return_t
3646S_default_pager_paging_file (pager, mdport, file_name, add)
3647 mach_port_t pager;
3648 mach_port_t mdport;
3649 default_pager_filename_t file_name;
3650 boolean_t add;
3651{
3652 kern_return_t kr;
3653
3654 if (pager != default_pager_default_port)
3655 return KERN_INVALID_ARGUMENT4;
3656
3657#if 0
3658dprintf("bmd %x md %x\n", bootstrap_master_device_port, mdport)(void) 0;
3659#endif
3660 if (add) {
3661 kr = add_paging_file(bootstrap_master_device_port,
3662 file_name, 0);
3663 } else {
3664 kr = remove_paging_file(file_name);
3665 }
3666
3667 /* XXXX more code needed */
3668 if (mdport != bootstrap_master_device_port)
3669 mach_port_deallocate( mach_task_self()((__mach_task_self_ + 0)), mdport);
3670
3671 return kr;
3672}
3673
3674kern_return_t
3675default_pager_register_fileserver(pager, fileserver)
3676 mach_port_t pager;
3677 mach_port_t fileserver;
3678{
3679 if (pager != default_pager_default_port)
3680 return KERN_INVALID_ARGUMENT4;
3681#if notyet
3682 mach_port_deallocate(mach_task_self()((__mach_task_self_ + 0)), fileserver);
3683 if (0) dp_helper_paging_space(0,0,0);/*just linkit*/
3684#endif
3685 return KERN_SUCCESS0;
3686}
3687
3688/*
3689 * When things do not quite workout...
3690 */
3691void no_paging_space(out_of_memory)
3692 boolean_t out_of_memory;
3693{
3694 static char here[] = "%s *** NOT ENOUGH PAGING SPACE ***";
3695
3696 if (out_of_memory)
3697 dprintf("*** OUT OF MEMORY *** ")(void) 0;
3698 panic(here, my_name);
3699}
3700
3701void overcommitted(got_more_space, space)
3702 boolean_t got_more_space;
3703 vm_size_t space; /* in pages */
3704{
3705 vm_size_t pages_free, pages_total;
3706
3707 static boolean_t user_warned = FALSE((boolean_t) 0);
3708 static vm_size_t pages_shortage = 0;
3709
3710 paging_space_info(&pages_total, &pages_free);
3711
3712 /*
3713 * If user added more space, see if it is enough
3714 */
3715 if (got_more_space) {
3716 pages_free -= pages_shortage;
3717 if (pages_free > 0) {
3718 pages_shortage = 0;
3719 if (user_warned)
3720 dprintf("%s paging space ok now.\n", my_name)(void) 0;
3721 } else
3722 pages_shortage = pages_free;
3723 user_warned = FALSE((boolean_t) 0);
3724 return;
3725 }
3726 /*
3727 * We ran out of gas, let user know.
3728 */
3729 pages_free -= space;
3730 pages_shortage = (pages_free > 0) ? 0 : -pages_free;
3731 if (!user_warned && pages_shortage) {
3732 user_warned = TRUE((boolean_t) 1);
3733 dprintf("%s paging space over-committed.\n", my_name)(void) 0;
3734 }
3735#if debug0
3736 user_warned = FALSE((boolean_t) 0);
3737 dprintf("%s paging space over-committed [+%d (%d) pages].\n",(void) 0
3738 my_name, space, pages_shortage)(void) 0;
3739#endif
3740}
3741
3742void paging_space_info(totp, freep)
3743 vm_size_t *totp, *freep;
3744{
3745 vm_size_t total, free;
3746 partition_t part;
3747 int i;
3748
3749 total = free = 0;
3750 for (i = 0; i < all_partitions.n_partitions; i++) {
3751
3752 if ((part = partition_of(i)) == 0) continue;
3753
3754 /* no need to lock: by the time this data
3755 gets back to any remote requestor it
3756 will be obsolete anyways */
3757 total += part->total_size;
3758 free += part->free;
3759#if debug0
3760 dprintf("Partition %d: x%x total, x%x free\n",(void) 0
3761 i, part->total_size, part->free)(void) 0;
3762#endif
3763 }
3764 *totp = total;
3765 *freep = free;
3766}
3767
3768/*
3769 * Catch exceptions.
3770 */
3771
3772kern_return_t
3773catch_exception_raise(exception_port, thread, task, exception, code, subcode)
3774 mach_port_t exception_port;
3775 mach_port_t thread, task;
3776 int exception, code, subcode;
3777{
3778 ddprintf ("(default_pager)catch_exception_raise(%d,%d,%d)\n",(void) 0
3779 exception, code, subcode)(void) 0;
3780 panic(my_name);
3781
3782 /* mach_msg_server will deallocate thread/task for us */
3783
3784 return KERN_FAILURE5;
3785}