Bug Summary

File:obj/../i386/intel/pmap.c
Location:line 1570, column 11
Description:Dereference of null pointer (loaded from variable 'pte')

Annotated Source Code

1/*
2 * Mach Operating System
3 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
4 * All Rights Reserved.
5 *
6 * Permission to use, copy, modify and distribute this software and its
7 * documentation is hereby granted, provided that both the copyright
8 * notice and this permission notice appear in all copies of the
9 * software, derivative works or modified versions, and any portions
10 * thereof, and that both notices appear in supporting documentation.
11 *
12 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
13 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
14 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
15 *
16 * Carnegie Mellon requests users of this software to return to
17 *
18 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
19 * School of Computer Science
20 * Carnegie Mellon University
21 * Pittsburgh PA 15213-3890
22 *
23 * any improvements or extensions that they make and grant Carnegie Mellon
24 * the rights to redistribute these changes.
25 */
26/*
27 * File: pmap.c
28 * Author: Avadis Tevanian, Jr., Michael Wayne Young
29 * (These guys wrote the Vax version)
30 *
31 * Physical Map management code for Intel i386, and i486.
32 *
33 * Manages physical address maps.
34 *
35 * In addition to hardware address maps, this
36 * module is called upon to provide software-use-only
37 * maps which may or may not be stored in the same
38 * form as hardware maps. These pseudo-maps are
39 * used to store intermediate results from copy
40 * operations to and from address spaces.
41 *
42 * Since the information managed by this module is
43 * also stored by the logical address mapping module,
44 * this module may throw away valid virtual-to-physical
45 * mappings at almost any time. However, invalidations
46 * of virtual-to-physical mappings must be done as
47 * requested.
48 *
49 * In order to cope with hardware architectures which
50 * make virtual-to-physical map invalidates expensive,
51 * this module may delay invalidate or reduced protection
52 * operations until such time as they are actually
53 * necessary. This module is given full information as
54 * to which processors are currently using which maps,
55 * and to when physical maps must be made correct.
56 */
57
58#include <string.h>
59
60#include <mach/machine/vm_types.h>
61
62#include <mach/boolean.h>
63#include <kern/debug.h>
64#include <kern/printf.h>
65#include <kern/thread.h>
66#include <kern/slab.h>
67
68#include <kern/lock.h>
69
70#include <vm/pmap.h>
71#include <vm/vm_map.h>
72#include <vm/vm_kern.h>
73#include <i3861/vm_param.h>
74#include <mach/vm_prot.h>
75#include <vm/vm_object.h>
76#include <vm/vm_page.h>
77#include <vm/vm_user.h>
78
79#include <mach/machine/vm_param.h>
80#include <mach/xen.h>
81#include <machine/thread.h>
82#include <i3861/cpu_number.h>
83#include <i3861/proc_reg.h>
84#include <i3861/locore.h>
85#include <i3861/model_dep.h>
86
87#ifdef MACH_PSEUDO_PHYS
88#define WRITE_PTE(pte_p, pte_entry)*(pte_p) = (pte_entry); *(pte_p) = pte_entry?pa_to_ma(pte_entry):0;
89#else /* MACH_PSEUDO_PHYS */
90#define WRITE_PTE(pte_p, pte_entry)*(pte_p) = (pte_entry); *(pte_p) = (pte_entry);
91#endif /* MACH_PSEUDO_PHYS */
92
93/*
94 * Private data structures.
95 */
96
97/*
98 * For each vm_page_t, there is a list of all currently
99 * valid virtual mappings of that page. An entry is
100 * a pv_entry_t; the list is the pv_table.
101 */
102
103typedef struct pv_entry {
104 struct pv_entry *next; /* next pv_entry */
105 pmap_t pmap; /* pmap where mapping lies */
106 vm_offset_t va; /* virtual address for mapping */
107} *pv_entry_t;
108
109#define PV_ENTRY_NULL((pv_entry_t) 0) ((pv_entry_t) 0)
110
111pv_entry_t pv_head_table; /* array of entries, one per page */
112
113/*
114 * pv_list entries are kept on a list that can only be accessed
115 * with the pmap system locked (at SPLVM, not in the cpus_active set).
116 * The list is refilled from the pv_list_cache if it becomes empty.
117 */
118pv_entry_t pv_free_list; /* free list at SPLVM */
119decl_simple_lock_data(, pv_free_list_lock)
120
121#define PV_ALLOC(pv_e){ ; if ((pv_e = pv_free_list) != 0) { pv_free_list = pv_e->
next; } ; } { \
122 simple_lock(&pv_free_list_lock); \
123 if ((pv_e = pv_free_list) != 0) { \
124 pv_free_list = pv_e->next; \
125 } \
126 simple_unlock(&pv_free_list_lock); \
127}
128
129#define PV_FREE(pv_e){ ; pv_e->next = pv_free_list; pv_free_list = pv_e; ; } { \
130 simple_lock(&pv_free_list_lock); \
131 pv_e->next = pv_free_list; \
132 pv_free_list = pv_e; \
133 simple_unlock(&pv_free_list_lock); \
134}
135
136struct kmem_cache pv_list_cache; /* cache of pv_entry structures */
137
138/*
139 * Each entry in the pv_head_table is locked by a bit in the
140 * pv_lock_table. The lock bits are accessed by the physical
141 * address of the page they lock.
142 */
143
144char *pv_lock_table; /* pointer to array of bits */
145#define pv_lock_table_size(n)(((n)+8 -1)/8) (((n)+BYTE_SIZE8-1)/BYTE_SIZE8)
146
147/* Has pmap_init completed? */
148boolean_t pmap_initialized = FALSE((boolean_t) 0);
149
150/*
151 * Range of kernel virtual addresses available for kernel memory mapping.
152 * Does not include the virtual addresses used to map physical memory 1-1.
153 * Initialized by pmap_bootstrap.
154 */
155vm_offset_t kernel_virtual_start;
156vm_offset_t kernel_virtual_end;
157
158/*
159 * Index into pv_head table, its lock bits, and the modify/reference
160 * bits starting at phys_first_addr.
161 */
162#define pa_index(pa)((((vm_size_t)(pa - phys_first_addr)) >> 12)) (atop(pa - phys_first_addr)(((vm_size_t)(pa - phys_first_addr)) >> 12))
163
164#define pai_to_pvh(pai)(&pv_head_table[pai]) (&pv_head_table[pai])
165#define lock_pvh_pai(pai)(bit_lock(pai, pv_lock_table)) (bit_lock(pai, pv_lock_table))
166#define unlock_pvh_pai(pai)(bit_unlock(pai, pv_lock_table)) (bit_unlock(pai, pv_lock_table))
167
168/*
169 * Array of physical page attribites for managed pages.
170 * One byte per physical page.
171 */
172char *pmap_phys_attributes;
173
174/*
175 * Physical page attributes. Copy bits from PTE definition.
176 */
177#define PHYS_MODIFIED0x00000040 INTEL_PTE_MOD0x00000040 /* page modified */
178#define PHYS_REFERENCED0x00000020 INTEL_PTE_REF0x00000020 /* page referenced */
179
180/*
181 * Amount of virtual memory mapped by one
182 * page-directory entry.
183 */
184#define PDE_MAPPED_SIZE(((vm_offset_t)(1) << 22)) (pdenum2lin(1)((vm_offset_t)(1) << 22))
185
186/*
187 * We allocate page table pages directly from the VM system
188 * through this object. It maps physical memory.
189 */
190vm_object_t pmap_object = VM_OBJECT_NULL((vm_object_t) 0);
191
192/*
193 * Locking and TLB invalidation
194 */
195
196/*
197 * Locking Protocols:
198 *
199 * There are two structures in the pmap module that need locking:
200 * the pmaps themselves, and the per-page pv_lists (which are locked
201 * by locking the pv_lock_table entry that corresponds to the pv_head
202 * for the list in question.) Most routines want to lock a pmap and
203 * then do operations in it that require pv_list locking -- however
204 * pmap_remove_all and pmap_copy_on_write operate on a physical page
205 * basis and want to do the locking in the reverse order, i.e. lock
206 * a pv_list and then go through all the pmaps referenced by that list.
207 * To protect against deadlock between these two cases, the pmap_lock
208 * is used. There are three different locking protocols as a result:
209 *
210 * 1. pmap operations only (pmap_extract, pmap_access, ...) Lock only
211 * the pmap.
212 *
213 * 2. pmap-based operations (pmap_enter, pmap_remove, ...) Get a read
214 * lock on the pmap_lock (shared read), then lock the pmap
215 * and finally the pv_lists as needed [i.e. pmap lock before
216 * pv_list lock.]
217 *
218 * 3. pv_list-based operations (pmap_remove_all, pmap_copy_on_write, ...)
219 * Get a write lock on the pmap_lock (exclusive write); this
220 * also guaranteees exclusive access to the pv_lists. Lock the
221 * pmaps as needed.
222 *
223 * At no time may any routine hold more than one pmap lock or more than
224 * one pv_list lock. Because interrupt level routines can allocate
225 * mbufs and cause pmap_enter's, the pmap_lock and the lock on the
226 * kernel_pmap can only be held at splvm.
227 */
228
229#if NCPUS1 > 1
230/*
231 * We raise the interrupt level to splvm, to block interprocessor
232 * interrupts during pmap operations. We must take the CPU out of
233 * the cpus_active set while interrupts are blocked.
234 */
235#define SPLVM(spl)((void)(spl)) { \
236 spl = splvm(); \
237 i_bit_clear(cpu_number()(0), &cpus_active); \
238}
239
240#define SPLX(spl)((void)(spl)) { \
241 i_bit_set(cpu_number()(0), &cpus_active); \
242 splx(spl); \
243}
244
245/*
246 * Lock on pmap system
247 */
248lock_data_t pmap_system_lock;
249
250#define PMAP_READ_LOCK(pmap, spl)((void)(spl)) { \
251 SPLVM(spl)((void)(spl)); \
252 lock_read(&pmap_system_lock); \
253 simple_lock(&(pmap)->lock); \
254}
255
256#define PMAP_WRITE_LOCK(spl)((void)(spl)) { \
257 SPLVM(spl)((void)(spl)); \
258 lock_write(&pmap_system_lock); \
259}
260
261#define PMAP_READ_UNLOCK(pmap, spl)((void)(spl)) { \
262 simple_unlock(&(pmap)->lock); \
263 lock_read_done(&pmap_system_lock)lock_done(&pmap_system_lock); \
264 SPLX(spl)((void)(spl)); \
265}
266
267#define PMAP_WRITE_UNLOCK(spl)((void)(spl)) { \
268 lock_write_done(&pmap_system_lock)lock_done(&pmap_system_lock); \
269 SPLX(spl)((void)(spl)); \
270}
271
272#define PMAP_WRITE_TO_READ_LOCK(pmap) { \
273 simple_lock(&(pmap)->lock); \
274 lock_write_to_read(&pmap_system_lock); \
275}
276
277#define LOCK_PVH(index) (lock_pvh_pai(index)(bit_lock(index, pv_lock_table)))
278
279#define UNLOCK_PVH(index) (unlock_pvh_pai(index)(bit_unlock(index, pv_lock_table)))
280
281#define PMAP_UPDATE_TLBS(pmap, s, e){ if ((pmap)->cpus_using) { { (void) ((pmap)); (void) ((s)
); (void) ((e)); ({ register unsigned long _temp__ = (({ register
unsigned long _temp__; asm volatile("mov %%cr3, %0" : "=r" (
_temp__)); _temp__; })); asm volatile("mov %0, %%cr3" : : "r"
(_temp__) : "memory"); }); }; } } \
282{ \
283 cpu_set cpu_mask = 1 << cpu_number()(0); \
284 cpu_set users; \
285 \
286 /* Since the pmap is locked, other updates are locked */ \
287 /* out, and any pmap_activate has finished. */ \
288 \
289 /* find other cpus using the pmap */ \
290 users = (pmap)->cpus_using & ~cpu_mask; \
291 if (users) { \
292 /* signal them, and wait for them to finish */ \
293 /* using the pmap */ \
294 signal_cpus(users, (pmap), (s), (e)); \
295 while ((pmap)->cpus_using & cpus_active & ~cpu_mask) \
296 continue; \
297 } \
298 \
299 /* invalidate our own TLB if pmap is in use */ \
300 if ((pmap)->cpus_using & cpu_mask) { \
301 INVALIDATE_TLB((pmap), (s), (e)){ (void) ((pmap)); (void) ((s)); (void) ((e)); ({ register unsigned
long _temp__ = (({ register unsigned long _temp__; asm volatile
("mov %%cr3, %0" : "=r" (_temp__)); _temp__; })); asm volatile
("mov %0, %%cr3" : : "r" (_temp__) : "memory"); }); }; \
302 } \
303}
304
305#else /* NCPUS > 1 */
306
307#define SPLVM(spl)((void)(spl)) ((void)(spl))
308#define SPLX(spl)((void)(spl)) ((void)(spl))
309
310#define PMAP_READ_LOCK(pmap, spl)((void)(spl)) SPLVM(spl)((void)(spl))
311#define PMAP_WRITE_LOCK(spl)((void)(spl)) SPLVM(spl)((void)(spl))
312#define PMAP_READ_UNLOCK(pmap, spl)((void)(spl)) SPLX(spl)((void)(spl))
313#define PMAP_WRITE_UNLOCK(spl)((void)(spl)) SPLX(spl)((void)(spl))
314#define PMAP_WRITE_TO_READ_LOCK(pmap)
315
316#define LOCK_PVH(index)
317#define UNLOCK_PVH(index)
318
319#define PMAP_UPDATE_TLBS(pmap, s, e){ if ((pmap)->cpus_using) { { (void) ((pmap)); (void) ((s)
); (void) ((e)); ({ register unsigned long _temp__ = (({ register
unsigned long _temp__; asm volatile("mov %%cr3, %0" : "=r" (
_temp__)); _temp__; })); asm volatile("mov %0, %%cr3" : : "r"
(_temp__) : "memory"); }); }; } } { \
320 /* invalidate our own TLB if pmap is in use */ \
321 if ((pmap)->cpus_using) { \
322 INVALIDATE_TLB((pmap), (s), (e)){ (void) ((pmap)); (void) ((s)); (void) ((e)); ({ register unsigned
long _temp__ = (({ register unsigned long _temp__; asm volatile
("mov %%cr3, %0" : "=r" (_temp__)); _temp__; })); asm volatile
("mov %0, %%cr3" : : "r" (_temp__) : "memory"); }); }; \
323 } \
324}
325
326#endif /* NCPUS > 1 */
327
328#define MAX_TBIS_SIZE32 32 /* > this -> TBIA */ /* XXX */
329
330#ifdef MACH_PV_PAGETABLES
331#if 1
332#define INVALIDATE_TLB(pmap, s, e){ (void) (pmap); (void) (s); (void) (e); ({ register unsigned
long _temp__ = (({ register unsigned long _temp__; asm volatile
("mov %%cr3, %0" : "=r" (_temp__)); _temp__; })); asm volatile
("mov %0, %%cr3" : : "r" (_temp__) : "memory"); }); } hyp_mmuext_op_void(MMUEXT_TLB_FLUSH_LOCAL)
333#else
334#define INVALIDATE_TLB(pmap, s, e){ (void) (pmap); (void) (s); (void) (e); ({ register unsigned
long _temp__ = (({ register unsigned long _temp__; asm volatile
("mov %%cr3, %0" : "=r" (_temp__)); _temp__; })); asm volatile
("mov %0, %%cr3" : : "r" (_temp__) : "memory"); }); } do { \
335 if (__builtin_constant_p((e) - (s)) \
336 && (e) - (s) == PAGE_SIZE(1 << 12)) \
337 hyp_invlpg((pmap) == kernel_pmap ? kvtolin(s)((vm_offset_t)(s) - 0xC0000000UL + ((0xc0000000UL))) : (s)); \
338 else \
339 hyp_mmuext_op_void(MMUEXT_TLB_FLUSH_LOCAL); \
340} while(0)
341#endif
342#else /* MACH_PV_PAGETABLES */
343#if 0
344/* It is hard to know when a TLB flush becomes less expensive than a bunch of
345 * invlpgs. But it surely is more expensive than just one invlpg. */
346#define INVALIDATE_TLB(pmap, s, e){ (void) (pmap); (void) (s); (void) (e); ({ register unsigned
long _temp__ = (({ register unsigned long _temp__; asm volatile
("mov %%cr3, %0" : "=r" (_temp__)); _temp__; })); asm volatile
("mov %0, %%cr3" : : "r" (_temp__) : "memory"); }); } { \
347 if (__builtin_constant_p((e) - (s)) \
348 && (e) - (s) == PAGE_SIZE(1 << 12)) \
349 invlpg_linear(s)({ asm volatile( "movw %w1,%%es\n" "\tinvlpg %%es:(%0)\n" "\tmovw %w2,%%es"
:: "r" (s), "q" (0x38), "q" ((0x10 | 0))); }); \
350 else \
351 flush_tlb()({ register unsigned long _temp__ = (({ register unsigned long
_temp__; asm volatile("mov %%cr3, %0" : "=r" (_temp__)); _temp__
; })); asm volatile("mov %0, %%cr3" : : "r" (_temp__) : "memory"
); }); \
352}
353#else
354#define INVALIDATE_TLB(pmap, s, e){ (void) (pmap); (void) (s); (void) (e); ({ register unsigned
long _temp__ = (({ register unsigned long _temp__; asm volatile
("mov %%cr3, %0" : "=r" (_temp__)); _temp__; })); asm volatile
("mov %0, %%cr3" : : "r" (_temp__) : "memory"); }); } { \
355 (void) (pmap); \
356 (void) (s); \
357 (void) (e); \
358 flush_tlb()({ register unsigned long _temp__ = (({ register unsigned long
_temp__; asm volatile("mov %%cr3, %0" : "=r" (_temp__)); _temp__
; })); asm volatile("mov %0, %%cr3" : : "r" (_temp__) : "memory"
); }); \
359}
360#endif
361#endif /* MACH_PV_PAGETABLES */
362
363
364#if NCPUS1 > 1
365/*
366 * Structures to keep track of pending TLB invalidations
367 */
368
369#define UPDATE_LIST_SIZE 4
370
371struct pmap_update_item {
372 pmap_t pmap; /* pmap to invalidate */
373 vm_offset_t start; /* start address to invalidate */
374 vm_offset_t end; /* end address to invalidate */
375} ;
376
377typedef struct pmap_update_item *pmap_update_item_t;
378
379/*
380 * List of pmap updates. If the list overflows,
381 * the last entry is changed to invalidate all.
382 */
383struct pmap_update_list {
384 decl_simple_lock_data(, lock)
385 int count;
386 struct pmap_update_item item[UPDATE_LIST_SIZE];
387} ;
388typedef struct pmap_update_list *pmap_update_list_t;
389
390struct pmap_update_list cpu_update_list[NCPUS1];
391
392#endif /* NCPUS > 1 */
393
394/*
395 * Other useful macros.
396 */
397#define current_pmap()((((active_threads[(0)])->task->map)->pmap)) (vm_map_pmap(current_thread()->task->map)(((active_threads[(0)])->task->map)->pmap))
398#define pmap_in_use(pmap, cpu)(((pmap)->cpus_using & (1 << (cpu))) != 0) (((pmap)->cpus_using & (1 << (cpu))) != 0)
399
400struct pmap kernel_pmap_store;
401pmap_t kernel_pmap;
402
403struct kmem_cache pmap_cache; /* cache of pmap structures */
404
405int pmap_debug = 0; /* flag for debugging prints */
406
407#if 0
408int ptes_per_vm_page1; /* number of hardware ptes needed
409 to map one VM page. */
410#else
411#define ptes_per_vm_page1 1
412#endif
413
414unsigned int inuse_ptepages_count = 0; /* debugging */
415
416extern char end;
417
418/*
419 * Pointer to the basic page directory for the kernel.
420 * Initialized by pmap_bootstrap().
421 */
422pt_entry_t *kernel_page_dir;
423
424void pmap_remove_range(); /* forward */
425#if NCPUS1 > 1
426void signal_cpus(); /* forward */
427#endif /* NCPUS > 1 */
428
429static inline pt_entry_t *
430pmap_pde(pmap_t pmap, vm_offset_t addr)
431{
432 if (pmap == kernel_pmap)
433 addr = kvtolin(addr)((vm_offset_t)(addr) - 0xC0000000UL + ((0xc0000000UL)));
434 return &pmap->dirbase[lin2pdenum(addr)(((addr) >> 22) & 0x3ff)];
435}
436
437/*
438 * Given an offset and a map, compute the address of the
439 * pte. If the address is invalid with respect to the map
440 * then PT_ENTRY_NULL is returned (and the map may need to grow).
441 *
442 * This is only used internally.
443 */
444pt_entry_t *
445pmap_pte(pmap_t pmap, vm_offset_t addr)
446{
447 pt_entry_t *ptp;
448 pt_entry_t pte;
449
450 if (pmap->dirbase == 0)
451 return(PT_ENTRY_NULL((pt_entry_t *) 0));
452 pte = *pmap_pde(pmap, addr);
453 if ((pte & INTEL_PTE_VALID0x00000001) == 0)
454 return(PT_ENTRY_NULL((pt_entry_t *) 0));
455 ptp = (pt_entry_t *)ptetokv(pte)(((vm_offset_t)(((pte) & 0xfffff000)) + 0xC0000000UL));
456 return(&ptp[ptenum(addr)(((addr) >> 12) & 0x3ff)]);
457}
458
459#define DEBUG_PTE_PAGE0 0
460
461#if DEBUG_PTE_PAGE0
462void ptep_check(ptep)
463 ptep_t ptep;
464{
465 register pt_entry_t *pte, *epte;
466 int ctu, ctw;
467
468 /* check the use and wired counts */
469 if (ptep == PTE_PAGE_NULL)
470 return;
471 pte = pmap_pte(ptep->pmap, ptep->va);
472 epte = pte + INTEL_PGBYTES4096/sizeof(pt_entry_t);
473 ctu = 0;
474 ctw = 0;
475 while (pte < epte) {
476 if (pte->pfn != 0) {
477 ctu++;
478 if (pte->wired)
479 ctw++;
480 }
481 pte += ptes_per_vm_page1;
482 }
483
484 if (ctu != ptep->use_count || ctw != ptep->wired_count) {
485 printf("use %d wired %d - actual use %d wired %d\n",
486 ptep->use_count, ptep->wired_count, ctu, ctw);
487 panic("pte count");
488 }
489}
490#endif /* DEBUG_PTE_PAGE */
491
492/*
493 * Map memory at initialization. The physical addresses being
494 * mapped are not managed and are never unmapped.
495 *
496 * For now, VM is already on, we only need to map the
497 * specified memory.
498 */
499vm_offset_t pmap_map(virt, start, end, prot)
500 register vm_offset_t virt;
501 register vm_offset_t start;
502 register vm_offset_t end;
503 register int prot;
504{
505 register int ps;
506
507 ps = PAGE_SIZE(1 << 12);
508 while (start < end) {
509 pmap_enter(kernel_pmap, virt, start, prot, FALSE((boolean_t) 0));
510 virt += ps;
511 start += ps;
512 }
513 return(virt);
514}
515
516/*
517 * Back-door routine for mapping kernel VM at initialization.
518 * Useful for mapping memory outside the range
519 * [phys_first_addr, phys_last_addr) (i.e., devices).
520 * Otherwise like pmap_map.
521 */
522vm_offset_t pmap_map_bd(virt, start, end, prot)
523 register vm_offset_t virt;
524 register vm_offset_t start;
525 register vm_offset_t end;
526 vm_prot_t prot;
527{
528 register pt_entry_t template;
529 register pt_entry_t *pte;
530 int spl;
531#ifdef MACH_PV_PAGETABLES
532 int n, i = 0;
533 struct mmu_update update[HYP_BATCH_MMU_UPDATES];
534#endif /* MACH_PV_PAGETABLES */
535
536 template = pa_to_pte(start)((start) & 0xfffff000)
537 | INTEL_PTE_NCACHE0x00000010|INTEL_PTE_WTHRU0x00000008
538 | INTEL_PTE_VALID0x00000001;
539 if (CPU_HAS_FEATURE(CPU_FEATURE_PGE)(cpu_features[(13) / 32] & (1 << ((13) % 32))))
540 template |= INTEL_PTE_GLOBAL0x00000100;
541 if (prot & VM_PROT_WRITE((vm_prot_t) 0x02))
542 template |= INTEL_PTE_WRITE0x00000002;
543
544 PMAP_READ_LOCK(pmap, spl)((void)(spl));
545 while (start < end) {
546 pte = pmap_pte(kernel_pmap, virt);
547 if (pte == PT_ENTRY_NULL((pt_entry_t *) 0))
548 panic("pmap_map_bd: Invalid kernel address\n");
549#ifdef MACH_PV_PAGETABLES
550 update[i].ptr = kv_to_ma(pte);
551 update[i].val = pa_to_ma(template);
552 i++;
553 if (i == HYP_BATCH_MMU_UPDATES) {
554 hyp_mmu_update(kvtolin(&update)((vm_offset_t)(&update) - 0xC0000000UL + ((0xc0000000UL))
), i, kvtolin(&n)((vm_offset_t)(&n) - 0xC0000000UL + ((0xc0000000UL))), DOMID_SELF);
555 if (n != i)
556 panic("couldn't pmap_map_bd\n");
557 i = 0;
558 }
559#else /* MACH_PV_PAGETABLES */
560 WRITE_PTE(pte, template)*(pte) = (template);
561#endif /* MACH_PV_PAGETABLES */
562 pte_increment_pa(template)((template) += 0xfff +1);
563 virt += PAGE_SIZE(1 << 12);
564 start += PAGE_SIZE(1 << 12);
565 }
566#ifdef MACH_PV_PAGETABLES
567 if (i > HYP_BATCH_MMU_UPDATES)
568 panic("overflowed array in pmap_map_bd");
569 hyp_mmu_update(kvtolin(&update)((vm_offset_t)(&update) - 0xC0000000UL + ((0xc0000000UL))
), i, kvtolin(&n)((vm_offset_t)(&n) - 0xC0000000UL + ((0xc0000000UL))), DOMID_SELF);
570 if (n != i)
571 panic("couldn't pmap_map_bd\n");
572#endif /* MACH_PV_PAGETABLES */
573 PMAP_READ_UNLOCK(pmap, spl)((void)(spl));
574 return(virt);
575}
576
577/*
578 * Bootstrap the system enough to run with virtual memory.
579 * Allocate the kernel page directory and page tables,
580 * and direct-map all physical memory.
581 * Called with mapping off.
582 */
583void pmap_bootstrap()
584{
585 /*
586 * Mapping is turned off; we must reference only physical addresses.
587 * The load image of the system is to be mapped 1-1 physical = virtual.
588 */
589
590 /*
591 * Set ptes_per_vm_page for general use.
592 */
593#if 0
594 ptes_per_vm_page1 = PAGE_SIZE(1 << 12) / INTEL_PGBYTES4096;
595#endif
596
597 /*
598 * The kernel's pmap is statically allocated so we don't
599 * have to use pmap_create, which is unlikely to work
600 * correctly at this part of the boot sequence.
601 */
602
603 kernel_pmap = &kernel_pmap_store;
604
605#if NCPUS1 > 1
606 lock_init(&pmap_system_lock, FALSE((boolean_t) 0)); /* NOT a sleep lock */
607#endif /* NCPUS > 1 */
608
609 simple_lock_init(&kernel_pmap->lock);
610
611 kernel_pmap->ref_count = 1;
612
613 /*
614 * Determine the kernel virtual address range.
615 * It starts at the end of the physical memory
616 * mapped into the kernel address space,
617 * and extends to a stupid arbitrary limit beyond that.
618 */
619 kernel_virtual_start = phystokv(phys_last_addr)((vm_offset_t)(phys_last_addr) + 0xC0000000UL);
620 kernel_virtual_end = phystokv(phys_last_addr)((vm_offset_t)(phys_last_addr) + 0xC0000000UL) + VM_KERNEL_MAP_SIZE(192 * 1024 * 1024);
621
622 if (kernel_virtual_end < kernel_virtual_start
623 || kernel_virtual_end > VM_MAX_KERNEL_ADDRESS((0xffffffffUL) - ((0xc0000000UL)) + 0xC0000000UL))
624 kernel_virtual_end = VM_MAX_KERNEL_ADDRESS((0xffffffffUL) - ((0xc0000000UL)) + 0xC0000000UL);
625
626 /*
627 * Allocate and clear a kernel page directory.
628 */
629 /* Note: initial Xen mapping holds at least 512kB free mapped page.
630 * We use that for directly building our linear mapping. */
631#if PAE
632 {
633 vm_offset_t addr;
634 init_alloc_aligned(PDPNUM1 * INTEL_PGBYTES4096, &addr);
635 kernel_pmap->dirbase = kernel_page_dir = (pt_entry_t*)phystokv(addr)((vm_offset_t)(addr) + 0xC0000000UL);
636 }
637 kernel_pmap->pdpbase = (pt_entry_t*)phystokv(pmap_grab_page())((vm_offset_t)(pmap_grab_page()) + 0xC0000000UL);
638 {
639 int i;
640 for (i = 0; i < PDPNUM1; i++)
641 WRITE_PTE(&kernel_pmap->pdpbase[i], pa_to_pte(_kvtophys((void *) kernel_pmap->dirbase + i * INTEL_PGBYTES)) | INTEL_PTE_VALID)*(&kernel_pmap->pdpbase[i]) = (((((vm_offset_t)((void *
) kernel_pmap->dirbase + i * 4096) - 0xC0000000UL)) & 0xfffff000
) | 0x00000001);;
642 }
643#else /* PAE */
644 kernel_pmap->dirbase = kernel_page_dir = (pt_entry_t*)phystokv(pmap_grab_page())((vm_offset_t)(pmap_grab_page()) + 0xC0000000UL);
645#endif /* PAE */
646 {
647 int i;
648 for (i = 0; i < NPDES(1 * ((((unsigned long)(1)) << 12)/sizeof(pt_entry_t))); i++)
649 kernel_pmap->dirbase[i] = 0;
650 }
651
652#ifdef MACH_PV_PAGETABLES
653 /* We don't actually deal with the CR3 register content at all */
654 hyp_vm_assist(VMASST_CMD_enable, VMASST_TYPE_pae_extended_cr3);
655 /*
656 * Xen may only provide as few as 512KB extra bootstrap linear memory,
657 * which is far from enough to map all available memory, so we need to
658 * map more bootstrap linear memory. We here map 1 (resp. 4 for PAE)
659 * other L1 table(s), thus 4MiB extra memory (resp. 8MiB), which is
660 * enough for a pagetable mapping 4GiB.
661 */
662#ifdef PAE
663#define NSUP_L1 4
664#else
665#define NSUP_L1 1
666#endif
667 pt_entry_t *l1_map[NSUP_L1];
668 {
669 pt_entry_t *base = (pt_entry_t*) boot_info.pt_base;
670 vm_offset_t la;
671 int n_l1map;
672 for (n_l1map = 0, la = VM_MIN_KERNEL_ADDRESS0xC0000000UL; la >= VM_MIN_KERNEL_ADDRESS0xC0000000UL; la += NPTES((((unsigned long)(1)) << 12)/sizeof(pt_entry_t)) * PAGE_SIZE(1 << 12)) {
673#ifdef PAE
674 pt_entry_t *l2_map = (pt_entry_t*) ptetokv(base[lin2pdpnum(la)])(((vm_offset_t)(((base[lin2pdpnum(la)]) & 0xfffff000)) + 0xC0000000UL
));
675#else /* PAE */
676 pt_entry_t *l2_map = base;
677#endif /* PAE */
678 /* Like lin2pdenum, but works with non-contiguous boot L3 */
679 l2_map += (la >> PDESHIFT22) & PDEMASK0x3ff;
680 if (!(*l2_map & INTEL_PTE_VALID0x00000001)) {
681 struct mmu_update update;
682 int j, n;
683
684 l1_map[n_l1map] = (pt_entry_t*) phystokv(pmap_grab_page())((vm_offset_t)(pmap_grab_page()) + 0xC0000000UL);
685 for (j = 0; j < NPTES((((unsigned long)(1)) << 12)/sizeof(pt_entry_t)); j++)
686 l1_map[n_l1map][j] = (((pt_entry_t)pfn_to_mfn(lin2pdenum(la - VM_MIN_KERNEL_ADDRESS)(((la - 0xC0000000UL) >> 22) & 0x3ff) * NPTES((((unsigned long)(1)) << 12)/sizeof(pt_entry_t)) + j)) << PAGE_SHIFT12) | INTEL_PTE_VALID0x00000001 | INTEL_PTE_WRITE0x00000002;
687 pmap_set_page_readonly_init(l1_map[n_l1map]);
688 if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, kv_to_mfn (l1_map[n_l1map])))
689 panic("couldn't pin page %p(%p)", l1_map[n_l1map], (vm_offset_t) kv_to_ma (l1_map[n_l1map]));
690 update.ptr = kv_to_ma(l2_map);
691 update.val = kv_to_ma(l1_map[n_l1map]) | INTEL_PTE_VALID0x00000001 | INTEL_PTE_WRITE0x00000002;
692 hyp_mmu_update(kv_to_la(&update), 1, kv_to_la(&n), DOMID_SELF);
693 if (n != 1)
694 panic("couldn't complete bootstrap map");
695 /* added the last L1 table, can stop */
696 if (++n_l1map >= NSUP_L1)
697 break;
698 }
699 }
700 }
701#endif /* MACH_PV_PAGETABLES */
702
703 /*
704 * Allocate and set up the kernel page tables.
705 */
706 {
707 vm_offset_t va;
708 pt_entry_t global = CPU_HAS_FEATURE(CPU_FEATURE_PGE)(cpu_features[(13) / 32] & (1 << ((13) % 32))) ? INTEL_PTE_GLOBAL0x00000100 : 0;
709
710 /*
711 * Map virtual memory for all known physical memory, 1-1,
712 * from phys_first_addr to phys_last_addr.
713 * Make any mappings completely in the kernel's text segment read-only.
714 *
715 * Also allocate some additional all-null page tables afterwards
716 * for kernel virtual memory allocation,
717 * because this PMAP module is too stupid
718 * to allocate new kernel page tables later.
719 * XX fix this
720 */
721 for (va = phystokv(phys_first_addr)((vm_offset_t)(phys_first_addr) + 0xC0000000UL); va >= phystokv(phys_first_addr)((vm_offset_t)(phys_first_addr) + 0xC0000000UL) && va < kernel_virtual_end; )
722 {
723 pt_entry_t *pde = kernel_page_dir + lin2pdenum(kvtolin(va))(((((vm_offset_t)(va) - 0xC0000000UL + ((0xc0000000UL)))) >>
22) & 0x3ff);
724 pt_entry_t *ptable = (pt_entry_t*)phystokv(pmap_grab_page())((vm_offset_t)(pmap_grab_page()) + 0xC0000000UL);
725 pt_entry_t *pte;
726
727 /* Initialize the page directory entry. */
728 WRITE_PTE(pde, pa_to_pte((vm_offset_t)_kvtophys(ptable))*(pde) = ((((vm_offset_t)((vm_offset_t)(ptable) - 0xC0000000UL
)) & 0xfffff000) | 0x00000001 | 0x00000002);
729 | INTEL_PTE_VALID | INTEL_PTE_WRITE)*(pde) = ((((vm_offset_t)((vm_offset_t)(ptable) - 0xC0000000UL
)) & 0xfffff000) | 0x00000001 | 0x00000002);;
730
731 /* Initialize the page table. */
732 for (pte = ptable; (va < phystokv(phys_last_addr)((vm_offset_t)(phys_last_addr) + 0xC0000000UL)) && (pte < ptable+NPTES((((unsigned long)(1)) << 12)/sizeof(pt_entry_t))); pte++)
733 {
734 if ((pte - ptable) < ptenum(va)(((va) >> 12) & 0x3ff))
735 {
736 WRITE_PTE(pte, 0)*(pte) = (0);;
737 }
738 else
739#ifdef MACH_PV_PAGETABLES
740 if (va == (vm_offset_t) &hyp_shared_info)
741 {
742 *pte = boot_info.shared_info | INTEL_PTE_VALID0x00000001 | INTEL_PTE_WRITE0x00000002;
743 va += INTEL_PGBYTES4096;
744 }
745 else
746#endif /* MACH_PV_PAGETABLES */
747 {
748 extern char _start[], etext[];
749
750 if (((va >= (vm_offset_t) _start)
751 && (va + INTEL_PGBYTES4096 <= (vm_offset_t)etext))
752#ifdef MACH_PV_PAGETABLES
753 || (va >= (vm_offset_t) boot_info.pt_base
754 && (va + INTEL_PGBYTES4096 <=
755 (vm_offset_t) ptable + INTEL_PGBYTES4096))
756#endif /* MACH_PV_PAGETABLES */
757 )
758 {
759 WRITE_PTE(pte, pa_to_pte(_kvtophys(va))*(pte) = (((((vm_offset_t)(va) - 0xC0000000UL)) & 0xfffff000
) | 0x00000001 | global);
760 | INTEL_PTE_VALID | global)*(pte) = (((((vm_offset_t)(va) - 0xC0000000UL)) & 0xfffff000
) | 0x00000001 | global);;
761 }
762 else
763 {
764#ifdef MACH_PV_PAGETABLES
765 /* Keep supplementary L1 pages read-only */
766 int i;
767 for (i = 0; i < NSUP_L1; i++)
768 if (va == (vm_offset_t) l1_map[i]) {
769 WRITE_PTE(pte, pa_to_pte(_kvtophys(va))*(pte) = (((((vm_offset_t)(va) - 0xC0000000UL)) & 0xfffff000
) | 0x00000001 | global);
770 | INTEL_PTE_VALID | global)*(pte) = (((((vm_offset_t)(va) - 0xC0000000UL)) & 0xfffff000
) | 0x00000001 | global);;
771 break;
772 }
773 if (i == NSUP_L1)
774#endif /* MACH_PV_PAGETABLES */
775 WRITE_PTE(pte, pa_to_pte(_kvtophys(va))*(pte) = (((((vm_offset_t)(va) - 0xC0000000UL)) & 0xfffff000
) | 0x00000001 | 0x00000002 | global);
776 | INTEL_PTE_VALID | INTEL_PTE_WRITE | global)*(pte) = (((((vm_offset_t)(va) - 0xC0000000UL)) & 0xfffff000
) | 0x00000001 | 0x00000002 | global);
777
778 }
779 va += INTEL_PGBYTES4096;
780 }
781 }
782 for (; pte < ptable+NPTES((((unsigned long)(1)) << 12)/sizeof(pt_entry_t)); pte++)
783 {
784 WRITE_PTE(pte, 0)*(pte) = (0);;
785 va += INTEL_PGBYTES4096;
786 }
787#ifdef MACH_PV_PAGETABLES
788 pmap_set_page_readonly_init(ptable);
789 if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, kv_to_mfn (ptable)))
790 panic("couldn't pin page %p(%p)\n", ptable, (vm_offset_t) kv_to_ma (ptable));
791#endif /* MACH_PV_PAGETABLES */
792 }
793 }
794
795 /* Architecture-specific code will turn on paging
796 soon after we return from here. */
797}
798
799#ifdef MACH_PV_PAGETABLES
800/* These are only required because of Xen security policies */
801
802/* Set back a page read write */
803void pmap_set_page_readwrite(void *_vaddr) {
804 vm_offset_t vaddr = (vm_offset_t) _vaddr;
805 vm_offset_t paddr = kvtophys(vaddr);
806 vm_offset_t canon_vaddr = phystokv(paddr)((vm_offset_t)(paddr) + 0xC0000000UL);
807 if (hyp_do_update_va_mapping (kvtolin(vaddr)((vm_offset_t)(vaddr) - 0xC0000000UL + ((0xc0000000UL))), pa_to_pte (pa_to_ma(paddr))((pa_to_ma(paddr)) & 0xfffff000) | INTEL_PTE_VALID0x00000001 | INTEL_PTE_WRITE0x00000002, UVMF_NONE))
808 panic("couldn't set hiMMU readwrite for addr %p(%p)\n", vaddr, (vm_offset_t) pa_to_ma (paddr));
809 if (canon_vaddr != vaddr)
810 if (hyp_do_update_va_mapping (kvtolin(canon_vaddr)((vm_offset_t)(canon_vaddr) - 0xC0000000UL + ((0xc0000000UL))
), pa_to_pte (pa_to_ma(paddr))((pa_to_ma(paddr)) & 0xfffff000) | INTEL_PTE_VALID0x00000001 | INTEL_PTE_WRITE0x00000002, UVMF_NONE))
811 panic("couldn't set hiMMU readwrite for paddr %p(%p)\n", canon_vaddr, (vm_offset_t) pa_to_ma (paddr));
812}
813
814/* Set a page read only (so as to pin it for instance) */
815void pmap_set_page_readonly(void *_vaddr) {
816 vm_offset_t vaddr = (vm_offset_t) _vaddr;
817 vm_offset_t paddr = kvtophys(vaddr);
818 vm_offset_t canon_vaddr = phystokv(paddr)((vm_offset_t)(paddr) + 0xC0000000UL);
819 if (*pmap_pde(kernel_pmap, vaddr) & INTEL_PTE_VALID0x00000001) {
820 if (hyp_do_update_va_mapping (kvtolin(vaddr)((vm_offset_t)(vaddr) - 0xC0000000UL + ((0xc0000000UL))), pa_to_pte (pa_to_ma(paddr))((pa_to_ma(paddr)) & 0xfffff000) | INTEL_PTE_VALID0x00000001, UVMF_NONE))
821 panic("couldn't set hiMMU readonly for vaddr %p(%p)\n", vaddr, (vm_offset_t) pa_to_ma (paddr));
822 }
823 if (canon_vaddr != vaddr &&
824 *pmap_pde(kernel_pmap, canon_vaddr) & INTEL_PTE_VALID0x00000001) {
825 if (hyp_do_update_va_mapping (kvtolin(canon_vaddr)((vm_offset_t)(canon_vaddr) - 0xC0000000UL + ((0xc0000000UL))
), pa_to_pte (pa_to_ma(paddr))((pa_to_ma(paddr)) & 0xfffff000) | INTEL_PTE_VALID0x00000001, UVMF_NONE))
826 panic("couldn't set hiMMU readonly for vaddr %p canon_vaddr %p paddr %p (%p)\n", vaddr, canon_vaddr, paddr, (vm_offset_t) pa_to_ma (paddr));
827 }
828}
829
830/* This needs to be called instead of pmap_set_page_readonly as long as RC3
831 * still points to the bootstrap dirbase, to also fix the bootstrap table. */
832void pmap_set_page_readonly_init(void *_vaddr) {
833 vm_offset_t vaddr = (vm_offset_t) _vaddr;
834#if PAE
835 pt_entry_t *pdpbase = (void*) boot_info.pt_base;
836 /* The bootstrap table does not necessarily use contiguous pages for the pde tables */
837 pt_entry_t *dirbase = (void*) ptetokv(pdpbase[lin2pdpnum(vaddr)])(((vm_offset_t)(((pdpbase[lin2pdpnum(vaddr)]) & 0xfffff000
)) + 0xC0000000UL));
838#else
839 pt_entry_t *dirbase = (void*) boot_info.pt_base;
840#endif
841 pt_entry_t *pte = &dirbase[lin2pdenum(vaddr)(((vaddr) >> 22) & 0x3ff) & PTEMASK0x3ff];
842 /* Modify our future kernel map (can't use update_va_mapping for this)... */
843 if (*pmap_pde(kernel_pmap, vaddr) & INTEL_PTE_VALID0x00000001) {
844 if (!hyp_mmu_update_la (kvtolin(vaddr)((vm_offset_t)(vaddr) - 0xC0000000UL + ((0xc0000000UL))), pa_to_pte (kv_to_ma(vaddr))((kv_to_ma(vaddr)) & 0xfffff000) | INTEL_PTE_VALID0x00000001))
845 panic("couldn't set hiMMU readonly for vaddr %p(%p)\n", vaddr, (vm_offset_t) kv_to_ma (vaddr));
846 }
847 /* ... and the bootstrap map. */
848 if (*pte & INTEL_PTE_VALID0x00000001) {
849 if (hyp_do_update_va_mapping (vaddr, pa_to_pte (kv_to_ma(vaddr))((kv_to_ma(vaddr)) & 0xfffff000) | INTEL_PTE_VALID0x00000001, UVMF_NONE))
850 panic("couldn't set MMU readonly for vaddr %p(%p)\n", vaddr, (vm_offset_t) kv_to_ma (vaddr));
851 }
852}
853
854void pmap_clear_bootstrap_pagetable(pt_entry_t *base) {
855 int i;
856 pt_entry_t *dir;
857 vm_offset_t va = 0;
858#if PAE
859 int j;
860#endif /* PAE */
861 if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, kv_to_mfn(base)))
862 panic("pmap_clear_bootstrap_pagetable: couldn't unpin page %p(%p)\n", base, (vm_offset_t) kv_to_ma(base));
863#if PAE
864 for (j = 0; j < PDPNUM1; j++)
865 {
866 pt_entry_t pdpe = base[j];
867 if (pdpe & INTEL_PTE_VALID0x00000001) {
868 dir = (pt_entry_t *) ptetokv(pdpe)(((vm_offset_t)(((pdpe) & 0xfffff000)) + 0xC0000000UL));
869#else /* PAE */
870 dir = base;
871#endif /* PAE */
872 for (i = 0; i < NPTES((((unsigned long)(1)) << 12)/sizeof(pt_entry_t)); i++) {
873 pt_entry_t pde = dir[i];
874 unsigned long pfn = atop(pte_to_pa(pde))(((vm_size_t)(((pde) & 0xfffff000))) >> 12);
875 void *pgt = (void*) phystokv(ptoa(pfn))((vm_offset_t)(((vm_offset_t)((pfn) << 12))) + 0xC0000000UL
);
876 if (pde & INTEL_PTE_VALID0x00000001)
877 hyp_free_page(pfn, pgt);
878 va += NPTES((((unsigned long)(1)) << 12)/sizeof(pt_entry_t)) * INTEL_PGBYTES4096;
879 if (va >= HYP_VIRT_START)
880 break;
881 }
882#if PAE
883 hyp_free_page(atop(_kvtophys(dir))(((vm_size_t)(((vm_offset_t)(dir) - 0xC0000000UL))) >> 12
), dir);
884 } else
885 va += NPTES((((unsigned long)(1)) << 12)/sizeof(pt_entry_t)) * NPTES((((unsigned long)(1)) << 12)/sizeof(pt_entry_t)) * INTEL_PGBYTES4096;
886 if (va >= HYP_VIRT_START)
887 break;
888 }
889#endif /* PAE */
890 hyp_free_page(atop(_kvtophys(base))(((vm_size_t)(((vm_offset_t)(base) - 0xC0000000UL))) >>
12), base);
891}
892#endif /* MACH_PV_PAGETABLES */
893
894void pmap_virtual_space(startp, endp)
895 vm_offset_t *startp;
896 vm_offset_t *endp;
897{
898 *startp = kernel_virtual_start;
899 *endp = kernel_virtual_end;
900}
901
902/*
903 * Initialize the pmap module.
904 * Called by vm_init, to initialize any structures that the pmap
905 * system needs to map virtual memory.
906 */
907void pmap_init()
908{
909 register long npages;
910 vm_offset_t addr;
911 register vm_size_t s;
912#if NCPUS1 > 1
913 int i;
914#endif /* NCPUS > 1 */
915
916 /*
917 * Allocate memory for the pv_head_table and its lock bits,
918 * the modify bit array, and the pte_page table.
919 */
920
921 npages = atop(phys_last_addr - phys_first_addr)(((vm_size_t)(phys_last_addr - phys_first_addr)) >> 12);
922 s = (vm_size_t) (sizeof(struct pv_entry) * npages
923 + pv_lock_table_size(npages)(((npages)+8 -1)/8)
924 + npages);
925
926 s = round_page(s)((vm_offset_t)((((vm_offset_t)(s)) + ((1 << 12)-1)) &
~((1 << 12)-1)));
927 if (kmem_alloc_wired(kernel_map, &addr, s) != KERN_SUCCESS0)
928 panic("pmap_init");
929 memset((char *) addr, 0, s);
930
931 /*
932 * Allocate the structures first to preserve word-alignment.
933 */
934 pv_head_table = (pv_entry_t) addr;
935 addr = (vm_offset_t) (pv_head_table + npages);
936
937 pv_lock_table = (char *) addr;
938 addr = (vm_offset_t) (pv_lock_table + pv_lock_table_size(npages)(((npages)+8 -1)/8));
939
940 pmap_phys_attributes = (char *) addr;
941
942 /*
943 * Create the cache of physical maps,
944 * and of the physical-to-virtual entries.
945 */
946 s = (vm_size_t) sizeof(struct pmap);
947 kmem_cache_init(&pmap_cache, "pmap", s, 0, NULL((void *) 0), NULL((void *) 0), NULL((void *) 0), 0);
948 s = (vm_size_t) sizeof(struct pv_entry);
949 kmem_cache_init(&pv_list_cache, "pv_entry", s, 0, NULL((void *) 0), NULL((void *) 0), NULL((void *) 0), 0);
950
951#if NCPUS1 > 1
952 /*
953 * Set up the pmap request lists
954 */
955 for (i = 0; i < NCPUS1; i++) {
956 pmap_update_list_t up = &cpu_update_list[i];
957
958 simple_lock_init(&up->lock);
959 up->count = 0;
960 }
961#endif /* NCPUS > 1 */
962
963 /*
964 * Indicate that the PMAP module is now fully initialized.
965 */
966 pmap_initialized = TRUE((boolean_t) 1);
967}
968
969#define valid_page(x)(pmap_initialized && pmap_valid_page(x)) (pmap_initialized && pmap_valid_page(x))
970
971boolean_t pmap_verify_free(phys)
972 vm_offset_t phys;
973{
974 pv_entry_t pv_h;
975 int pai;
976 int spl;
977 boolean_t result;
978
979 assert(phys != vm_page_fictitious_addr)({ if (!(phys != vm_page_fictitious_addr)) Assert("phys != vm_page_fictitious_addr"
, "../i386/intel/pmap.c", 979); });
980 if (!pmap_initialized)
981 return(TRUE((boolean_t) 1));
982
983 if (!pmap_valid_page(phys))
984 return(FALSE((boolean_t) 0));
985
986 PMAP_WRITE_LOCK(spl)((void)(spl));
987
988 pai = pa_index(phys)((((vm_size_t)(phys - phys_first_addr)) >> 12));
989 pv_h = pai_to_pvh(pai)(&pv_head_table[pai]);
990
991 result = (pv_h->pmap == PMAP_NULL((pmap_t) 0));
992 PMAP_WRITE_UNLOCK(spl)((void)(spl));
993
994 return(result);
995}
996
997/*
998 * Routine: pmap_page_table_page_alloc
999 *
1000 * Allocates a new physical page to be used as a page-table page.
1001 *
1002 * Must be called with the pmap system and the pmap unlocked,
1003 * since these must be unlocked to use vm_page_grab.
1004 */
1005vm_offset_t
1006pmap_page_table_page_alloc()
1007{
1008 register vm_page_t m;
1009 register vm_offset_t pa;
1010
1011 check_simple_locks();
1012
1013 /*
1014 * We cannot allocate the pmap_object in pmap_init,
1015 * because it is called before the cache package is up.
1016 * Allocate it now if it is missing.
1017 */
1018 if (pmap_object == VM_OBJECT_NULL((vm_object_t) 0))
1019 pmap_object = vm_object_allocate(phys_last_addr - phys_first_addr);
1020
1021 /*
1022 * Allocate a VM page for the level 2 page table entries.
1023 */
1024 while ((m = vm_page_grab(FALSE((boolean_t) 0))) == VM_PAGE_NULL((vm_page_t) 0))
1025 VM_PAGE_WAIT((void (*)()) 0)vm_page_wait((void (*)()) 0);
1026
1027 /*
1028 * Map the page to its physical address so that it
1029 * can be found later.
1030 */
1031 pa = m->phys_addr;
1032 vm_object_lock(pmap_object);
1033 vm_page_insert(m, pmap_object, pa);
1034 vm_page_lock_queues();
1035 vm_page_wire(m);
1036 inuse_ptepages_count++;
1037 vm_page_unlock_queues();
1038 vm_object_unlock(pmap_object);
1039
1040 /*
1041 * Zero the page.
1042 */
1043 memset((void *)phystokv(pa)((vm_offset_t)(pa) + 0xC0000000UL), 0, PAGE_SIZE(1 << 12));
1044
1045 return pa;
1046}
1047
1048#ifdef MACH_XEN
1049void pmap_map_mfn(void *_addr, unsigned long mfn) {
1050 vm_offset_t addr = (vm_offset_t) _addr;
1051 pt_entry_t *pte, *pdp;
1052 vm_offset_t ptp;
1053 pt_entry_t ma = ((pt_entry_t) mfn) << PAGE_SHIFT12;
1054
1055 /* Add a ptp if none exist yet for this pte */
1056 if ((pte = pmap_pte(kernel_pmap, addr)) == PT_ENTRY_NULL((pt_entry_t *) 0)) {
1057 ptp = phystokv(pmap_page_table_page_alloc())((vm_offset_t)(pmap_page_table_page_alloc()) + 0xC0000000UL);
1058#ifdef MACH_PV_PAGETABLES
1059 pmap_set_page_readonly((void*) ptp);
1060 if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, pa_to_mfn(ptp)))
1061 panic("couldn't pin page %p(%p)\n",ptp,(vm_offset_t) kv_to_ma(ptp));
1062#endif /* MACH_PV_PAGETABLES */
1063 pdp = pmap_pde(kernel_pmap, addr);
1064
1065#ifdef MACH_PV_PAGETABLES
1066 if (!hyp_mmu_update_pte(kv_to_ma(pdp),
1067 pa_to_pte(kv_to_ma(ptp))((kv_to_ma(ptp)) & 0xfffff000) | INTEL_PTE_VALID0x00000001
1068 | INTEL_PTE_USER0x00000004
1069 | INTEL_PTE_WRITE0x00000002))
1070 panic("%s:%d could not set pde %p(%p) to %p(%p)\n",__FILE__"../i386/intel/pmap.c",__LINE__1070,kvtophys((vm_offset_t)pdp),(vm_offset_t) kv_to_ma(pdp), ptp, (vm_offset_t) pa_to_ma(ptp));
1071#else /* MACH_PV_PAGETABLES */
1072 *pdp = pa_to_pte(kvtophys(ptp))((kvtophys(ptp)) & 0xfffff000) | INTEL_PTE_VALID0x00000001
1073 | INTEL_PTE_USER0x00000004
1074 | INTEL_PTE_WRITE0x00000002;
1075#endif /* MACH_PV_PAGETABLES */
1076 pte = pmap_pte(kernel_pmap, addr);
1077 }
1078
1079#ifdef MACH_PV_PAGETABLES
1080 if (!hyp_mmu_update_pte(kv_to_ma(pte), ma | INTEL_PTE_VALID0x00000001 | INTEL_PTE_WRITE0x00000002))
1081 panic("%s:%d could not set pte %p(%p) to %p(%p)\n",__FILE__"../i386/intel/pmap.c",__LINE__1081,pte,(vm_offset_t) kv_to_ma(pte), ma, ma_to_pa(ma));
1082#else /* MACH_PV_PAGETABLES */
1083 /* Note: in this case, mfn is actually a pfn. */
1084 WRITE_PTE(pte, ma | INTEL_PTE_VALID | INTEL_PTE_WRITE)*(pte) = (ma | 0x00000001 | 0x00000002);;
1085#endif /* MACH_PV_PAGETABLES */
1086}
1087#endif /* MACH_XEN */
1088
1089/*
1090 * Deallocate a page-table page.
1091 * The page-table page must have all mappings removed,
1092 * and be removed from its page directory.
1093 */
1094void
1095pmap_page_table_page_dealloc(pa)
1096 vm_offset_t pa;
1097{
1098 vm_page_t m;
1099
1100 vm_object_lock(pmap_object);
1101 m = vm_page_lookup(pmap_object, pa);
1102 vm_page_lock_queues();
1103 vm_page_free(m);
1104 inuse_ptepages_count--;
1105 vm_page_unlock_queues();
1106 vm_object_unlock(pmap_object);
1107}
1108
1109/*
1110 * Create and return a physical map.
1111 *
1112 * If the size specified for the map
1113 * is zero, the map is an actual physical
1114 * map, and may be referenced by the
1115 * hardware.
1116 *
1117 * If the size specified is non-zero,
1118 * the map will be used in software only, and
1119 * is bounded by that size.
1120 */
1121pmap_t pmap_create(size)
1122 vm_size_t size;
1123{
1124 register pmap_t p;
1125 register pmap_statistics_t stats;
1126
1127 /*
1128 * A software use-only map doesn't even need a map.
1129 */
1130
1131 if (size != 0) {
1132 return(PMAP_NULL((pmap_t) 0));
1133 }
1134
1135/*
1136 * Allocate a pmap struct from the pmap_cache. Then allocate
1137 * the page descriptor table.
1138 */
1139
1140 p = (pmap_t) kmem_cache_alloc(&pmap_cache);
1141 if (p == PMAP_NULL((pmap_t) 0))
1142 panic("pmap_create");
1143
1144 if (kmem_alloc_wired(kernel_map,
1145 (vm_offset_t *)&p->dirbase, PDPNUM1 * INTEL_PGBYTES4096)
1146 != KERN_SUCCESS0)
1147 panic("pmap_create");
1148
1149 memcpy(p->dirbase, kernel_page_dir, PDPNUM1 * INTEL_PGBYTES4096);
1150#ifdef LINUX_DEV
1151#if VM_MIN_KERNEL_ADDRESS0xC0000000UL != 0
1152 /* Do not map BIOS in user tasks */
1153 p->dirbase[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS)(((((0xc0000000UL)) - 0xC0000000UL) >> 22) & 0x3ff)] = 0;
1154#endif
1155#endif
1156#ifdef MACH_PV_PAGETABLES
1157 {
1158 int i;
1159 for (i = 0; i < PDPNUM1; i++)
1160 pmap_set_page_readonly((void*) p->dirbase + i * INTEL_PGBYTES4096);
1161 }
1162#endif /* MACH_PV_PAGETABLES */
1163
1164#if PAE
1165 if (kmem_alloc_wired(kernel_map,
1166 (vm_offset_t *)&p->pdpbase, INTEL_PGBYTES4096)
1167 != KERN_SUCCESS0)
1168 panic("pmap_create");
1169 {
1170 int i;
1171 for (i = 0; i < PDPNUM1; i++)
1172 WRITE_PTE(&p->pdpbase[i], pa_to_pte(kvtophys((vm_offset_t) p->dirbase + i * INTEL_PGBYTES)) | INTEL_PTE_VALID)*(&p->pdpbase[i]) = (((kvtophys((vm_offset_t) p->dirbase
+ i * 4096)) & 0xfffff000) | 0x00000001);;
1173 }
1174#ifdef MACH_PV_PAGETABLES
1175 pmap_set_page_readonly(p->pdpbase);
1176#endif /* MACH_PV_PAGETABLES */
1177#endif /* PAE */
1178
1179 p->ref_count = 1;
1180
1181 simple_lock_init(&p->lock);
1182 p->cpus_using = 0;
1183
1184 /*
1185 * Initialize statistics.
1186 */
1187
1188 stats = &p->stats;
1189 stats->resident_count = 0;
1190 stats->wired_count = 0;
1191
1192 return(p);
1193}
1194
1195/*
1196 * Retire the given physical map from service.
1197 * Should only be called if the map contains
1198 * no valid mappings.
1199 */
1200
1201void pmap_destroy(p)
1202 register pmap_t p;
1203{
1204 register pt_entry_t *pdep;
1205 register vm_offset_t pa;
1206 register int c, s;
1207 register vm_page_t m;
1208
1209 if (p == PMAP_NULL((pmap_t) 0))
1210 return;
1211
1212 SPLVM(s)((void)(s));
1213 simple_lock(&p->lock);
1214 c = --p->ref_count;
1215 simple_unlock(&p->lock);
1216 SPLX(s)((void)(s));
1217
1218 if (c != 0) {
1219 return; /* still in use */
1220 }
1221
1222 /*
1223 * Free the memory maps, then the
1224 * pmap structure.
1225 */
1226 for (pdep = p->dirbase;
1227 pdep < &p->dirbase[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS)(((((0xc0000000UL))) >> 22) & 0x3ff)];
1228 pdep += ptes_per_vm_page1) {
1229 if (*pdep & INTEL_PTE_VALID0x00000001) {
1230 pa = pte_to_pa(*pdep)((*pdep) & 0xfffff000);
1231 vm_object_lock(pmap_object);
1232 m = vm_page_lookup(pmap_object, pa);
1233 if (m == VM_PAGE_NULL((vm_page_t) 0))
1234 panic("pmap_destroy: pte page not in object");
1235 vm_page_lock_queues();
1236#ifdef MACH_PV_PAGETABLES
1237 if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, pa_to_mfn(pa)))
1238 panic("pmap_destroy: couldn't unpin page %p(%p)\n", pa, (vm_offset_t) kv_to_ma(pa));
1239 pmap_set_page_readwrite((void*) phystokv(pa)((vm_offset_t)(pa) + 0xC0000000UL));
1240#endif /* MACH_PV_PAGETABLES */
1241 vm_page_free(m);
1242 inuse_ptepages_count--;
1243 vm_page_unlock_queues();
1244 vm_object_unlock(pmap_object);
1245 }
1246 }
1247#ifdef MACH_PV_PAGETABLES
1248 {
1249 int i;
1250 for (i = 0; i < PDPNUM1; i++)
1251 pmap_set_page_readwrite((void*) p->dirbase + i * INTEL_PGBYTES4096);
1252 }
1253#endif /* MACH_PV_PAGETABLES */
1254 kmem_free(kernel_map, (vm_offset_t)p->dirbase, PDPNUM1 * INTEL_PGBYTES4096);
1255#if PAE
1256#ifdef MACH_PV_PAGETABLES
1257 pmap_set_page_readwrite(p->pdpbase);
1258#endif /* MACH_PV_PAGETABLES */
1259 kmem_free(kernel_map, (vm_offset_t)p->pdpbase, INTEL_PGBYTES4096);
1260#endif /* PAE */
1261 kmem_cache_free(&pmap_cache, (vm_offset_t) p);
1262}
1263
1264/*
1265 * Add a reference to the specified pmap.
1266 */
1267
1268void pmap_reference(p)
1269 register pmap_t p;
1270{
1271 int s;
1272 if (p != PMAP_NULL((pmap_t) 0)) {
1273 SPLVM(s)((void)(s));
1274 simple_lock(&p->lock);
1275 p->ref_count++;
1276 simple_unlock(&p->lock);
1277 SPLX(s)((void)(s));
1278 }
1279}
1280
1281/*
1282 * Remove a range of hardware page-table entries.
1283 * The entries given are the first (inclusive)
1284 * and last (exclusive) entries for the VM pages.
1285 * The virtual address is the va for the first pte.
1286 *
1287 * The pmap must be locked.
1288 * If the pmap is not the kernel pmap, the range must lie
1289 * entirely within one pte-page. This is NOT checked.
1290 * Assumes that the pte-page exists.
1291 */
1292
1293/* static */
1294void pmap_remove_range(pmap, va, spte, epte)
1295 pmap_t pmap;
1296 vm_offset_t va;
1297 pt_entry_t *spte;
1298 pt_entry_t *epte;
1299{
1300 register pt_entry_t *cpte;
1301 int num_removed, num_unwired;
1302 int pai;
1303 vm_offset_t pa;
1304#ifdef MACH_PV_PAGETABLES
1305 int n, ii = 0;
1306 struct mmu_update update[HYP_BATCH_MMU_UPDATES];
1307#endif /* MACH_PV_PAGETABLES */
1308
1309#if DEBUG_PTE_PAGE0
1310 if (pmap != kernel_pmap)
1311 ptep_check(get_pte_page(spte));
1312#endif /* DEBUG_PTE_PAGE */
1313 num_removed = 0;
1314 num_unwired = 0;
1315
1316 for (cpte = spte; cpte < epte;
1317 cpte += ptes_per_vm_page1, va += PAGE_SIZE(1 << 12)) {
1318
1319 if (*cpte == 0)
1320 continue;
1321 pa = pte_to_pa(*cpte)((*cpte) & 0xfffff000);
1322
1323 num_removed++;
1324 if (*cpte & INTEL_PTE_WIRED0x00000200)
1325 num_unwired++;
1326
1327 if (!valid_page(pa)(pmap_initialized && pmap_valid_page(pa))) {
1328
1329 /*
1330 * Outside range of managed physical memory.
1331 * Just remove the mappings.
1332 */
1333 register int i = ptes_per_vm_page1;
1334 register pt_entry_t *lpte = cpte;
1335 do {
1336#ifdef MACH_PV_PAGETABLES
1337 update[ii].ptr = kv_to_ma(lpte);
1338 update[ii].val = 0;
1339 ii++;
1340 if (ii == HYP_BATCH_MMU_UPDATES) {
1341 hyp_mmu_update(kvtolin(&update)((vm_offset_t)(&update) - 0xC0000000UL + ((0xc0000000UL))
), ii, kvtolin(&n)((vm_offset_t)(&n) - 0xC0000000UL + ((0xc0000000UL))), DOMID_SELF);
1342 if (n != ii)
1343 panic("couldn't pmap_remove_range\n");
1344 ii = 0;
1345 }
1346#else /* MACH_PV_PAGETABLES */
1347 *lpte = 0;
1348#endif /* MACH_PV_PAGETABLES */
1349 lpte++;
1350 } while (--i > 0);
1351 continue;
1352 }
1353
1354 pai = pa_index(pa)((((vm_size_t)(pa - phys_first_addr)) >> 12));
1355 LOCK_PVH(pai);
1356
1357 /*
1358 * Get the modify and reference bits.
1359 */
1360 {
1361 register int i;
1362 register pt_entry_t *lpte;
1363
1364 i = ptes_per_vm_page1;
1365 lpte = cpte;
1366 do {
1367 pmap_phys_attributes[pai] |=
1368 *lpte & (PHYS_MODIFIED0x00000040|PHYS_REFERENCED0x00000020);
1369#ifdef MACH_PV_PAGETABLES
1370 update[ii].ptr = kv_to_ma(lpte);
1371 update[ii].val = 0;
1372 ii++;
1373 if (ii == HYP_BATCH_MMU_UPDATES) {
1374 hyp_mmu_update(kvtolin(&update)((vm_offset_t)(&update) - 0xC0000000UL + ((0xc0000000UL))
), ii, kvtolin(&n)((vm_offset_t)(&n) - 0xC0000000UL + ((0xc0000000UL))), DOMID_SELF);
1375 if (n != ii)
1376 panic("couldn't pmap_remove_range\n");
1377 ii = 0;
1378 }
1379#else /* MACH_PV_PAGETABLES */
1380 *lpte = 0;
1381#endif /* MACH_PV_PAGETABLES */
1382 lpte++;
1383 } while (--i > 0);
1384 }
1385
1386 /*
1387 * Remove the mapping from the pvlist for
1388 * this physical page.
1389 */
1390 {
1391 register pv_entry_t pv_h, prev, cur;
1392
1393 pv_h = pai_to_pvh(pai)(&pv_head_table[pai]);
1394 if (pv_h->pmap == PMAP_NULL((pmap_t) 0)) {
1395 panic("pmap_remove: null pv_list!");
1396 }
1397 if (pv_h->va == va && pv_h->pmap == pmap) {
1398 /*
1399 * Header is the pv_entry. Copy the next one
1400 * to header and free the next one (we cannot
1401 * free the header)
1402 */
1403 cur = pv_h->next;
1404 if (cur != PV_ENTRY_NULL((pv_entry_t) 0)) {
1405 *pv_h = *cur;
1406 PV_FREE(cur){ ; cur->next = pv_free_list; pv_free_list = cur; ; };
1407 }
1408 else {
1409 pv_h->pmap = PMAP_NULL((pmap_t) 0);
1410 }
1411 }
1412 else {
1413 cur = pv_h;
1414 do {
1415 prev = cur;
1416 if ((cur = prev->next) == PV_ENTRY_NULL((pv_entry_t) 0)) {
1417 panic("pmap-remove: mapping not in pv_list!");
1418 }
1419 } while (cur->va != va || cur->pmap != pmap);
1420 prev->next = cur->next;
1421 PV_FREE(cur){ ; cur->next = pv_free_list; pv_free_list = cur; ; };
1422 }
1423 UNLOCK_PVH(pai);
1424 }
1425 }
1426
1427#ifdef MACH_PV_PAGETABLES
1428 if (ii > HYP_BATCH_MMU_UPDATES)
1429 panic("overflowed array in pmap_remove_range");
1430 hyp_mmu_update(kvtolin(&update)((vm_offset_t)(&update) - 0xC0000000UL + ((0xc0000000UL))
), ii, kvtolin(&n)((vm_offset_t)(&n) - 0xC0000000UL + ((0xc0000000UL))), DOMID_SELF);
1431 if (n != ii)
1432 panic("couldn't pmap_remove_range\n");
1433#endif /* MACH_PV_PAGETABLES */
1434
1435 /*
1436 * Update the counts
1437 */
1438 pmap->stats.resident_count -= num_removed;
1439 pmap->stats.wired_count -= num_unwired;
1440}
1441
1442/*
1443 * Remove the given range of addresses
1444 * from the specified map.
1445 *
1446 * It is assumed that the start and end are properly
1447 * rounded to the hardware page size.
1448 */
1449
1450void pmap_remove(map, s, e)
1451 pmap_t map;
1452 vm_offset_t s, e;
1453{
1454 int spl;
1455 register pt_entry_t *pde;
1456 register pt_entry_t *spte, *epte;
1457 vm_offset_t l;
1458 vm_offset_t _s = s;
1459
1460 if (map == PMAP_NULL((pmap_t) 0))
1461 return;
1462
1463 PMAP_READ_LOCK(map, spl)((void)(spl));
1464
1465 pde = pmap_pde(map, s);
1466 while (s < e) {
1467 l = (s + PDE_MAPPED_SIZE(((vm_offset_t)(1) << 22))) & ~(PDE_MAPPED_SIZE(((vm_offset_t)(1) << 22))-1);
1468 if (l > e)
1469 l = e;
1470 if (*pde & INTEL_PTE_VALID0x00000001) {
1471 spte = (pt_entry_t *)ptetokv(*pde)(((vm_offset_t)(((*pde) & 0xfffff000)) + 0xC0000000UL));
1472 spte = &spte[ptenum(s)(((s) >> 12) & 0x3ff)];
1473 epte = &spte[intel_btop(l-s)(((unsigned long)(l-s)) >> 12)];
1474 pmap_remove_range(map, s, spte, epte);
1475 }
1476 s = l;
1477 pde++;
1478 }
1479 PMAP_UPDATE_TLBS(map, _s, e){ if ((map)->cpus_using) { { (void) ((map)); (void) ((_s))
; (void) ((e)); ({ register unsigned long _temp__ = (({ register
unsigned long _temp__; asm volatile("mov %%cr3, %0" : "=r" (
_temp__)); _temp__; })); asm volatile("mov %0, %%cr3" : : "r"
(_temp__) : "memory"); }); }; } };
1480
1481 PMAP_READ_UNLOCK(map, spl)((void)(spl));
1482}
1483
1484/*
1485 * Routine: pmap_page_protect
1486 *
1487 * Function:
1488 * Lower the permission for all mappings to a given
1489 * page.
1490 */
1491void pmap_page_protect(phys, prot)
1492 vm_offset_t phys;
1493 vm_prot_t prot;
1494{
1495 pv_entry_t pv_h, prev;
1496 register pv_entry_t pv_e;
1497 register pt_entry_t *pte;
1498 int pai;
1499 register pmap_t pmap;
1500 int spl;
1501 boolean_t remove;
1502
1503 assert(phys != vm_page_fictitious_addr)({ if (!(phys != vm_page_fictitious_addr)) Assert("phys != vm_page_fictitious_addr"
, "../i386/intel/pmap.c", 1503); });
1504 if (!valid_page(phys)(pmap_initialized && pmap_valid_page(phys))) {
1
Taking false branch
→
1505 /*
1506 * Not a managed page.
1507 */
1508 return;
1509 }
1510
1511 /*
1512 * Determine the new protection.
1513 */
1514 switch (prot) {
2
←
Control jumps to the 'default' case at line 1521
→
1515 case VM_PROT_READ((vm_prot_t) 0x01):
1516 case VM_PROT_READ((vm_prot_t) 0x01)|VM_PROT_EXECUTE((vm_prot_t) 0x04):
1517 remove = FALSE((boolean_t) 0);
1518 break;
1519 case VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04)):
1520 return; /* nothing to do */
1521 default:
1522 remove = TRUE((boolean_t) 1);
1523 break;
3
←
 Execution continues on line 1531
→
1524 }
1525
1526 /*
1527 * Lock the pmap system first, since we will be changing
1528 * several pmaps.
1529 */
1530
1531 PMAP_WRITE_LOCK(spl)((void)(spl));
1532
1533 pai = pa_index(phys)((((vm_size_t)(phys - phys_first_addr)) >> 12));
1534 pv_h = pai_to_pvh(pai)(&pv_head_table[pai]);
1535
1536 /*
1537 * Walk down PV list, changing or removing all mappings.
1538 * We do not have to lock the pv_list because we have
1539 * the entire pmap system locked.
1540 */
1541 if (pv_h->pmap != PMAP_NULL((pmap_t) 0)) {
4
←
Taking true branch
→
1542
1543 prev = pv_e = pv_h;
1544 do {
8
←
Loop condition is true. Execution continues on line 1545
→
12
←
Loop condition is true. Execution continues on line 1545
→
16
←
Loop condition is true. Execution continues on line 1545
→
1545 register vm_offset_t va;
1546
1547 pmap = pv_e->pmap;
1548 /*
1549 * Lock the pmap to block pmap_extract and similar routines.
1550 */
1551 simple_lock(&pmap->lock);
1552
1553 va = pv_e->va;
1554 pte = pmap_pte(pmap, va);
1555
1556 /*
1557 * Consistency checks.
1558 */
1559 /* assert(*pte & INTEL_PTE_VALID); XXX */
1560 /* assert(pte_to_phys(*pte) == phys); */
1561
1562 /*
1563 * Remove the mapping if new protection is NONE
1564 * or if write-protecting a kernel mapping.
1565 */
1566 if (remove || pmap == kernel_pmap) {
1567 /*
1568 * Remove the mapping, collecting any modify bits.
1569 */
1570 if (*pte & INTEL_PTE_WIRED0x00000200)
5
←
Taking false branch
→
9
←
Taking false branch
→
13
←
Taking false branch
→
17
←
Dereference of null pointer (loaded from variable 'pte')
1571 panic("pmap_remove_all removing a wired page");
1572
1573 {
1574 register int i = ptes_per_vm_page1;
1575
1576 do {
6
←
Loop condition is false. Exiting loop
→
10
←
Loop condition is false. Exiting loop
→
14
←
Loop condition is false. Exiting loop
→
1577 pmap_phys_attributes[pai] |=
1578 *pte & (PHYS_MODIFIED0x00000040|PHYS_REFERENCED0x00000020);
1579#ifdef MACH_PV_PAGETABLES
1580 if (!hyp_mmu_update_pte(kv_to_ma(pte++), 0))
1581 panic("%s:%d could not clear pte %p\n",__FILE__"../i386/intel/pmap.c",__LINE__1581,pte-1);
1582#else /* MACH_PV_PAGETABLES */
1583 *pte++ = 0;
1584#endif /* MACH_PV_PAGETABLES */
1585 } while (--i > 0);
1586 }
1587
1588 pmap->stats.resident_count--;
1589
1590 /*
1591 * Remove the pv_entry.
1592 */
1593 if (pv_e == pv_h) {
7
←
Taking true branch
→
11
←
Taking true branch
→
15
←
Taking true branch
→
1594 /*
1595 * Fix up head later.
1596 */
1597 pv_h->pmap = PMAP_NULL((pmap_t) 0);
1598 }
1599 else {
1600 /*
1601 * Delete this entry.
1602 */
1603 prev->next = pv_e->next;
1604 PV_FREE(pv_e){ ; pv_e->next = pv_free_list; pv_free_list = pv_e; ; };
1605 }
1606 }
1607 else {
1608 /*
1609 * Write-protect.
1610 */
1611 register int i = ptes_per_vm_page1;
1612
1613 do {
1614#ifdef MACH_PV_PAGETABLES
1615 if (!hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~INTEL_PTE_WRITE0x00000002))
1616 panic("%s:%d could not disable write on pte %p\n",__FILE__"../i386/intel/pmap.c",__LINE__1616,pte);
1617#else /* MACH_PV_PAGETABLES */
1618 *pte &= ~INTEL_PTE_WRITE0x00000002;
1619#endif /* MACH_PV_PAGETABLES */
1620 pte++;
1621 } while (--i > 0);
1622
1623 /*
1624 * Advance prev.
1625 */
1626 prev = pv_e;
1627 }
1628 PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE){ if ((pmap)->cpus_using) { { (void) ((pmap)); (void) ((va
)); (void) ((va + (1 << 12))); ({ register unsigned long
_temp__ = (({ register unsigned long _temp__; asm volatile("mov %%cr3, %0"
: "=r" (_temp__)); _temp__; })); asm volatile("mov %0, %%cr3"
: : "r" (_temp__) : "memory"); }); }; } };
1629
1630 simple_unlock(&pmap->lock);
1631
1632 } while ((pv_e = prev->next) != PV_ENTRY_NULL((pv_entry_t) 0));
1633
1634 /*
1635 * If pv_head mapping was removed, fix it up.
1636 */
1637 if (pv_h->pmap == PMAP_NULL((pmap_t) 0)) {
1638 pv_e = pv_h->next;
1639 if (pv_e != PV_ENTRY_NULL((pv_entry_t) 0)) {
1640 *pv_h = *pv_e;
1641 PV_FREE(pv_e){ ; pv_e->next = pv_free_list; pv_free_list = pv_e; ; };
1642 }
1643 }
1644 }
1645
1646 PMAP_WRITE_UNLOCK(spl)((void)(spl));
1647}
1648
1649/*
1650 * Set the physical protection on the
1651 * specified range of this map as requested.
1652 * Will not increase permissions.
1653 */
1654void pmap_protect(map, s, e, prot)
1655 pmap_t map;
1656 vm_offset_t s, e;
1657 vm_prot_t prot;
1658{
1659 register pt_entry_t *pde;
1660 register pt_entry_t *spte, *epte;
1661 vm_offset_t l;
1662 int spl;
1663 vm_offset_t _s = s;
1664
1665 if (map == PMAP_NULL((pmap_t) 0))
1666 return;
1667
1668 /*
1669 * Determine the new protection.
1670 */
1671 switch (prot) {
1672 case VM_PROT_READ((vm_prot_t) 0x01):
1673 case VM_PROT_READ((vm_prot_t) 0x01)|VM_PROT_EXECUTE((vm_prot_t) 0x04):
1674 break;
1675 case VM_PROT_READ((vm_prot_t) 0x01)|VM_PROT_WRITE((vm_prot_t) 0x02):
1676 case VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04)):
1677 return; /* nothing to do */
1678 default:
1679 pmap_remove(map, s, e);
1680 return;
1681 }
1682
1683 /*
1684 * If write-protecting in the kernel pmap,
1685 * remove the mappings; the i386 ignores
1686 * the write-permission bit in kernel mode.
1687 *
1688 * XXX should be #if'd for i386
1689 */
1690 if (map == kernel_pmap) {
1691 pmap_remove(map, s, e);
1692 return;
1693 }
1694
1695 SPLVM(spl)((void)(spl));
1696 simple_lock(&map->lock);
1697
1698 pde = pmap_pde(map, s);
1699 while (s < e) {
1700 l = (s + PDE_MAPPED_SIZE(((vm_offset_t)(1) << 22))) & ~(PDE_MAPPED_SIZE(((vm_offset_t)(1) << 22))-1);
1701 if (l > e)
1702 l = e;
1703 if (*pde & INTEL_PTE_VALID0x00000001) {
1704 spte = (pt_entry_t *)ptetokv(*pde)(((vm_offset_t)(((*pde) & 0xfffff000)) + 0xC0000000UL));
1705 spte = &spte[ptenum(s)(((s) >> 12) & 0x3ff)];
1706 epte = &spte[intel_btop(l-s)(((unsigned long)(l-s)) >> 12)];
1707
1708#ifdef MACH_PV_PAGETABLES
1709 int n, i = 0;
1710 struct mmu_update update[HYP_BATCH_MMU_UPDATES];
1711#endif /* MACH_PV_PAGETABLES */
1712
1713 while (spte < epte) {
1714 if (*spte & INTEL_PTE_VALID0x00000001) {
1715#ifdef MACH_PV_PAGETABLES
1716 update[i].ptr = kv_to_ma(spte);
1717 update[i].val = *spte & ~INTEL_PTE_WRITE0x00000002;
1718 i++;
1719 if (i == HYP_BATCH_MMU_UPDATES) {
1720 hyp_mmu_update(kvtolin(&update)((vm_offset_t)(&update) - 0xC0000000UL + ((0xc0000000UL))
), i, kvtolin(&n)((vm_offset_t)(&n) - 0xC0000000UL + ((0xc0000000UL))), DOMID_SELF);
1721 if (n != i)
1722 panic("couldn't pmap_protect\n");
1723 i = 0;
1724 }
1725#else /* MACH_PV_PAGETABLES */
1726 *spte &= ~INTEL_PTE_WRITE0x00000002;
1727#endif /* MACH_PV_PAGETABLES */
1728 }
1729 spte++;
1730 }
1731#ifdef MACH_PV_PAGETABLES
1732 if (i > HYP_BATCH_MMU_UPDATES)
1733 panic("overflowed array in pmap_protect");
1734 hyp_mmu_update(kvtolin(&update)((vm_offset_t)(&update) - 0xC0000000UL + ((0xc0000000UL))
), i, kvtolin(&n)((vm_offset_t)(&n) - 0xC0000000UL + ((0xc0000000UL))), DOMID_SELF);
1735 if (n != i)
1736 panic("couldn't pmap_protect\n");
1737#endif /* MACH_PV_PAGETABLES */
1738 }
1739 s = l;
1740 pde++;
1741 }
1742 PMAP_UPDATE_TLBS(map, _s, e){ if ((map)->cpus_using) { { (void) ((map)); (void) ((_s))
; (void) ((e)); ({ register unsigned long _temp__ = (({ register
unsigned long _temp__; asm volatile("mov %%cr3, %0" : "=r" (
_temp__)); _temp__; })); asm volatile("mov %0, %%cr3" : : "r"
(_temp__) : "memory"); }); }; } };
1743
1744 simple_unlock(&map->lock);
1745 SPLX(spl)((void)(spl));
1746}
1747
1748/*
1749 * Insert the given physical page (p) at
1750 * the specified virtual address (v) in the
1751 * target physical map with the protection requested.
1752 *
1753 * If specified, the page will be wired down, meaning
1754 * that the related pte can not be reclaimed.
1755 *
1756 * NB: This is the only routine which MAY NOT lazy-evaluate
1757 * or lose information. That is, this routine must actually
1758 * insert this page into the given map NOW.
1759 */
1760void pmap_enter(pmap, v, pa, prot, wired)
1761 register pmap_t pmap;
1762 vm_offset_t v;
1763 register vm_offset_t pa;
1764 vm_prot_t prot;
1765 boolean_t wired;
1766{
1767 register pt_entry_t *pte;
1768 register pv_entry_t pv_h;
1769 register int i, pai;
1770 pv_entry_t pv_e;
1771 pt_entry_t template;
1772 int spl;
1773 vm_offset_t old_pa;
1774
1775 assert(pa != vm_page_fictitious_addr)({ if (!(pa != vm_page_fictitious_addr)) Assert("pa != vm_page_fictitious_addr"
, "../i386/intel/pmap.c", 1775); });
1776if (pmap_debug) printf("pmap(%lx, %lx)\n", v, pa);
1777 if (pmap == PMAP_NULL((pmap_t) 0))
1778 return;
1779
1780#if !MACH_KDB0
1781 if (pmap == kernel_pmap && (v < kernel_virtual_start || v >= kernel_virtual_end))
1782 panic("pmap_enter(%p, %p) falls in physical memory area!\n", v, pa);
1783#endif
1784 if (pmap == kernel_pmap && (prot & VM_PROT_WRITE((vm_prot_t) 0x02)) == 0
1785 && !wired /* hack for io_wire */ ) {
1786 /*
1787 * Because the 386 ignores write protection in kernel mode,
1788 * we cannot enter a read-only kernel mapping, and must
1789 * remove an existing mapping if changing it.
1790 *
1791 * XXX should be #if'd for i386
1792 */
1793 PMAP_READ_LOCK(pmap, spl)((void)(spl));
1794
1795 pte = pmap_pte(pmap, v);
1796 if (pte != PT_ENTRY_NULL((pt_entry_t *) 0) && *pte != 0) {
1797 /*
1798 * Invalidate the translation buffer,
1799 * then remove the mapping.
1800 */
1801 pmap_remove_range(pmap, v, pte,
1802 pte + ptes_per_vm_page1);
1803 PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE){ if ((pmap)->cpus_using) { { (void) ((pmap)); (void) ((v)
); (void) ((v + (1 << 12))); ({ register unsigned long _temp__
= (({ register unsigned long _temp__; asm volatile("mov %%cr3, %0"
: "=r" (_temp__)); _temp__; })); asm volatile("mov %0, %%cr3"
: : "r" (_temp__) : "memory"); }); }; } };
1804 }
1805 PMAP_READ_UNLOCK(pmap, spl)((void)(spl));
1806 return;
1807 }
1808
1809 /*
1810 * Must allocate a new pvlist entry while we're unlocked;
1811 * Allocating may cause pageout (which will lock the pmap system).
1812 * If we determine we need a pvlist entry, we will unlock
1813 * and allocate one. Then we will retry, throughing away
1814 * the allocated entry later (if we no longer need it).
1815 */
1816 pv_e = PV_ENTRY_NULL((pv_entry_t) 0);
1817Retry:
1818 PMAP_READ_LOCK(pmap, spl)((void)(spl));
1819
1820 /*
1821 * Expand pmap to include this pte. Assume that
1822 * pmap is always expanded to include enough hardware
1823 * pages to map one VM page.
1824 */
1825
1826 while ((pte = pmap_pte(pmap, v)) == PT_ENTRY_NULL((pt_entry_t *) 0)) {
1827 /*
1828 * Need to allocate a new page-table page.
1829 */
1830 vm_offset_t ptp;
1831 pt_entry_t *pdp;
1832 int i;
1833
1834 if (pmap == kernel_pmap) {
1835 /*
1836 * Would have to enter the new page-table page in
1837 * EVERY pmap.
1838 */
1839 panic("pmap_expand kernel pmap to %#x", v);
1840 }
1841
1842 /*
1843 * Unlock the pmap and allocate a new page-table page.
1844 */
1845 PMAP_READ_UNLOCK(pmap, spl)((void)(spl));
1846
1847 ptp = phystokv(pmap_page_table_page_alloc())((vm_offset_t)(pmap_page_table_page_alloc()) + 0xC0000000UL);
1848
1849 /*
1850 * Re-lock the pmap and check that another thread has
1851 * not already allocated the page-table page. If it
1852 * has, discard the new page-table page (and try
1853 * again to make sure).
1854 */
1855 PMAP_READ_LOCK(pmap, spl)((void)(spl));
1856
1857 if (pmap_pte(pmap, v) != PT_ENTRY_NULL((pt_entry_t *) 0)) {
1858 /*
1859 * Oops...
1860 */
1861 PMAP_READ_UNLOCK(pmap, spl)((void)(spl));
1862 pmap_page_table_page_dealloc(kvtophys(ptp));
1863 PMAP_READ_LOCK(pmap, spl)((void)(spl));
1864 continue;
1865 }
1866
1867 /*
1868 * Enter the new page table page in the page directory.
1869 */
1870 i = ptes_per_vm_page1;
1871 /*XX pdp = &pmap->dirbase[pdenum(v) & ~(i-1)];*/
1872 pdp = pmap_pde(pmap, v);
1873 do {
1874#ifdef MACH_PV_PAGETABLES
1875 pmap_set_page_readonly((void *) ptp);
1876 if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, kv_to_mfn(ptp)))
1877 panic("couldn't pin page %p(%p)\n",ptp,(vm_offset_t) kv_to_ma(ptp));
1878 if (!hyp_mmu_update_pte(pa_to_ma(kvtophys((vm_offset_t)pdp)),
1879 pa_to_pte(pa_to_ma(kvtophys(ptp)))((pa_to_ma(kvtophys(ptp))) & 0xfffff000) | INTEL_PTE_VALID0x00000001
1880 | INTEL_PTE_USER0x00000004
1881 | INTEL_PTE_WRITE0x00000002))
1882 panic("%s:%d could not set pde %p(%p,%p) to %p(%p,%p) %p\n",__FILE__"../i386/intel/pmap.c",__LINE__1882, pdp, kvtophys((vm_offset_t)pdp), (vm_offset_t) pa_to_ma(kvtophys((vm_offset_t)pdp)), ptp, kvtophys(ptp), (vm_offset_t) pa_to_ma(kvtophys(ptp)), (vm_offset_t) pa_to_pte(kv_to_ma(ptp))((kv_to_ma(ptp)) & 0xfffff000));
1883#else /* MACH_PV_PAGETABLES */
1884 *pdp = pa_to_pte(kvtophys(ptp))((kvtophys(ptp)) & 0xfffff000) | INTEL_PTE_VALID0x00000001
1885 | INTEL_PTE_USER0x00000004
1886 | INTEL_PTE_WRITE0x00000002;
1887#endif /* MACH_PV_PAGETABLES */
1888 pdp++;
1889 ptp += INTEL_PGBYTES4096;
1890 } while (--i > 0);
1891
1892 /*
1893 * Now, get the address of the page-table entry.
1894 */
1895 continue;
1896 }
1897
1898 /*
1899 * Special case if the physical page is already mapped
1900 * at this address.
1901 */
1902 old_pa = pte_to_pa(*pte)((*pte) & 0xfffff000);
1903 if (*pte && old_pa == pa) {
1904 /*
1905 * May be changing its wired attribute or protection
1906 */
1907
1908 if (wired && !(*pte & INTEL_PTE_WIRED0x00000200))
1909 pmap->stats.wired_count++;
1910 else if (!wired && (*pte & INTEL_PTE_WIRED0x00000200))
1911 pmap->stats.wired_count--;
1912
1913 template = pa_to_pte(pa)((pa) & 0xfffff000) | INTEL_PTE_VALID0x00000001;
1914 if (pmap != kernel_pmap)
1915 template |= INTEL_PTE_USER0x00000004;
1916 if (prot & VM_PROT_WRITE((vm_prot_t) 0x02))
1917 template |= INTEL_PTE_WRITE0x00000002;
1918 if (machine_slot[cpu_number()(0)].cpu_type >= CPU_TYPE_I486((cpu_type_t) 17)
1919 && pa >= phys_last_addr)
1920 template |= INTEL_PTE_NCACHE0x00000010|INTEL_PTE_WTHRU0x00000008;
1921 if (wired)
1922 template |= INTEL_PTE_WIRED0x00000200;
1923 i = ptes_per_vm_page1;
1924 do {
1925 if (*pte & INTEL_PTE_MOD0x00000040)
1926 template |= INTEL_PTE_MOD0x00000040;
1927#ifdef MACH_PV_PAGETABLES
1928 if (!hyp_mmu_update_pte(kv_to_ma(pte), pa_to_ma(template)))
1929 panic("%s:%d could not set pte %p to %p\n",__FILE__"../i386/intel/pmap.c",__LINE__1929,pte,template);
1930#else /* MACH_PV_PAGETABLES */
1931 WRITE_PTE(pte, template)*(pte) = (template);
1932#endif /* MACH_PV_PAGETABLES */
1933 pte++;
1934 pte_increment_pa(template)((template) += 0xfff +1);
1935 } while (--i > 0);
1936 PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE){ if ((pmap)->cpus_using) { { (void) ((pmap)); (void) ((v)
); (void) ((v + (1 << 12))); ({ register unsigned long _temp__
= (({ register unsigned long _temp__; asm volatile("mov %%cr3, %0"
: "=r" (_temp__)); _temp__; })); asm volatile("mov %0, %%cr3"
: : "r" (_temp__) : "memory"); }); }; } };
1937 }
1938 else {
1939
1940 /*
1941 * Remove old mapping from the PV list if necessary.
1942 */
1943 if (*pte) {
1944 /*
1945 * Don't free the pte page if removing last
1946 * mapping - we will immediately replace it.
1947 */
1948 pmap_remove_range(pmap, v, pte,
1949 pte + ptes_per_vm_page1);
1950 PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE){ if ((pmap)->cpus_using) { { (void) ((pmap)); (void) ((v)
); (void) ((v + (1 << 12))); ({ register unsigned long _temp__
= (({ register unsigned long _temp__; asm volatile("mov %%cr3, %0"
: "=r" (_temp__)); _temp__; })); asm volatile("mov %0, %%cr3"
: : "r" (_temp__) : "memory"); }); }; } };
1951 }
1952
1953 if (valid_page(pa)(pmap_initialized && pmap_valid_page(pa))) {
1954
1955 /*
1956 * Enter the mapping in the PV list for this
1957 * physical page.
1958 */
1959
1960 pai = pa_index(pa)((((vm_size_t)(pa - phys_first_addr)) >> 12));
1961 LOCK_PVH(pai);
1962 pv_h = pai_to_pvh(pai)(&pv_head_table[pai]);
1963
1964 if (pv_h->pmap == PMAP_NULL((pmap_t) 0)) {
1965 /*
1966 * No mappings yet
1967 */
1968 pv_h->va = v;
1969 pv_h->pmap = pmap;
1970 pv_h->next = PV_ENTRY_NULL((pv_entry_t) 0);
1971 }
1972 else {
1973#if DEBUG
1974 {
1975 /* check that this mapping is not already there */
1976 pv_entry_t e = pv_h;
1977 while (e != PV_ENTRY_NULL((pv_entry_t) 0)) {
1978 if (e->pmap == pmap && e->va == v)
1979 panic("pmap_enter: already in pv_list");
1980 e = e->next;
1981 }
1982 }
1983#endif /* DEBUG */
1984
1985 /*
1986 * Add new pv_entry after header.
1987 */
1988 if (pv_e == PV_ENTRY_NULL((pv_entry_t) 0)) {
1989 PV_ALLOC(pv_e){ ; if ((pv_e = pv_free_list) != 0) { pv_free_list = pv_e->
next; } ; };
1990 if (pv_e == PV_ENTRY_NULL((pv_entry_t) 0)) {
1991 UNLOCK_PVH(pai);
1992 PMAP_READ_UNLOCK(pmap, spl)((void)(spl));
1993
1994 /*
1995 * Refill from cache.
1996 */
1997 pv_e = (pv_entry_t) kmem_cache_alloc(&pv_list_cache);
1998 goto Retry;
1999 }
2000 }
2001 pv_e->va = v;
2002 pv_e->pmap = pmap;
2003 pv_e->next = pv_h->next;
2004 pv_h->next = pv_e;
2005 /*
2006 * Remember that we used the pvlist entry.
2007 */
2008 pv_e = PV_ENTRY_NULL((pv_entry_t) 0);
2009 }
2010 UNLOCK_PVH(pai);
2011 }
2012
2013 /*
2014 * And count the mapping.
2015 */
2016
2017 pmap->stats.resident_count++;
2018 if (wired)
2019 pmap->stats.wired_count++;
2020
2021 /*
2022 * Build a template to speed up entering -
2023 * only the pfn changes.
2024 */
2025 template = pa_to_pte(pa)((pa) & 0xfffff000) | INTEL_PTE_VALID0x00000001;
2026 if (pmap != kernel_pmap)
2027 template |= INTEL_PTE_USER0x00000004;
2028 if (prot & VM_PROT_WRITE((vm_prot_t) 0x02))
2029 template |= INTEL_PTE_WRITE0x00000002;
2030 if (machine_slot[cpu_number()(0)].cpu_type >= CPU_TYPE_I486((cpu_type_t) 17)
2031 && pa >= phys_last_addr)
2032 template |= INTEL_PTE_NCACHE0x00000010|INTEL_PTE_WTHRU0x00000008;
2033 if (wired)
2034 template |= INTEL_PTE_WIRED0x00000200;
2035 i = ptes_per_vm_page1;
2036 do {
2037#ifdef MACH_PV_PAGETABLES
2038 if (!(hyp_mmu_update_pte(kv_to_ma(pte), pa_to_ma(template))))
2039 panic("%s:%d could not set pte %p to %p\n",__FILE__"../i386/intel/pmap.c",__LINE__2039,pte,template);
2040#else /* MACH_PV_PAGETABLES */
2041 WRITE_PTE(pte, template)*(pte) = (template);
2042#endif /* MACH_PV_PAGETABLES */
2043 pte++;
2044 pte_increment_pa(template)((template) += 0xfff +1);
2045 } while (--i > 0);
2046 }
2047
2048 if (pv_e != PV_ENTRY_NULL((pv_entry_t) 0)) {
2049 PV_FREE(pv_e){ ; pv_e->next = pv_free_list; pv_free_list = pv_e; ; };
2050 }
2051
2052 PMAP_READ_UNLOCK(pmap, spl)((void)(spl));
2053}
2054
2055/*
2056 * Routine: pmap_change_wiring
2057 * Function: Change the wiring attribute for a map/virtual-address
2058 * pair.
2059 * In/out conditions:
2060 * The mapping must already exist in the pmap.
2061 */
2062void pmap_change_wiring(map, v, wired)
2063 register pmap_t map;
2064 vm_offset_t v;
2065 boolean_t wired;
2066{
2067 register pt_entry_t *pte;
2068 register int i;
2069 int spl;
2070
2071 /*
2072 * We must grab the pmap system lock because we may
2073 * change a pte_page queue.
2074 */
2075 PMAP_READ_LOCK(map, spl)((void)(spl));
2076
2077 if ((pte = pmap_pte(map, v)) == PT_ENTRY_NULL((pt_entry_t *) 0))
2078 panic("pmap_change_wiring: pte missing");
2079
2080 if (wired && !(*pte & INTEL_PTE_WIRED0x00000200)) {
2081 /*
2082 * wiring down mapping
2083 */
2084 map->stats.wired_count++;
2085 i = ptes_per_vm_page1;
2086 do {
2087 *pte++ |= INTEL_PTE_WIRED0x00000200;
2088 } while (--i > 0);
2089 }
2090 else if (!wired && (*pte & INTEL_PTE_WIRED0x00000200)) {
2091 /*
2092 * unwiring mapping
2093 */
2094 map->stats.wired_count--;
2095 i = ptes_per_vm_page1;
2096 do {
2097#ifdef MACH_PV_PAGETABLES
2098 if (!(hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~INTEL_PTE_WIRED0x00000200)))
2099 panic("%s:%d could not wire down pte %p\n",__FILE__"../i386/intel/pmap.c",__LINE__2099,pte);
2100#else /* MACH_PV_PAGETABLES */
2101 *pte &= ~INTEL_PTE_WIRED0x00000200;
2102#endif /* MACH_PV_PAGETABLES */
2103 pte++;
2104 } while (--i > 0);
2105 }
2106
2107 PMAP_READ_UNLOCK(map, spl)((void)(spl));
2108}
2109
2110/*
2111 * Routine: pmap_extract
2112 * Function:
2113 * Extract the physical page address associated
2114 * with the given map/virtual_address pair.
2115 */
2116
2117vm_offset_t pmap_extract(pmap, va)
2118 register pmap_t pmap;
2119 vm_offset_t va;
2120{
2121 register pt_entry_t *pte;
2122 register vm_offset_t pa;
2123 int spl;
2124
2125 SPLVM(spl)((void)(spl));
2126 simple_lock(&pmap->lock);
2127 if ((pte = pmap_pte(pmap, va)) == PT_ENTRY_NULL((pt_entry_t *) 0))
2128 pa = (vm_offset_t) 0;
2129 else if (!(*pte & INTEL_PTE_VALID0x00000001))
2130 pa = (vm_offset_t) 0;
2131 else
2132 pa = pte_to_pa(*pte)((*pte) & 0xfffff000) + (va & INTEL_OFFMASK0xfff);
2133 simple_unlock(&pmap->lock);
2134 SPLX(spl)((void)(spl));
2135 return(pa);
2136}
2137
2138/*
2139 * Copy the range specified by src_addr/len
2140 * from the source map to the range dst_addr/len
2141 * in the destination map.
2142 *
2143 * This routine is only advisory and need not do anything.
2144 */
2145#if 0
2146void pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr)
2147 pmap_t dst_pmap;
2148 pmap_t src_pmap;
2149 vm_offset_t dst_addr;
2150 vm_size_t len;
2151 vm_offset_t src_addr;
2152{
2153#ifdef lint
2154 dst_pmap++; src_pmap++; dst_addr++; len++; src_addr++;
2155#endif /* lint */
2156}
2157#endif /* 0 */
2158
2159/*
2160 * Routine: pmap_collect
2161 * Function:
2162 * Garbage collects the physical map system for
2163 * pages which are no longer used.
2164 * Success need not be guaranteed -- that is, there
2165 * may well be pages which are not referenced, but
2166 * others may be collected.
2167 * Usage:
2168 * Called by the pageout daemon when pages are scarce.
2169 */
2170void pmap_collect(p)
2171 pmap_t p;
2172{
2173 register pt_entry_t *pdp, *ptp;
2174 pt_entry_t *eptp;
2175 vm_offset_t pa;
2176 int spl, wired;
2177
2178 if (p == PMAP_NULL((pmap_t) 0))
2179 return;
2180
2181 if (p == kernel_pmap)
2182 return;
2183
2184 /*
2185 * Garbage collect map.
2186 */
2187 PMAP_READ_LOCK(p, spl)((void)(spl));
2188 for (pdp = p->dirbase;
2189 pdp < &p->dirbase[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS)(((((0xc0000000UL))) >> 22) & 0x3ff)];
2190 pdp += ptes_per_vm_page1)
2191 {
2192 if (*pdp & INTEL_PTE_VALID0x00000001) {
2193
2194 pa = pte_to_pa(*pdp)((*pdp) & 0xfffff000);
2195 ptp = (pt_entry_t *)phystokv(pa)((vm_offset_t)(pa) + 0xC0000000UL);
2196 eptp = ptp + NPTES((((unsigned long)(1)) << 12)/sizeof(pt_entry_t))*ptes_per_vm_page1;
2197
2198 /*
2199 * If the pte page has any wired mappings, we cannot
2200 * free it.
2201 */
2202 wired = 0;
2203 {
2204 register pt_entry_t *ptep;
2205 for (ptep = ptp; ptep < eptp; ptep++) {
2206 if (*ptep & INTEL_PTE_WIRED0x00000200) {
2207 wired = 1;
2208 break;
2209 }
2210 }
2211 }
2212 if (!wired) {
2213 /*
2214 * Remove the virtual addresses mapped by this pte page.
2215 */
2216 { /*XXX big hack*/
2217 vm_offset_t va = pdenum2lin(pdp - p->dirbase)((vm_offset_t)(pdp - p->dirbase) << 22);
2218 if (p == kernel_pmap)
2219 va = lintokv(va)((vm_offset_t)(va) - ((0xc0000000UL)) + 0xC0000000UL);
2220 pmap_remove_range(p,
2221 va,
2222 ptp,
2223 eptp);
2224 }
2225
2226 /*
2227 * Invalidate the page directory pointer.
2228 */
2229 {
2230 register int i = ptes_per_vm_page1;
2231 register pt_entry_t *pdep = pdp;
2232 do {
2233#ifdef MACH_PV_PAGETABLES
2234 unsigned long pte = *pdep;
2235 void *ptable = (void*) ptetokv(pte)(((vm_offset_t)(((pte) & 0xfffff000)) + 0xC0000000UL));
2236 if (!(hyp_mmu_update_pte(pa_to_ma(kvtophys((vm_offset_t)pdep++)), 0)))
2237 panic("%s:%d could not clear pde %p\n",__FILE__"../i386/intel/pmap.c",__LINE__2237,pdep-1);
2238 if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, kv_to_mfn(ptable)))
2239 panic("couldn't unpin page %p(%p)\n", ptable, (vm_offset_t) pa_to_ma(kvtophys((vm_offset_t)ptable)));
2240 pmap_set_page_readwrite(ptable);
2241#else /* MACH_PV_PAGETABLES */
2242 *pdep++ = 0;
2243#endif /* MACH_PV_PAGETABLES */
2244 } while (--i > 0);
2245 }
2246
2247 PMAP_READ_UNLOCK(p, spl)((void)(spl));
2248
2249 /*
2250 * And free the pte page itself.
2251 */
2252 {
2253 register vm_page_t m;
2254
2255 vm_object_lock(pmap_object);
2256 m = vm_page_lookup(pmap_object, pa);
2257 if (m == VM_PAGE_NULL((vm_page_t) 0))
2258 panic("pmap_collect: pte page not in object");
2259 vm_page_lock_queues();
2260 vm_page_free(m);
2261 inuse_ptepages_count--;
2262 vm_page_unlock_queues();
2263 vm_object_unlock(pmap_object);
2264 }
2265
2266 PMAP_READ_LOCK(p, spl)((void)(spl));
2267 }
2268 }
2269 }
2270 PMAP_UPDATE_TLBS(p, VM_MIN_ADDRESS, VM_MAX_ADDRESS){ if ((p)->cpus_using) { { (void) ((p)); (void) (((0))); (
void) (((0xc0000000UL))); ({ register unsigned long _temp__ =
(({ register unsigned long _temp__; asm volatile("mov %%cr3, %0"
: "=r" (_temp__)); _temp__; })); asm volatile("mov %0, %%cr3"
: : "r" (_temp__) : "memory"); }); }; } };
2271
2272 PMAP_READ_UNLOCK(p, spl)((void)(spl));
2273 return;
2274
2275}
2276
2277/*
2278 * Routine: pmap_activate
2279 * Function:
2280 * Binds the given physical map to the given
2281 * processor, and returns a hardware map description.
2282 */
2283#if 0
2284void pmap_activate(my_pmap, th, my_cpu)
2285 register pmap_t my_pmap;
2286 thread_t th;
2287 int my_cpu;
2288{
2289 PMAP_ACTIVATE(my_pmap, th, my_cpu);
2290}
2291#endif /* 0 */
2292
2293/*
2294 * Routine: pmap_deactivate
2295 * Function:
2296 * Indicates that the given physical map is no longer
2297 * in use on the specified processor. (This is a macro
2298 * in pmap.h)
2299 */
2300#if 0
2301void pmap_deactivate(pmap, th, which_cpu)
2302 pmap_t pmap;
2303 thread_t th;
2304 int which_cpu;
2305{
2306#ifdef lint
2307 pmap++; th++; which_cpu++;
2308#endif /* lint */
2309 PMAP_DEACTIVATE(pmap, th, which_cpu);
2310}
2311#endif /* 0 */
2312
2313/*
2314 * Routine: pmap_kernel
2315 * Function:
2316 * Returns the physical map handle for the kernel.
2317 */
2318#if 0
2319pmap_t pmap_kernel()(kernel_pmap)
2320{
2321 return (kernel_pmap);
2322}
2323#endif /* 0 */
2324
2325/*
2326 * pmap_zero_page zeros the specified (machine independent) page.
2327 * See machine/phys.c or machine/phys.s for implementation.
2328 */
2329#if 0
2330pmap_zero_page(phys)
2331 register vm_offset_t phys;
2332{
2333 register int i;
2334
2335 assert(phys != vm_page_fictitious_addr)({ if (!(phys != vm_page_fictitious_addr)) Assert("phys != vm_page_fictitious_addr"
, "../i386/intel/pmap.c", 2335); });
2336 i = PAGE_SIZE(1 << 12) / INTEL_PGBYTES4096;
2337 phys = intel_pfn(phys);
2338
2339 while (i--)
2340 zero_phys(phys++);
2341}
2342#endif /* 0 */
2343
2344/*
2345 * pmap_copy_page copies the specified (machine independent) page.
2346 * See machine/phys.c or machine/phys.s for implementation.
2347 */
2348#if 0
2349pmap_copy_page(src, dst)
2350 vm_offset_t src, dst;
2351{
2352 int i;
2353
2354 assert(src != vm_page_fictitious_addr)({ if (!(src != vm_page_fictitious_addr)) Assert("src != vm_page_fictitious_addr"
, "../i386/intel/pmap.c", 2354); });
2355 assert(dst != vm_page_fictitious_addr)({ if (!(dst != vm_page_fictitious_addr)) Assert("dst != vm_page_fictitious_addr"
, "../i386/intel/pmap.c", 2355); });
2356 i = PAGE_SIZE(1 << 12) / INTEL_PGBYTES4096;
2357
2358 while (i--) {
2359 copy_phys(intel_pfn(src), intel_pfn(dst));
2360 src += INTEL_PGBYTES4096;
2361 dst += INTEL_PGBYTES4096;
2362 }
2363}
2364#endif /* 0 */
2365
2366/*
2367 * Routine: pmap_pageable
2368 * Function:
2369 * Make the specified pages (by pmap, offset)
2370 * pageable (or not) as requested.
2371 *
2372 * A page which is not pageable may not take
2373 * a fault; therefore, its page table entry
2374 * must remain valid for the duration.
2375 *
2376 * This routine is merely advisory; pmap_enter
2377 * will specify that these pages are to be wired
2378 * down (or not) as appropriate.
2379 */
2380void
2381pmap_pageable(pmap, start, end, pageable)
2382 pmap_t pmap;
2383 vm_offset_t start;
2384 vm_offset_t end;
2385 boolean_t pageable;
2386{
2387#ifdef lint
2388 pmap++; start++; end++; pageable++;
2389#endif /* lint */
2390}
2391
2392/*
2393 * Clear specified attribute bits.
2394 */
2395void
2396phys_attribute_clear(phys, bits)
2397 vm_offset_t phys;
2398 int bits;
2399{
2400 pv_entry_t pv_h;
2401 register pv_entry_t pv_e;
2402 register pt_entry_t *pte;
2403 int pai;
2404 register pmap_t pmap;
2405 int spl;
2406
2407 assert(phys != vm_page_fictitious_addr)({ if (!(phys != vm_page_fictitious_addr)) Assert("phys != vm_page_fictitious_addr"
, "../i386/intel/pmap.c", 2407); });
2408 if (!valid_page(phys)(pmap_initialized && pmap_valid_page(phys))) {
2409 /*
2410 * Not a managed page.
2411 */
2412 return;
2413 }
2414
2415 /*
2416 * Lock the pmap system first, since we will be changing
2417 * several pmaps.
2418 */
2419
2420 PMAP_WRITE_LOCK(spl)((void)(spl));
2421
2422 pai = pa_index(phys)((((vm_size_t)(phys - phys_first_addr)) >> 12));
2423 pv_h = pai_to_pvh(pai)(&pv_head_table[pai]);
2424
2425 /*
2426 * Walk down PV list, clearing all modify or reference bits.
2427 * We do not have to lock the pv_list because we have
2428 * the entire pmap system locked.
2429 */
2430 if (pv_h->pmap != PMAP_NULL((pmap_t) 0)) {
2431 /*
2432 * There are some mappings.
2433 */
2434 for (pv_e = pv_h; pv_e != PV_ENTRY_NULL((pv_entry_t) 0); pv_e = pv_e->next) {
2435 register vm_offset_t va;
2436
2437 pmap = pv_e->pmap;
2438 /*
2439 * Lock the pmap to block pmap_extract and similar routines.
2440 */
2441 simple_lock(&pmap->lock);
2442
2443 va = pv_e->va;
2444 pte = pmap_pte(pmap, va);
2445
2446#if 0
2447 /*
2448 * Consistency checks.
2449 */
2450 assert(*pte & INTEL_PTE_VALID)({ if (!(*pte & 0x00000001)) Assert("*pte & INTEL_PTE_VALID"
, "../i386/intel/pmap.c", 2450); });
2451 /* assert(pte_to_phys(*pte) == phys); */
2452#endif
2453
2454 /*
2455 * Clear modify or reference bits.
2456 */
2457 {
2458 register int i = ptes_per_vm_page1;
2459 do {
2460#ifdef MACH_PV_PAGETABLES
2461 if (!(hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~bits)))
2462 panic("%s:%d could not clear bits %lx from pte %p\n",__FILE__"../i386/intel/pmap.c",__LINE__2462,bits,pte);
2463#else /* MACH_PV_PAGETABLES */
2464 *pte &= ~bits;
2465#endif /* MACH_PV_PAGETABLES */
2466 } while (--i > 0);
2467 }
2468 PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE){ if ((pmap)->cpus_using) { { (void) ((pmap)); (void) ((va
)); (void) ((va + (1 << 12))); ({ register unsigned long
_temp__ = (({ register unsigned long _temp__; asm volatile("mov %%cr3, %0"
: "=r" (_temp__)); _temp__; })); asm volatile("mov %0, %%cr3"
: : "r" (_temp__) : "memory"); }); }; } };
2469 simple_unlock(&pmap->lock);
2470 }
2471 }
2472
2473 pmap_phys_attributes[pai] &= ~bits;
2474
2475 PMAP_WRITE_UNLOCK(spl)((void)(spl));
2476}
2477
2478/*
2479 * Check specified attribute bits.
2480 */
2481boolean_t
2482phys_attribute_test(phys, bits)
2483 vm_offset_t phys;
2484 int bits;
2485{
2486 pv_entry_t pv_h;
2487 register pv_entry_t pv_e;
2488 register pt_entry_t *pte;
2489 int pai;
2490 register pmap_t pmap;
2491 int spl;
2492
2493 assert(phys != vm_page_fictitious_addr)({ if (!(phys != vm_page_fictitious_addr)) Assert("phys != vm_page_fictitious_addr"
, "../i386/intel/pmap.c", 2493); });
2494 if (!valid_page(phys)(pmap_initialized && pmap_valid_page(phys))) {
2495 /*
2496 * Not a managed page.
2497 */
2498 return (FALSE((boolean_t) 0));
2499 }
2500
2501 /*
2502 * Lock the pmap system first, since we will be checking
2503 * several pmaps.
2504 */
2505
2506 PMAP_WRITE_LOCK(spl)((void)(spl));
2507
2508 pai = pa_index(phys)((((vm_size_t)(phys - phys_first_addr)) >> 12));
2509 pv_h = pai_to_pvh(pai)(&pv_head_table[pai]);
2510
2511 if (pmap_phys_attributes[pai] & bits) {
2512 PMAP_WRITE_UNLOCK(spl)((void)(spl));
2513 return (TRUE((boolean_t) 1));
2514 }
2515
2516 /*
2517 * Walk down PV list, checking all mappings.
2518 * We do not have to lock the pv_list because we have
2519 * the entire pmap system locked.
2520 */
2521 if (pv_h->pmap != PMAP_NULL((pmap_t) 0)) {
2522 /*
2523 * There are some mappings.
2524 */
2525 for (pv_e = pv_h; pv_e != PV_ENTRY_NULL((pv_entry_t) 0); pv_e = pv_e->next) {
2526
2527 pmap = pv_e->pmap;
2528 /*
2529 * Lock the pmap to block pmap_extract and similar routines.
2530 */
2531 simple_lock(&pmap->lock);
2532
2533 {
2534 register vm_offset_t va;
2535
2536 va = pv_e->va;
2537 pte = pmap_pte(pmap, va);
2538
2539#if 0
2540 /*
2541 * Consistency checks.
2542 */
2543 assert(*pte & INTEL_PTE_VALID)({ if (!(*pte & 0x00000001)) Assert("*pte & INTEL_PTE_VALID"
, "../i386/intel/pmap.c", 2543); });
2544 /* assert(pte_to_phys(*pte) == phys); */
2545#endif
2546 }
2547
2548 /*
2549 * Check modify or reference bits.
2550 */
2551 {
2552 register int i = ptes_per_vm_page1;
2553
2554 do {
2555 if (*pte & bits) {
2556 simple_unlock(&pmap->lock);
2557 PMAP_WRITE_UNLOCK(spl)((void)(spl));
2558 return (TRUE((boolean_t) 1));
2559 }
2560 } while (--i > 0);
2561 }
2562 simple_unlock(&pmap->lock);
2563 }
2564 }
2565 PMAP_WRITE_UNLOCK(spl)((void)(spl));
2566 return (FALSE((boolean_t) 0));
2567}
2568
2569/*
2570 * Clear the modify bits on the specified physical page.
2571 */
2572
2573void pmap_clear_modify(phys)
2574 register vm_offset_t phys;
2575{
2576 phys_attribute_clear(phys, PHYS_MODIFIED0x00000040);
2577}
2578
2579/*
2580 * pmap_is_modified:
2581 *
2582 * Return whether or not the specified physical page is modified
2583 * by any physical maps.
2584 */
2585
2586boolean_t pmap_is_modified(phys)
2587 register vm_offset_t phys;
2588{
2589 return (phys_attribute_test(phys, PHYS_MODIFIED0x00000040));
2590}
2591
2592/*
2593 * pmap_clear_reference:
2594 *
2595 * Clear the reference bit on the specified physical page.
2596 */
2597
2598void pmap_clear_reference(phys)
2599 vm_offset_t phys;
2600{
2601 phys_attribute_clear(phys, PHYS_REFERENCED0x00000020);
2602}
2603
2604/*
2605 * pmap_is_referenced:
2606 *
2607 * Return whether or not the specified physical page is referenced
2608 * by any physical maps.
2609 */
2610
2611boolean_t pmap_is_referenced(phys)
2612 vm_offset_t phys;
2613{
2614 return (phys_attribute_test(phys, PHYS_REFERENCED0x00000020));
2615}
2616
2617#if NCPUS1 > 1
2618/*
2619* TLB Coherence Code (TLB "shootdown" code)
2620*
2621* Threads that belong to the same task share the same address space and
2622* hence share a pmap. However, they may run on distinct cpus and thus
2623* have distinct TLBs that cache page table entries. In order to guarantee
2624* the TLBs are consistent, whenever a pmap is changed, all threads that
2625* are active in that pmap must have their TLB updated. To keep track of
2626* this information, the set of cpus that are currently using a pmap is
2627* maintained within each pmap structure (cpus_using). Pmap_activate() and
2628* pmap_deactivate add and remove, respectively, a cpu from this set.
2629* Since the TLBs are not addressable over the bus, each processor must
2630* flush its own TLB; a processor that needs to invalidate another TLB
2631* needs to interrupt the processor that owns that TLB to signal the
2632* update.
2633*
2634* Whenever a pmap is updated, the lock on that pmap is locked, and all
2635* cpus using the pmap are signaled to invalidate. All threads that need
2636* to activate a pmap must wait for the lock to clear to await any updates
2637* in progress before using the pmap. They must ACQUIRE the lock to add
2638* their cpu to the cpus_using set. An implicit assumption made
2639* throughout the TLB code is that all kernel code that runs at or higher
2640* than splvm blocks out update interrupts, and that such code does not
2641* touch pageable pages.
2642*
2643* A shootdown interrupt serves another function besides signaling a
2644* processor to invalidate. The interrupt routine (pmap_update_interrupt)
2645* waits for the both the pmap lock (and the kernel pmap lock) to clear,
2646* preventing user code from making implicit pmap updates while the
2647* sending processor is performing its update. (This could happen via a
2648* user data write reference that turns on the modify bit in the page
2649* table). It must wait for any kernel updates that may have started
2650* concurrently with a user pmap update because the IPC code
2651* changes mappings.
2652* Spinning on the VALUES of the locks is sufficient (rather than
2653* having to acquire the locks) because any updates that occur subsequent
2654* to finding the lock unlocked will be signaled via another interrupt.
2655* (This assumes the interrupt is cleared before the low level interrupt code
2656* calls pmap_update_interrupt()).
2657*
2658* The signaling processor must wait for any implicit updates in progress
2659* to terminate before continuing with its update. Thus it must wait for an
2660* acknowledgement of the interrupt from each processor for which such
2661* references could be made. For maintaining this information, a set
2662* cpus_active is used. A cpu is in this set if and only if it can
2663* use a pmap. When pmap_update_interrupt() is entered, a cpu is removed from
2664* this set; when all such cpus are removed, it is safe to update.
2665*
2666* Before attempting to acquire the update lock on a pmap, a cpu (A) must
2667* be at least at the priority of the interprocessor interrupt
2668* (splip<=splvm). Otherwise, A could grab a lock and be interrupted by a
2669* kernel update; it would spin forever in pmap_update_interrupt() trying
2670* to acquire the user pmap lock it had already acquired. Furthermore A
2671* must remove itself from cpus_active. Otherwise, another cpu holding
2672* the lock (B) could be in the process of sending an update signal to A,
2673* and thus be waiting for A to remove itself from cpus_active. If A is
2674* spinning on the lock at priority this will never happen and a deadlock
2675* will result.
2676*/
2677
2678/*
2679 * Signal another CPU that it must flush its TLB
2680 */
2681void signal_cpus(use_list, pmap, start, end)
2682 cpu_set use_list;
2683 pmap_t pmap;
2684 vm_offset_t start, end;
2685{
2686 register int which_cpu, j;
2687 register pmap_update_list_t update_list_p;
2688
2689 while ((which_cpu = ffs(use_list)) != 0) {
2690 which_cpu -= 1; /* convert to 0 origin */
2691
2692 update_list_p = &cpu_update_list[which_cpu];
2693 simple_lock(&update_list_p->lock);
2694
2695 j = update_list_p->count;
2696 if (j >= UPDATE_LIST_SIZE) {
2697 /*
2698 * list overflowed. Change last item to
2699 * indicate overflow.
2700 */
2701 update_list_p->item[UPDATE_LIST_SIZE-1].pmap = kernel_pmap;
2702 update_list_p->item[UPDATE_LIST_SIZE-1].start = VM_MIN_ADDRESS(0);
2703 update_list_p->item[UPDATE_LIST_SIZE-1].end = VM_MAX_KERNEL_ADDRESS((0xffffffffUL) - ((0xc0000000UL)) + 0xC0000000UL);
2704 }
2705 else {
2706 update_list_p->item[j].pmap = pmap;
2707 update_list_p->item[j].start = start;
2708 update_list_p->item[j].end = end;
2709 update_list_p->count = j+1;
2710 }
2711 cpu_update_needed[which_cpu] = TRUE((boolean_t) 1);
2712 simple_unlock(&update_list_p->lock);
2713
2714 if ((cpus_idle & (1 << which_cpu)) == 0)
2715 interrupt_processor(which_cpu);
2716 use_list &= ~(1 << which_cpu);
2717 }
2718}
2719
2720void process_pmap_updates(my_pmap)
2721 register pmap_t my_pmap;
2722{
2723 register int my_cpu = cpu_number()(0);
2724 register pmap_update_list_t update_list_p;
2725 register int j;
2726 register pmap_t pmap;
2727
2728 update_list_p = &cpu_update_list[my_cpu];
2729 simple_lock(&update_list_p->lock);
2730
2731 for (j = 0; j < update_list_p->count; j++) {
2732 pmap = update_list_p->item[j].pmap;
2733 if (pmap == my_pmap ||
2734 pmap == kernel_pmap) {
2735
2736 INVALIDATE_TLB(pmap,{ (void) (pmap); (void) (update_list_p->item[j].start); (void
) (update_list_p->item[j].end); ({ register unsigned long _temp__
= (({ register unsigned long _temp__; asm volatile("mov %%cr3, %0"
: "=r" (_temp__)); _temp__; })); asm volatile("mov %0, %%cr3"
: : "r" (_temp__) : "memory"); }); }
2737 update_list_p->item[j].start,{ (void) (pmap); (void) (update_list_p->item[j].start); (void
) (update_list_p->item[j].end); ({ register unsigned long _temp__
= (({ register unsigned long _temp__; asm volatile("mov %%cr3, %0"
: "=r" (_temp__)); _temp__; })); asm volatile("mov %0, %%cr3"
: : "r" (_temp__) : "memory"); }); }
2738 update_list_p->item[j].end){ (void) (pmap); (void) (update_list_p->item[j].start); (void
) (update_list_p->item[j].end); ({ register unsigned long _temp__
= (({ register unsigned long _temp__; asm volatile("mov %%cr3, %0"
: "=r" (_temp__)); _temp__; })); asm volatile("mov %0, %%cr3"
: : "r" (_temp__) : "memory"); }); };
2739 }
2740 }
2741 update_list_p->count = 0;
2742 cpu_update_needed[my_cpu] = FALSE((boolean_t) 0);
2743 simple_unlock(&update_list_p->lock);
2744}
2745
2746/*
2747 * Interrupt routine for TBIA requested from other processor.
2748 */
2749void pmap_update_interrupt(void)
2750{
2751 register int my_cpu;
2752 register pmap_t my_pmap;
2753 int s;
2754
2755 my_cpu = cpu_number()(0);
2756
2757 /*
2758 * Exit now if we're idle. We'll pick up the update request
2759 * when we go active, and we must not put ourselves back in
2760 * the active set because we'll never process the interrupt
2761 * while we're idle (thus hanging the system).
2762 */
2763 if (cpus_idle & (1 << my_cpu))
2764 return;
2765
2766 if (current_thread()(active_threads[(0)]) == THREAD_NULL((thread_t) 0))
2767 my_pmap = kernel_pmap;
2768 else {
2769 my_pmap = current_pmap()((((active_threads[(0)])->task->map)->pmap));
2770 if (!pmap_in_use(my_pmap, my_cpu)(((my_pmap)->cpus_using & (1 << (my_cpu))) != 0))
2771 my_pmap = kernel_pmap;
2772 }
2773
2774 /*
2775 * Raise spl to splvm (above splip) to block out pmap_extract
2776 * from IO code (which would put this cpu back in the active
2777 * set).
2778 */
2779 s = splvm();
2780
2781 do {
2782
2783 /*
2784 * Indicate that we're not using either user or kernel
2785 * pmap.
2786 */
2787 i_bit_clear(my_cpu, &cpus_active);
2788
2789 /*
2790 * Wait for any pmap updates in progress, on either user
2791 * or kernel pmap.
2792 */
2793 while (*(volatile int *)&my_pmap->lock.lock_data ||
2794 *(volatile int *)&kernel_pmap->lock.lock_data)
2795 continue;
2796
2797 process_pmap_updates(my_pmap);
2798
2799 i_bit_set(my_cpu, &cpus_active);
2800
2801 } while (cpu_update_needed[my_cpu]);
2802
2803 splx(s);
2804}
2805#else /* NCPUS > 1 */
2806/*
2807 * Dummy routine to satisfy external reference.
2808 */
2809void pmap_update_interrupt()
2810{
2811 /* should never be called. */
2812}
2813#endif /* NCPUS > 1 */
2814
2815#if defined(__i386__1)
2816/* Unmap page 0 to trap NULL references. */
2817void
2818pmap_unmap_page_zero ()
2819{
2820 int *pte;
2821
2822 pte = (int *) pmap_pte (kernel_pmap, 0);
2823 if (!pte)
2824 return;
2825 assert (pte)({ if (!(pte)) Assert("pte", "../i386/intel/pmap.c", 2825); }
);
2826#ifdef MACH_PV_PAGETABLES
2827 if (!hyp_mmu_update_pte(kv_to_ma(pte), 0))
2828 printf("couldn't unmap page 0\n");
2829#else /* MACH_PV_PAGETABLES */
2830 *pte = 0;
2831 INVALIDATE_TLB(kernel_pmap, 0, PAGE_SIZE){ (void) (kernel_pmap); (void) (0); (void) ((1 << 12));
({ register unsigned long _temp__ = (({ register unsigned long
_temp__; asm volatile("mov %%cr3, %0" : "=r" (_temp__)); _temp__
; })); asm volatile("mov %0, %%cr3" : : "r" (_temp__) : "memory"
); }); };
2832#endif /* MACH_PV_PAGETABLES */
2833}
2834#endif /* __i386__ */