Bug Summary

File:obj/../i386/intel/pmap.c
Location:line 1570, column 11
Description:Dereference of null pointer (loaded from variable 'pte')

Annotated Source Code

1/*
2 * Mach Operating System
3 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
4 * All Rights Reserved.
5 *
6 * Permission to use, copy, modify and distribute this software and its
7 * documentation is hereby granted, provided that both the copyright
8 * notice and this permission notice appear in all copies of the
9 * software, derivative works or modified versions, and any portions
10 * thereof, and that both notices appear in supporting documentation.
11 *
12 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
13 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
14 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
15 *
16 * Carnegie Mellon requests users of this software to return to
17 *
18 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
19 * School of Computer Science
20 * Carnegie Mellon University
21 * Pittsburgh PA 15213-3890
22 *
23 * any improvements or extensions that they make and grant Carnegie Mellon
24 * the rights to redistribute these changes.
25 */
26/*
27 * File: pmap.c
28 * Author: Avadis Tevanian, Jr., Michael Wayne Young
29 * (These guys wrote the Vax version)
30 *
31 * Physical Map management code for Intel i386, and i486.
32 *
33 * Manages physical address maps.
34 *
35 * In addition to hardware address maps, this
36 * module is called upon to provide software-use-only
37 * maps which may or may not be stored in the same
38 * form as hardware maps. These pseudo-maps are
39 * used to store intermediate results from copy
40 * operations to and from address spaces.
41 *
42 * Since the information managed by this module is
43 * also stored by the logical address mapping module,
44 * this module may throw away valid virtual-to-physical
45 * mappings at almost any time. However, invalidations
46 * of virtual-to-physical mappings must be done as
47 * requested.
48 *
49 * In order to cope with hardware architectures which
50 * make virtual-to-physical map invalidates expensive,
51 * this module may delay invalidate or reduced protection
52 * operations until such time as they are actually
53 * necessary. This module is given full information as
54 * to which processors are currently using which maps,
55 * and to when physical maps must be made correct.
56 */
57
58#include <string.h>
59
60#include <mach/machine/vm_types.h>
61
62#include <mach/boolean.h>
63#include <kern/debug.h>
64#include <kern/printf.h>
65#include <kern/thread.h>
66#include <kern/slab.h>
67
68#include <kern/lock.h>
69
70#include <vm/pmap.h>
71#include <vm/vm_map.h>
72#include <vm/vm_kern.h>
73#include <i3861/vm_param.h>
74#include <mach/vm_prot.h>
75#include <vm/vm_object.h>
76#include <vm/vm_page.h>
77#include <vm/vm_user.h>
78
79#include <mach/machine/vm_param.h>
80#include <mach/xen.h>
81#include <machine/thread.h>
82#include <i3861/cpu_number.h>
83#include <i3861/proc_reg.h>
84#include <i3861/locore.h>
85#include <i3861/model_dep.h>
86
87#ifdef MACH_PSEUDO_PHYS
88#define WRITE_PTE(pte_p, pte_entry)*(pte_p) = (pte_entry); *(pte_p) = pte_entry?pa_to_ma(pte_entry):0;
89#else /* MACH_PSEUDO_PHYS */
90#define WRITE_PTE(pte_p, pte_entry)*(pte_p) = (pte_entry); *(pte_p) = (pte_entry);
91#endif /* MACH_PSEUDO_PHYS */
92
93/*
94 * Private data structures.
95 */
96
97/*
98 * For each vm_page_t, there is a list of all currently
99 * valid virtual mappings of that page. An entry is
100 * a pv_entry_t; the list is the pv_table.
101 */
102
103typedef struct pv_entry {
104 struct pv_entry *next; /* next pv_entry */
105 pmap_t pmap; /* pmap where mapping lies */
106 vm_offset_t va; /* virtual address for mapping */
107} *pv_entry_t;
108
109#define PV_ENTRY_NULL((pv_entry_t) 0) ((pv_entry_t) 0)
110
111pv_entry_t pv_head_table; /* array of entries, one per page */
112
113/*
114 * pv_list entries are kept on a list that can only be accessed
115 * with the pmap system locked (at SPLVM, not in the cpus_active set).
116 * The list is refilled from the pv_list_cache if it becomes empty.
117 */
118pv_entry_t pv_free_list; /* free list at SPLVM */
119decl_simple_lock_data(, pv_free_list_lock)
120
121#define PV_ALLOC(pv_e){ ; if ((pv_e = pv_free_list) != 0) { pv_free_list = pv_e->
next; } ; }
{ \
122 simple_lock(&pv_free_list_lock); \
123 if ((pv_e = pv_free_list) != 0) { \
124 pv_free_list = pv_e->next; \
125 } \
126 simple_unlock(&pv_free_list_lock); \
127}
128
129#define PV_FREE(pv_e){ ; pv_e->next = pv_free_list; pv_free_list = pv_e; ; } { \
130 simple_lock(&pv_free_list_lock); \
131 pv_e->next = pv_free_list; \
132 pv_free_list = pv_e; \
133 simple_unlock(&pv_free_list_lock); \
134}
135
136struct kmem_cache pv_list_cache; /* cache of pv_entry structures */
137
138/*
139 * Each entry in the pv_head_table is locked by a bit in the
140 * pv_lock_table. The lock bits are accessed by the physical
141 * address of the page they lock.
142 */
143
144char *pv_lock_table; /* pointer to array of bits */
145#define pv_lock_table_size(n)(((n)+8 -1)/8) (((n)+BYTE_SIZE8-1)/BYTE_SIZE8)
146
147/* Has pmap_init completed? */
148boolean_t pmap_initialized = FALSE((boolean_t) 0);
149
150/*
151 * Range of kernel virtual addresses available for kernel memory mapping.
152 * Does not include the virtual addresses used to map physical memory 1-1.
153 * Initialized by pmap_bootstrap.
154 */
155vm_offset_t kernel_virtual_start;
156vm_offset_t kernel_virtual_end;
157
158/*
159 * Index into pv_head table, its lock bits, and the modify/reference
160 * bits starting at phys_first_addr.
161 */
162#define pa_index(pa)((((vm_size_t)(pa - phys_first_addr)) >> 12)) (atop(pa - phys_first_addr)(((vm_size_t)(pa - phys_first_addr)) >> 12))
163
164#define pai_to_pvh(pai)(&pv_head_table[pai]) (&pv_head_table[pai])
165#define lock_pvh_pai(pai)(bit_lock(pai, pv_lock_table)) (bit_lock(pai, pv_lock_table))
166#define unlock_pvh_pai(pai)(bit_unlock(pai, pv_lock_table)) (bit_unlock(pai, pv_lock_table))
167
168/*
169 * Array of physical page attribites for managed pages.
170 * One byte per physical page.
171 */
172char *pmap_phys_attributes;
173
174/*
175 * Physical page attributes. Copy bits from PTE definition.
176 */
177#define PHYS_MODIFIED0x00000040 INTEL_PTE_MOD0x00000040 /* page modified */
178#define PHYS_REFERENCED0x00000020 INTEL_PTE_REF0x00000020 /* page referenced */
179
180/*
181 * Amount of virtual memory mapped by one
182 * page-directory entry.
183 */
184#define PDE_MAPPED_SIZE(((vm_offset_t)(1) << 22)) (pdenum2lin(1)((vm_offset_t)(1) << 22))
185
186/*
187 * We allocate page table pages directly from the VM system
188 * through this object. It maps physical memory.
189 */
190vm_object_t pmap_object = VM_OBJECT_NULL((vm_object_t) 0);
191
192/*
193 * Locking and TLB invalidation
194 */
195
196/*
197 * Locking Protocols:
198 *
199 * There are two structures in the pmap module that need locking:
200 * the pmaps themselves, and the per-page pv_lists (which are locked
201 * by locking the pv_lock_table entry that corresponds to the pv_head
202 * for the list in question.) Most routines want to lock a pmap and
203 * then do operations in it that require pv_list locking -- however
204 * pmap_remove_all and pmap_copy_on_write operate on a physical page
205 * basis and want to do the locking in the reverse order, i.e. lock
206 * a pv_list and then go through all the pmaps referenced by that list.
207 * To protect against deadlock between these two cases, the pmap_lock
208 * is used. There are three different locking protocols as a result:
209 *
210 * 1. pmap operations only (pmap_extract, pmap_access, ...) Lock only
211 * the pmap.
212 *
213 * 2. pmap-based operations (pmap_enter, pmap_remove, ...) Get a read
214 * lock on the pmap_lock (shared read), then lock the pmap
215 * and finally the pv_lists as needed [i.e. pmap lock before
216 * pv_list lock.]
217 *
218 * 3. pv_list-based operations (pmap_remove_all, pmap_copy_on_write, ...)
219 * Get a write lock on the pmap_lock (exclusive write); this
220 * also guaranteees exclusive access to the pv_lists. Lock the
221 * pmaps as needed.
222 *
223 * At no time may any routine hold more than one pmap lock or more than
224 * one pv_list lock. Because interrupt level routines can allocate
225 * mbufs and cause pmap_enter's, the pmap_lock and the lock on the
226 * kernel_pmap can only be held at splvm.
227 */
228
229#if NCPUS1 > 1
230/*
231 * We raise the interrupt level to splvm, to block interprocessor
232 * interrupts during pmap operations. We must take the CPU out of
233 * the cpus_active set while interrupts are blocked.
234 */
235#define SPLVM(spl)((void)(spl)) { \
236 spl = splvm(); \
237 i_bit_clear(cpu_number()(0), &cpus_active); \
238}
239
240#define SPLX(spl)((void)(spl)) { \
241 i_bit_set(cpu_number()(0), &cpus_active); \
242 splx(spl); \
243}
244
245/*
246 * Lock on pmap system
247 */
248lock_data_t pmap_system_lock;
249
250#define PMAP_READ_LOCK(pmap, spl)((void)(spl)) { \
251 SPLVM(spl)((void)(spl)); \
252 lock_read(&pmap_system_lock); \
253 simple_lock(&(pmap)->lock); \
254}
255
256#define PMAP_WRITE_LOCK(spl)((void)(spl)) { \
257 SPLVM(spl)((void)(spl)); \
258 lock_write(&pmap_system_lock); \
259}
260
261#define PMAP_READ_UNLOCK(pmap, spl)((void)(spl)) { \
262 simple_unlock(&(pmap)->lock); \
263 lock_read_done(&pmap_system_lock)lock_done(&pmap_system_lock); \
264 SPLX(spl)((void)(spl)); \
265}
266
267#define PMAP_WRITE_UNLOCK(spl)((void)(spl)) { \
268 lock_write_done(&pmap_system_lock)lock_done(&pmap_system_lock); \
269 SPLX(spl)((void)(spl)); \
270}
271
272#define PMAP_WRITE_TO_READ_LOCK(pmap) { \
273 simple_lock(&(pmap)->lock); \
274 lock_write_to_read(&pmap_system_lock); \
275}
276
277#define LOCK_PVH(index) (lock_pvh_pai(index)(bit_lock(index, pv_lock_table)))
278
279#define UNLOCK_PVH(index) (unlock_pvh_pai(index)(bit_unlock(index, pv_lock_table)))
280
281#define PMAP_UPDATE_TLBS(pmap, s, e){ if ((pmap)->cpus_using) { { (void) ((pmap)); (void) ((s)
); (void) ((e)); ({ register unsigned long _temp__ = (({ register
unsigned long _temp__; asm volatile("mov %%cr3, %0" : "=r" (
_temp__)); _temp__; })); asm volatile("mov %0, %%cr3" : : "r"
(_temp__) : "memory"); }); }; } }
\
282{ \
283 cpu_set cpu_mask = 1 << cpu_number()(0); \
284 cpu_set users; \
285 \
286 /* Since the pmap is locked, other updates are locked */ \
287 /* out, and any pmap_activate has finished. */ \
288 \
289 /* find other cpus using the pmap */ \
290 users = (pmap)->cpus_using & ~cpu_mask; \
291 if (users) { \
292 /* signal them, and wait for them to finish */ \
293 /* using the pmap */ \
294 signal_cpus(users, (pmap), (s), (e)); \
295 while ((pmap)->cpus_using & cpus_active & ~cpu_mask) \
296 continue; \
297 } \
298 \
299 /* invalidate our own TLB if pmap is in use */ \
300 if ((pmap)->cpus_using & cpu_mask) { \
301 INVALIDATE_TLB((pmap), (s), (e)){ (void) ((pmap)); (void) ((s)); (void) ((e)); ({ register unsigned
long _temp__ = (({ register unsigned long _temp__; asm volatile
("mov %%cr3, %0" : "=r" (_temp__)); _temp__; })); asm volatile
("mov %0, %%cr3" : : "r" (_temp__) : "memory"); }); }
; \
302 } \
303}
304
305#else /* NCPUS > 1 */
306
307#define SPLVM(spl)((void)(spl)) ((void)(spl))
308#define SPLX(spl)((void)(spl)) ((void)(spl))
309
310#define PMAP_READ_LOCK(pmap, spl)((void)(spl)) SPLVM(spl)((void)(spl))
311#define PMAP_WRITE_LOCK(spl)((void)(spl)) SPLVM(spl)((void)(spl))
312#define PMAP_READ_UNLOCK(pmap, spl)((void)(spl)) SPLX(spl)((void)(spl))
313#define PMAP_WRITE_UNLOCK(spl)((void)(spl)) SPLX(spl)((void)(spl))
314#define PMAP_WRITE_TO_READ_LOCK(pmap)
315
316#define LOCK_PVH(index)
317#define UNLOCK_PVH(index)
318
319#define PMAP_UPDATE_TLBS(pmap, s, e){ if ((pmap)->cpus_using) { { (void) ((pmap)); (void) ((s)
); (void) ((e)); ({ register unsigned long _temp__ = (({ register
unsigned long _temp__; asm volatile("mov %%cr3, %0" : "=r" (
_temp__)); _temp__; })); asm volatile("mov %0, %%cr3" : : "r"
(_temp__) : "memory"); }); }; } }
{ \
320 /* invalidate our own TLB if pmap is in use */ \
321 if ((pmap)->cpus_using) { \
322 INVALIDATE_TLB((pmap), (s), (e)){ (void) ((pmap)); (void) ((s)); (void) ((e)); ({ register unsigned
long _temp__ = (({ register unsigned long _temp__; asm volatile
("mov %%cr3, %0" : "=r" (_temp__)); _temp__; })); asm volatile
("mov %0, %%cr3" : : "r" (_temp__) : "memory"); }); }
; \
323 } \
324}
325
326#endif /* NCPUS > 1 */
327
328#define MAX_TBIS_SIZE32 32 /* > this -> TBIA */ /* XXX */
329
330#ifdef MACH_PV_PAGETABLES
331#if 1
332#define INVALIDATE_TLB(pmap, s, e){ (void) (pmap); (void) (s); (void) (e); ({ register unsigned
long _temp__ = (({ register unsigned long _temp__; asm volatile
("mov %%cr3, %0" : "=r" (_temp__)); _temp__; })); asm volatile
("mov %0, %%cr3" : : "r" (_temp__) : "memory"); }); }
hyp_mmuext_op_void(MMUEXT_TLB_FLUSH_LOCAL)
333#else
334#define INVALIDATE_TLB(pmap, s, e){ (void) (pmap); (void) (s); (void) (e); ({ register unsigned
long _temp__ = (({ register unsigned long _temp__; asm volatile
("mov %%cr3, %0" : "=r" (_temp__)); _temp__; })); asm volatile
("mov %0, %%cr3" : : "r" (_temp__) : "memory"); }); }
do { \
335 if (__builtin_constant_p((e) - (s)) \
336 && (e) - (s) == PAGE_SIZE(1 << 12)) \
337 hyp_invlpg((pmap) == kernel_pmap ? kvtolin(s)((vm_offset_t)(s) - 0xC0000000UL + ((0xc0000000UL))) : (s)); \
338 else \
339 hyp_mmuext_op_void(MMUEXT_TLB_FLUSH_LOCAL); \
340} while(0)
341#endif
342#else /* MACH_PV_PAGETABLES */
343#if 0
344/* It is hard to know when a TLB flush becomes less expensive than a bunch of
345 * invlpgs. But it surely is more expensive than just one invlpg. */
346#define INVALIDATE_TLB(pmap, s, e){ (void) (pmap); (void) (s); (void) (e); ({ register unsigned
long _temp__ = (({ register unsigned long _temp__; asm volatile
("mov %%cr3, %0" : "=r" (_temp__)); _temp__; })); asm volatile
("mov %0, %%cr3" : : "r" (_temp__) : "memory"); }); }
{ \
347 if (__builtin_constant_p((e) - (s)) \
348 && (e) - (s) == PAGE_SIZE(1 << 12)) \
349 invlpg_linear(s)({ asm volatile( "movw %w1,%%es\n" "\tinvlpg %%es:(%0)\n" "\tmovw %w2,%%es"
:: "r" (s), "q" (0x38), "q" ((0x10 | 0))); })
; \
350 else \
351 flush_tlb()({ register unsigned long _temp__ = (({ register unsigned long
_temp__; asm volatile("mov %%cr3, %0" : "=r" (_temp__)); _temp__
; })); asm volatile("mov %0, %%cr3" : : "r" (_temp__) : "memory"
); })
; \
352}
353#else
354#define INVALIDATE_TLB(pmap, s, e){ (void) (pmap); (void) (s); (void) (e); ({ register unsigned
long _temp__ = (({ register unsigned long _temp__; asm volatile
("mov %%cr3, %0" : "=r" (_temp__)); _temp__; })); asm volatile
("mov %0, %%cr3" : : "r" (_temp__) : "memory"); }); }
{ \
355 (void) (pmap); \
356 (void) (s); \
357 (void) (e); \
358 flush_tlb()({ register unsigned long _temp__ = (({ register unsigned long
_temp__; asm volatile("mov %%cr3, %0" : "=r" (_temp__)); _temp__
; })); asm volatile("mov %0, %%cr3" : : "r" (_temp__) : "memory"
); })
; \
359}
360#endif
361#endif /* MACH_PV_PAGETABLES */
362
363
364#if NCPUS1 > 1
365/*
366 * Structures to keep track of pending TLB invalidations
367 */
368
369#define UPDATE_LIST_SIZE 4
370
371struct pmap_update_item {
372 pmap_t pmap; /* pmap to invalidate */
373 vm_offset_t start; /* start address to invalidate */
374 vm_offset_t end; /* end address to invalidate */
375} ;
376
377typedef struct pmap_update_item *pmap_update_item_t;
378
379/*
380 * List of pmap updates. If the list overflows,
381 * the last entry is changed to invalidate all.
382 */
383struct pmap_update_list {
384 decl_simple_lock_data(, lock)
385 int count;
386 struct pmap_update_item item[UPDATE_LIST_SIZE];
387} ;
388typedef struct pmap_update_list *pmap_update_list_t;
389
390struct pmap_update_list cpu_update_list[NCPUS1];
391
392#endif /* NCPUS > 1 */
393
394/*
395 * Other useful macros.
396 */
397#define current_pmap()((((active_threads[(0)])->task->map)->pmap)) (vm_map_pmap(current_thread()->task->map)(((active_threads[(0)])->task->map)->pmap))
398#define pmap_in_use(pmap, cpu)(((pmap)->cpus_using & (1 << (cpu))) != 0) (((pmap)->cpus_using & (1 << (cpu))) != 0)
399
400struct pmap kernel_pmap_store;
401pmap_t kernel_pmap;
402
403struct kmem_cache pmap_cache; /* cache of pmap structures */
404
405int pmap_debug = 0; /* flag for debugging prints */
406
407#if 0
408int ptes_per_vm_page1; /* number of hardware ptes needed
409 to map one VM page. */
410#else
411#define ptes_per_vm_page1 1
412#endif
413
414unsigned int inuse_ptepages_count = 0; /* debugging */
415
416extern char end;
417
418/*
419 * Pointer to the basic page directory for the kernel.
420 * Initialized by pmap_bootstrap().
421 */
422pt_entry_t *kernel_page_dir;
423
424void pmap_remove_range(); /* forward */
425#if NCPUS1 > 1
426void signal_cpus(); /* forward */
427#endif /* NCPUS > 1 */
428
429static inline pt_entry_t *
430pmap_pde(pmap_t pmap, vm_offset_t addr)
431{
432 if (pmap == kernel_pmap)
433 addr = kvtolin(addr)((vm_offset_t)(addr) - 0xC0000000UL + ((0xc0000000UL)));
434 return &pmap->dirbase[lin2pdenum(addr)(((addr) >> 22) & 0x3ff)];
435}
436
437/*
438 * Given an offset and a map, compute the address of the
439 * pte. If the address is invalid with respect to the map
440 * then PT_ENTRY_NULL is returned (and the map may need to grow).
441 *
442 * This is only used internally.
443 */
444pt_entry_t *
445pmap_pte(pmap_t pmap, vm_offset_t addr)
446{
447 pt_entry_t *ptp;
448 pt_entry_t pte;
449
450 if (pmap->dirbase == 0)
451 return(PT_ENTRY_NULL((pt_entry_t *) 0));
452 pte = *pmap_pde(pmap, addr);
453 if ((pte & INTEL_PTE_VALID0x00000001) == 0)
454 return(PT_ENTRY_NULL((pt_entry_t *) 0));
455 ptp = (pt_entry_t *)ptetokv(pte)(((vm_offset_t)(((pte) & 0xfffff000)) + 0xC0000000UL));
456 return(&ptp[ptenum(addr)(((addr) >> 12) & 0x3ff)]);
457}
458
459#define DEBUG_PTE_PAGE0 0
460
461#if DEBUG_PTE_PAGE0
462void ptep_check(ptep)
463 ptep_t ptep;
464{
465 register pt_entry_t *pte, *epte;
466 int ctu, ctw;
467
468 /* check the use and wired counts */
469 if (ptep == PTE_PAGE_NULL)
470 return;
471 pte = pmap_pte(ptep->pmap, ptep->va);
472 epte = pte + INTEL_PGBYTES4096/sizeof(pt_entry_t);
473 ctu = 0;
474 ctw = 0;
475 while (pte < epte) {
476 if (pte->pfn != 0) {
477 ctu++;
478 if (pte->wired)
479 ctw++;
480 }
481 pte += ptes_per_vm_page1;
482 }
483
484 if (ctu != ptep->use_count || ctw != ptep->wired_count) {
485 printf("use %d wired %d - actual use %d wired %d\n",
486 ptep->use_count, ptep->wired_count, ctu, ctw);
487 panic("pte count");
488 }
489}
490#endif /* DEBUG_PTE_PAGE */
491
492/*
493 * Map memory at initialization. The physical addresses being
494 * mapped are not managed and are never unmapped.
495 *
496 * For now, VM is already on, we only need to map the
497 * specified memory.
498 */
499vm_offset_t pmap_map(virt, start, end, prot)
500 register vm_offset_t virt;
501 register vm_offset_t start;
502 register vm_offset_t end;
503 register int prot;
504{
505 register int ps;
506
507 ps = PAGE_SIZE(1 << 12);
508 while (start < end) {
509 pmap_enter(kernel_pmap, virt, start, prot, FALSE((boolean_t) 0));
510 virt += ps;
511 start += ps;
512 }
513 return(virt);
514}
515
516/*
517 * Back-door routine for mapping kernel VM at initialization.
518 * Useful for mapping memory outside the range
519 * [phys_first_addr, phys_last_addr) (i.e., devices).
520 * Otherwise like pmap_map.
521 */
522vm_offset_t pmap_map_bd(virt, start, end, prot)
523 register vm_offset_t virt;
524 register vm_offset_t start;
525 register vm_offset_t end;
526 vm_prot_t prot;
527{
528 register pt_entry_t template;
529 register pt_entry_t *pte;
530 int spl;
531#ifdef MACH_PV_PAGETABLES
532 int n, i = 0;
533 struct mmu_update update[HYP_BATCH_MMU_UPDATES];
534#endif /* MACH_PV_PAGETABLES */
535
536 template = pa_to_pte(start)((start) & 0xfffff000)
537 | INTEL_PTE_NCACHE0x00000010|INTEL_PTE_WTHRU0x00000008
538 | INTEL_PTE_VALID0x00000001;
539 if (CPU_HAS_FEATURE(CPU_FEATURE_PGE)(cpu_features[(13) / 32] & (1 << ((13) % 32))))
540 template |= INTEL_PTE_GLOBAL0x00000100;
541 if (prot & VM_PROT_WRITE((vm_prot_t) 0x02))
542 template |= INTEL_PTE_WRITE0x00000002;
543
544 PMAP_READ_LOCK(pmap, spl)((void)(spl));
545 while (start < end) {
546 pte = pmap_pte(kernel_pmap, virt);
547 if (pte == PT_ENTRY_NULL((pt_entry_t *) 0))
548 panic("pmap_map_bd: Invalid kernel address\n");
549#ifdef MACH_PV_PAGETABLES
550 update[i].ptr = kv_to_ma(pte);
551 update[i].val = pa_to_ma(template);
552 i++;
553 if (i == HYP_BATCH_MMU_UPDATES) {
554 hyp_mmu_update(kvtolin(&update)((vm_offset_t)(&update) - 0xC0000000UL + ((0xc0000000UL))
)
, i, kvtolin(&n)((vm_offset_t)(&n) - 0xC0000000UL + ((0xc0000000UL))), DOMID_SELF);
555 if (n != i)
556 panic("couldn't pmap_map_bd\n");
557 i = 0;
558 }
559#else /* MACH_PV_PAGETABLES */
560 WRITE_PTE(pte, template)*(pte) = (template);
561#endif /* MACH_PV_PAGETABLES */
562 pte_increment_pa(template)((template) += 0xfff +1);
563 virt += PAGE_SIZE(1 << 12);
564 start += PAGE_SIZE(1 << 12);
565 }
566#ifdef MACH_PV_PAGETABLES
567 if (i > HYP_BATCH_MMU_UPDATES)
568 panic("overflowed array in pmap_map_bd");
569 hyp_mmu_update(kvtolin(&update)((vm_offset_t)(&update) - 0xC0000000UL + ((0xc0000000UL))
)
, i, kvtolin(&n)((vm_offset_t)(&n) - 0xC0000000UL + ((0xc0000000UL))), DOMID_SELF);
570 if (n != i)
571 panic("couldn't pmap_map_bd\n");
572#endif /* MACH_PV_PAGETABLES */
573 PMAP_READ_UNLOCK(pmap, spl)((void)(spl));
574 return(virt);
575}
576
577/*
578 * Bootstrap the system enough to run with virtual memory.
579 * Allocate the kernel page directory and page tables,
580 * and direct-map all physical memory.
581 * Called with mapping off.
582 */
583void pmap_bootstrap()
584{
585 /*
586 * Mapping is turned off; we must reference only physical addresses.
587 * The load image of the system is to be mapped 1-1 physical = virtual.
588 */
589
590 /*
591 * Set ptes_per_vm_page for general use.
592 */
593#if 0
594 ptes_per_vm_page1 = PAGE_SIZE(1 << 12) / INTEL_PGBYTES4096;
595#endif
596
597 /*
598 * The kernel's pmap is statically allocated so we don't
599 * have to use pmap_create, which is unlikely to work
600 * correctly at this part of the boot sequence.
601 */
602
603 kernel_pmap = &kernel_pmap_store;
604
605#if NCPUS1 > 1
606 lock_init(&pmap_system_lock, FALSE((boolean_t) 0)); /* NOT a sleep lock */
607#endif /* NCPUS > 1 */
608
609 simple_lock_init(&kernel_pmap->lock);
610
611 kernel_pmap->ref_count = 1;
612
613 /*
614 * Determine the kernel virtual address range.
615 * It starts at the end of the physical memory
616 * mapped into the kernel address space,
617 * and extends to a stupid arbitrary limit beyond that.
618 */
619 kernel_virtual_start = phystokv(phys_last_addr)((vm_offset_t)(phys_last_addr) + 0xC0000000UL);
620 kernel_virtual_end = phystokv(phys_last_addr)((vm_offset_t)(phys_last_addr) + 0xC0000000UL) + VM_KERNEL_MAP_SIZE(192 * 1024 * 1024);
621
622 if (kernel_virtual_end < kernel_virtual_start
623 || kernel_virtual_end > VM_MAX_KERNEL_ADDRESS((0xffffffffUL) - ((0xc0000000UL)) + 0xC0000000UL))
624 kernel_virtual_end = VM_MAX_KERNEL_ADDRESS((0xffffffffUL) - ((0xc0000000UL)) + 0xC0000000UL);
625
626 /*
627 * Allocate and clear a kernel page directory.
628 */
629 /* Note: initial Xen mapping holds at least 512kB free mapped page.
630 * We use that for directly building our linear mapping. */
631#if PAE
632 {
633 vm_offset_t addr;
634 init_alloc_aligned(PDPNUM1 * INTEL_PGBYTES4096, &addr);
635 kernel_pmap->dirbase = kernel_page_dir = (pt_entry_t*)phystokv(addr)((vm_offset_t)(addr) + 0xC0000000UL);
636 }
637 kernel_pmap->pdpbase = (pt_entry_t*)phystokv(pmap_grab_page())((vm_offset_t)(pmap_grab_page()) + 0xC0000000UL);
638 {
639 int i;
640 for (i = 0; i < PDPNUM1; i++)
641 WRITE_PTE(&kernel_pmap->pdpbase[i], pa_to_pte(_kvtophys((void *) kernel_pmap->dirbase + i * INTEL_PGBYTES)) | INTEL_PTE_VALID)*(&kernel_pmap->pdpbase[i]) = (((((vm_offset_t)((void *
) kernel_pmap->dirbase + i * 4096) - 0xC0000000UL)) & 0xfffff000
) | 0x00000001);
;
642 }
643#else /* PAE */
644 kernel_pmap->dirbase = kernel_page_dir = (pt_entry_t*)phystokv(pmap_grab_page())((vm_offset_t)(pmap_grab_page()) + 0xC0000000UL);
645#endif /* PAE */
646 {
647 int i;
648 for (i = 0; i < NPDES(1 * ((((unsigned long)(1)) << 12)/sizeof(pt_entry_t))); i++)
649 kernel_pmap->dirbase[i] = 0;
650 }
651
652#ifdef MACH_PV_PAGETABLES
653 /* We don't actually deal with the CR3 register content at all */
654 hyp_vm_assist(VMASST_CMD_enable, VMASST_TYPE_pae_extended_cr3);
655 /*
656 * Xen may only provide as few as 512KB extra bootstrap linear memory,
657 * which is far from enough to map all available memory, so we need to
658 * map more bootstrap linear memory. We here map 1 (resp. 4 for PAE)
659 * other L1 table(s), thus 4MiB extra memory (resp. 8MiB), which is
660 * enough for a pagetable mapping 4GiB.
661 */
662#ifdef PAE
663#define NSUP_L1 4
664#else
665#define NSUP_L1 1
666#endif
667 pt_entry_t *l1_map[NSUP_L1];
668 {
669 pt_entry_t *base = (pt_entry_t*) boot_info.pt_base;
670 vm_offset_t la;
671 int n_l1map;
672 for (n_l1map = 0, la = VM_MIN_KERNEL_ADDRESS0xC0000000UL; la >= VM_MIN_KERNEL_ADDRESS0xC0000000UL; la += NPTES((((unsigned long)(1)) << 12)/sizeof(pt_entry_t)) * PAGE_SIZE(1 << 12)) {
673#ifdef PAE
674 pt_entry_t *l2_map = (pt_entry_t*) ptetokv(base[lin2pdpnum(la)])(((vm_offset_t)(((base[lin2pdpnum(la)]) & 0xfffff000)) + 0xC0000000UL
))
;
675#else /* PAE */
676 pt_entry_t *l2_map = base;
677#endif /* PAE */
678 /* Like lin2pdenum, but works with non-contiguous boot L3 */
679 l2_map += (la >> PDESHIFT22) & PDEMASK0x3ff;
680 if (!(*l2_map & INTEL_PTE_VALID0x00000001)) {
681 struct mmu_update update;
682 int j, n;
683
684 l1_map[n_l1map] = (pt_entry_t*) phystokv(pmap_grab_page())((vm_offset_t)(pmap_grab_page()) + 0xC0000000UL);
685 for (j = 0; j < NPTES((((unsigned long)(1)) << 12)/sizeof(pt_entry_t)); j++)
686 l1_map[n_l1map][j] = (((pt_entry_t)pfn_to_mfn(lin2pdenum(la - VM_MIN_KERNEL_ADDRESS)(((la - 0xC0000000UL) >> 22) & 0x3ff) * NPTES((((unsigned long)(1)) << 12)/sizeof(pt_entry_t)) + j)) << PAGE_SHIFT12) | INTEL_PTE_VALID0x00000001 | INTEL_PTE_WRITE0x00000002;
687 pmap_set_page_readonly_init(l1_map[n_l1map]);
688 if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, kv_to_mfn (l1_map[n_l1map])))
689 panic("couldn't pin page %p(%p)", l1_map[n_l1map], (vm_offset_t) kv_to_ma (l1_map[n_l1map]));
690 update.ptr = kv_to_ma(l2_map);
691 update.val = kv_to_ma(l1_map[n_l1map]) | INTEL_PTE_VALID0x00000001 | INTEL_PTE_WRITE0x00000002;
692 hyp_mmu_update(kv_to_la(&update), 1, kv_to_la(&n), DOMID_SELF);
693 if (n != 1)
694 panic("couldn't complete bootstrap map");
695 /* added the last L1 table, can stop */
696 if (++n_l1map >= NSUP_L1)
697 break;
698 }
699 }
700 }
701#endif /* MACH_PV_PAGETABLES */
702
703 /*
704 * Allocate and set up the kernel page tables.
705 */
706 {
707 vm_offset_t va;
708 pt_entry_t global = CPU_HAS_FEATURE(CPU_FEATURE_PGE)(cpu_features[(13) / 32] & (1 << ((13) % 32))) ? INTEL_PTE_GLOBAL0x00000100 : 0;
709
710 /*
711 * Map virtual memory for all known physical memory, 1-1,
712 * from phys_first_addr to phys_last_addr.
713 * Make any mappings completely in the kernel's text segment read-only.
714 *
715 * Also allocate some additional all-null page tables afterwards
716 * for kernel virtual memory allocation,
717 * because this PMAP module is too stupid
718 * to allocate new kernel page tables later.
719 * XX fix this
720 */
721 for (va = phystokv(phys_first_addr)((vm_offset_t)(phys_first_addr) + 0xC0000000UL); va >= phystokv(phys_first_addr)((vm_offset_t)(phys_first_addr) + 0xC0000000UL) && va < kernel_virtual_end; )
722 {
723 pt_entry_t *pde = kernel_page_dir + lin2pdenum(kvtolin(va))(((((vm_offset_t)(va) - 0xC0000000UL + ((0xc0000000UL)))) >>
22) & 0x3ff)
;
724 pt_entry_t *ptable = (pt_entry_t*)phystokv(pmap_grab_page())((vm_offset_t)(pmap_grab_page()) + 0xC0000000UL);
725 pt_entry_t *pte;
726
727 /* Initialize the page directory entry. */
728 WRITE_PTE(pde, pa_to_pte((vm_offset_t)_kvtophys(ptable))*(pde) = ((((vm_offset_t)((vm_offset_t)(ptable) - 0xC0000000UL
)) & 0xfffff000) | 0x00000001 | 0x00000002);
729 | INTEL_PTE_VALID | INTEL_PTE_WRITE)*(pde) = ((((vm_offset_t)((vm_offset_t)(ptable) - 0xC0000000UL
)) & 0xfffff000) | 0x00000001 | 0x00000002);
;
730
731 /* Initialize the page table. */
732 for (pte = ptable; (va < phystokv(phys_last_addr)((vm_offset_t)(phys_last_addr) + 0xC0000000UL)) && (pte < ptable+NPTES((((unsigned long)(1)) << 12)/sizeof(pt_entry_t))); pte++)
733 {
734 if ((pte - ptable) < ptenum(va)(((va) >> 12) & 0x3ff))
735 {
736 WRITE_PTE(pte, 0)*(pte) = (0);;
737 }
738 else
739#ifdef MACH_PV_PAGETABLES
740 if (va == (vm_offset_t) &hyp_shared_info)
741 {
742 *pte = boot_info.shared_info | INTEL_PTE_VALID0x00000001 | INTEL_PTE_WRITE0x00000002;
743 va += INTEL_PGBYTES4096;
744 }
745 else
746#endif /* MACH_PV_PAGETABLES */
747 {
748 extern char _start[], etext[];
749
750 if (((va >= (vm_offset_t) _start)
751 && (va + INTEL_PGBYTES4096 <= (vm_offset_t)etext))
752#ifdef MACH_PV_PAGETABLES
753 || (va >= (vm_offset_t) boot_info.pt_base
754 && (va + INTEL_PGBYTES4096 <=
755 (vm_offset_t) ptable + INTEL_PGBYTES4096))
756#endif /* MACH_PV_PAGETABLES */
757 )
758 {
759 WRITE_PTE(pte, pa_to_pte(_kvtophys(va))*(pte) = (((((vm_offset_t)(va) - 0xC0000000UL)) & 0xfffff000
) | 0x00000001 | global);
760 | INTEL_PTE_VALID | global)*(pte) = (((((vm_offset_t)(va) - 0xC0000000UL)) & 0xfffff000
) | 0x00000001 | global);
;
761 }
762 else
763 {
764#ifdef MACH_PV_PAGETABLES
765 /* Keep supplementary L1 pages read-only */
766 int i;
767 for (i = 0; i < NSUP_L1; i++)
768 if (va == (vm_offset_t) l1_map[i]) {
769 WRITE_PTE(pte, pa_to_pte(_kvtophys(va))*(pte) = (((((vm_offset_t)(va) - 0xC0000000UL)) & 0xfffff000
) | 0x00000001 | global);
770 | INTEL_PTE_VALID | global)*(pte) = (((((vm_offset_t)(va) - 0xC0000000UL)) & 0xfffff000
) | 0x00000001 | global);
;
771 break;
772 }
773 if (i == NSUP_L1)
774#endif /* MACH_PV_PAGETABLES */
775 WRITE_PTE(pte, pa_to_pte(_kvtophys(va))*(pte) = (((((vm_offset_t)(va) - 0xC0000000UL)) & 0xfffff000
) | 0x00000001 | 0x00000002 | global);
776 | INTEL_PTE_VALID | INTEL_PTE_WRITE | global)*(pte) = (((((vm_offset_t)(va) - 0xC0000000UL)) & 0xfffff000
) | 0x00000001 | 0x00000002 | global);
777
778 }
779 va += INTEL_PGBYTES4096;
780 }
781 }
782 for (; pte < ptable+NPTES((((unsigned long)(1)) << 12)/sizeof(pt_entry_t)); pte++)
783 {
784 WRITE_PTE(pte, 0)*(pte) = (0);;
785 va += INTEL_PGBYTES4096;
786 }
787#ifdef MACH_PV_PAGETABLES
788 pmap_set_page_readonly_init(ptable);
789 if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, kv_to_mfn (ptable)))
790 panic("couldn't pin page %p(%p)\n", ptable, (vm_offset_t) kv_to_ma (ptable));
791#endif /* MACH_PV_PAGETABLES */
792 }
793 }
794
795 /* Architecture-specific code will turn on paging
796 soon after we return from here. */
797}
798
799#ifdef MACH_PV_PAGETABLES
800/* These are only required because of Xen security policies */
801
802/* Set back a page read write */
803void pmap_set_page_readwrite(void *_vaddr) {
804 vm_offset_t vaddr = (vm_offset_t) _vaddr;
805 vm_offset_t paddr = kvtophys(vaddr);
806 vm_offset_t canon_vaddr = phystokv(paddr)((vm_offset_t)(paddr) + 0xC0000000UL);
807 if (hyp_do_update_va_mapping (kvtolin(vaddr)((vm_offset_t)(vaddr) - 0xC0000000UL + ((0xc0000000UL))), pa_to_pte (pa_to_ma(paddr))((pa_to_ma(paddr)) & 0xfffff000) | INTEL_PTE_VALID0x00000001 | INTEL_PTE_WRITE0x00000002, UVMF_NONE))
808 panic("couldn't set hiMMU readwrite for addr %p(%p)\n", vaddr, (vm_offset_t) pa_to_ma (paddr));
809 if (canon_vaddr != vaddr)
810 if (hyp_do_update_va_mapping (kvtolin(canon_vaddr)((vm_offset_t)(canon_vaddr) - 0xC0000000UL + ((0xc0000000UL))
)
, pa_to_pte (pa_to_ma(paddr))((pa_to_ma(paddr)) & 0xfffff000) | INTEL_PTE_VALID0x00000001 | INTEL_PTE_WRITE0x00000002, UVMF_NONE))
811 panic("couldn't set hiMMU readwrite for paddr %p(%p)\n", canon_vaddr, (vm_offset_t) pa_to_ma (paddr));
812}
813
814/* Set a page read only (so as to pin it for instance) */
815void pmap_set_page_readonly(void *_vaddr) {
816 vm_offset_t vaddr = (vm_offset_t) _vaddr;
817 vm_offset_t paddr = kvtophys(vaddr);
818 vm_offset_t canon_vaddr = phystokv(paddr)((vm_offset_t)(paddr) + 0xC0000000UL);
819 if (*pmap_pde(kernel_pmap, vaddr) & INTEL_PTE_VALID0x00000001) {
820 if (hyp_do_update_va_mapping (kvtolin(vaddr)((vm_offset_t)(vaddr) - 0xC0000000UL + ((0xc0000000UL))), pa_to_pte (pa_to_ma(paddr))((pa_to_ma(paddr)) & 0xfffff000) | INTEL_PTE_VALID0x00000001, UVMF_NONE))
821 panic("couldn't set hiMMU readonly for vaddr %p(%p)\n", vaddr, (vm_offset_t) pa_to_ma (paddr));
822 }
823 if (canon_vaddr != vaddr &&
824 *pmap_pde(kernel_pmap, canon_vaddr) & INTEL_PTE_VALID0x00000001) {
825 if (hyp_do_update_va_mapping (kvtolin(canon_vaddr)((vm_offset_t)(canon_vaddr) - 0xC0000000UL + ((0xc0000000UL))
)
, pa_to_pte (pa_to_ma(paddr))((pa_to_ma(paddr)) & 0xfffff000) | INTEL_PTE_VALID0x00000001, UVMF_NONE))
826 panic("couldn't set hiMMU readonly for vaddr %p canon_vaddr %p paddr %p (%p)\n", vaddr, canon_vaddr, paddr, (vm_offset_t) pa_to_ma (paddr));
827 }
828}
829
830/* This needs to be called instead of pmap_set_page_readonly as long as RC3
831 * still points to the bootstrap dirbase, to also fix the bootstrap table. */
832void pmap_set_page_readonly_init(void *_vaddr) {
833 vm_offset_t vaddr = (vm_offset_t) _vaddr;
834#if PAE
835 pt_entry_t *pdpbase = (void*) boot_info.pt_base;
836 /* The bootstrap table does not necessarily use contiguous pages for the pde tables */
837 pt_entry_t *dirbase = (void*) ptetokv(pdpbase[lin2pdpnum(vaddr)])(((vm_offset_t)(((pdpbase[lin2pdpnum(vaddr)]) & 0xfffff000
)) + 0xC0000000UL))
;
838#else
839 pt_entry_t *dirbase = (void*) boot_info.pt_base;
840#endif
841 pt_entry_t *pte = &dirbase[lin2pdenum(vaddr)(((vaddr) >> 22) & 0x3ff) & PTEMASK0x3ff];
842 /* Modify our future kernel map (can't use update_va_mapping for this)... */
843 if (*pmap_pde(kernel_pmap, vaddr) & INTEL_PTE_VALID0x00000001) {
844 if (!hyp_mmu_update_la (kvtolin(vaddr)((vm_offset_t)(vaddr) - 0xC0000000UL + ((0xc0000000UL))), pa_to_pte (kv_to_ma(vaddr))((kv_to_ma(vaddr)) & 0xfffff000) | INTEL_PTE_VALID0x00000001))
845 panic("couldn't set hiMMU readonly for vaddr %p(%p)\n", vaddr, (vm_offset_t) kv_to_ma (vaddr));
846 }
847 /* ... and the bootstrap map. */
848 if (*pte & INTEL_PTE_VALID0x00000001) {
849 if (hyp_do_update_va_mapping (vaddr, pa_to_pte (kv_to_ma(vaddr))((kv_to_ma(vaddr)) & 0xfffff000) | INTEL_PTE_VALID0x00000001, UVMF_NONE))
850 panic("couldn't set MMU readonly for vaddr %p(%p)\n", vaddr, (vm_offset_t) kv_to_ma (vaddr));
851 }
852}
853
854void pmap_clear_bootstrap_pagetable(pt_entry_t *base) {
855 int i;
856 pt_entry_t *dir;
857 vm_offset_t va = 0;
858#if PAE
859 int j;
860#endif /* PAE */
861 if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, kv_to_mfn(base)))
862 panic("pmap_clear_bootstrap_pagetable: couldn't unpin page %p(%p)\n", base, (vm_offset_t) kv_to_ma(base));
863#if PAE
864 for (j = 0; j < PDPNUM1; j++)
865 {
866 pt_entry_t pdpe = base[j];
867 if (pdpe & INTEL_PTE_VALID0x00000001) {
868 dir = (pt_entry_t *) ptetokv(pdpe)(((vm_offset_t)(((pdpe) & 0xfffff000)) + 0xC0000000UL));
869#else /* PAE */
870 dir = base;
871#endif /* PAE */
872 for (i = 0; i < NPTES((((unsigned long)(1)) << 12)/sizeof(pt_entry_t)); i++) {
873 pt_entry_t pde = dir[i];
874 unsigned long pfn = atop(pte_to_pa(pde))(((vm_size_t)(((pde) & 0xfffff000))) >> 12);
875 void *pgt = (void*) phystokv(ptoa(pfn))((vm_offset_t)(((vm_offset_t)((pfn) << 12))) + 0xC0000000UL
)
;
876 if (pde & INTEL_PTE_VALID0x00000001)
877 hyp_free_page(pfn, pgt);
878 va += NPTES((((unsigned long)(1)) << 12)/sizeof(pt_entry_t)) * INTEL_PGBYTES4096;
879 if (va >= HYP_VIRT_START)
880 break;
881 }
882#if PAE
883 hyp_free_page(atop(_kvtophys(dir))(((vm_size_t)(((vm_offset_t)(dir) - 0xC0000000UL))) >> 12
)
, dir);
884 } else
885 va += NPTES((((unsigned long)(1)) << 12)/sizeof(pt_entry_t)) * NPTES((((unsigned long)(1)) << 12)/sizeof(pt_entry_t)) * INTEL_PGBYTES4096;
886 if (va >= HYP_VIRT_START)
887 break;
888 }
889#endif /* PAE */
890 hyp_free_page(atop(_kvtophys(base))(((vm_size_t)(((vm_offset_t)(base) - 0xC0000000UL))) >>
12)
, base);
891}
892#endif /* MACH_PV_PAGETABLES */
893
894void pmap_virtual_space(startp, endp)
895 vm_offset_t *startp;
896 vm_offset_t *endp;
897{
898 *startp = kernel_virtual_start;
899 *endp = kernel_virtual_end;
900}
901
902/*
903 * Initialize the pmap module.
904 * Called by vm_init, to initialize any structures that the pmap
905 * system needs to map virtual memory.
906 */
907void pmap_init()
908{
909 register long npages;
910 vm_offset_t addr;
911 register vm_size_t s;
912#if NCPUS1 > 1
913 int i;
914#endif /* NCPUS > 1 */
915
916 /*
917 * Allocate memory for the pv_head_table and its lock bits,
918 * the modify bit array, and the pte_page table.
919 */
920
921 npages = atop(phys_last_addr - phys_first_addr)(((vm_size_t)(phys_last_addr - phys_first_addr)) >> 12);
922 s = (vm_size_t) (sizeof(struct pv_entry) * npages
923 + pv_lock_table_size(npages)(((npages)+8 -1)/8)
924 + npages);
925
926 s = round_page(s)((vm_offset_t)((((vm_offset_t)(s)) + ((1 << 12)-1)) &
~((1 << 12)-1)))
;
927 if (kmem_alloc_wired(kernel_map, &addr, s) != KERN_SUCCESS0)
928 panic("pmap_init");
929 memset((char *) addr, 0, s);
930
931 /*
932 * Allocate the structures first to preserve word-alignment.
933 */
934 pv_head_table = (pv_entry_t) addr;
935 addr = (vm_offset_t) (pv_head_table + npages);
936
937 pv_lock_table = (char *) addr;
938 addr = (vm_offset_t) (pv_lock_table + pv_lock_table_size(npages)(((npages)+8 -1)/8));
939
940 pmap_phys_attributes = (char *) addr;
941
942 /*
943 * Create the cache of physical maps,
944 * and of the physical-to-virtual entries.
945 */
946 s = (vm_size_t) sizeof(struct pmap);
947 kmem_cache_init(&pmap_cache, "pmap", s, 0, NULL((void *) 0), NULL((void *) 0), NULL((void *) 0), 0);
948 s = (vm_size_t) sizeof(struct pv_entry);
949 kmem_cache_init(&pv_list_cache, "pv_entry", s, 0, NULL((void *) 0), NULL((void *) 0), NULL((void *) 0), 0);
950
951#if NCPUS1 > 1
952 /*
953 * Set up the pmap request lists
954 */
955 for (i = 0; i < NCPUS1; i++) {
956 pmap_update_list_t up = &cpu_update_list[i];
957
958 simple_lock_init(&up->lock);
959 up->count = 0;
960 }
961#endif /* NCPUS > 1 */
962
963 /*
964 * Indicate that the PMAP module is now fully initialized.
965 */
966 pmap_initialized = TRUE((boolean_t) 1);
967}
968
969#define valid_page(x)(pmap_initialized && pmap_valid_page(x)) (pmap_initialized && pmap_valid_page(x))
970
971boolean_t pmap_verify_free(phys)
972 vm_offset_t phys;
973{
974 pv_entry_t pv_h;
975 int pai;
976 int spl;
977 boolean_t result;
978
979 assert(phys != vm_page_fictitious_addr)({ if (!(phys != vm_page_fictitious_addr)) Assert("phys != vm_page_fictitious_addr"
, "../i386/intel/pmap.c", 979); })
;
980 if (!pmap_initialized)
981 return(TRUE((boolean_t) 1));
982
983 if (!pmap_valid_page(phys))
984 return(FALSE((boolean_t) 0));
985
986 PMAP_WRITE_LOCK(spl)((void)(spl));
987
988 pai = pa_index(phys)((((vm_size_t)(phys - phys_first_addr)) >> 12));
989 pv_h = pai_to_pvh(pai)(&pv_head_table[pai]);
990
991 result = (pv_h->pmap == PMAP_NULL((pmap_t) 0));
992 PMAP_WRITE_UNLOCK(spl)((void)(spl));
993
994 return(result);
995}
996
997/*
998 * Routine: pmap_page_table_page_alloc
999 *
1000 * Allocates a new physical page to be used as a page-table page.
1001 *
1002 * Must be called with the pmap system and the pmap unlocked,
1003 * since these must be unlocked to use vm_page_grab.
1004 */
1005vm_offset_t
1006pmap_page_table_page_alloc()
1007{
1008 register vm_page_t m;
1009 register vm_offset_t pa;
1010
1011 check_simple_locks();
1012
1013 /*
1014 * We cannot allocate the pmap_object in pmap_init,
1015 * because it is called before the cache package is up.
1016 * Allocate it now if it is missing.
1017 */
1018 if (pmap_object == VM_OBJECT_NULL((vm_object_t) 0))
1019 pmap_object = vm_object_allocate(phys_last_addr - phys_first_addr);
1020
1021 /*
1022 * Allocate a VM page for the level 2 page table entries.
1023 */
1024 while ((m = vm_page_grab(FALSE((boolean_t) 0))) == VM_PAGE_NULL((vm_page_t) 0))
1025 VM_PAGE_WAIT((void (*)()) 0)vm_page_wait((void (*)()) 0);
1026
1027 /*
1028 * Map the page to its physical address so that it
1029 * can be found later.
1030 */
1031 pa = m->phys_addr;
1032 vm_object_lock(pmap_object);
1033 vm_page_insert(m, pmap_object, pa);
1034 vm_page_lock_queues();
1035 vm_page_wire(m);
1036 inuse_ptepages_count++;
1037 vm_page_unlock_queues();
1038 vm_object_unlock(pmap_object);
1039
1040 /*
1041 * Zero the page.
1042 */
1043 memset((void *)phystokv(pa)((vm_offset_t)(pa) + 0xC0000000UL), 0, PAGE_SIZE(1 << 12));
1044
1045 return pa;
1046}
1047
1048#ifdef MACH_XEN
1049void pmap_map_mfn(void *_addr, unsigned long mfn) {
1050 vm_offset_t addr = (vm_offset_t) _addr;
1051 pt_entry_t *pte, *pdp;
1052 vm_offset_t ptp;
1053 pt_entry_t ma = ((pt_entry_t) mfn) << PAGE_SHIFT12;
1054
1055 /* Add a ptp if none exist yet for this pte */
1056 if ((pte = pmap_pte(kernel_pmap, addr)) == PT_ENTRY_NULL((pt_entry_t *) 0)) {
1057 ptp = phystokv(pmap_page_table_page_alloc())((vm_offset_t)(pmap_page_table_page_alloc()) + 0xC0000000UL);
1058#ifdef MACH_PV_PAGETABLES
1059 pmap_set_page_readonly((void*) ptp);
1060 if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, pa_to_mfn(ptp)))
1061 panic("couldn't pin page %p(%p)\n",ptp,(vm_offset_t) kv_to_ma(ptp));
1062#endif /* MACH_PV_PAGETABLES */
1063 pdp = pmap_pde(kernel_pmap, addr);
1064
1065#ifdef MACH_PV_PAGETABLES
1066 if (!hyp_mmu_update_pte(kv_to_ma(pdp),
1067 pa_to_pte(kv_to_ma(ptp))((kv_to_ma(ptp)) & 0xfffff000) | INTEL_PTE_VALID0x00000001
1068 | INTEL_PTE_USER0x00000004
1069 | INTEL_PTE_WRITE0x00000002))
1070 panic("%s:%d could not set pde %p(%p) to %p(%p)\n",__FILE__"../i386/intel/pmap.c",__LINE__1070,kvtophys((vm_offset_t)pdp),(vm_offset_t) kv_to_ma(pdp), ptp, (vm_offset_t) pa_to_ma(ptp));
1071#else /* MACH_PV_PAGETABLES */
1072 *pdp = pa_to_pte(kvtophys(ptp))((kvtophys(ptp)) & 0xfffff000) | INTEL_PTE_VALID0x00000001
1073 | INTEL_PTE_USER0x00000004
1074 | INTEL_PTE_WRITE0x00000002;
1075#endif /* MACH_PV_PAGETABLES */
1076 pte = pmap_pte(kernel_pmap, addr);
1077 }
1078
1079#ifdef MACH_PV_PAGETABLES
1080 if (!hyp_mmu_update_pte(kv_to_ma(pte), ma | INTEL_PTE_VALID0x00000001 | INTEL_PTE_WRITE0x00000002))
1081 panic("%s:%d could not set pte %p(%p) to %p(%p)\n",__FILE__"../i386/intel/pmap.c",__LINE__1081,pte,(vm_offset_t) kv_to_ma(pte), ma, ma_to_pa(ma));
1082#else /* MACH_PV_PAGETABLES */
1083 /* Note: in this case, mfn is actually a pfn. */
1084 WRITE_PTE(pte, ma | INTEL_PTE_VALID | INTEL_PTE_WRITE)*(pte) = (ma | 0x00000001 | 0x00000002);;
1085#endif /* MACH_PV_PAGETABLES */
1086}
1087#endif /* MACH_XEN */
1088
1089/*
1090 * Deallocate a page-table page.
1091 * The page-table page must have all mappings removed,
1092 * and be removed from its page directory.
1093 */
1094void
1095pmap_page_table_page_dealloc(pa)
1096 vm_offset_t pa;
1097{
1098 vm_page_t m;
1099
1100 vm_object_lock(pmap_object);
1101 m = vm_page_lookup(pmap_object, pa);
1102 vm_page_lock_queues();
1103 vm_page_free(m);
1104 inuse_ptepages_count--;
1105 vm_page_unlock_queues();
1106 vm_object_unlock(pmap_object);
1107}
1108
1109/*
1110 * Create and return a physical map.
1111 *
1112 * If the size specified for the map
1113 * is zero, the map is an actual physical
1114 * map, and may be referenced by the
1115 * hardware.
1116 *
1117 * If the size specified is non-zero,
1118 * the map will be used in software only, and
1119 * is bounded by that size.
1120 */
1121pmap_t pmap_create(size)
1122 vm_size_t size;
1123{
1124 register pmap_t p;
1125 register pmap_statistics_t stats;
1126
1127 /*
1128 * A software use-only map doesn't even need a map.
1129 */
1130
1131 if (size != 0) {
1132 return(PMAP_NULL((pmap_t) 0));
1133 }
1134
1135/*
1136 * Allocate a pmap struct from the pmap_cache. Then allocate
1137 * the page descriptor table.
1138 */
1139
1140 p = (pmap_t) kmem_cache_alloc(&pmap_cache);
1141 if (p == PMAP_NULL((pmap_t) 0))
1142 panic("pmap_create");
1143
1144 if (kmem_alloc_wired(kernel_map,
1145 (vm_offset_t *)&p->dirbase, PDPNUM1 * INTEL_PGBYTES4096)
1146 != KERN_SUCCESS0)
1147 panic("pmap_create");
1148
1149 memcpy(p->dirbase, kernel_page_dir, PDPNUM1 * INTEL_PGBYTES4096);
1150#ifdef LINUX_DEV
1151#if VM_MIN_KERNEL_ADDRESS0xC0000000UL != 0
1152 /* Do not map BIOS in user tasks */
1153 p->dirbase[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS)(((((0xc0000000UL)) - 0xC0000000UL) >> 22) & 0x3ff)] = 0;
1154#endif
1155#endif
1156#ifdef MACH_PV_PAGETABLES
1157 {
1158 int i;
1159 for (i = 0; i < PDPNUM1; i++)
1160 pmap_set_page_readonly((void*) p->dirbase + i * INTEL_PGBYTES4096);
1161 }
1162#endif /* MACH_PV_PAGETABLES */
1163
1164#if PAE
1165 if (kmem_alloc_wired(kernel_map,
1166 (vm_offset_t *)&p->pdpbase, INTEL_PGBYTES4096)
1167 != KERN_SUCCESS0)
1168 panic("pmap_create");
1169 {
1170 int i;
1171 for (i = 0; i < PDPNUM1; i++)
1172 WRITE_PTE(&p->pdpbase[i], pa_to_pte(kvtophys((vm_offset_t) p->dirbase + i * INTEL_PGBYTES)) | INTEL_PTE_VALID)*(&p->pdpbase[i]) = (((kvtophys((vm_offset_t) p->dirbase
+ i * 4096)) & 0xfffff000) | 0x00000001);
;
1173 }
1174#ifdef MACH_PV_PAGETABLES
1175 pmap_set_page_readonly(p->pdpbase);
1176#endif /* MACH_PV_PAGETABLES */
1177#endif /* PAE */
1178
1179 p->ref_count = 1;
1180
1181 simple_lock_init(&p->lock);
1182 p->cpus_using = 0;
1183
1184 /*
1185 * Initialize statistics.
1186 */
1187
1188 stats = &p->stats;
1189 stats->resident_count = 0;
1190 stats->wired_count = 0;
1191
1192 return(p);
1193}
1194
1195/*
1196 * Retire the given physical map from service.
1197 * Should only be called if the map contains
1198 * no valid mappings.
1199 */
1200
1201void pmap_destroy(p)
1202 register pmap_t p;
1203{
1204 register pt_entry_t *pdep;
1205 register vm_offset_t pa;
1206 register int c, s;
1207 register vm_page_t m;
1208
1209 if (p == PMAP_NULL((pmap_t) 0))
1210 return;
1211
1212 SPLVM(s)((void)(s));
1213 simple_lock(&p->lock);
1214 c = --p->ref_count;
1215 simple_unlock(&p->lock);
1216 SPLX(s)((void)(s));
1217
1218 if (c != 0) {
1219 return; /* still in use */
1220 }
1221
1222 /*
1223 * Free the memory maps, then the
1224 * pmap structure.
1225 */
1226 for (pdep = p->dirbase;
1227 pdep < &p->dirbase[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS)(((((0xc0000000UL))) >> 22) & 0x3ff)];
1228 pdep += ptes_per_vm_page1) {
1229 if (*pdep & INTEL_PTE_VALID0x00000001) {
1230 pa = pte_to_pa(*pdep)((*pdep) & 0xfffff000);
1231 vm_object_lock(pmap_object);
1232 m = vm_page_lookup(pmap_object, pa);
1233 if (m == VM_PAGE_NULL((vm_page_t) 0))
1234 panic("pmap_destroy: pte page not in object");
1235 vm_page_lock_queues();
1236#ifdef MACH_PV_PAGETABLES
1237 if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, pa_to_mfn(pa)))
1238 panic("pmap_destroy: couldn't unpin page %p(%p)\n", pa, (vm_offset_t) kv_to_ma(pa));
1239 pmap_set_page_readwrite((void*) phystokv(pa)((vm_offset_t)(pa) + 0xC0000000UL));
1240#endif /* MACH_PV_PAGETABLES */
1241 vm_page_free(m);
1242 inuse_ptepages_count--;
1243 vm_page_unlock_queues();
1244 vm_object_unlock(pmap_object);
1245 }
1246 }
1247#ifdef MACH_PV_PAGETABLES
1248 {
1249 int i;
1250 for (i = 0; i < PDPNUM1; i++)
1251 pmap_set_page_readwrite((void*) p->dirbase + i * INTEL_PGBYTES4096);
1252 }
1253#endif /* MACH_PV_PAGETABLES */
1254 kmem_free(kernel_map, (vm_offset_t)p->dirbase, PDPNUM1 * INTEL_PGBYTES4096);
1255#if PAE
1256#ifdef MACH_PV_PAGETABLES
1257 pmap_set_page_readwrite(p->pdpbase);
1258#endif /* MACH_PV_PAGETABLES */
1259 kmem_free(kernel_map, (vm_offset_t)p->pdpbase, INTEL_PGBYTES4096);
1260#endif /* PAE */
1261 kmem_cache_free(&pmap_cache, (vm_offset_t) p);
1262}
1263
1264/*
1265 * Add a reference to the specified pmap.
1266 */
1267
1268void pmap_reference(p)
1269 register pmap_t p;
1270{
1271 int s;
1272 if (p != PMAP_NULL((pmap_t) 0)) {
1273 SPLVM(s)((void)(s));
1274 simple_lock(&p->lock);
1275 p->ref_count++;
1276 simple_unlock(&p->lock);
1277 SPLX(s)((void)(s));
1278 }
1279}
1280
1281/*
1282 * Remove a range of hardware page-table entries.
1283 * The entries given are the first (inclusive)
1284 * and last (exclusive) entries for the VM pages.
1285 * The virtual address is the va for the first pte.
1286 *
1287 * The pmap must be locked.
1288 * If the pmap is not the kernel pmap, the range must lie
1289 * entirely within one pte-page. This is NOT checked.
1290 * Assumes that the pte-page exists.
1291 */
1292
1293/* static */
1294void pmap_remove_range(pmap, va, spte, epte)
1295 pmap_t pmap;
1296 vm_offset_t va;
1297 pt_entry_t *spte;
1298 pt_entry_t *epte;
1299{
1300 register pt_entry_t *cpte;
1301 int num_removed, num_unwired;
1302 int pai;
1303 vm_offset_t pa;
1304#ifdef MACH_PV_PAGETABLES
1305 int n, ii = 0;
1306 struct mmu_update update[HYP_BATCH_MMU_UPDATES];
1307#endif /* MACH_PV_PAGETABLES */
1308
1309#if DEBUG_PTE_PAGE0
1310 if (pmap != kernel_pmap)
1311 ptep_check(get_pte_page(spte));
1312#endif /* DEBUG_PTE_PAGE */
1313 num_removed = 0;
1314 num_unwired = 0;
1315
1316 for (cpte = spte; cpte < epte;
1317 cpte += ptes_per_vm_page1, va += PAGE_SIZE(1 << 12)) {
1318
1319 if (*cpte == 0)
1320 continue;
1321 pa = pte_to_pa(*cpte)((*cpte) & 0xfffff000);
1322
1323 num_removed++;
1324 if (*cpte & INTEL_PTE_WIRED0x00000200)
1325 num_unwired++;
1326
1327 if (!valid_page(pa)(pmap_initialized && pmap_valid_page(pa))) {
1328
1329 /*
1330 * Outside range of managed physical memory.
1331 * Just remove the mappings.
1332 */
1333 register int i = ptes_per_vm_page1;
1334 register pt_entry_t *lpte = cpte;
1335 do {
1336#ifdef MACH_PV_PAGETABLES
1337 update[ii].ptr = kv_to_ma(lpte);
1338 update[ii].val = 0;
1339 ii++;
1340 if (ii == HYP_BATCH_MMU_UPDATES) {
1341 hyp_mmu_update(kvtolin(&update)((vm_offset_t)(&update) - 0xC0000000UL + ((0xc0000000UL))
)
, ii, kvtolin(&n)((vm_offset_t)(&n) - 0xC0000000UL + ((0xc0000000UL))), DOMID_SELF);
1342 if (n != ii)
1343 panic("couldn't pmap_remove_range\n");
1344 ii = 0;
1345 }
1346#else /* MACH_PV_PAGETABLES */
1347 *lpte = 0;
1348#endif /* MACH_PV_PAGETABLES */
1349 lpte++;
1350 } while (--i > 0);
1351 continue;
1352 }
1353
1354 pai = pa_index(pa)((((vm_size_t)(pa - phys_first_addr)) >> 12));
1355 LOCK_PVH(pai);
1356
1357 /*
1358 * Get the modify and reference bits.
1359 */
1360 {
1361 register int i;
1362 register pt_entry_t *lpte;
1363
1364 i = ptes_per_vm_page1;
1365 lpte = cpte;
1366 do {
1367 pmap_phys_attributes[pai] |=
1368 *lpte & (PHYS_MODIFIED0x00000040|PHYS_REFERENCED0x00000020);
1369#ifdef MACH_PV_PAGETABLES
1370 update[ii].ptr = kv_to_ma(lpte);
1371 update[ii].val = 0;
1372 ii++;
1373 if (ii == HYP_BATCH_MMU_UPDATES) {
1374 hyp_mmu_update(kvtolin(&update)((vm_offset_t)(&update) - 0xC0000000UL + ((0xc0000000UL))
)
, ii, kvtolin(&n)((vm_offset_t)(&n) - 0xC0000000UL + ((0xc0000000UL))), DOMID_SELF);
1375 if (n != ii)
1376 panic("couldn't pmap_remove_range\n");
1377 ii = 0;
1378 }
1379#else /* MACH_PV_PAGETABLES */
1380 *lpte = 0;
1381#endif /* MACH_PV_PAGETABLES */
1382 lpte++;
1383 } while (--i > 0);
1384 }
1385
1386 /*
1387 * Remove the mapping from the pvlist for
1388 * this physical page.
1389 */
1390 {
1391 register pv_entry_t pv_h, prev, cur;
1392
1393 pv_h = pai_to_pvh(pai)(&pv_head_table[pai]);
1394 if (pv_h->pmap == PMAP_NULL((pmap_t) 0)) {
1395 panic("pmap_remove: null pv_list!");
1396 }
1397 if (pv_h->va == va && pv_h->pmap == pmap) {
1398 /*
1399 * Header is the pv_entry. Copy the next one
1400 * to header and free the next one (we cannot
1401 * free the header)
1402 */
1403 cur = pv_h->next;
1404 if (cur != PV_ENTRY_NULL((pv_entry_t) 0)) {
1405 *pv_h = *cur;
1406 PV_FREE(cur){ ; cur->next = pv_free_list; pv_free_list = cur; ; };
1407 }
1408 else {
1409 pv_h->pmap = PMAP_NULL((pmap_t) 0);
1410 }
1411 }
1412 else {
1413 cur = pv_h;
1414 do {
1415 prev = cur;
1416 if ((cur = prev->next) == PV_ENTRY_NULL((pv_entry_t) 0)) {
1417 panic("pmap-remove: mapping not in pv_list!");
1418 }
1419 } while (cur->va != va || cur->pmap != pmap);
1420 prev->next = cur->next;
1421 PV_FREE(cur){ ; cur->next = pv_free_list; pv_free_list = cur; ; };
1422 }
1423 UNLOCK_PVH(pai);
1424 }
1425 }
1426
1427#ifdef MACH_PV_PAGETABLES
1428 if (ii > HYP_BATCH_MMU_UPDATES)
1429 panic("overflowed array in pmap_remove_range");
1430 hyp_mmu_update(kvtolin(&update)((vm_offset_t)(&update) - 0xC0000000UL + ((0xc0000000UL))
)
, ii, kvtolin(&n)((vm_offset_t)(&n) - 0xC0000000UL + ((0xc0000000UL))), DOMID_SELF);
1431 if (n != ii)
1432 panic("couldn't pmap_remove_range\n");
1433#endif /* MACH_PV_PAGETABLES */
1434
1435 /*
1436 * Update the counts
1437 */
1438 pmap->stats.resident_count -= num_removed;
1439 pmap->stats.wired_count -= num_unwired;
1440}
1441
1442/*
1443 * Remove the given range of addresses
1444 * from the specified map.
1445 *
1446 * It is assumed that the start and end are properly
1447 * rounded to the hardware page size.
1448 */
1449
1450void pmap_remove(map, s, e)
1451 pmap_t map;
1452 vm_offset_t s, e;
1453{
1454 int spl;
1455 register pt_entry_t *pde;
1456 register pt_entry_t *spte, *epte;
1457 vm_offset_t l;
1458 vm_offset_t _s = s;
1459
1460 if (map == PMAP_NULL((pmap_t) 0))
1461 return;
1462
1463 PMAP_READ_LOCK(map, spl)((void)(spl));
1464
1465 pde = pmap_pde(map, s);
1466 while (s < e) {
1467 l = (s + PDE_MAPPED_SIZE(((vm_offset_t)(1) << 22))) & ~(PDE_MAPPED_SIZE(((vm_offset_t)(1) << 22))-1);
1468 if (l > e)
1469 l = e;
1470 if (*pde & INTEL_PTE_VALID0x00000001) {
1471 spte = (pt_entry_t *)ptetokv(*pde)(((vm_offset_t)(((*pde) & 0xfffff000)) + 0xC0000000UL));
1472 spte = &spte[ptenum(s)(((s) >> 12) & 0x3ff)];
1473 epte = &spte[intel_btop(l-s)(((unsigned long)(l-s)) >> 12)];
1474 pmap_remove_range(map, s, spte, epte);
1475 }
1476 s = l;
1477 pde++;
1478 }
1479 PMAP_UPDATE_TLBS(map, _s, e){ if ((map)->cpus_using) { { (void) ((map)); (void) ((_s))
; (void) ((e)); ({ register unsigned long _temp__ = (({ register
unsigned long _temp__; asm volatile("mov %%cr3, %0" : "=r" (
_temp__)); _temp__; })); asm volatile("mov %0, %%cr3" : : "r"
(_temp__) : "memory"); }); }; } }
;
1480
1481 PMAP_READ_UNLOCK(map, spl)((void)(spl));
1482}
1483
1484/*
1485 * Routine: pmap_page_protect
1486 *
1487 * Function:
1488 * Lower the permission for all mappings to a given
1489 * page.
1490 */
1491void pmap_page_protect(phys, prot)
1492 vm_offset_t phys;
1493 vm_prot_t prot;
1494{
1495 pv_entry_t pv_h, prev;
1496 register pv_entry_t pv_e;
1497 register pt_entry_t *pte;
1498 int pai;
1499 register pmap_t pmap;
1500 int spl;
1501 boolean_t remove;
1502
1503 assert(phys != vm_page_fictitious_addr)({ if (!(phys != vm_page_fictitious_addr)) Assert("phys != vm_page_fictitious_addr"
, "../i386/intel/pmap.c", 1503); })
;
1504 if (!valid_page(phys)(pmap_initialized && pmap_valid_page(phys))) {
1
Taking false branch
1505 /*
1506 * Not a managed page.
1507 */
1508 return;
1509 }
1510
1511 /*
1512 * Determine the new protection.
1513 */
1514 switch (prot) {
2
Control jumps to the 'default' case at line 1521
1515 case VM_PROT_READ((vm_prot_t) 0x01):
1516 case VM_PROT_READ((vm_prot_t) 0x01)|VM_PROT_EXECUTE((vm_prot_t) 0x04):
1517 remove = FALSE((boolean_t) 0);
1518 break;
1519 case VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04)):
1520 return; /* nothing to do */
1521 default:
1522 remove = TRUE((boolean_t) 1);
1523 break;
3
Execution continues on line 1531
1524 }
1525
1526 /*
1527 * Lock the pmap system first, since we will be changing
1528 * several pmaps.
1529 */
1530
1531 PMAP_WRITE_LOCK(spl)((void)(spl));
1532
1533 pai = pa_index(phys)((((vm_size_t)(phys - phys_first_addr)) >> 12));
1534 pv_h = pai_to_pvh(pai)(&pv_head_table[pai]);
1535
1536 /*
1537 * Walk down PV list, changing or removing all mappings.
1538 * We do not have to lock the pv_list because we have
1539 * the entire pmap system locked.
1540 */
1541 if (pv_h->pmap != PMAP_NULL((pmap_t) 0)) {
4
Taking true branch
1542
1543 prev = pv_e = pv_h;
1544 do {
8
Loop condition is true. Execution continues on line 1545
12
Loop condition is true. Execution continues on line 1545
16
Loop condition is true. Execution continues on line 1545
1545 register vm_offset_t va;
1546
1547 pmap = pv_e->pmap;
1548 /*
1549 * Lock the pmap to block pmap_extract and similar routines.
1550 */
1551 simple_lock(&pmap->lock);
1552
1553 va = pv_e->va;
1554 pte = pmap_pte(pmap, va);
1555
1556 /*
1557 * Consistency checks.
1558 */
1559 /* assert(*pte & INTEL_PTE_VALID); XXX */
1560 /* assert(pte_to_phys(*pte) == phys); */
1561
1562 /*
1563 * Remove the mapping if new protection is NONE
1564 * or if write-protecting a kernel mapping.
1565 */
1566 if (remove || pmap == kernel_pmap) {
1567 /*
1568 * Remove the mapping, collecting any modify bits.
1569 */
1570 if (*pte & INTEL_PTE_WIRED0x00000200)
5
Taking false branch
9
Taking false branch
13
Taking false branch
17
Dereference of null pointer (loaded from variable 'pte')
1571 panic("pmap_remove_all removing a wired page");
1572
1573 {
1574 register int i = ptes_per_vm_page1;
1575
1576 do {
6
Loop condition is false. Exiting loop
10
Loop condition is false. Exiting loop
14
Loop condition is false. Exiting loop
1577 pmap_phys_attributes[pai] |=
1578 *pte & (PHYS_MODIFIED0x00000040|PHYS_REFERENCED0x00000020);
1579#ifdef MACH_PV_PAGETABLES
1580 if (!hyp_mmu_update_pte(kv_to_ma(pte++), 0))
1581 panic("%s:%d could not clear pte %p\n",__FILE__"../i386/intel/pmap.c",__LINE__1581,pte-1);
1582#else /* MACH_PV_PAGETABLES */
1583 *pte++ = 0;
1584#endif /* MACH_PV_PAGETABLES */
1585 } while (--i > 0);
1586 }
1587
1588 pmap->stats.resident_count--;
1589
1590 /*
1591 * Remove the pv_entry.
1592 */
1593 if (pv_e == pv_h) {
7
Taking true branch
11
Taking true branch
15
Taking true branch
1594 /*
1595 * Fix up head later.
1596 */
1597 pv_h->pmap = PMAP_NULL((pmap_t) 0);
1598 }
1599 else {
1600 /*
1601 * Delete this entry.
1602 */
1603 prev->next = pv_e->next;
1604 PV_FREE(pv_e){ ; pv_e->next = pv_free_list; pv_free_list = pv_e; ; };
1605 }
1606 }
1607 else {
1608 /*
1609 * Write-protect.
1610 */
1611 register int i = ptes_per_vm_page1;
1612
1613 do {
1614#ifdef MACH_PV_PAGETABLES
1615 if (!hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~INTEL_PTE_WRITE0x00000002))
1616 panic("%s:%d could not disable write on pte %p\n",__FILE__"../i386/intel/pmap.c",__LINE__1616,pte);
1617#else /* MACH_PV_PAGETABLES */
1618 *pte &= ~INTEL_PTE_WRITE0x00000002;
1619#endif /* MACH_PV_PAGETABLES */
1620 pte++;
1621 } while (--i > 0);
1622
1623 /*
1624 * Advance prev.
1625 */
1626 prev = pv_e;
1627 }
1628 PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE){ if ((pmap)->cpus_using) { { (void) ((pmap)); (void) ((va
)); (void) ((va + (1 << 12))); ({ register unsigned long
_temp__ = (({ register unsigned long _temp__; asm volatile("mov %%cr3, %0"
: "=r" (_temp__)); _temp__; })); asm volatile("mov %0, %%cr3"
: : "r" (_temp__) : "memory"); }); }; } }
;
1629
1630 simple_unlock(&pmap->lock);
1631
1632 } while ((pv_e = prev->next) != PV_ENTRY_NULL((pv_entry_t) 0));
1633
1634 /*
1635 * If pv_head mapping was removed, fix it up.
1636 */
1637 if (pv_h->pmap == PMAP_NULL((pmap_t) 0)) {
1638 pv_e = pv_h->next;
1639 if (pv_e != PV_ENTRY_NULL((pv_entry_t) 0)) {
1640 *pv_h = *pv_e;
1641 PV_FREE(pv_e){ ; pv_e->next = pv_free_list; pv_free_list = pv_e; ; };
1642 }
1643 }
1644 }
1645
1646 PMAP_WRITE_UNLOCK(spl)((void)(spl));
1647}
1648
1649/*
1650 * Set the physical protection on the
1651 * specified range of this map as requested.
1652 * Will not increase permissions.
1653 */
1654void pmap_protect(map, s, e, prot)
1655 pmap_t map;
1656 vm_offset_t s, e;
1657 vm_prot_t prot;
1658{
1659 register pt_entry_t *pde;
1660 register pt_entry_t *spte, *epte;
1661 vm_offset_t l;
1662 int spl;
1663 vm_offset_t _s = s;
1664
1665 if (map == PMAP_NULL((pmap_t) 0))
1666 return;
1667
1668 /*
1669 * Determine the new protection.
1670 */
1671 switch (prot) {
1672 case VM_PROT_READ((vm_prot_t) 0x01):
1673 case VM_PROT_READ((vm_prot_t) 0x01)|VM_PROT_EXECUTE((vm_prot_t) 0x04):
1674 break;
1675 case VM_PROT_READ((vm_prot_t) 0x01)|VM_PROT_WRITE((vm_prot_t) 0x02):
1676 case VM_PROT_ALL(((vm_prot_t) 0x01)|((vm_prot_t) 0x02)|((vm_prot_t) 0x04)):
1677 return; /* nothing to do */
1678 default:
1679 pmap_remove(map, s, e);
1680 return;
1681 }
1682
1683 /*
1684 * If write-protecting in the kernel pmap,
1685 * remove the mappings; the i386 ignores
1686 * the write-permission bit in kernel mode.
1687 *
1688 * XXX should be #if'd for i386
1689 */
1690 if (map == kernel_pmap) {
1691 pmap_remove(map, s, e);
1692 return;
1693 }
1694
1695 SPLVM(spl)((void)(spl));
1696 simple_lock(&map->lock);
1697
1698 pde = pmap_pde(map, s);
1699 while (s < e) {
1700 l = (s + PDE_MAPPED_SIZE(((vm_offset_t)(1) << 22))) & ~(PDE_MAPPED_SIZE(((vm_offset_t)(1) << 22))-1);
1701 if (l > e)
1702 l = e;
1703 if (*pde & INTEL_PTE_VALID0x00000001) {
1704 spte = (pt_entry_t *)ptetokv(*pde)(((vm_offset_t)(((*pde) & 0xfffff000)) + 0xC0000000UL));
1705 spte = &spte[ptenum(s)(((s) >> 12) & 0x3ff)];
1706 epte = &spte[intel_btop(l-s)(((unsigned long)(l-s)) >> 12)];
1707
1708#ifdef MACH_PV_PAGETABLES
1709 int n, i = 0;
1710 struct mmu_update update[HYP_BATCH_MMU_UPDATES];
1711#endif /* MACH_PV_PAGETABLES */
1712
1713 while (spte < epte) {
1714 if (*spte & INTEL_PTE_VALID0x00000001) {
1715#ifdef MACH_PV_PAGETABLES
1716 update[i].ptr = kv_to_ma(spte);
1717 update[i].val = *spte & ~INTEL_PTE_WRITE0x00000002;
1718 i++;
1719 if (i == HYP_BATCH_MMU_UPDATES) {
1720 hyp_mmu_update(kvtolin(&update)((vm_offset_t)(&update) - 0xC0000000UL + ((0xc0000000UL))
)
, i, kvtolin(&n)((vm_offset_t)(&n) - 0xC0000000UL + ((0xc0000000UL))), DOMID_SELF);
1721 if (n != i)
1722 panic("couldn't pmap_protect\n");
1723 i = 0;
1724 }
1725#else /* MACH_PV_PAGETABLES */
1726 *spte &= ~INTEL_PTE_WRITE0x00000002;
1727#endif /* MACH_PV_PAGETABLES */
1728 }
1729 spte++;
1730 }
1731#ifdef MACH_PV_PAGETABLES
1732 if (i > HYP_BATCH_MMU_UPDATES)
1733 panic("overflowed array in pmap_protect");
1734 hyp_mmu_update(kvtolin(&update)((vm_offset_t)(&update) - 0xC0000000UL + ((0xc0000000UL))
)
, i, kvtolin(&n)((vm_offset_t)(&n) - 0xC0000000UL + ((0xc0000000UL))), DOMID_SELF);
1735 if (n != i)
1736 panic("couldn't pmap_protect\n");
1737#endif /* MACH_PV_PAGETABLES */
1738 }
1739 s = l;
1740 pde++;
1741 }
1742 PMAP_UPDATE_TLBS(map, _s, e){ if ((map)->cpus_using) { { (void) ((map)); (void) ((_s))
; (void) ((e)); ({ register unsigned long _temp__ = (({ register
unsigned long _temp__; asm volatile("mov %%cr3, %0" : "=r" (
_temp__)); _temp__; })); asm volatile("mov %0, %%cr3" : : "r"
(_temp__) : "memory"); }); }; } }
;
1743
1744 simple_unlock(&map->lock);
1745 SPLX(spl)((void)(spl));
1746}
1747
1748/*
1749 * Insert the given physical page (p) at
1750 * the specified virtual address (v) in the
1751 * target physical map with the protection requested.
1752 *
1753 * If specified, the page will be wired down, meaning
1754 * that the related pte can not be reclaimed.
1755 *
1756 * NB: This is the only routine which MAY NOT lazy-evaluate
1757 * or lose information. That is, this routine must actually
1758 * insert this page into the given map NOW.
1759 */
1760void pmap_enter(pmap, v, pa, prot, wired)
1761 register pmap_t pmap;
1762 vm_offset_t v;
1763 register vm_offset_t pa;
1764 vm_prot_t prot;
1765 boolean_t wired;
1766{
1767 register pt_entry_t *pte;
1768 register pv_entry_t pv_h;
1769 register int i, pai;
1770 pv_entry_t pv_e;
1771 pt_entry_t template;
1772 int spl;
1773 vm_offset_t old_pa;
1774
1775 assert(pa != vm_page_fictitious_addr)({ if (!(pa != vm_page_fictitious_addr)) Assert("pa != vm_page_fictitious_addr"
, "../i386/intel/pmap.c", 1775); })
;
1776if (pmap_debug) printf("pmap(%lx, %lx)\n", v, pa);
1777 if (pmap == PMAP_NULL((pmap_t) 0))
1778 return;
1779
1780#if !MACH_KDB0
1781 if (pmap == kernel_pmap && (v < kernel_virtual_start || v >= kernel_virtual_end))
1782 panic("pmap_enter(%p, %p) falls in physical memory area!\n", v, pa);
1783#endif
1784 if (pmap == kernel_pmap && (prot & VM_PROT_WRITE((vm_prot_t) 0x02)) == 0
1785 && !wired /* hack for io_wire */ ) {
1786 /*
1787 * Because the 386 ignores write protection in kernel mode,
1788 * we cannot enter a read-only kernel mapping, and must
1789 * remove an existing mapping if changing it.
1790 *
1791 * XXX should be #if'd for i386
1792 */
1793 PMAP_READ_LOCK(pmap, spl)((void)(spl));
1794
1795 pte = pmap_pte(pmap, v);
1796 if (pte != PT_ENTRY_NULL((pt_entry_t *) 0) && *pte != 0) {
1797 /*
1798 * Invalidate the translation buffer,
1799 * then remove the mapping.
1800 */
1801 pmap_remove_range(pmap, v, pte,
1802 pte + ptes_per_vm_page1);
1803 PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE){ if ((pmap)->cpus_using) { { (void) ((pmap)); (void) ((v)
); (void) ((v + (1 << 12))); ({ register unsigned long _temp__
= (({ register unsigned long _temp__; asm volatile("mov %%cr3, %0"
: "=r" (_temp__)); _temp__; })); asm volatile("mov %0, %%cr3"
: : "r" (_temp__) : "memory"); }); }; } }
;
1804 }
1805 PMAP_READ_UNLOCK(pmap, spl)((void)(spl));
1806 return;
1807 }
1808
1809 /*
1810 * Must allocate a new pvlist entry while we're unlocked;
1811 * Allocating may cause pageout (which will lock the pmap system).
1812 * If we determine we need a pvlist entry, we will unlock
1813 * and allocate one. Then we will retry, throughing away
1814 * the allocated entry later (if we no longer need it).
1815 */
1816 pv_e = PV_ENTRY_NULL((pv_entry_t) 0);
1817Retry:
1818 PMAP_READ_LOCK(pmap, spl)((void)(spl));
1819
1820 /*
1821 * Expand pmap to include this pte. Assume that
1822 * pmap is always expanded to include enough hardware
1823 * pages to map one VM page.
1824 */
1825
1826 while ((pte = pmap_pte(pmap, v)) == PT_ENTRY_NULL((pt_entry_t *) 0)) {
1827 /*
1828 * Need to allocate a new page-table page.
1829 */
1830 vm_offset_t ptp;
1831 pt_entry_t *pdp;
1832 int i;
1833
1834 if (pmap == kernel_pmap) {
1835 /*
1836 * Would have to enter the new page-table page in
1837 * EVERY pmap.
1838 */
1839 panic("pmap_expand kernel pmap to %#x", v);
1840 }
1841
1842 /*
1843 * Unlock the pmap and allocate a new page-table page.
1844 */
1845 PMAP_READ_UNLOCK(pmap, spl)((void)(spl));
1846
1847 ptp = phystokv(pmap_page_table_page_alloc())((vm_offset_t)(pmap_page_table_page_alloc()) + 0xC0000000UL);
1848
1849 /*
1850 * Re-lock the pmap and check that another thread has
1851 * not already allocated the page-table page. If it
1852 * has, discard the new page-table page (and try
1853 * again to make sure).
1854 */
1855 PMAP_READ_LOCK(pmap, spl)((void)(spl));
1856
1857 if (pmap_pte(pmap, v) != PT_ENTRY_NULL((pt_entry_t *) 0)) {
1858 /*
1859 * Oops...
1860 */
1861 PMAP_READ_UNLOCK(pmap, spl)((void)(spl));
1862 pmap_page_table_page_dealloc(kvtophys(ptp));
1863 PMAP_READ_LOCK(pmap, spl)((void)(spl));
1864 continue;
1865 }
1866
1867 /*
1868 * Enter the new page table page in the page directory.
1869 */
1870 i = ptes_per_vm_page1;
1871 /*XX pdp = &pmap->dirbase[pdenum(v) & ~(i-1)];*/
1872 pdp = pmap_pde(pmap, v);
1873 do {
1874#ifdef MACH_PV_PAGETABLES
1875 pmap_set_page_readonly((void *) ptp);
1876 if (!hyp_mmuext_op_mfn (MMUEXT_PIN_L1_TABLE, kv_to_mfn(ptp)))
1877 panic("couldn't pin page %p(%p)\n",ptp,(vm_offset_t) kv_to_ma(ptp));
1878 if (!hyp_mmu_update_pte(pa_to_ma(kvtophys((vm_offset_t)pdp)),
1879 pa_to_pte(pa_to_ma(kvtophys(ptp)))((pa_to_ma(kvtophys(ptp))) & 0xfffff000) | INTEL_PTE_VALID0x00000001
1880 | INTEL_PTE_USER0x00000004
1881 | INTEL_PTE_WRITE0x00000002))
1882 panic("%s:%d could not set pde %p(%p,%p) to %p(%p,%p) %p\n",__FILE__"../i386/intel/pmap.c",__LINE__1882, pdp, kvtophys((vm_offset_t)pdp), (vm_offset_t) pa_to_ma(kvtophys((vm_offset_t)pdp)), ptp, kvtophys(ptp), (vm_offset_t) pa_to_ma(kvtophys(ptp)), (vm_offset_t) pa_to_pte(kv_to_ma(ptp))((kv_to_ma(ptp)) & 0xfffff000));
1883#else /* MACH_PV_PAGETABLES */
1884 *pdp = pa_to_pte(kvtophys(ptp))((kvtophys(ptp)) & 0xfffff000) | INTEL_PTE_VALID0x00000001
1885 | INTEL_PTE_USER0x00000004
1886 | INTEL_PTE_WRITE0x00000002;
1887#endif /* MACH_PV_PAGETABLES */
1888 pdp++;
1889 ptp += INTEL_PGBYTES4096;
1890 } while (--i > 0);
1891
1892 /*
1893 * Now, get the address of the page-table entry.
1894 */
1895 continue;
1896 }
1897
1898 /*
1899 * Special case if the physical page is already mapped
1900 * at this address.
1901 */
1902 old_pa = pte_to_pa(*pte)((*pte) & 0xfffff000);
1903 if (*pte && old_pa == pa) {
1904 /*
1905 * May be changing its wired attribute or protection
1906 */
1907
1908 if (wired && !(*pte & INTEL_PTE_WIRED0x00000200))
1909 pmap->stats.wired_count++;
1910 else if (!wired && (*pte & INTEL_PTE_WIRED0x00000200))
1911 pmap->stats.wired_count--;
1912
1913 template = pa_to_pte(pa)((pa) & 0xfffff000) | INTEL_PTE_VALID0x00000001;
1914 if (pmap != kernel_pmap)
1915 template |= INTEL_PTE_USER0x00000004;
1916 if (prot & VM_PROT_WRITE((vm_prot_t) 0x02))
1917 template |= INTEL_PTE_WRITE0x00000002;
1918 if (machine_slot[cpu_number()(0)].cpu_type >= CPU_TYPE_I486((cpu_type_t) 17)
1919 && pa >= phys_last_addr)
1920 template |= INTEL_PTE_NCACHE0x00000010|INTEL_PTE_WTHRU0x00000008;
1921 if (wired)
1922 template |= INTEL_PTE_WIRED0x00000200;
1923 i = ptes_per_vm_page1;
1924 do {
1925 if (*pte & INTEL_PTE_MOD0x00000040)
1926 template |= INTEL_PTE_MOD0x00000040;
1927#ifdef MACH_PV_PAGETABLES
1928 if (!hyp_mmu_update_pte(kv_to_ma(pte), pa_to_ma(template)))
1929 panic("%s:%d could not set pte %p to %p\n",__FILE__"../i386/intel/pmap.c",__LINE__1929,pte,template);
1930#else /* MACH_PV_PAGETABLES */
1931 WRITE_PTE(pte, template)*(pte) = (template);
1932#endif /* MACH_PV_PAGETABLES */
1933 pte++;
1934 pte_increment_pa(template)((template) += 0xfff +1);
1935 } while (--i > 0);
1936 PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE){ if ((pmap)->cpus_using) { { (void) ((pmap)); (void) ((v)
); (void) ((v + (1 << 12))); ({ register unsigned long _temp__
= (({ register unsigned long _temp__; asm volatile("mov %%cr3, %0"
: "=r" (_temp__)); _temp__; })); asm volatile("mov %0, %%cr3"
: : "r" (_temp__) : "memory"); }); }; } }
;
1937 }
1938 else {
1939
1940 /*
1941 * Remove old mapping from the PV list if necessary.
1942 */
1943 if (*pte) {
1944 /*
1945 * Don't free the pte page if removing last
1946 * mapping - we will immediately replace it.
1947 */
1948 pmap_remove_range(pmap, v, pte,
1949 pte + ptes_per_vm_page1);
1950 PMAP_UPDATE_TLBS(pmap, v, v + PAGE_SIZE){ if ((pmap)->cpus_using) { { (void) ((pmap)); (void) ((v)
); (void) ((v + (1 << 12))); ({ register unsigned long _temp__
= (({ register unsigned long _temp__; asm volatile("mov %%cr3, %0"
: "=r" (_temp__)); _temp__; })); asm volatile("mov %0, %%cr3"
: : "r" (_temp__) : "memory"); }); }; } }
;
1951 }
1952
1953 if (valid_page(pa)(pmap_initialized && pmap_valid_page(pa))) {
1954
1955 /*
1956 * Enter the mapping in the PV list for this
1957 * physical page.
1958 */
1959
1960 pai = pa_index(pa)((((vm_size_t)(pa - phys_first_addr)) >> 12));
1961 LOCK_PVH(pai);
1962 pv_h = pai_to_pvh(pai)(&pv_head_table[pai]);
1963
1964 if (pv_h->pmap == PMAP_NULL((pmap_t) 0)) {
1965 /*
1966 * No mappings yet
1967 */
1968 pv_h->va = v;
1969 pv_h->pmap = pmap;
1970 pv_h->next = PV_ENTRY_NULL((pv_entry_t) 0);
1971 }
1972 else {
1973#if DEBUG
1974 {
1975 /* check that this mapping is not already there */
1976 pv_entry_t e = pv_h;
1977 while (e != PV_ENTRY_NULL((pv_entry_t) 0)) {
1978 if (e->pmap == pmap && e->va == v)
1979 panic("pmap_enter: already in pv_list");
1980 e = e->next;
1981 }
1982 }
1983#endif /* DEBUG */
1984
1985 /*
1986 * Add new pv_entry after header.
1987 */
1988 if (pv_e == PV_ENTRY_NULL((pv_entry_t) 0)) {
1989 PV_ALLOC(pv_e){ ; if ((pv_e = pv_free_list) != 0) { pv_free_list = pv_e->
next; } ; }
;
1990 if (pv_e == PV_ENTRY_NULL((pv_entry_t) 0)) {
1991 UNLOCK_PVH(pai);
1992 PMAP_READ_UNLOCK(pmap, spl)((void)(spl));
1993
1994 /*
1995 * Refill from cache.
1996 */
1997 pv_e = (pv_entry_t) kmem_cache_alloc(&pv_list_cache);
1998 goto Retry;
1999 }
2000 }
2001 pv_e->va = v;
2002 pv_e->pmap = pmap;
2003 pv_e->next = pv_h->next;
2004 pv_h->next = pv_e;
2005 /*
2006 * Remember that we used the pvlist entry.
2007 */
2008 pv_e = PV_ENTRY_NULL((pv_entry_t) 0);
2009 }
2010 UNLOCK_PVH(pai);
2011 }
2012
2013 /*
2014 * And count the mapping.
2015 */
2016
2017 pmap->stats.resident_count++;
2018 if (wired)
2019 pmap->stats.wired_count++;
2020
2021 /*
2022 * Build a template to speed up entering -
2023 * only the pfn changes.
2024 */
2025 template = pa_to_pte(pa)((pa) & 0xfffff000) | INTEL_PTE_VALID0x00000001;
2026 if (pmap != kernel_pmap)
2027 template |= INTEL_PTE_USER0x00000004;
2028 if (prot & VM_PROT_WRITE((vm_prot_t) 0x02))
2029 template |= INTEL_PTE_WRITE0x00000002;
2030 if (machine_slot[cpu_number()(0)].cpu_type >= CPU_TYPE_I486((cpu_type_t) 17)
2031 && pa >= phys_last_addr)
2032 template |= INTEL_PTE_NCACHE0x00000010|INTEL_PTE_WTHRU0x00000008;
2033 if (wired)
2034 template |= INTEL_PTE_WIRED0x00000200;
2035 i = ptes_per_vm_page1;
2036 do {
2037#ifdef MACH_PV_PAGETABLES
2038 if (!(hyp_mmu_update_pte(kv_to_ma(pte), pa_to_ma(template))))
2039 panic("%s:%d could not set pte %p to %p\n",__FILE__"../i386/intel/pmap.c",__LINE__2039,pte,template);
2040#else /* MACH_PV_PAGETABLES */
2041 WRITE_PTE(pte, template)*(pte) = (template);
2042#endif /* MACH_PV_PAGETABLES */
2043 pte++;
2044 pte_increment_pa(template)((template) += 0xfff +1);
2045 } while (--i > 0);
2046 }
2047
2048 if (pv_e != PV_ENTRY_NULL((pv_entry_t) 0)) {
2049 PV_FREE(pv_e){ ; pv_e->next = pv_free_list; pv_free_list = pv_e; ; };
2050 }
2051
2052 PMAP_READ_UNLOCK(pmap, spl)((void)(spl));
2053}
2054
2055/*
2056 * Routine: pmap_change_wiring
2057 * Function: Change the wiring attribute for a map/virtual-address
2058 * pair.
2059 * In/out conditions:
2060 * The mapping must already exist in the pmap.
2061 */
2062void pmap_change_wiring(map, v, wired)
2063 register pmap_t map;
2064 vm_offset_t v;
2065 boolean_t wired;
2066{
2067 register pt_entry_t *pte;
2068 register int i;
2069 int spl;
2070
2071 /*
2072 * We must grab the pmap system lock because we may
2073 * change a pte_page queue.
2074 */
2075 PMAP_READ_LOCK(map, spl)((void)(spl));
2076
2077 if ((pte = pmap_pte(map, v)) == PT_ENTRY_NULL((pt_entry_t *) 0))
2078 panic("pmap_change_wiring: pte missing");
2079
2080 if (wired && !(*pte & INTEL_PTE_WIRED0x00000200)) {
2081 /*
2082 * wiring down mapping
2083 */
2084 map->stats.wired_count++;
2085 i = ptes_per_vm_page1;
2086 do {
2087 *pte++ |= INTEL_PTE_WIRED0x00000200;
2088 } while (--i > 0);
2089 }
2090 else if (!wired && (*pte & INTEL_PTE_WIRED0x00000200)) {
2091 /*
2092 * unwiring mapping
2093 */
2094 map->stats.wired_count--;
2095 i = ptes_per_vm_page1;
2096 do {
2097#ifdef MACH_PV_PAGETABLES
2098 if (!(hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~INTEL_PTE_WIRED0x00000200)))
2099 panic("%s:%d could not wire down pte %p\n",__FILE__"../i386/intel/pmap.c",__LINE__2099,pte);
2100#else /* MACH_PV_PAGETABLES */
2101 *pte &= ~INTEL_PTE_WIRED0x00000200;
2102#endif /* MACH_PV_PAGETABLES */
2103 pte++;
2104 } while (--i > 0);
2105 }
2106
2107 PMAP_READ_UNLOCK(map, spl)((void)(spl));
2108}
2109
2110/*
2111 * Routine: pmap_extract
2112 * Function:
2113 * Extract the physical page address associated
2114 * with the given map/virtual_address pair.
2115 */
2116
2117vm_offset_t pmap_extract(pmap, va)
2118 register pmap_t pmap;
2119 vm_offset_t va;
2120{
2121 register pt_entry_t *pte;
2122 register vm_offset_t pa;
2123 int spl;
2124
2125 SPLVM(spl)((void)(spl));
2126 simple_lock(&pmap->lock);
2127 if ((pte = pmap_pte(pmap, va)) == PT_ENTRY_NULL((pt_entry_t *) 0))
2128 pa = (vm_offset_t) 0;
2129 else if (!(*pte & INTEL_PTE_VALID0x00000001))
2130 pa = (vm_offset_t) 0;
2131 else
2132 pa = pte_to_pa(*pte)((*pte) & 0xfffff000) + (va & INTEL_OFFMASK0xfff);
2133 simple_unlock(&pmap->lock);
2134 SPLX(spl)((void)(spl));
2135 return(pa);
2136}
2137
2138/*
2139 * Copy the range specified by src_addr/len
2140 * from the source map to the range dst_addr/len
2141 * in the destination map.
2142 *
2143 * This routine is only advisory and need not do anything.
2144 */
2145#if 0
2146void pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr)
2147 pmap_t dst_pmap;
2148 pmap_t src_pmap;
2149 vm_offset_t dst_addr;
2150 vm_size_t len;
2151 vm_offset_t src_addr;
2152{
2153#ifdef lint
2154 dst_pmap++; src_pmap++; dst_addr++; len++; src_addr++;
2155#endif /* lint */
2156}
2157#endif /* 0 */
2158
2159/*
2160 * Routine: pmap_collect
2161 * Function:
2162 * Garbage collects the physical map system for
2163 * pages which are no longer used.
2164 * Success need not be guaranteed -- that is, there
2165 * may well be pages which are not referenced, but
2166 * others may be collected.
2167 * Usage:
2168 * Called by the pageout daemon when pages are scarce.
2169 */
2170void pmap_collect(p)
2171 pmap_t p;
2172{
2173 register pt_entry_t *pdp, *ptp;
2174 pt_entry_t *eptp;
2175 vm_offset_t pa;
2176 int spl, wired;
2177
2178 if (p == PMAP_NULL((pmap_t) 0))
2179 return;
2180
2181 if (p == kernel_pmap)
2182 return;
2183
2184 /*
2185 * Garbage collect map.
2186 */
2187 PMAP_READ_LOCK(p, spl)((void)(spl));
2188 for (pdp = p->dirbase;
2189 pdp < &p->dirbase[lin2pdenum(LINEAR_MIN_KERNEL_ADDRESS)(((((0xc0000000UL))) >> 22) & 0x3ff)];
2190 pdp += ptes_per_vm_page1)
2191 {
2192 if (*pdp & INTEL_PTE_VALID0x00000001) {
2193
2194 pa = pte_to_pa(*pdp)((*pdp) & 0xfffff000);
2195 ptp = (pt_entry_t *)phystokv(pa)((vm_offset_t)(pa) + 0xC0000000UL);
2196 eptp = ptp + NPTES((((unsigned long)(1)) << 12)/sizeof(pt_entry_t))*ptes_per_vm_page1;
2197
2198 /*
2199 * If the pte page has any wired mappings, we cannot
2200 * free it.
2201 */
2202 wired = 0;
2203 {
2204 register pt_entry_t *ptep;
2205 for (ptep = ptp; ptep < eptp; ptep++) {
2206 if (*ptep & INTEL_PTE_WIRED0x00000200) {
2207 wired = 1;
2208 break;
2209 }
2210 }
2211 }
2212 if (!wired) {
2213 /*
2214 * Remove the virtual addresses mapped by this pte page.
2215 */
2216 { /*XXX big hack*/
2217 vm_offset_t va = pdenum2lin(pdp - p->dirbase)((vm_offset_t)(pdp - p->dirbase) << 22);
2218 if (p == kernel_pmap)
2219 va = lintokv(va)((vm_offset_t)(va) - ((0xc0000000UL)) + 0xC0000000UL);
2220 pmap_remove_range(p,
2221 va,
2222 ptp,
2223 eptp);
2224 }
2225
2226 /*
2227 * Invalidate the page directory pointer.
2228 */
2229 {
2230 register int i = ptes_per_vm_page1;
2231 register pt_entry_t *pdep = pdp;
2232 do {
2233#ifdef MACH_PV_PAGETABLES
2234 unsigned long pte = *pdep;
2235 void *ptable = (void*) ptetokv(pte)(((vm_offset_t)(((pte) & 0xfffff000)) + 0xC0000000UL));
2236 if (!(hyp_mmu_update_pte(pa_to_ma(kvtophys((vm_offset_t)pdep++)), 0)))
2237 panic("%s:%d could not clear pde %p\n",__FILE__"../i386/intel/pmap.c",__LINE__2237,pdep-1);
2238 if (!hyp_mmuext_op_mfn (MMUEXT_UNPIN_TABLE, kv_to_mfn(ptable)))
2239 panic("couldn't unpin page %p(%p)\n", ptable, (vm_offset_t) pa_to_ma(kvtophys((vm_offset_t)ptable)));
2240 pmap_set_page_readwrite(ptable);
2241#else /* MACH_PV_PAGETABLES */
2242 *pdep++ = 0;
2243#endif /* MACH_PV_PAGETABLES */
2244 } while (--i > 0);
2245 }
2246
2247 PMAP_READ_UNLOCK(p, spl)((void)(spl));
2248
2249 /*
2250 * And free the pte page itself.
2251 */
2252 {
2253 register vm_page_t m;
2254
2255 vm_object_lock(pmap_object);
2256 m = vm_page_lookup(pmap_object, pa);
2257 if (m == VM_PAGE_NULL((vm_page_t) 0))
2258 panic("pmap_collect: pte page not in object");
2259 vm_page_lock_queues();
2260 vm_page_free(m);
2261 inuse_ptepages_count--;
2262 vm_page_unlock_queues();
2263 vm_object_unlock(pmap_object);
2264 }
2265
2266 PMAP_READ_LOCK(p, spl)((void)(spl));
2267 }
2268 }
2269 }
2270 PMAP_UPDATE_TLBS(p, VM_MIN_ADDRESS, VM_MAX_ADDRESS){ if ((p)->cpus_using) { { (void) ((p)); (void) (((0))); (
void) (((0xc0000000UL))); ({ register unsigned long _temp__ =
(({ register unsigned long _temp__; asm volatile("mov %%cr3, %0"
: "=r" (_temp__)); _temp__; })); asm volatile("mov %0, %%cr3"
: : "r" (_temp__) : "memory"); }); }; } }
;
2271
2272 PMAP_READ_UNLOCK(p, spl)((void)(spl));
2273 return;
2274
2275}
2276
2277/*
2278 * Routine: pmap_activate
2279 * Function:
2280 * Binds the given physical map to the given
2281 * processor, and returns a hardware map description.
2282 */
2283#if 0
2284void pmap_activate(my_pmap, th, my_cpu)
2285 register pmap_t my_pmap;
2286 thread_t th;
2287 int my_cpu;
2288{
2289 PMAP_ACTIVATE(my_pmap, th, my_cpu);
2290}
2291#endif /* 0 */
2292
2293/*
2294 * Routine: pmap_deactivate
2295 * Function:
2296 * Indicates that the given physical map is no longer
2297 * in use on the specified processor. (This is a macro
2298 * in pmap.h)
2299 */
2300#if 0
2301void pmap_deactivate(pmap, th, which_cpu)
2302 pmap_t pmap;
2303 thread_t th;
2304 int which_cpu;
2305{
2306#ifdef lint
2307 pmap++; th++; which_cpu++;
2308#endif /* lint */
2309 PMAP_DEACTIVATE(pmap, th, which_cpu);
2310}
2311#endif /* 0 */
2312
2313/*
2314 * Routine: pmap_kernel
2315 * Function:
2316 * Returns the physical map handle for the kernel.
2317 */
2318#if 0
2319pmap_t pmap_kernel()(kernel_pmap)
2320{
2321 return (kernel_pmap);
2322}
2323#endif /* 0 */
2324
2325/*
2326 * pmap_zero_page zeros the specified (machine independent) page.
2327 * See machine/phys.c or machine/phys.s for implementation.
2328 */
2329#if 0
2330pmap_zero_page(phys)
2331 register vm_offset_t phys;
2332{
2333 register int i;
2334
2335 assert(phys != vm_page_fictitious_addr)({ if (!(phys != vm_page_fictitious_addr)) Assert("phys != vm_page_fictitious_addr"
, "../i386/intel/pmap.c", 2335); })
;
2336 i = PAGE_SIZE(1 << 12) / INTEL_PGBYTES4096;
2337 phys = intel_pfn(phys);
2338
2339 while (i--)
2340 zero_phys(phys++);
2341}
2342#endif /* 0 */
2343
2344/*
2345 * pmap_copy_page copies the specified (machine independent) page.
2346 * See machine/phys.c or machine/phys.s for implementation.
2347 */
2348#if 0
2349pmap_copy_page(src, dst)
2350 vm_offset_t src, dst;
2351{
2352 int i;
2353
2354 assert(src != vm_page_fictitious_addr)({ if (!(src != vm_page_fictitious_addr)) Assert("src != vm_page_fictitious_addr"
, "../i386/intel/pmap.c", 2354); })
;
2355 assert(dst != vm_page_fictitious_addr)({ if (!(dst != vm_page_fictitious_addr)) Assert("dst != vm_page_fictitious_addr"
, "../i386/intel/pmap.c", 2355); })
;
2356 i = PAGE_SIZE(1 << 12) / INTEL_PGBYTES4096;
2357
2358 while (i--) {
2359 copy_phys(intel_pfn(src), intel_pfn(dst));
2360 src += INTEL_PGBYTES4096;
2361 dst += INTEL_PGBYTES4096;
2362 }
2363}
2364#endif /* 0 */
2365
2366/*
2367 * Routine: pmap_pageable
2368 * Function:
2369 * Make the specified pages (by pmap, offset)
2370 * pageable (or not) as requested.
2371 *
2372 * A page which is not pageable may not take
2373 * a fault; therefore, its page table entry
2374 * must remain valid for the duration.
2375 *
2376 * This routine is merely advisory; pmap_enter
2377 * will specify that these pages are to be wired
2378 * down (or not) as appropriate.
2379 */
2380void
2381pmap_pageable(pmap, start, end, pageable)
2382 pmap_t pmap;
2383 vm_offset_t start;
2384 vm_offset_t end;
2385 boolean_t pageable;
2386{
2387#ifdef lint
2388 pmap++; start++; end++; pageable++;
2389#endif /* lint */
2390}
2391
2392/*
2393 * Clear specified attribute bits.
2394 */
2395void
2396phys_attribute_clear(phys, bits)
2397 vm_offset_t phys;
2398 int bits;
2399{
2400 pv_entry_t pv_h;
2401 register pv_entry_t pv_e;
2402 register pt_entry_t *pte;
2403 int pai;
2404 register pmap_t pmap;
2405 int spl;
2406
2407 assert(phys != vm_page_fictitious_addr)({ if (!(phys != vm_page_fictitious_addr)) Assert("phys != vm_page_fictitious_addr"
, "../i386/intel/pmap.c", 2407); })
;
2408 if (!valid_page(phys)(pmap_initialized && pmap_valid_page(phys))) {
2409 /*
2410 * Not a managed page.
2411 */
2412 return;
2413 }
2414
2415 /*
2416 * Lock the pmap system first, since we will be changing
2417 * several pmaps.
2418 */
2419
2420 PMAP_WRITE_LOCK(spl)((void)(spl));
2421
2422 pai = pa_index(phys)((((vm_size_t)(phys - phys_first_addr)) >> 12));
2423 pv_h = pai_to_pvh(pai)(&pv_head_table[pai]);
2424
2425 /*
2426 * Walk down PV list, clearing all modify or reference bits.
2427 * We do not have to lock the pv_list because we have
2428 * the entire pmap system locked.
2429 */
2430 if (pv_h->pmap != PMAP_NULL((pmap_t) 0)) {
2431 /*
2432 * There are some mappings.
2433 */
2434 for (pv_e = pv_h; pv_e != PV_ENTRY_NULL((pv_entry_t) 0); pv_e = pv_e->next) {
2435 register vm_offset_t va;
2436
2437 pmap = pv_e->pmap;
2438 /*
2439 * Lock the pmap to block pmap_extract and similar routines.
2440 */
2441 simple_lock(&pmap->lock);
2442
2443 va = pv_e->va;
2444 pte = pmap_pte(pmap, va);
2445
2446#if 0
2447 /*
2448 * Consistency checks.
2449 */
2450 assert(*pte & INTEL_PTE_VALID)({ if (!(*pte & 0x00000001)) Assert("*pte & INTEL_PTE_VALID"
, "../i386/intel/pmap.c", 2450); })
;
2451 /* assert(pte_to_phys(*pte) == phys); */
2452#endif
2453
2454 /*
2455 * Clear modify or reference bits.
2456 */
2457 {
2458 register int i = ptes_per_vm_page1;
2459 do {
2460#ifdef MACH_PV_PAGETABLES
2461 if (!(hyp_mmu_update_pte(kv_to_ma(pte), *pte & ~bits)))
2462 panic("%s:%d could not clear bits %lx from pte %p\n",__FILE__"../i386/intel/pmap.c",__LINE__2462,bits,pte);
2463#else /* MACH_PV_PAGETABLES */
2464 *pte &= ~bits;
2465#endif /* MACH_PV_PAGETABLES */
2466 } while (--i > 0);
2467 }
2468 PMAP_UPDATE_TLBS(pmap, va, va + PAGE_SIZE){ if ((pmap)->cpus_using) { { (void) ((pmap)); (void) ((va
)); (void) ((va + (1 << 12))); ({ register unsigned long
_temp__ = (({ register unsigned long _temp__; asm volatile("mov %%cr3, %0"
: "=r" (_temp__)); _temp__; })); asm volatile("mov %0, %%cr3"
: : "r" (_temp__) : "memory"); }); }; } }
;
2469 simple_unlock(&pmap->lock);
2470 }
2471 }
2472
2473 pmap_phys_attributes[pai] &= ~bits;
2474
2475 PMAP_WRITE_UNLOCK(spl)((void)(spl));
2476}
2477
2478/*
2479 * Check specified attribute bits.
2480 */
2481boolean_t
2482phys_attribute_test(phys, bits)
2483 vm_offset_t phys;
2484 int bits;
2485{
2486 pv_entry_t pv_h;
2487 register pv_entry_t pv_e;
2488 register pt_entry_t *pte;
2489 int pai;
2490 register pmap_t pmap;
2491 int spl;
2492
2493 assert(phys != vm_page_fictitious_addr)({ if (!(phys != vm_page_fictitious_addr)) Assert("phys != vm_page_fictitious_addr"
, "../i386/intel/pmap.c", 2493); })
;
2494 if (!valid_page(phys)(pmap_initialized && pmap_valid_page(phys))) {
2495 /*
2496 * Not a managed page.
2497 */
2498 return (FALSE((boolean_t) 0));
2499 }
2500
2501 /*
2502 * Lock the pmap system first, since we will be checking
2503 * several pmaps.
2504 */
2505
2506 PMAP_WRITE_LOCK(spl)((void)(spl));
2507
2508 pai = pa_index(phys)((((vm_size_t)(phys - phys_first_addr)) >> 12));
2509 pv_h = pai_to_pvh(pai)(&pv_head_table[pai]);
2510
2511 if (pmap_phys_attributes[pai] & bits) {
2512 PMAP_WRITE_UNLOCK(spl)((void)(spl));
2513 return (TRUE((boolean_t) 1));
2514 }
2515
2516 /*
2517 * Walk down PV list, checking all mappings.
2518 * We do not have to lock the pv_list because we have
2519 * the entire pmap system locked.
2520 */
2521 if (pv_h->pmap != PMAP_NULL((pmap_t) 0)) {
2522 /*
2523 * There are some mappings.
2524 */
2525 for (pv_e = pv_h; pv_e != PV_ENTRY_NULL((pv_entry_t) 0); pv_e = pv_e->next) {
2526
2527 pmap = pv_e->pmap;
2528 /*
2529 * Lock the pmap to block pmap_extract and similar routines.
2530 */
2531 simple_lock(&pmap->lock);
2532
2533 {
2534 register vm_offset_t va;
2535
2536 va = pv_e->va;
2537 pte = pmap_pte(pmap, va);
2538
2539#if 0
2540 /*
2541 * Consistency checks.
2542 */
2543 assert(*pte & INTEL_PTE_VALID)({ if (!(*pte & 0x00000001)) Assert("*pte & INTEL_PTE_VALID"
, "../i386/intel/pmap.c", 2543); })
;
2544 /* assert(pte_to_phys(*pte) == phys); */
2545#endif
2546 }
2547
2548 /*
2549 * Check modify or reference bits.
2550 */
2551 {
2552 register int i = ptes_per_vm_page1;
2553
2554 do {
2555 if (*pte & bits) {
2556 simple_unlock(&pmap->lock);
2557 PMAP_WRITE_UNLOCK(spl)((void)(spl));
2558 return (TRUE((boolean_t) 1));
2559 }
2560 } while (--i > 0);
2561 }
2562 simple_unlock(&pmap->lock);
2563 }
2564 }
2565 PMAP_WRITE_UNLOCK(spl)((void)(spl));
2566 return (FALSE((boolean_t) 0));
2567}
2568
2569/*
2570 * Clear the modify bits on the specified physical page.
2571 */
2572
2573void pmap_clear_modify(phys)
2574 register vm_offset_t phys;
2575{
2576 phys_attribute_clear(phys, PHYS_MODIFIED0x00000040);
2577}
2578
2579/*
2580 * pmap_is_modified:
2581 *
2582 * Return whether or not the specified physical page is modified
2583 * by any physical maps.
2584 */
2585
2586boolean_t pmap_is_modified(phys)
2587 register vm_offset_t phys;
2588{
2589 return (phys_attribute_test(phys, PHYS_MODIFIED0x00000040));
2590}
2591
2592/*
2593 * pmap_clear_reference:
2594 *
2595 * Clear the reference bit on the specified physical page.
2596 */
2597
2598void pmap_clear_reference(phys)
2599 vm_offset_t phys;
2600{
2601 phys_attribute_clear(phys, PHYS_REFERENCED0x00000020);
2602}
2603
2604/*
2605 * pmap_is_referenced:
2606 *
2607 * Return whether or not the specified physical page is referenced
2608 * by any physical maps.
2609 */
2610
2611boolean_t pmap_is_referenced(phys)
2612 vm_offset_t phys;
2613{
2614 return (phys_attribute_test(phys, PHYS_REFERENCED0x00000020));
2615}
2616
2617#if NCPUS1 > 1
2618/*
2619* TLB Coherence Code (TLB "shootdown" code)
2620*
2621* Threads that belong to the same task share the same address space and
2622* hence share a pmap. However, they may run on distinct cpus and thus
2623* have distinct TLBs that cache page table entries. In order to guarantee
2624* the TLBs are consistent, whenever a pmap is changed, all threads that
2625* are active in that pmap must have their TLB updated. To keep track of
2626* this information, the set of cpus that are currently using a pmap is
2627* maintained within each pmap structure (cpus_using). Pmap_activate() and
2628* pmap_deactivate add and remove, respectively, a cpu from this set.
2629* Since the TLBs are not addressable over the bus, each processor must
2630* flush its own TLB; a processor that needs to invalidate another TLB
2631* needs to interrupt the processor that owns that TLB to signal the
2632* update.
2633*
2634* Whenever a pmap is updated, the lock on that pmap is locked, and all
2635* cpus using the pmap are signaled to invalidate. All threads that need
2636* to activate a pmap must wait for the lock to clear to await any updates
2637* in progress before using the pmap. They must ACQUIRE the lock to add
2638* their cpu to the cpus_using set. An implicit assumption made
2639* throughout the TLB code is that all kernel code that runs at or higher
2640* than splvm blocks out update interrupts, and that such code does not
2641* touch pageable pages.
2642*
2643* A shootdown interrupt serves another function besides signaling a
2644* processor to invalidate. The interrupt routine (pmap_update_interrupt)
2645* waits for the both the pmap lock (and the kernel pmap lock) to clear,
2646* preventing user code from making implicit pmap updates while the
2647* sending processor is performing its update. (This could happen via a
2648* user data write reference that turns on the modify bit in the page
2649* table). It must wait for any kernel updates that may have started
2650* concurrently with a user pmap update because the IPC code
2651* changes mappings.
2652* Spinning on the VALUES of the locks is sufficient (rather than
2653* having to acquire the locks) because any updates that occur subsequent
2654* to finding the lock unlocked will be signaled via another interrupt.
2655* (This assumes the interrupt is cleared before the low level interrupt code
2656* calls pmap_update_interrupt()).
2657*
2658* The signaling processor must wait for any implicit updates in progress
2659* to terminate before continuing with its update. Thus it must wait for an
2660* acknowledgement of the interrupt from each processor for which such
2661* references could be made. For maintaining this information, a set
2662* cpus_active is used. A cpu is in this set if and only if it can
2663* use a pmap. When pmap_update_interrupt() is entered, a cpu is removed from
2664* this set; when all such cpus are removed, it is safe to update.
2665*
2666* Before attempting to acquire the update lock on a pmap, a cpu (A) must
2667* be at least at the priority of the interprocessor interrupt
2668* (splip<=splvm). Otherwise, A could grab a lock and be interrupted by a
2669* kernel update; it would spin forever in pmap_update_interrupt() trying
2670* to acquire the user pmap lock it had already acquired. Furthermore A
2671* must remove itself from cpus_active. Otherwise, another cpu holding
2672* the lock (B) could be in the process of sending an update signal to A,
2673* and thus be waiting for A to remove itself from cpus_active. If A is
2674* spinning on the lock at priority this will never happen and a deadlock
2675* will result.
2676*/
2677
2678/*
2679 * Signal another CPU that it must flush its TLB
2680 */
2681void signal_cpus(use_list, pmap, start, end)
2682 cpu_set use_list;
2683 pmap_t pmap;
2684 vm_offset_t start, end;
2685{
2686 register int which_cpu, j;
2687 register pmap_update_list_t update_list_p;
2688
2689 while ((which_cpu = ffs(use_list)) != 0) {
2690 which_cpu -= 1; /* convert to 0 origin */
2691
2692 update_list_p = &cpu_update_list[which_cpu];
2693 simple_lock(&update_list_p->lock);
2694
2695 j = update_list_p->count;
2696 if (j >= UPDATE_LIST_SIZE) {
2697 /*
2698 * list overflowed. Change last item to
2699 * indicate overflow.
2700 */
2701 update_list_p->item[UPDATE_LIST_SIZE-1].pmap = kernel_pmap;
2702 update_list_p->item[UPDATE_LIST_SIZE-1].start = VM_MIN_ADDRESS(0);
2703 update_list_p->item[UPDATE_LIST_SIZE-1].end = VM_MAX_KERNEL_ADDRESS((0xffffffffUL) - ((0xc0000000UL)) + 0xC0000000UL);
2704 }
2705 else {
2706 update_list_p->item[j].pmap = pmap;
2707 update_list_p->item[j].start = start;
2708 update_list_p->item[j].end = end;
2709 update_list_p->count = j+1;
2710 }
2711 cpu_update_needed[which_cpu] = TRUE((boolean_t) 1);
2712 simple_unlock(&update_list_p->lock);
2713
2714 if ((cpus_idle & (1 << which_cpu)) == 0)
2715 interrupt_processor(which_cpu);
2716 use_list &= ~(1 << which_cpu);
2717 }
2718}
2719
2720void process_pmap_updates(my_pmap)
2721 register pmap_t my_pmap;
2722{
2723 register int my_cpu = cpu_number()(0);
2724 register pmap_update_list_t update_list_p;
2725 register int j;
2726 register pmap_t pmap;
2727
2728 update_list_p = &cpu_update_list[my_cpu];
2729 simple_lock(&update_list_p->lock);
2730
2731 for (j = 0; j < update_list_p->count; j++) {
2732 pmap = update_list_p->item[j].pmap;
2733 if (pmap == my_pmap ||
2734 pmap == kernel_pmap) {
2735
2736 INVALIDATE_TLB(pmap,{ (void) (pmap); (void) (update_list_p->item[j].start); (void
) (update_list_p->item[j].end); ({ register unsigned long _temp__
= (({ register unsigned long _temp__; asm volatile("mov %%cr3, %0"
: "=r" (_temp__)); _temp__; })); asm volatile("mov %0, %%cr3"
: : "r" (_temp__) : "memory"); }); }
2737 update_list_p->item[j].start,{ (void) (pmap); (void) (update_list_p->item[j].start); (void
) (update_list_p->item[j].end); ({ register unsigned long _temp__
= (({ register unsigned long _temp__; asm volatile("mov %%cr3, %0"
: "=r" (_temp__)); _temp__; })); asm volatile("mov %0, %%cr3"
: : "r" (_temp__) : "memory"); }); }
2738 update_list_p->item[j].end){ (void) (pmap); (void) (update_list_p->item[j].start); (void
) (update_list_p->item[j].end); ({ register unsigned long _temp__
= (({ register unsigned long _temp__; asm volatile("mov %%cr3, %0"
: "=r" (_temp__)); _temp__; })); asm volatile("mov %0, %%cr3"
: : "r" (_temp__) : "memory"); }); }
;
2739 }
2740 }
2741 update_list_p->count = 0;
2742 cpu_update_needed[my_cpu] = FALSE((boolean_t) 0);
2743 simple_unlock(&update_list_p->lock);
2744}
2745
2746/*
2747 * Interrupt routine for TBIA requested from other processor.
2748 */
2749void pmap_update_interrupt(void)
2750{
2751 register int my_cpu;
2752 register pmap_t my_pmap;
2753 int s;
2754
2755 my_cpu = cpu_number()(0);
2756
2757 /*
2758 * Exit now if we're idle. We'll pick up the update request
2759 * when we go active, and we must not put ourselves back in
2760 * the active set because we'll never process the interrupt
2761 * while we're idle (thus hanging the system).
2762 */
2763 if (cpus_idle & (1 << my_cpu))
2764 return;
2765
2766 if (current_thread()(active_threads[(0)]) == THREAD_NULL((thread_t) 0))
2767 my_pmap = kernel_pmap;
2768 else {
2769 my_pmap = current_pmap()((((active_threads[(0)])->task->map)->pmap));
2770 if (!pmap_in_use(my_pmap, my_cpu)(((my_pmap)->cpus_using & (1 << (my_cpu))) != 0))
2771 my_pmap = kernel_pmap;
2772 }
2773
2774 /*
2775 * Raise spl to splvm (above splip) to block out pmap_extract
2776 * from IO code (which would put this cpu back in the active
2777 * set).
2778 */
2779 s = splvm();
2780
2781 do {
2782
2783 /*
2784 * Indicate that we're not using either user or kernel
2785 * pmap.
2786 */
2787 i_bit_clear(my_cpu, &cpus_active);
2788
2789 /*
2790 * Wait for any pmap updates in progress, on either user
2791 * or kernel pmap.
2792 */
2793 while (*(volatile int *)&my_pmap->lock.lock_data ||
2794 *(volatile int *)&kernel_pmap->lock.lock_data)
2795 continue;
2796
2797 process_pmap_updates(my_pmap);
2798
2799 i_bit_set(my_cpu, &cpus_active);
2800
2801 } while (cpu_update_needed[my_cpu]);
2802
2803 splx(s);
2804}
2805#else /* NCPUS > 1 */
2806/*
2807 * Dummy routine to satisfy external reference.
2808 */
2809void pmap_update_interrupt()
2810{
2811 /* should never be called. */
2812}
2813#endif /* NCPUS > 1 */
2814
2815#if defined(__i386__1)
2816/* Unmap page 0 to trap NULL references. */
2817void
2818pmap_unmap_page_zero ()
2819{
2820 int *pte;
2821
2822 pte = (int *) pmap_pte (kernel_pmap, 0);
2823 if (!pte)
2824 return;
2825 assert (pte)({ if (!(pte)) Assert("pte", "../i386/intel/pmap.c", 2825); }
)
;
2826#ifdef MACH_PV_PAGETABLES
2827 if (!hyp_mmu_update_pte(kv_to_ma(pte), 0))
2828 printf("couldn't unmap page 0\n");
2829#else /* MACH_PV_PAGETABLES */
2830 *pte = 0;
2831 INVALIDATE_TLB(kernel_pmap, 0, PAGE_SIZE){ (void) (kernel_pmap); (void) (0); (void) ((1 << 12));
({ register unsigned long _temp__ = (({ register unsigned long
_temp__; asm volatile("mov %%cr3, %0" : "=r" (_temp__)); _temp__
; })); asm volatile("mov %0, %%cr3" : : "r" (_temp__) : "memory"
); }); }
;
2832#endif /* MACH_PV_PAGETABLES */
2833}
2834#endif /* __i386__ */