26 files changed, 193 insertions, 218 deletions

diff --git a/device/dev_lookup.c b/device/dev_lookup.c
index 297dcde..f6fc0d4 100644
--- a/device/dev_lookup.c
+++ b/device/dev_lookup.c
@@ -365,6 +365,8 @@ dev_lookup_init(void)
 	for (i = 0; i < NDEVHASH; i++)
 	    queue_init(&dev_number_hash_table[i]);
 
-	kmem_cache_init(&dev_hdr_cache, "mach_device",
-			sizeof(struct mach_device), 0, NULL, NULL, NULL, 0);
+	kmem_cache_init (&dev_hdr_cache,
+                 "mach_device",
+                 sizeof(struct mach_device), 0,
+                 NULL, 0);
 }
diff --git a/device/dev_pager.c b/device/dev_pager.c
index 815473a..0ffcdf5 100644
--- a/device/dev_pager.c
+++ b/device/dev_pager.c
@@ -172,8 +172,10 @@ void dev_pager_hash_init(void)
 	vm_size_t	size;
 
 	size = sizeof(struct dev_pager_entry);
-	kmem_cache_init(&dev_pager_hash_cache, "dev_pager_entry", size, 0,
-			NULL, NULL, NULL, 0);
+	kmem_cache_init (&dev_pager_hash_cache,
+                 "dev_pager_entry",
+                 size, 0,
+                 NULL, 0);
 	for (i = 0; i < DEV_PAGER_HASH_COUNT; i++)
 	    queue_init(&dev_pager_hashtable[i]);
 	simple_lock_init(&dev_pager_hash_lock);
@@ -704,8 +706,10 @@ void device_pager_init(void)
 	 * Initialize cache of paging structures.
 	 */
 	size = sizeof(struct dev_pager);
-	kmem_cache_init(&dev_pager_cache, "dev_pager", size, 0,
-			NULL, NULL, NULL, 0);
+	kmem_cache_init (&dev_pager_cache,
+                 "dev_pager",
+                 size, 0,
+                 NULL, 0);
 
 	/*
 	 *	Initialize the name port hashing stuff.
diff --git a/device/ds_routines.c b/device/ds_routines.c
index 33cfd89..e3502f7 100644
--- a/device/ds_routines.c
+++ b/device/ds_routines.c
@@ -1553,8 +1553,10 @@ void mach_device_init(void)
 	 */
 	device_io_map->wait_for_space = TRUE;
 
-	kmem_cache_init(&io_inband_cache, "io_buf_ptr_inband",
-			sizeof(io_buf_ptr_inband_t), 0, NULL, NULL, NULL, 0);
+	kmem_cache_init (&io_inband_cache,
+                 "io_buf_ptr_inband",
+                 sizeof(io_buf_ptr_inband_t), 0,
+                 NULL, 0);
 
 	mach_device_trap_init();
 }
@@ -1597,8 +1599,10 @@ struct kmem_cache io_trap_cache;
 static void
 mach_device_trap_init(void)
 {
-	kmem_cache_init(&io_trap_cache, "io_req", IOTRAP_REQSIZE, 0,
-			NULL, NULL, NULL, 0);
+	kmem_cache_init (&io_trap_cache,
+                 "io_req",
+                 IOTRAP_REQSIZE, 0,
+                 NULL, 0);
 }
 
 /*
diff --git a/device/net_io.c b/device/net_io.c
index 12a1e9c..67d849a 100644
--- a/device/net_io.c
+++ b/device/net_io.c
@@ -1494,12 +1494,16 @@ net_io_init(void)
 	vm_size_t		size;
 
 	size = sizeof(struct net_rcv_port);
-	kmem_cache_init(&net_rcv_cache, "net_rcv_port", size, 0,
-			NULL, NULL, NULL, 0);
+	kmem_cache_init (&net_rcv_cache,
+                 "net_rcv_port",
+                 size, 0,
+                 NULL, 0);
 
  	size = sizeof(struct net_hash_entry);
-	kmem_cache_init(&net_hash_entry_cache, "net_hash_entry", size, 0,
-			NULL, NULL, NULL, 0);
+	kmem_cache_init (&net_hash_entry_cache,
+                 "net_hash_entry",
+                 size, 0,
+                 NULL, 0);
 
 	size = ikm_plus_overhead(sizeof(struct net_rcv_msg));
 	net_kmsg_size = round_page(size);
diff --git a/i386/i386/fpu.c b/i386/i386/fpu.c
index 0f34833..476e1de 100644
--- a/i386/i386/fpu.c
+++ b/i386/i386/fpu.c
@@ -187,9 +187,10 @@ init_fpu(void)
 void
 fpu_module_init(void)
 {
-	kmem_cache_init(&ifps_cache, "i386_fpsave_state",
-			sizeof(struct i386_fpsave_state), 16,
-			NULL, NULL, NULL, 0);
+	kmem_cache_init (&ifps_cache,
+                 "i386_fpsave_state",
+                 sizeof(struct i386_fpsave_state), 16,
+                 NULL, 0);
 }
 
 /*
diff --git a/i386/i386/machine_task.c b/i386/i386/machine_task.c
index 490b102..a514086 100644
--- a/i386/i386/machine_task.c
+++ b/i386/i386/machine_task.c
@@ -37,8 +37,10 @@ struct kmem_cache machine_task_iopb_cache;
 void
 machine_task_module_init (void)
 {
-  kmem_cache_init (&machine_task_iopb_cache, "i386_task_iopb", IOPB_BYTES, 0,
-		   NULL, NULL, NULL, 0);
+  kmem_cache_init (&machine_task_iopb_cache,
+                 "i386_task_iopb",
+                 IOPB_BYTES, 0,
+                 NULL, 0);
 }
 
 
diff --git a/i386/i386/pcb.c b/i386/i386/pcb.c
index 3a0eba0..4199f78 100644
--- a/i386/i386/pcb.c
+++ b/i386/i386/pcb.c
@@ -370,8 +370,10 @@ thread_t switch_context(
 
 void pcb_module_init(void)
 {
-	kmem_cache_init(&pcb_cache, "pcb", sizeof(struct pcb), 0,
-			NULL, NULL, NULL, 0);
+	kmem_cache_init (&pcb_cache,
+                 "pcb",
+                 sizeof(struct pcb), 0,
+                 NULL, 0);
 
 	fpu_module_init();
 }
diff --git a/i386/intel/pmap.c b/i386/intel/pmap.c
index dc9f360..102309f 100644
--- a/i386/intel/pmap.c
+++ b/i386/intel/pmap.c
@@ -977,9 +977,15 @@ void pmap_init(void)
 	 *	and of the physical-to-virtual entries.
 	 */
 	s = (vm_size_t) sizeof(struct pmap);
-	kmem_cache_init(&pmap_cache, "pmap", s, 0, NULL, NULL, NULL, 0);
+	kmem_cache_init (&pmap_cache,
+                 "pmap",
+                 s, 0,
+                 NULL, 0);
 	s = (vm_size_t) sizeof(struct pv_entry);
-	kmem_cache_init(&pv_list_cache, "pv_entry", s, 0, NULL, NULL, NULL, 0);
+	kmem_cache_init (&pv_list_cache,
+                 "pv_entry",
+                 s, 0,
+                 NULL, 0);
 
 #if	NCPUS > 1
 	/*
diff --git a/ipc/ipc_init.c b/ipc/ipc_init.c
index 2c58a6e..b7483d4 100644
--- a/ipc/ipc_init.c
+++ b/ipc/ipc_init.c
@@ -72,17 +72,25 @@ ipc_bootstrap(void)
 	ipc_port_timestamp_lock_init();
 	ipc_port_timestamp_data = 0;
 
-	kmem_cache_init(&ipc_space_cache, "ipc_space",
-			sizeof(struct ipc_space), 0, NULL, NULL, NULL, 0);
-
-	kmem_cache_init(&ipc_entry_cache, "ipc_entry",
-			sizeof(struct ipc_entry), 0, NULL, NULL, NULL, 0);
-
-	kmem_cache_init(&ipc_object_caches[IOT_PORT], "ipc_port",
-			sizeof(struct ipc_port), 0, NULL, NULL, NULL, 0);
-
-	kmem_cache_init(&ipc_object_caches[IOT_PORT_SET], "ipc_pset",
-			sizeof(struct ipc_pset), 0, NULL, NULL, NULL, 0);
+	kmem_cache_init (&ipc_space_cache,
+                 "ipc_space",
+                 sizeof(struct ipc_space), 0,
+                 NULL, 0);
+
+	kmem_cache_init (&ipc_entry_cache,
+                 "ipc_entry",
+                 sizeof(struct ipc_entry), 0,
+                 NULL, 0);
+
+	kmem_cache_init (&ipc_object_caches[IOT_PORT],
+                 "ipc_port",
+                 sizeof(struct ipc_port), 0,
+                 NULL, 0);
+
+	kmem_cache_init (&ipc_object_caches[IOT_PORT_SET],
+                 "ipc_pset",
+                 sizeof(struct ipc_pset), 0,
+                 NULL, 0);
 
 	/* create special spaces */
 
diff --git a/ipc/ipc_marequest.c b/ipc/ipc_marequest.c
index ded1711..ab4d8fe 100644
--- a/ipc/ipc_marequest.c
+++ b/ipc/ipc_marequest.c
@@ -136,8 +136,10 @@ ipc_marequest_init(void)
 		bucket->imarb_head = IMAR_NULL;
 	}
 
-	kmem_cache_init(&ipc_marequest_cache, "ipc_marequest",
-			sizeof(struct ipc_marequest), 0, NULL, NULL, NULL, 0);
+	kmem_cache_init (&ipc_marequest_cache,
+                 "ipc_marequest",
+                 sizeof(struct ipc_marequest), 0,
+                 NULL, 0);
 }
 
 /*
diff --git a/kern/act.c b/kern/act.c
index 3186f7e..1e786ff 100644
--- a/kern/act.c
+++ b/kern/act.c
@@ -67,8 +67,10 @@ void
 global_act_init(void)
 {
 #ifndef ACT_STATIC_KLUDGE
-	kmem_cache_init(&act_cache, "Act", sizeof(struct Act), 0,
-			NULL, NULL, NULL, 0);
+	kmem_cache_init (&act_cache,
+                 "Act",
+                 sizeof(struct Act), 0,
+                 NULL, 0);
 #else
 	int i;
 
diff --git a/kern/kalloc.h b/kern/kalloc.h
index 004e3a6..c378873 100644
--- a/kern/kalloc.h
+++ b/kern/kalloc.h
@@ -33,6 +33,4 @@
 extern vm_offset_t kalloc (vm_size_t size);
 extern void kfree (vm_offset_t data, vm_size_t size);
 
-extern void kalloc_init (void);
-
 #endif	/* _KERN_KALLOC_H_ */
diff --git a/kern/processor.c b/kern/processor.c
index bc70dde..f2f9a2d 100644
--- a/kern/processor.c
+++ b/kern/processor.c
@@ -108,8 +108,10 @@ void pset_sys_init(void)
 	/*
 	 * Allocate the cache for processor sets.
 	 */
-	kmem_cache_init(&pset_cache, "processor_set",
-			sizeof(struct processor_set), 0, NULL, NULL, NULL, 0);
+	kmem_cache_init (&pset_cache,
+                 "processor_set",
+                 sizeof(struct processor_set), 0,
+                 NULL, 0);
 
 	/*
 	 * Give each processor a control port.
diff --git a/kern/rdxtree.c b/kern/rdxtree.c
index 78868b1..83ba750 100644
--- a/kern/rdxtree.c
+++ b/kern/rdxtree.c
@@ -123,8 +123,10 @@ static struct kmem_cache rdxtree_node_cache;
 void
 rdxtree_cache_init(void)
 {
-    kmem_cache_init(&rdxtree_node_cache, "rdxtree_node",
-		    sizeof(struct rdxtree_node), 0, NULL, NULL, NULL, 0);
+    kmem_cache_init (&rdxtree_node_cache,
+                 "rdxtree_node",
+                 sizeof(struct rdxtree_node), 0,
+                 NULL, 0);
 }
 
 #ifdef RDXTREE_ENABLE_NODE_CREATION_FAILURES
diff --git a/kern/slab.c b/kern/slab.c
index ca8b44a..2084eb7 100644
--- a/kern/slab.c
+++ b/kern/slab.c
@@ -53,14 +53,11 @@
  *
  * The per-cache self-scaling hash table for buffer-to-bufctl conversion,
  * described in 3.2.3 "Slab Layout for Large Objects", has been replaced with
- * a constant time buffer-to-slab lookup that relies on the VM system. When
- * slabs are created, their backing virtual pages are mapped to physical pages
- * that are never aliased by other virtual addresses (unless explicitely by
- * other kernel code). The lookup operation provides the associated physical
- * page descriptor which can store data private to this allocator. The main
- * drawback of this method is that it needs to walk the low level page tables,
- * but it's expected that these have already been cached by the CPU due to
- * prior access by the allocator user, depending on the hardware properties.
+ * a constant time buffer-to-slab lookup that relies on the VM system. Slabs
+ * are allocated from the direct mapping of physical memory, which enables
+ * the retrieval of physical addresses backing slabs with a simple shift.
+ * Physical addresses are then used to find page descriptors, which store
+ * data private to this allocator.
  *
  * This implementation uses per-cpu pools of objects, which service most
  * allocation requests. These pools act as caches (but are named differently
@@ -81,6 +78,7 @@
 #include <mach/vm_param.h>
 #include <mach/machine/vm_types.h>
 #include <vm/vm_kern.h>
+#include <vm/vm_page.h>
 #include <vm/vm_types.h>
 #include <sys/types.h>
 
@@ -135,9 +133,9 @@
 #define KMEM_CPU_POOL_TRANSFER_RATIO 2
 
 /*
- * Size of the VM submap from which default backend functions allocate.
+ * Logarithm of the size of the smallest general cache.
  */
-#define KMEM_MAP_SIZE (96 * 1024 * 1024)
+#define KMEM_CACHES_FIRST_ORDER 5
 
 /*
  * Shift for the first kalloc cache size.
@@ -204,12 +202,6 @@ static unsigned int kmem_nr_caches;
 static simple_lock_data_t __attribute__((used)) kmem_cache_list_lock;
 
 /*
- * VM submap for slab caches.
- */
-static struct vm_map kmem_map_store;
-vm_map_t kmem_map = &kmem_map_store;
-
-/*
  * Time of the last memory reclaim, in clock ticks.
  */
 static unsigned long kmem_gc_last_tick;
@@ -310,30 +302,12 @@ static inline void * kmem_bufctl_to_buf(union kmem_bufctl *bufctl,
     return (void *)bufctl - cache->bufctl_dist;
 }
 
-static vm_offset_t kmem_pagealloc(vm_size_t size)
-{
-    vm_offset_t addr;
-    kern_return_t kr;
-
-    kr = kmem_alloc_wired(kmem_map, &addr, size);
-
-    if (kr != KERN_SUCCESS)
-        return 0;
-
-    return addr;
-}
-
-static void kmem_pagefree(vm_offset_t ptr, vm_size_t size)
-{
-    kmem_free(kmem_map, ptr, size);
-}
-
 static void kmem_slab_create_verify(struct kmem_slab *slab,
                                     struct kmem_cache *cache)
 {
     struct kmem_buftag *buftag;
-    size_t buf_size;
     unsigned long buffers;
+    size_t buf_size;
     void *buf;
 
     buf_size = cache->buf_size;
@@ -356,29 +330,26 @@ static void kmem_slab_create_verify(struct kmem_slab *slab,
 static struct kmem_slab * kmem_slab_create(struct kmem_cache *cache,
                                            size_t color)
 {
+    struct vm_page *page;
     struct kmem_slab *slab;
     union kmem_bufctl *bufctl;
     size_t buf_size;
     unsigned long buffers;
     void *slab_buf;
 
-    if (cache->slab_alloc_fn == NULL)
-        slab_buf = (void *)kmem_pagealloc(cache->slab_size);
-    else
-        slab_buf = (void *)cache->slab_alloc_fn(cache->slab_size);
+    page = vm_page_alloc_p(cache->slab_order, VM_PAGE_SEL_DIRECTMAP,
+                           VM_PAGE_KMEM);
 
-    if (slab_buf == NULL)
+    if (page == NULL)
         return NULL;
 
+    slab_buf = vm_page_direct_ptr(page);
+
     if (cache->flags & KMEM_CF_SLAB_EXTERNAL) {
         slab = (struct kmem_slab *)kmem_cache_alloc(&kmem_slab_cache);
 
         if (slab == NULL) {
-            if (cache->slab_free_fn == NULL)
-                kmem_pagefree((vm_offset_t)slab_buf, cache->slab_size);
-            else
-                cache->slab_free_fn((vm_offset_t)slab_buf, cache->slab_size);
-
+            vm_page_free_p(page, cache->slab_order);
             return NULL;
         }
     } else {
@@ -438,21 +409,12 @@ static void kmem_slab_destroy_verify(struct kmem_slab *slab,
  */
 static void kmem_slab_destroy(struct kmem_slab *slab, struct kmem_cache *cache)
 {
-    vm_offset_t slab_buf;
-
     assert(slab->nr_refs == 0);
     assert(slab->first_free != NULL);
 
     if (cache->flags & KMEM_CF_VERIFY)
         kmem_slab_destroy_verify(slab, cache);
 
-    slab_buf = (vm_offset_t)P2ALIGN((unsigned long)slab->addr, PAGE_SIZE);
-
-    if (cache->slab_free_fn == NULL)
-        kmem_pagefree(slab_buf, cache->slab_size);
-    else
-        cache->slab_free_fn(slab_buf, cache->slab_size);
-
     if (cache->flags & KMEM_CF_SLAB_EXTERNAL)
         kmem_cache_free(&kmem_slab_cache, (vm_offset_t)slab);
 }
@@ -473,7 +435,7 @@ kmem_slab_vmref(struct kmem_slab *slab, size_t size)
     end = va + size;
 
     do {
-        page = vm_kmem_lookup_page(va);
+        page = vm_page_lookup_pa(vm_page_direct_pa(va));
         assert(page != NULL);
         assert(page->slab_priv == NULL);
         page->slab_priv = slab;
@@ -628,7 +590,8 @@ static void kmem_cache_compute_sizes(struct kmem_cache *cache, int flags)
 {
     size_t i, buffers, buf_size, slab_size, free_slab_size, optimal_size = 0;
     size_t waste, waste_min;
-    int embed, optimal_embed = 0;
+    int embed, optimal_embed = optimal_embed;
+    unsigned int slab_order, optimal_order = optimal_order;
 
     buf_size = cache->buf_size;
 
@@ -640,7 +603,9 @@ static void kmem_cache_compute_sizes(struct kmem_cache *cache, int flags)
 
     do {
         i++;
-        slab_size = P2ROUND(i * buf_size, PAGE_SIZE);
+
+        slab_order = vm_page_order(i * buf_size);
+        slab_size = PAGE_SIZE << slab_order;
         free_slab_size = slab_size;
 
         if (flags & KMEM_CACHE_NOOFFSLAB)
@@ -663,6 +628,7 @@ static void kmem_cache_compute_sizes(struct kmem_cache *cache, int flags)
 
         if (waste <= waste_min) {
             waste_min = waste;
+            optimal_order = slab_order;
             optimal_size = slab_size;
             optimal_embed = embed;
         }
@@ -672,10 +638,10 @@ static void kmem_cache_compute_sizes(struct kmem_cache *cache, int flags)
     assert(optimal_size > 0);
     assert(!(flags & KMEM_CACHE_NOOFFSLAB) || optimal_embed);
 
+    cache->slab_order = optimal_order;
     cache->slab_size = optimal_size;
-    slab_size = cache->slab_size - (optimal_embed
-                ? sizeof(struct kmem_slab)
-                : 0);
+    slab_size = cache->slab_size
+                - (optimal_embed ? sizeof(struct kmem_slab) : 0);
     cache->bufs_per_slab = slab_size / buf_size;
     cache->color_max = slab_size % buf_size;
 
@@ -690,10 +656,9 @@ static void kmem_cache_compute_sizes(struct kmem_cache *cache, int flags)
     }
 }
 
-void kmem_cache_init(struct kmem_cache *cache, const char *name,
-                     size_t obj_size, size_t align, kmem_ctor_fn_t ctor,
-                     kmem_slab_alloc_fn_t slab_alloc_fn,
-                     kmem_slab_free_fn_t slab_free_fn, int flags)
+void
+kmem_cache_init(struct kmem_cache *cache, const char *name, size_t obj_size,
+                size_t align, kmem_ctor_fn_t ctor, int flags)
 {
 #if SLAB_USE_CPU_POOLS
     struct kmem_cpu_pool_type *cpu_pool_type;
@@ -735,8 +700,6 @@ void kmem_cache_init(struct kmem_cache *cache, const char *name,
     cache->nr_slabs = 0;
     cache->nr_free_slabs = 0;
     cache->ctor = ctor;
-    cache->slab_alloc_fn = slab_alloc_fn;
-    cache->slab_free_fn = slab_free_fn;
     strncpy(cache->name, name, sizeof(cache->name));
     cache->name[sizeof(cache->name) - 1] = '\0';
     cache->buftag_dist = 0;
@@ -905,7 +868,7 @@ static void kmem_cache_free_to_slab(struct kmem_cache *cache, void *buf)
     } else {
         struct vm_page *page;
 
-        page = vm_kmem_lookup_page((unsigned long)buf);
+        page = vm_page_lookup_pa(vm_page_direct_pa((unsigned long)buf));
         assert(page != NULL);
         slab = page->slab_priv;
         assert(slab != NULL);
@@ -1194,16 +1157,13 @@ void slab_bootstrap(void)
 
 void slab_init(void)
 {
-    vm_offset_t min, max;
+    size_t i, size;
+    char name[KMEM_CACHE_NAME_SIZE];
 
 #if SLAB_USE_CPU_POOLS
     struct kmem_cpu_pool_type *cpu_pool_type;
-    char name[KMEM_CACHE_NAME_SIZE];
-    size_t i, size;
 #endif /* SLAB_USE_CPU_POOLS */
 
-    kmem_submap(kmem_map, kernel_map, &min, &max, KMEM_MAP_SIZE, FALSE);
-
 #if SLAB_USE_CPU_POOLS
     for (i = 0; i < ARRAY_SIZE(kmem_cpu_pool_types); i++) {
         cpu_pool_type = &kmem_cpu_pool_types[i];
@@ -1211,7 +1171,7 @@ void slab_init(void)
         sprintf(name, "kmem_cpu_array_%d", cpu_pool_type->array_size);
         size = sizeof(void *) * cpu_pool_type->array_size;
         kmem_cache_init(cpu_pool_type->array_cache, name, size,
-                        cpu_pool_type->array_align, NULL, NULL, NULL, 0);
+                        cpu_pool_type->array_align, NULL, 0);
     }
 #endif /* SLAB_USE_CPU_POOLS */
 
@@ -1219,56 +1179,21 @@ void slab_init(void)
      * Prevent off slab data for the slab cache to avoid infinite recursion.
      */
     kmem_cache_init(&kmem_slab_cache, "kmem_slab", sizeof(struct kmem_slab),
-                    0, NULL, NULL, NULL, KMEM_CACHE_NOOFFSLAB);
-}
+                    0, NULL, KMEM_CACHE_NOOFFSLAB);
 
-static vm_offset_t kalloc_pagealloc(vm_size_t size)
-{
-    vm_offset_t addr;
-    kern_return_t kr;
-
-    kr = kmem_alloc_wired(kmem_map, &addr, size);
-
-    if (kr != KERN_SUCCESS)
-        return 0;
-
-    return addr;
-}
-
-static void kalloc_pagefree(vm_offset_t ptr, vm_size_t size)
-{
-    kmem_free(kmem_map, ptr, size);
-}
-
-void kalloc_init(void)
-{
-    char name[KMEM_CACHE_NAME_SIZE];
-    size_t i, size;
-
-    size = 1 << KALLOC_FIRST_SHIFT;
+    size = 1 << KMEM_CACHES_FIRST_ORDER;
 
     for (i = 0; i < ARRAY_SIZE(kalloc_caches); i++) {
-        sprintf(name, "kalloc_%lu", size);
-        kmem_cache_init(&kalloc_caches[i], name, size, 0, NULL,
-                        kalloc_pagealloc, kalloc_pagefree, 0);
+        sprintf(name, "kmem_%zu", size);
+        kmem_cache_init(&kalloc_caches[i], name, size, 0, NULL, 0);
         size <<= 1;
     }
 }
 
-/*
- * Return the kalloc cache index matching the given allocation size, which
- * must be strictly greater than 0.
- */
-static inline size_t kalloc_get_index(unsigned long size)
+static inline size_t
+kalloc_get_index(unsigned long size)
 {
-    assert(size != 0);
-
-    size = (size - 1) >> KALLOC_FIRST_SHIFT;
-
-    if (size == 0)
-        return 0;
-    else
-        return (sizeof(long) * 8) - __builtin_clzl(size);
+    return iorder2(size) - KMEM_CACHES_FIRST_ORDER;
 }
 
 static void kalloc_verify(struct kmem_cache *cache, void *buf, size_t size)
@@ -1301,8 +1226,17 @@ vm_offset_t kalloc(vm_size_t size)
 
         if ((buf != 0) && (cache->flags & KMEM_CF_VERIFY))
             kalloc_verify(cache, buf, size);
-    } else
-        buf = (void *)kalloc_pagealloc(size);
+    } else {
+        struct vm_page *page;
+
+        page = vm_page_alloc_p(vm_page_order(size), VM_PAGE_SEL_DIRECTMAP,
+                               VM_PAGE_KERNEL);
+
+        if (page == NULL)
+            return NULL;
+
+        buf = vm_page_direct_ptr(page);
+    }
 
     return (vm_offset_t)buf;
 }
@@ -1343,7 +1277,10 @@ void kfree(vm_offset_t data, vm_size_t size)
 
         kmem_cache_free(cache, data);
     } else {
-        kalloc_pagefree(data, size);
+        struct vm_page *page;
+
+        page = vm_page_lookup_pa(vm_page_direct_pa((unsigned long)data));
+        vm_page_free_p(page, vm_page_order(size));
     }
 }
 
diff --git a/kern/slab.h b/kern/slab.h
index 52aa11b..6f5cc18 100644
--- a/kern/slab.h
+++ b/kern/slab.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011 Free Software Foundation.
+ * Copyright (c) 2011-2015 Free Software Foundation.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -69,14 +69,6 @@ struct kmem_cache;
  */
 typedef void (*kmem_ctor_fn_t)(void *);
 
-/*
- * Types for slab allocation/free functions.
- *
- * All addresses and sizes must be page-aligned.
- */
-typedef vm_offset_t (*kmem_slab_alloc_fn_t)(vm_size_t);
-typedef void (*kmem_slab_free_fn_t)(vm_offset_t, vm_size_t);
-
 #include <kern/slab_i.h>
 
 /*
@@ -86,11 +78,6 @@ typedef struct kmem_cache *kmem_cache_t;
 #define KMEM_CACHE_NULL ((kmem_cache_t) 0)
 
 /*
- * VM submap for slab allocations.
- */
-extern vm_map_t kmem_map;
-
-/*
  * Cache initialization flags.
  */
 #define KMEM_CACHE_NOCPUPOOL    0x1 /* Don't use the per-cpu pools */
@@ -99,14 +86,10 @@ extern vm_map_t kmem_map;
 
 /*
  * Initialize a cache.
- *
- * If a slab allocation/free function pointer is NULL, the default backend
- * (vm_kmem on the kernel map) is used for the allocation/free action.
  */
 void kmem_cache_init(struct kmem_cache *cache, const char *name,
                      size_t obj_size, size_t align, kmem_ctor_fn_t ctor,
-                     kmem_slab_alloc_fn_t slab_alloc_fn,
-                     kmem_slab_free_fn_t slab_free_fn, int flags);
+                     int flags);
 
 /*
  * Allocate an object from a cache.
diff --git a/kern/slab_i.h b/kern/slab_i.h
index 06da0ea..f8dc901 100644
--- a/kern/slab_i.h
+++ b/kern/slab_i.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2010, 2011, 2012, 2013 Richard Braun.
+ * Copyright (c) 2010-2014 Richard Braun.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -197,6 +197,7 @@ struct kmem_cache {
     int flags;
     size_t bufctl_dist; /* Distance from buffer to bufctl   */
     size_t slab_size;
+    unsigned int slab_order;
     unsigned long bufs_per_slab;
     unsigned long nr_objs;  /* Number of allocated objects */
     unsigned long nr_free_slabs;
@@ -210,8 +211,6 @@ struct kmem_cache {
     size_t color_max;
     unsigned long nr_bufs;  /* Total number of buffers */
     unsigned long nr_slabs;
-    kmem_slab_alloc_fn_t slab_alloc_fn;
-    kmem_slab_free_fn_t slab_free_fn;
     char name[KMEM_CACHE_NAME_SIZE];
     size_t buftag_dist; /* Distance from buffer to buftag */
     size_t redzone_pad; /* Bytes from end of object to redzone word */
diff --git a/kern/task.c b/kern/task.c
index 3b91de2..66f86e9 100644
--- a/kern/task.c
+++ b/kern/task.c
@@ -62,8 +62,10 @@ ipc_port_t new_task_notification = NULL;
 
 void task_init(void)
 {
-	kmem_cache_init(&task_cache, "task", sizeof(struct task), 0,
-			NULL, NULL, NULL, 0);
+	kmem_cache_init (&task_cache,
+                 "task",
+                 sizeof(struct task), 0,
+                 NULL, 0);
 
 	eml_init();
 	machine_task_module_init ();
diff --git a/kern/thread.c b/kern/thread.c
index 5b51ec1..720711c 100644
--- a/kern/thread.c
+++ b/kern/thread.c
@@ -223,7 +223,7 @@ void stack_privilege(
 void thread_init(void)
 {
 	kmem_cache_init(&thread_cache, "thread", sizeof(struct thread), 0,
-			NULL, NULL, NULL, 0);
+			NULL, 0);
 
 	/*
 	 *	Kernel stacks should be naturally aligned, so that it
@@ -232,7 +232,7 @@ void thread_init(void)
 	 */
 	kmem_cache_init(&stack_cache, "stack",
 			KERNEL_STACK_SIZE, KERNEL_STACK_SIZE,
-			NULL, NULL, NULL, 0);
+			NULL, 0);
 
 	/*
 	 *	Fill in a template thread for fast initialization.
diff --git a/vm/memory_object_proxy.c b/vm/memory_object_proxy.c
index a64bfcc..6346b5b 100644
--- a/vm/memory_object_proxy.c
+++ b/vm/memory_object_proxy.c
@@ -63,8 +63,10 @@ typedef struct memory_object_proxy *memory_object_proxy_t;
 void
 memory_object_proxy_init (void)
 {
-  kmem_cache_init (&memory_object_proxy_cache, "memory_object_proxy",
-		   sizeof (struct memory_object_proxy), 0, NULL, NULL, NULL, 0);
+  kmem_cache_init (&memory_object_proxy_cache,
+                 "memory_object_proxy",
+                 sizeof(struct memory_object_proxy), 0,
+                 NULL, 0);
 }
   
 /* Lookup a proxy memory object by its port.  */
diff --git a/vm/vm_external.c b/vm/vm_external.c
index 2e2593b..03ebcfb 100644
--- a/vm/vm_external.c
+++ b/vm/vm_external.c
@@ -137,14 +137,18 @@ void		vm_external_module_initialize(void)
 {
 	vm_size_t	size = (vm_size_t) sizeof(struct vm_external);
 
-	kmem_cache_init(&vm_external_cache, "vm_external", size, 0,
-			NULL, NULL, NULL, 0);
-
-	kmem_cache_init(&vm_object_small_existence_map_cache,
-			"small_existence_map", SMALL_SIZE, 0,
-			NULL, NULL, NULL, 0);
-
-	kmem_cache_init(&vm_object_large_existence_map_cache,
-			"large_existence_map", LARGE_SIZE, 0,
-			NULL, NULL, NULL, 0);
+	kmem_cache_init (&vm_external_cache,
+                 "vm_external",
+                 size, 0,
+                 NULL, 0);
+
+	kmem_cache_init (&vm_object_small_existence_map_cache,
+                 "small_existence_map",
+                 SMALL_SIZE, 0,
+                 NULL, 0);
+
+	kmem_cache_init (&vm_object_large_existence_map_cache,
+                 "large_existence_map",
+                 LARGE_SIZE, 0,
+                 NULL, 0);
 }
diff --git a/vm/vm_fault.c b/vm/vm_fault.c
index 0fa4d6a..ccb3d0d 100644
--- a/vm/vm_fault.c
+++ b/vm/vm_fault.c
@@ -104,8 +104,10 @@ extern struct db_watchpoint *db_watchpoint_list;
  */
 void vm_fault_init(void)
 {
-	kmem_cache_init(&vm_fault_state_cache, "vm_fault_state",
-			sizeof(vm_fault_state_t), 0, NULL, NULL, NULL, 0);
+	kmem_cache_init (&vm_fault_state_cache,
+                 "vm_fault_state",
+                 sizeof(vm_fault_state_t), 0,
+                 NULL, 0);
 }
 
 /*
diff --git a/vm/vm_init.c b/vm/vm_init.c
index 3d1081c..4fdcd83 100644
--- a/vm/vm_init.c
+++ b/vm/vm_init.c
@@ -36,7 +36,6 @@
 
 #include <mach/machine/vm_types.h>
 #include <kern/slab.h>
-#include <kern/kalloc.h>
 #include <vm/vm_fault.h>
 #include <vm/vm_object.h>
 #include <vm/vm_map.h>
@@ -73,7 +72,6 @@ void vm_mem_bootstrap(void)
 	kmem_init(start, end);
 	pmap_init();
 	slab_init();
-	kalloc_init();
 	vm_fault_init();
 	vm_page_module_init();
 	memory_manager_default_init();
diff --git a/vm/vm_map.c b/vm/vm_map.c
index 4977bff..287bdfc 100644
--- a/vm/vm_map.c
+++ b/vm/vm_map.c
@@ -178,18 +178,23 @@ static vm_offset_t kentry_pagealloc(vm_size_t size)
 
 void vm_map_init(void)
 {
-	kmem_cache_init(&vm_map_cache, "vm_map", sizeof(struct vm_map), 0,
-			NULL, NULL, NULL, 0);
-	kmem_cache_init(&vm_map_entry_cache, "vm_map_entry",
-			sizeof(struct vm_map_entry), 0, NULL, NULL, NULL, 0);
-#if 0
+	kmem_cache_init (&vm_map_cache,
+                 "vm_map",
+                 sizeof(struct vm_map), 0,
+                 NULL, 0);
+	kmem_cache_init (&vm_map_entry_cache,
+                 "vm_map_entry",
+                 sizeof(struct vm_map_entry), 0,
+                 NULL, 0);
+#if 1
 	kmem_cache_init(&vm_map_kentry_cache, "vm_map_kentry",
-			sizeof(struct vm_map_entry), 0, NULL, kentry_pagealloc,
-			NULL, KMEM_CACHE_NOCPUPOOL | KMEM_CACHE_NOOFFSLAB
-			      | KMEM_CACHE_NORECLAIM);
+			sizeof(struct vm_map_entry), 0,
+			NULL, KMEM_CACHE_NOCPUPOOL | KMEM_CACHE_NOOFFSLAB);
 #endif
-	kmem_cache_init(&vm_map_copy_cache, "vm_map_copy",
-			sizeof(struct vm_map_copy), 0, NULL, NULL, NULL, 0);
+	kmem_cache_init (&vm_map_copy_cache,
+                 "vm_map_copy",
+                 sizeof(struct vm_map_copy), 0,
+                 NULL, 0);
 
 	/*
 	 *	Submap object is initialized by vm_object_init.
diff --git a/vm/vm_object.c b/vm/vm_object.c
index b403d30..ee09e3b 100644
--- a/vm/vm_object.c
+++ b/vm/vm_object.c
@@ -259,8 +259,10 @@ vm_object_t vm_object_allocate(
  */
 void vm_object_bootstrap(void)
 {
-	kmem_cache_init(&vm_object_cache, "vm_object",
-			sizeof(struct vm_object), 0, NULL, NULL, NULL, 0);
+	kmem_cache_init (&vm_object_cache,
+                 "vm_object",
+                 sizeof(struct vm_object), 0,
+                 NULL, 0);
 
 	queue_init(&vm_object_cached_list);
 	vm_object_cache_lock_init();
diff --git a/vm/vm_resident.c b/vm/vm_resident.c
index c70fa73..d3b5a8e 100644
--- a/vm/vm_resident.c
+++ b/vm/vm_resident.c
@@ -429,8 +429,10 @@ void pmap_startup(
  */
 void		vm_page_module_init(void)
 {
-	kmem_cache_init(&vm_page_cache, "vm_page", sizeof(struct vm_page), 0,
-			NULL, NULL, NULL, 0);
+	kmem_cache_init (&vm_page_cache,
+                 "vm_page",
+                 sizeof(struct vm_page), 0,
+                 NULL, 0);
 }
 
 /*