/*
* Copyright (c) 2010-2014 Richard Braun.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see .
*/
#include
#include
#include
#include
#include
#include
#include
#include
#include "biosmem.h"
#include "x15/elf.h"
#include "x15/multiboot.h"
/* Mach glue. */
#define __bootdata /* nothing */
#define __boot /* nothing */
#define __init /* nothing */
#define boot_memmove memmove
#define boot_memset(P,C,S) memset(phystokv(P), C, S)
#define boot_strlen(P) strlen(phystokv(P))
#define boot_panic panic
#define printk printf
#define BOOT_VTOP(addr) _kvtophys(addr)
/* XXX */
extern char _boot;
extern char _end;
/*
* Maximum number of entries in the BIOS memory map.
*
* Because of adjustments of overlapping ranges, the memory map can grow
* to twice this size.
*/
#define BIOSMEM_MAX_MAP_SIZE 128
/*
* Memory range types.
*/
#define BIOSMEM_TYPE_AVAILABLE 1
#define BIOSMEM_TYPE_RESERVED 2
#define BIOSMEM_TYPE_ACPI 3
#define BIOSMEM_TYPE_NVS 4
#define BIOSMEM_TYPE_UNUSABLE 5
#define BIOSMEM_TYPE_DISABLED 6
/*
* Memory map entry.
*/
struct biosmem_map_entry {
uint64_t base_addr;
uint64_t length;
unsigned int type;
};
/*
* Contiguous block of physical memory.
*
* Tha "available" range records what has been passed to the VM system as
* available inside the segment.
*/
struct biosmem_segment {
phys_addr_t start;
phys_addr_t end;
phys_addr_t avail_start;
phys_addr_t avail_end;
};
/*
* Memory map built from the information passed by the boot loader.
*
* If the boot loader didn't pass a valid memory map, a simple map is built
* based on the mem_lower and mem_upper multiboot fields.
*/
static struct biosmem_map_entry biosmem_map[BIOSMEM_MAX_MAP_SIZE * 2]
__bootdata;
static unsigned int biosmem_map_size __bootdata;
/*
* Physical segment boundaries.
*/
static struct biosmem_segment biosmem_segments[VM_PAGE_MAX_SEGS] __bootdata;
/*
* Boundaries of the simple bootstrap heap.
*
* This heap is located above BIOS memory.
*/
static uint32_t biosmem_heap_start __bootdata;
static uint32_t biosmem_heap_cur __bootdata;
static uint32_t biosmem_heap_end __bootdata;
static char biosmem_panic_toobig_msg[] __bootdata
= "biosmem: too many memory map entries";
static char biosmem_panic_setup_msg[] __bootdata
= "biosmem: unable to set up the early memory allocator";
static char biosmem_panic_noseg_msg[] __bootdata
= "biosmem: unable to find any memory segment";
static char biosmem_panic_inval_msg[] __bootdata
= "biosmem: attempt to allocate 0 page";
static char biosmem_panic_nomem_msg[] __bootdata
= "biosmem: unable to allocate memory";
static void __boot
biosmem_map_build(const struct multiboot_raw_info *mbi)
{
struct multiboot_raw_mmap_entry *mb_entry, *mb_end;
struct biosmem_map_entry *start, *entry, *end;
unsigned long addr;
addr = phystokv(mbi->mmap_addr);
mb_entry = (struct multiboot_raw_mmap_entry *)addr;
mb_end = (struct multiboot_raw_mmap_entry *)(addr + mbi->mmap_length);
start = biosmem_map;
entry = start;
end = entry + BIOSMEM_MAX_MAP_SIZE;
while ((mb_entry < mb_end) && (entry < end)) {
entry->base_addr = mb_entry->base_addr;
entry->length = mb_entry->length;
entry->type = mb_entry->type;
mb_entry = (void *)mb_entry + sizeof(mb_entry->size) + mb_entry->size;
entry++;
}
biosmem_map_size = entry - start;
}
static void __boot
biosmem_map_build_simple(const struct multiboot_raw_info *mbi)
{
struct biosmem_map_entry *entry;
entry = biosmem_map;
entry->base_addr = 0;
entry->length = mbi->mem_lower << 10;
entry->type = BIOSMEM_TYPE_AVAILABLE;
entry++;
entry->base_addr = BIOSMEM_END;
entry->length = mbi->mem_upper << 10;
entry->type = BIOSMEM_TYPE_AVAILABLE;
biosmem_map_size = 2;
}
static int __boot
biosmem_map_entry_is_invalid(const struct biosmem_map_entry *entry)
{
return (entry->base_addr + entry->length) <= entry->base_addr;
}
static void __boot
biosmem_map_filter(void)
{
struct biosmem_map_entry *entry;
unsigned int i;
i = 0;
while (i < biosmem_map_size) {
entry = &biosmem_map[i];
if (biosmem_map_entry_is_invalid(entry)) {
biosmem_map_size--;
boot_memmove(entry, entry + 1,
(biosmem_map_size - i) * sizeof(*entry));
continue;
}
i++;
}
}
static void __boot
biosmem_map_sort(void)
{
struct biosmem_map_entry tmp;
unsigned int i, j;
/*
* Simple insertion sort.
*/
for (i = 1; i < biosmem_map_size; i++) {
tmp = biosmem_map[i];
for (j = i - 1; j < i; j--) {
if (biosmem_map[j].base_addr < tmp.base_addr)
break;
biosmem_map[j + 1] = biosmem_map[j];
}
biosmem_map[j + 1] = tmp;
}
}
static void __boot
biosmem_map_adjust(void)
{
struct biosmem_map_entry tmp, *a, *b, *first, *second;
uint64_t a_end, b_end, last_end;
unsigned int i, j, last_type;
biosmem_map_filter();
/*
* Resolve overlapping areas, giving priority to most restrictive
* (i.e. numerically higher) types.
*/
for (i = 0; i < biosmem_map_size; i++) {
a = &biosmem_map[i];
a_end = a->base_addr + a->length;
j = i + 1;
while (j < biosmem_map_size) {
b = &biosmem_map[j];
b_end = b->base_addr + b->length;
if ((a->base_addr >= b_end) || (a_end <= b->base_addr)) {
j++;
continue;
}
if (a->base_addr < b->base_addr) {
first = a;
second = b;
} else {
first = b;
second = a;
}
if (a_end > b_end) {
last_end = a_end;
last_type = a->type;
} else {
last_end = b_end;
last_type = b->type;
}
tmp.base_addr = second->base_addr;
tmp.length = MIN(a_end, b_end) - tmp.base_addr;
tmp.type = MAX(a->type, b->type);
first->length = tmp.base_addr - first->base_addr;
second->base_addr += tmp.length;
second->length = last_end - second->base_addr;
second->type = last_type;
/*
* Filter out invalid entries.
*/
if (biosmem_map_entry_is_invalid(a)
&& biosmem_map_entry_is_invalid(b)) {
*a = tmp;
biosmem_map_size--;
memmove(b, b + 1, (biosmem_map_size - j) * sizeof(*b));
continue;
} else if (biosmem_map_entry_is_invalid(a)) {
*a = tmp;
j++;
continue;
} else if (biosmem_map_entry_is_invalid(b)) {
*b = tmp;
j++;
continue;
}
if (tmp.type == a->type)
first = a;
else if (tmp.type == b->type)
first = b;
else {
/*
* If the overlapping area can't be merged with one of its
* neighbors, it must be added as a new entry.
*/
if (biosmem_map_size >= ARRAY_SIZE(biosmem_map))
boot_panic(biosmem_panic_toobig_msg);
biosmem_map[biosmem_map_size] = tmp;
biosmem_map_size++;
j++;
continue;
}
if (first->base_addr > tmp.base_addr)
first->base_addr = tmp.base_addr;
first->length += tmp.length;
j++;
}
}
biosmem_map_sort();
}
static int __boot
biosmem_map_find_avail(phys_addr_t *phys_start, phys_addr_t *phys_end)
{
const struct biosmem_map_entry *entry, *map_end;
phys_addr_t seg_start, seg_end;
uint64_t start, end;
seg_start = (phys_addr_t)-1;
seg_end = (phys_addr_t)-1;
map_end = biosmem_map + biosmem_map_size;
for (entry = biosmem_map; entry < map_end; entry++) {
if (entry->type != BIOSMEM_TYPE_AVAILABLE)
continue;
start = vm_page_round(entry->base_addr);
if (start >= *phys_end)
break;
end = vm_page_trunc(entry->base_addr + entry->length);
if ((start < end) && (start < *phys_end) && (end > *phys_start)) {
if (seg_start == (phys_addr_t)-1)
seg_start = start;
seg_end = end;
}
}
if ((seg_start == (phys_addr_t)-1) || (seg_end == (phys_addr_t)-1))
return -1;
if (seg_start > *phys_start)
*phys_start = seg_start;
if (seg_end < *phys_end)
*phys_end = seg_end;
return 0;
}
static void __boot
biosmem_set_segment(unsigned int seg_index, phys_addr_t start, phys_addr_t end)
{
biosmem_segments[seg_index].start = start;
biosmem_segments[seg_index].end = end;
}
static phys_addr_t __boot
biosmem_segment_end(unsigned int seg_index)
{
return biosmem_segments[seg_index].end;
}
static phys_addr_t __boot
biosmem_segment_size(unsigned int seg_index)
{
return biosmem_segments[seg_index].end - biosmem_segments[seg_index].start;
}
static void __boot
biosmem_save_cmdline_sizes(struct multiboot_raw_info *mbi)
{
struct multiboot_raw_module *mod;
uint32_t i;
if (mbi->flags & MULTIBOOT_LOADER_CMDLINE)
mbi->unused0 = boot_strlen((char *)(unsigned long)mbi->cmdline) + 1;
if (mbi->flags & MULTIBOOT_LOADER_MODULES) {
unsigned long addr;
addr = phystokv(mbi->mods_addr);
for (i = 0; i < mbi->mods_count; i++) {
mod = (struct multiboot_raw_module *)addr + i;
mod->reserved = boot_strlen((char *)(unsigned long)mod->string) + 1;
}
}
}
static void __boot
biosmem_find_boot_data_update(uint32_t min, uint32_t *start, uint32_t *end,
uint32_t data_start, uint32_t data_end)
{
assert (data_start < data_end);
if ((min <= data_start) && (data_start < *start)) {
*start = data_start;
*end = data_end;
}
}
/*
* Find the first boot data in the given range, and return their containing
* area (start address is returned directly, end address is returned in end).
* The following are considered boot data :
* - the kernel
* - the kernel command line
* - the module table
* - the modules
* - the modules command lines
* - the ELF section header table
* - the ELF .shstrtab, .symtab and .strtab sections
*
* If no boot data was found, 0 is returned, and the end address isn't set.
*/
static uint32_t __boot
biosmem_find_boot_data(const struct multiboot_raw_info *mbi, uint32_t min,
uint32_t max, uint32_t *endp)
{
struct multiboot_raw_module *mod;
struct elf_shdr *shdr;
uint32_t i, start, end = end;
unsigned long tmp;
start = max;
biosmem_find_boot_data_update(min, &start, &end,
BOOT_VTOP((unsigned long)&_boot),
BOOT_VTOP((unsigned long)&_end));
if ((mbi->flags & MULTIBOOT_LOADER_CMDLINE) && (mbi->cmdline != 0))
biosmem_find_boot_data_update(min, &start, &end, mbi->cmdline,
mbi->cmdline + mbi->unused0);
if (mbi->flags & MULTIBOOT_LOADER_MODULES) {
i = mbi->mods_count * sizeof(struct multiboot_raw_module);
biosmem_find_boot_data_update(min, &start, &end, mbi->mods_addr,
mbi->mods_addr + i);
tmp = phystokv(mbi->mods_addr);
for (i = 0; i < mbi->mods_count; i++) {
mod = (struct multiboot_raw_module *)tmp + i;
biosmem_find_boot_data_update(min, &start, &end, mod->mod_start,
mod->mod_end);
if (mod->string != 0)
biosmem_find_boot_data_update(min, &start, &end, mod->string,
mod->string + mod->reserved);
}
}
if (mbi->flags & MULTIBOOT_LOADER_SHDR) {
tmp = mbi->shdr_num * mbi->shdr_size;
biosmem_find_boot_data_update(min, &start, &end, mbi->shdr_addr,
mbi->shdr_addr + tmp);
tmp = phystokv(mbi->shdr_addr);
for (i = 0; i < mbi->shdr_num; i++) {
shdr = (struct elf_shdr *)(tmp + (i * mbi->shdr_size));
if ((shdr->type != ELF_SHT_SYMTAB)
&& (shdr->type != ELF_SHT_STRTAB))
continue;
biosmem_find_boot_data_update(min, &start, &end, shdr->addr,
shdr->addr + shdr->size);
}
}
if (start == max)
return 0;
*endp = end;
return start;
}
static void __boot
biosmem_setup_allocator(struct multiboot_raw_info *mbi)
{
uint32_t heap_start, heap_end, max_heap_start, max_heap_end;
uint32_t mem_end, next;
/*
* Find some memory for the heap. Look for the largest unused area in
* upper memory, carefully avoiding all boot data.
*/
mem_end = vm_page_trunc((mbi->mem_upper + 1024) << 10);
#ifndef __LP64__
if (mem_end > VM_PAGE_DIRECTMAP_LIMIT)
mem_end = VM_PAGE_DIRECTMAP_LIMIT;
#endif /* __LP64__ */
max_heap_start = 0;
max_heap_end = 0;
next = BIOSMEM_END;
do {
heap_start = next;
heap_end = biosmem_find_boot_data(mbi, heap_start, mem_end, &next);
if (heap_end == 0) {
heap_end = mem_end;
next = 0;
}
if ((heap_end - heap_start) > (max_heap_end - max_heap_start)) {
max_heap_start = heap_start;
max_heap_end = heap_end;
}
} while (next != 0);
max_heap_start = vm_page_round(max_heap_start);
max_heap_end = vm_page_trunc(max_heap_end);
if (max_heap_start >= max_heap_end)
boot_panic(biosmem_panic_setup_msg);
biosmem_heap_start = max_heap_start;
biosmem_heap_end = max_heap_end;
biosmem_heap_cur = biosmem_heap_end;
/* Mach pmap glue. */
extern vm_offset_t phys_last_addr;
phys_last_addr = (vm_offset_t) max_heap_end;
}
void __boot
biosmem_bootstrap(struct multiboot_raw_info *mbi)
{
phys_addr_t phys_start, phys_end;
int error;
if (mbi->flags & MULTIBOOT_LOADER_MMAP)
biosmem_map_build(mbi);
else
biosmem_map_build_simple(mbi);
biosmem_map_adjust();
phys_start = BIOSMEM_BASE;
phys_end = VM_PAGE_DMA_LIMIT;
error = biosmem_map_find_avail(&phys_start, &phys_end);
if (error)
boot_panic(biosmem_panic_noseg_msg);
biosmem_set_segment(VM_PAGE_SEG_DMA, phys_start, phys_end);
phys_start = VM_PAGE_DMA_LIMIT;
#ifdef VM_PAGE_DMA32_LIMIT
phys_end = VM_PAGE_DMA32_LIMIT;
error = biosmem_map_find_avail(&phys_start, &phys_end);
if (error)
goto out;
biosmem_set_segment(VM_PAGE_SEG_DMA32, phys_start, phys_end);
phys_start = VM_PAGE_DMA32_LIMIT;
#endif /* VM_PAGE_DMA32_LIMIT */
phys_end = VM_PAGE_DIRECTMAP_LIMIT;
error = biosmem_map_find_avail(&phys_start, &phys_end);
if (error)
goto out;
biosmem_set_segment(VM_PAGE_SEG_DIRECTMAP, phys_start, phys_end);
phys_start = VM_PAGE_DIRECTMAP_LIMIT;
phys_end = VM_PAGE_HIGHMEM_LIMIT;
error = biosmem_map_find_avail(&phys_start, &phys_end);
if (error)
goto out;
biosmem_set_segment(VM_PAGE_SEG_HIGHMEM, phys_start, phys_end);
out:
/*
* The kernel and modules command lines will be memory mapped later
* during initialization. Their respective sizes must be saved.
*/
biosmem_save_cmdline_sizes(mbi);
biosmem_setup_allocator(mbi);
}
void * __boot
biosmem_bootalloc(unsigned int nr_pages)
{
unsigned long addr, size;
size = vm_page_ptoa(nr_pages);
if (size == 0)
boot_panic(biosmem_panic_inval_msg);
/* Top-down allocation to avoid unnecessarily filling DMA segments */
addr = biosmem_heap_cur - size;
if ((addr < biosmem_heap_start) || (addr > biosmem_heap_cur))
boot_panic(biosmem_panic_nomem_msg);
biosmem_heap_cur = addr;
return boot_memset((void *)addr, 0, size);
}
phys_addr_t __boot
biosmem_directmap_size(void)
{
if (biosmem_segment_size(VM_PAGE_SEG_DIRECTMAP) != 0)
return biosmem_segment_end(VM_PAGE_SEG_DIRECTMAP);
else if (biosmem_segment_size(VM_PAGE_SEG_DMA32) != 0)
return biosmem_segment_end(VM_PAGE_SEG_DMA32);
else
return biosmem_segment_end(VM_PAGE_SEG_DMA);
}
static const char * __init
biosmem_type_desc(unsigned int type)
{
switch (type) {
case BIOSMEM_TYPE_AVAILABLE:
return "available";
case BIOSMEM_TYPE_RESERVED:
return "reserved";
case BIOSMEM_TYPE_ACPI:
return "ACPI";
case BIOSMEM_TYPE_NVS:
return "ACPI NVS";
case BIOSMEM_TYPE_UNUSABLE:
return "unusable";
default:
return "unknown (reserved)";
}
}
static void __init
biosmem_map_show(void)
{
const struct biosmem_map_entry *entry, *end;
printk("biosmem: physical memory map:\n");
for (entry = biosmem_map, end = entry + biosmem_map_size;
entry < end;
entry++)
printk("biosmem: %018llx:%018llx, %s\n", entry->base_addr,
entry->base_addr + entry->length,
biosmem_type_desc(entry->type));
printk("biosmem: heap: %x-%x\n", biosmem_heap_start, biosmem_heap_end);
}
static void __init
biosmem_load_segment(struct biosmem_segment *seg, uint64_t max_phys_end,
phys_addr_t phys_start, phys_addr_t phys_end,
phys_addr_t avail_start, phys_addr_t avail_end)
{
unsigned int seg_index;
seg_index = seg - biosmem_segments;
if (phys_end > max_phys_end) {
if (max_phys_end <= phys_start) {
printk("biosmem: warning: segment %s physically unreachable, "
"not loaded\n", vm_page_seg_name(seg_index));
return;
}
printk("biosmem: warning: segment %s truncated to %#llx\n",
vm_page_seg_name(seg_index), max_phys_end);
phys_end = max_phys_end;
}
if ((avail_start < phys_start) || (avail_start > phys_end))
avail_start = phys_start;
if ((avail_end < phys_start) || (avail_end > phys_end))
avail_end = phys_end;
seg->avail_start = avail_start;
seg->avail_end = avail_end;
vm_page_load(seg_index, phys_start, phys_end, avail_start, avail_end);
}
void __init
biosmem_setup(void)
{
uint64_t max_phys_end;
struct biosmem_segment *seg;
struct cpu *cpu;
unsigned int i;
biosmem_map_show();
#if notyet
cpu = cpu_current();
max_phys_end = (cpu->phys_addr_width == 0)
? (uint64_t)-1
: (uint64_t)1 << cpu->phys_addr_width;
#else
max_phys_end = (uint64_t)1 << 32;
(void) cpu;
#endif
for (i = 0; i < ARRAY_SIZE(biosmem_segments); i++) {
if (biosmem_segment_size(i) == 0)
break;
seg = &biosmem_segments[i];
biosmem_load_segment(seg, max_phys_end, seg->start, seg->end,
biosmem_heap_start,
biosmem_heap_cur == seg->start
? seg->end: biosmem_heap_cur);
}
}
static void __init
biosmem_free_usable_range(phys_addr_t start, phys_addr_t end)
{
struct vm_page *page;
printk("biosmem: release to vm_page: %llx-%llx (%lluk)\n",
(unsigned long long)start, (unsigned long long)end,
(unsigned long long)((end - start) >> 10));
while (start < end) {
page = vm_page_lookup_pa(start);
assert(page != NULL);
vm_page_manage(page);
start += PAGE_SIZE;
}
}
static void __init
biosmem_free_usable_update_start(phys_addr_t *start, phys_addr_t res_start,
phys_addr_t res_end)
{
if ((*start >= res_start) && (*start < res_end))
*start = res_end;
}
static phys_addr_t __init
biosmem_free_usable_start(phys_addr_t start)
{
const struct biosmem_segment *seg;
unsigned int i;
biosmem_free_usable_update_start(&start, (unsigned long)&_boot,
BOOT_VTOP((unsigned long)&_end));
biosmem_free_usable_update_start(&start, biosmem_heap_start,
biosmem_heap_end);
for (i = 0; i < ARRAY_SIZE(biosmem_segments); i++) {
seg = &biosmem_segments[i];
biosmem_free_usable_update_start(&start, seg->avail_start,
seg->avail_end);
}
return start;
}
static int __init
biosmem_free_usable_reserved(phys_addr_t addr)
{
const struct biosmem_segment *seg;
unsigned int i;
if ((addr >= (unsigned long)&_boot)
&& (addr < BOOT_VTOP((unsigned long)&_end)))
return 1;
if ((addr >= biosmem_heap_start) && (addr < biosmem_heap_end))
return 1;
for (i = 0; i < ARRAY_SIZE(biosmem_segments); i++) {
seg = &biosmem_segments[i];
if ((addr >= seg->avail_start) && (addr < seg->avail_end))
return 1;
}
return 0;
}
static phys_addr_t __init
biosmem_free_usable_end(phys_addr_t start, phys_addr_t entry_end)
{
while (start < entry_end) {
if (biosmem_free_usable_reserved(start))
break;
start += PAGE_SIZE;
}
return start;
}
static void __init
biosmem_free_usable_entry(phys_addr_t start, phys_addr_t end)
{
phys_addr_t entry_end;
entry_end = end;
for (;;) {
start = biosmem_free_usable_start(start);
if (start >= entry_end)
return;
end = biosmem_free_usable_end(start, entry_end);
biosmem_free_usable_range(start, end);
start = end;
}
}
void __init
biosmem_free_usable(void)
{
struct biosmem_map_entry *entry;
uint64_t start, end;
unsigned int i;
for (i = 0; i < biosmem_map_size; i++) {
entry = &biosmem_map[i];
if (entry->type != BIOSMEM_TYPE_AVAILABLE)
continue;
start = vm_page_round(entry->base_addr);
if (start >= VM_PAGE_HIGHMEM_LIMIT)
break;
end = vm_page_trunc(entry->base_addr + entry->length);
if (start < BIOSMEM_BASE)
start = BIOSMEM_BASE;
biosmem_free_usable_entry(start, end);
}
}