diff options
Diffstat (limited to 'libpager')
| -rw-r--r-- | libpager/pager-memcpy.c | 196 |
1 files changed, 153 insertions, 43 deletions
diff --git a/libpager/pager-memcpy.c b/libpager/pager-memcpy.c index 1104ca0d..0a5a4b7b 100644 --- a/libpager/pager-memcpy.c +++ b/libpager/pager-memcpy.c @@ -1,5 +1,5 @@ /* Fault-safe copy into or out of pager-backed memory. - Copyright (C) 1996,97,99,2000,2001 Free Software Foundation, Inc. + Copyright (C) 1996,97,99, 2000,01,02 Free Software Foundation, Inc. Written by Roland McGrath. This program is free software; you can redistribute it and/or @@ -16,12 +16,17 @@ along with this program; if not, write to the Free Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ +#include "priv.h" #include "pager.h" #include <sys/mman.h> #include <hurd/sigpreempt.h> #include <assert.h> #include <string.h> +/* Start using vm_copy over memcpy when we have at a page. (This + value *cannot* be less than vm_page_size.) */ +#define VMCOPY_BETTER_THAN_MEMCPY (vm_page_size) + /* Try to copy *SIZE bytes between the region OTHER points to and the region at OFFSET in the pager indicated by PAGER and MEMOBJ. If PROT is VM_PROT_READ, copying is from the pager to OTHER; @@ -29,80 +34,185 @@ *SIZE is always filled in the actual number of bytes successfully copied. Returns an error code if the pager-backed memory faults; if there is no fault, returns 0 and *SIZE will be unchanged. */ - error_t pager_memcpy (struct pager *pager, memory_object_t memobj, vm_offset_t offset, void *other, size_t *size, vm_prot_t prot) { - vm_address_t window = 0; - vm_size_t windowsize = 8 * vm_page_size; - size_t to_copy = *size; error_t err; + size_t n = *size; - error_t copy (struct hurd_signal_preemptor *preemptor) +#define VMCOPY_WINDOW_DEFAULT_SIZE (16 * vm_page_size) +#define MEMCPY_WINDOW_DEFAULT_SIZE (8 * vm_page_size) + vm_address_t window; + vm_size_t window_size; + + error_t do_vm_copy (void) { - while (to_copy > 0) - { - size_t pageoff = offset & (vm_page_size - 1); + assert ((offset & (vm_page_size - 1)) == 0); + assert (((vm_address_t) other & (vm_page_size - 1)) == 0); + assert (n >= vm_page_size); - if (window) - /* Deallocate the old window. */ - munmap ((caddr_t) window, windowsize); + do + { + window_size = + VMCOPY_WINDOW_DEFAULT_SIZE > n + ? (n - (n & (vm_page_size - 1))) + : VMCOPY_WINDOW_DEFAULT_SIZE; + + assert (window_size >= VMCOPY_BETTER_THAN_MEMCPY); + assert ((window_size & (vm_page_size - 1)) == 0); + + err = vm_map (mach_task_self (), &window, window_size, 0, 1, + memobj, offset, 0, prot, prot, VM_INHERIT_NONE); + if (err) + return err; - /* Map in and copy a standard-sized window, unless that is - more than the total left to be copied. */ + if (prot == VM_PROT_READ) + err = vm_copy (mach_task_self (), window, window_size, + (vm_address_t) other); + else + err = vm_copy (mach_task_self (), (vm_address_t) other, + window_size, window); - if (windowsize > pageoff + to_copy) - windowsize = pageoff + to_copy; + vm_deallocate (mach_task_self (), window, window_size); - window = 0; - err = vm_map (mach_task_self (), &window, windowsize, 0, 1, - memobj, offset - pageoff, 0, - prot, prot, VM_INHERIT_NONE); if (err) - return 0; + return err; - /* Realign the fault preemptor for the new mapping window. */ - preemptor->first = window; - preemptor->last = window + windowsize; + other += window_size; + offset += window_size; + n -= window_size; + } + while (n >= VMCOPY_BETTER_THAN_MEMCPY); - if (prot == VM_PROT_READ) - memcpy (other, (const void *) window + pageoff, - windowsize - pageoff); - else - memcpy ((void *) window + pageoff, other, windowsize - pageoff); + return 0; + } - offset += windowsize - pageoff; - other += windowsize - pageoff; - to_copy -= windowsize - pageoff; + error_t do_copy (struct hurd_signal_preemptor *preemptor) + { + error_t do_memcpy (size_t to_copy) + { + window_size = MEMCPY_WINDOW_DEFAULT_SIZE; + + do + { + size_t pageoff = offset & (vm_page_size - 1); + size_t copy_count = window_size - pageoff; + + /* Map in and copy a standard-sized window, unless that is + more than the total left to be copied. */ + + if (window_size >= round_page (pageoff + to_copy)) + { + copy_count = to_copy; + window_size = round_page (pageoff + to_copy); + } + + err = vm_map (mach_task_self (), &window, window_size, 0, 1, + memobj, offset - pageoff, 0, + prot, prot, VM_INHERIT_NONE); + if (err) + return err; + + /* Realign the fault preemptor for the new mapping window. */ + preemptor->first = window; + preemptor->last = window + window_size; + + if (prot == VM_PROT_READ) + memcpy (other, (const void *) window + pageoff, copy_count); + else + memcpy ((void *) window + pageoff, other, copy_count); + + vm_deallocate (mach_task_self (), window, window_size); + + offset += copy_count; + other += copy_count; + to_copy -= copy_count; + n -= copy_count; + + assert (n >= 0); + assert (to_copy >= 0); + } + while (to_copy > 0); + + return 0; } - return 0; + + /* Can we use vm_copy? */ + if ((((vm_address_t) other & (vm_page_size - 1)) + == (offset & (vm_page_size - 1))) + && (n >= (VMCOPY_BETTER_THAN_MEMCPY + vm_page_size + - ((vm_address_t) other & (vm_page_size - 1))))) + /* 1) other and offset are aligned with repect to each other; + and 2) we have at least VMCOPY_BETTER_THAN_MEMCPY fully + aligned pages. */ + { + err = do_memcpy (vm_page_size + - ((vm_address_t) other & (vm_page_size - 1))); + if (err) + return err; + + assert (n >= VMCOPY_BETTER_THAN_MEMCPY); + + err = do_vm_copy (); + if (err || n == 0) + /* We failed or we finished. */ + return err; + + assert (n < VMCOPY_BETTER_THAN_MEMCPY); + } + + return do_memcpy (n); } jmp_buf buf; void fault (int signo, long int sigcode, struct sigcontext *scp) { assert (scp->sc_error == EKERN_MEMORY_ERROR); - err = pager_get_error (pager, sigcode - window + offset); - to_copy -= sigcode - window; + err = pager_get_error (pager, + (sigcode - window + offset) / vm_page_size); + n -= sigcode - window; + vm_deallocate (mach_task_self (), window, window_size); longjmp (buf, 1); } - if (to_copy == 0) - /* Short-circuit return if nothing to do. - ERR would not be initialized by the copy loop in this case. */ + if (n == 0) + /* Nothing to do. */ return 0; + if (((vm_address_t) other & (vm_page_size - 1)) == 0 + && (offset & (vm_page_size - 1)) == 0 + && n >= VMCOPY_BETTER_THAN_MEMCPY) + /* 1) the start address is page aligned; 2) the offset is page + aligned; and 3) we have more than VMCOPY_BETTER_THAN_MEMCPY + pages. */ + { + err = do_vm_copy (); + if (err || n == 0) + /* We failed or we finished. */ + { + *size -= n; + return err; + } + + assert (n < VMCOPY_BETTER_THAN_MEMCPY); + } + + /* Need to do it the hard way. */ + + window = 0; + window_size = 0; + if (setjmp (buf) == 0) hurd_catch_signal (sigmask (SIGSEGV) | sigmask (SIGBUS), - window, window + windowsize, - ©, (sighandler_t) &fault); + window, window + window_size, + &do_copy, (sighandler_t) &fault); - if (window) - munmap ((caddr_t) window, windowsize); + if (! err) + assert (n == 0); - *size -= to_copy; + *size -= n; return err; } |