Handle notification on page eviction

If requested by the user, make libpager call pager_notify_evict when a page
is flushed out by the kernel. Based on work by Ognyan Kulev.

* console/pager.c (pager_notify_evict): New function.
(user_pager_create): Update call to pager_create.
* ext2fs/pager.c: (pager_notify_evict): New function.
(create_disk_pager): Update call to diskfs_start_disk_pager.
(diskfs_get_filemap): Update call to pager_create.
* fatfs/pager.c: (pager_notify_evict): New function.
(create_fat_pager): Update call to diskfs_start_disk_pager.
(diskfs_get_filemap): Update call to pager_create.
* isofs/pager.c: (pager_notify_evict): New function.
(create_disk_pager): Update call to diskfs_start_disk_pager.
(diskfs_get_filemap): Update call to pager_create.
* libdiskfs/disk-pager.c (diskfs_start_disk_pager): Update definition and call
to pager_create.
* libdiskfs/diskfs-pager.h (diskfs_start_disk_pager): Update declaration.
* libpager/data-request.c (_pager_seqnos_memory_object_data_request): Take
pager's `notify_on_evict' member into account when calling
memory_object_data_supply.
* libpager/data-return.c (_pager_do_write_request): Handle user notification
on page flush.
* libpager/pager-create.c (pager_create): Update definition and set pager's
`notify_on_evict' member.
* libpager/pager.h (pager_create): Update declaration.
(pager_notify_evict): New declaration.
* libpager/priv.h (struct pager): New `notify_on_evict' member.
* storeio/pager.c: (pager_notify_evict): New function.
(dev_get_memory_object): Update call to pager_create.
* tmpfs/pager-stubs.c (pager_notify_evict): New function.
* ufs/pager.c (pager_notify_evict): New function.
(create_disk_pager): Update call to diskfs_start_disk_pager.
(diskfs_get_filemap): Update call to pager_create.

5 files changed, 88 insertions, 22 deletions

diff --git a/libpager/data-request.c b/libpager/data-request.c
index 4454facb..34b8b438 100644
--- a/libpager/data-request.c
+++ b/libpager/data-request.c
@@ -119,7 +119,8 @@ _pager_seqnos_memory_object_data_request (mach_port_t object,
     goto error_read;
 
   memory_object_data_supply (p->memobjcntl, offset, page, length, 1,
-			     write_lock ? VM_PROT_WRITE : VM_PROT_NONE, 0,
+			     write_lock ? VM_PROT_WRITE : VM_PROT_NONE,
+			     p->notify_on_evict ? 1 : 0,
 			     MACH_PORT_NULL);
   pthread_mutex_lock (&p->interlock);
   _pager_mark_object_error (p, offset, length, 0);
diff --git a/libpager/data-return.c b/libpager/data-return.c
index c70f0e8d..6a3b9037 100644
--- a/libpager/data-return.c
+++ b/libpager/data-return.c
@@ -39,6 +39,7 @@ _pager_do_write_request (mach_port_t object,
   struct pager *p;
   short *pm_entries;
   int npages, i;
+  char *notified;
   error_t *pagerrs;
   struct lock_request *lr;
   struct lock_list {struct lock_request *lr;
@@ -71,9 +72,6 @@ _pager_do_write_request (mach_port_t object,
       goto release_out;
     }
 
-  if (! dirty)
-    goto release_out;
-
   if (p->pager_state != NORMAL)
     {
       printf ("pager in wrong state for write\n");
@@ -83,6 +81,11 @@ _pager_do_write_request (mach_port_t object,
   npages = length / __vm_page_size;
   pagerrs = alloca (npages * sizeof (error_t));
 
+  notified = alloca (npages * (sizeof *notified));
+#ifndef NDEBUG
+  memset (notified, -1, npages * (sizeof *notified));
+#endif
+
   _pager_block_termination (p);	/* until we are done with the pagemap
 				   when the write completes. */
 
@@ -90,6 +93,24 @@ _pager_do_write_request (mach_port_t object,
 
   pm_entries = &p->pagemap[offset / __vm_page_size];
 
+  if (! dirty)
+    {
+      munmap ((void *) data, length);
+      if (!kcopy) {
+        /* Prepare notified array.  */
+        for (i = 0; i < npages; i++)
+          notified[i] = (p->notify_on_evict
+                         && ! (pm_entries[i] & PM_PAGEINWAIT));
+
+        _pager_release_seqno (p, seqno);
+        goto notify;
+      }
+      else {
+        _pager_allow_termination (p);
+        goto release_out;
+      }
+    }
+
   /* Make sure there are no other in-progress writes for any of these
      pages before we begin.  This imposes a little more serialization
      than we really have to require (because *all* future writes on
@@ -120,10 +141,6 @@ _pager_do_write_request (mach_port_t object,
     for (i = 0; i < npages; i++)
       pm_entries[i] |= PM_PAGINGOUT | PM_INIT;
 
-  if (!kcopy)
-    for (i = 0; i < npages; i++)
-      pm_entries[i] &= ~PM_INCORE;
-
   /* If this write occurs while a lock is pending, record
      it.  We have to keep this list because a lock request
      might come in while we do the I/O; in that case there
@@ -163,7 +180,10 @@ _pager_do_write_request (mach_port_t object,
   for (i = 0; i < npages; i++)
     {
       if (omitdata & (1 << i))
-	continue;
+	{
+	  notified[i] = 0;
+	  continue;
+	}
 
       if (pm_entries[i] & PM_WRITEWAIT)
 	wakeup = 1;
@@ -179,14 +199,22 @@ _pager_do_write_request (mach_port_t object,
 	pm_entries[i] |= PM_INVALID;
 
       if (pm_entries[i] & PM_PAGEINWAIT)
-	memory_object_data_supply (p->memobjcntl,
-				   offset + (vm_page_size * i),
-				   data + (vm_page_size * i),
-				   vm_page_size, 1,
-				   VM_PROT_NONE, 0, MACH_PORT_NULL);
+	{
+	  memory_object_data_supply (p->memobjcntl,
+				     offset + (vm_page_size * i),
+				     data + (vm_page_size * i),
+				     vm_page_size, 1,
+				     VM_PROT_NONE, 0, MACH_PORT_NULL);
+	  notified[i] = 0;
+	}
       else
-	munmap ((caddr_t) (data + (vm_page_size * i)),
-		vm_page_size);
+	{
+	  munmap ((void *) (data + (vm_page_size * i)),
+		  vm_page_size);
+	  notified[i] = (! kcopy && p->notify_on_evict);
+	  if (! kcopy)
+	    pm_entries[i] &= ~PM_INCORE;
+	}
 
       pm_entries[i] &= ~(PM_PAGINGOUT | PM_PAGEINWAIT | PM_WRITEWAIT);
     }
@@ -198,10 +226,29 @@ _pager_do_write_request (mach_port_t object,
   if (wakeup)
     pthread_cond_broadcast (&p->wakeup);
 
+ notify:
   _pager_allow_termination (p);
-
   pthread_mutex_unlock (&p->interlock);
 
+  for (i = 0; i < npages; i++)
+    {
+      assert (notified[i] == 0 || notified[i] == 1);
+      if (notified[i])
+	{
+	  short *pm_entry = &pm_entries[i];
+
+	  /* Do notify user.  */
+	  pager_notify_evict (p->upi, offset + (i * vm_page_size));
+
+	  /* Clear any error that is left.  Notification on eviction
+	     is used only to change association of page, so any
+	     error may no longer be valid.  */
+	  pthread_mutex_lock (&p->interlock);
+	  *pm_entry = SET_PM_ERROR (SET_PM_NEXTERROR (*pm_entry, 0), 0);
+	  pthread_mutex_unlock (&p->interlock);
+	}
+    }
+
   ports_port_deref (p);
   return 0;
 
diff --git a/libpager/pager-create.c b/libpager/pager-create.c
index 318c9f15..1aea6e96 100644
--- a/libpager/pager-create.c
+++ b/libpager/pager-create.c
@@ -22,7 +22,8 @@ struct pager *
 pager_create (struct user_pager_info *upi,
 	      struct port_bucket *bucket,
 	      boolean_t may_cache,
-	      memory_object_copy_strategy_t copy_strategy)
+	      memory_object_copy_strategy_t copy_strategy,
+	      boolean_t notify_on_evict)
 {
   struct pager *p;
 
@@ -38,6 +39,7 @@ pager_create (struct user_pager_info *upi,
   p->attribute_requests = 0;
   p->may_cache = may_cache;
   p->copy_strategy = copy_strategy;
+  p->notify_on_evict = notify_on_evict;
   p->memobjcntl = MACH_PORT_NULL;
   p->memobjname = MACH_PORT_NULL;
   p->seqno = -1;
diff --git a/libpager/pager.h b/libpager/pager.h
index 99fb3845..75ff108a 100644
--- a/libpager/pager.h
+++ b/libpager/pager.h
@@ -36,14 +36,17 @@ int pager_demuxer (mach_msg_header_t *inp,
    to receive requests.  U_PAGER will be provided to later calls to
    pager_find_address.  The pager will have one user reference
    created.  MAY_CACHE and COPY_STRATEGY are the original values of
-   those attributes as for memory_object_ready.  Users may create
-   references to pagers by use of the relevant ports library
-   functions.  On errors, return null and set errno.  */
+   those attributes as for memory_object_ready.  If NOTIFY_ON_EVICT is
+   non-zero, pager_notify_evict user callback will be called when page
+   is evicted.  Users may create references to pagers by use of the
+   relevant ports library functions.  On errors, return null and set
+   errno.  */
 struct pager *
 pager_create (struct user_pager_info *u_pager,
 	      struct port_bucket *bucket,
 	      boolean_t may_cache,
-	      memory_object_copy_strategy_t copy_strategy);
+	      memory_object_copy_strategy_t copy_strategy,
+	      boolean_t notify_on_evict);
 
 /* Return the user_pager_info struct associated with a pager. */
 struct user_pager_info *
@@ -172,6 +175,18 @@ error_t
 pager_unlock_page (struct user_pager_info *pager,
 		   vm_offset_t address);
 
+/* The user must define this function.  It is used when you want be
+   able to change association of pages to backing store.  To use it,
+   pass non-zero value in NOTIFY_ON_EVICT when pager is created with
+   pager_create.  You can change association of page only when
+   pager_notify_evict has been called and you haven't touched page
+   content after that.  Note there is a possibility that a page is
+   evicted, but user is not notified about that.  The user should be
+   able to handle this case.  */
+void
+pager_notify_evict (struct user_pager_info *pager,
+		    vm_offset_t page);
+
 /* The user must define this function.  It should report back (in
    *OFFSET and *SIZE the minimum valid address the pager will accept
    and the size of the object.   */
diff --git a/libpager/priv.h b/libpager/priv.h
index e6b2546b..7aa0fb44 100644
--- a/libpager/priv.h
+++ b/libpager/priv.h
@@ -49,6 +49,7 @@ struct pager
 
   boolean_t may_cache;
   memory_object_copy_strategy_t copy_strategy;
+  boolean_t notify_on_evict;
 
   /* Interface ports */
   memory_object_control_t memobjcntl;