Handle notification on page eviction

If requested by the user, make libpager call pager_notify_evict when a page
is flushed out by the kernel. Based on work by Ognyan Kulev.

* console/pager.c (pager_notify_evict): New function.
(user_pager_create): Update call to pager_create.
* ext2fs/pager.c: (pager_notify_evict): New function.
(create_disk_pager): Update call to diskfs_start_disk_pager.
(diskfs_get_filemap): Update call to pager_create.
* fatfs/pager.c: (pager_notify_evict): New function.
(create_fat_pager): Update call to diskfs_start_disk_pager.
(diskfs_get_filemap): Update call to pager_create.
* isofs/pager.c: (pager_notify_evict): New function.
(create_disk_pager): Update call to diskfs_start_disk_pager.
(diskfs_get_filemap): Update call to pager_create.
* libdiskfs/disk-pager.c (diskfs_start_disk_pager): Update definition and call
to pager_create.
* libdiskfs/diskfs-pager.h (diskfs_start_disk_pager): Update declaration.
* libpager/data-request.c (_pager_seqnos_memory_object_data_request): Take
pager's `notify_on_evict' member into account when calling
memory_object_data_supply.
* libpager/data-return.c (_pager_do_write_request): Handle user notification
on page flush.
* libpager/pager-create.c (pager_create): Update definition and set pager's
`notify_on_evict' member.
* libpager/pager.h (pager_create): Update declaration.
(pager_notify_evict): New declaration.
* libpager/priv.h (struct pager): New `notify_on_evict' member.
* storeio/pager.c: (pager_notify_evict): New function.
(dev_get_memory_object): Update call to pager_create.
* tmpfs/pager-stubs.c (pager_notify_evict): New function.
* ufs/pager.c (pager_notify_evict): New function.
(create_disk_pager): Update call to diskfs_start_disk_pager.
(diskfs_get_filemap): Update call to pager_create.

14 files changed, 158 insertions, 36 deletions

diff --git a/console/pager.c b/console/pager.c
index 781ba35e..4d0c5cdb 100644
--- a/console/pager.c
+++ b/console/pager.c
@@ -95,6 +95,14 @@ pager_unlock_page (struct user_pager_info *pager,
 }
 
 
+void
+pager_notify_evict (struct user_pager_info *pager,
+		    vm_offset_t page)
+{
+  assert (!"unrequested notification on eviction");
+}
+
+
 /* Tell how big the file is. */
 error_t
 pager_report_extent (struct user_pager_info *upi,
@@ -170,7 +178,7 @@ user_pager_create (struct user_pager *user_pager, unsigned int npages,
 
   /* XXX Are the values 1 and MEMORY_OBJECT_COPY_DELAY correct? */
   user_pager->pager = pager_create (upi, pager_bucket,
-				    1, MEMORY_OBJECT_COPY_DELAY);
+				    1, MEMORY_OBJECT_COPY_DELAY, 0);
   if (!user_pager->pager)
     {
       free (upi);
diff --git a/ext2fs/pager.c b/ext2fs/pager.c
index f740434a..92137112 100644
--- a/ext2fs/pager.c
+++ b/ext2fs/pager.c
@@ -511,6 +511,13 @@ pager_write_page (struct user_pager_info *pager, vm_offset_t page,
   else
     return file_pager_write_page (pager->node, page, (void *)buf);
 }
+
+void
+pager_notify_evict (struct user_pager_info *pager, vm_offset_t page)
+{
+  assert (!"unrequested notification on eviction");
+}
+
 
 /* Make page PAGE writable, at least up to ALLOCSIZE.  This function and
    diskfs_grow are the only places that blocks are actually added to the
@@ -776,8 +783,9 @@ create_disk_pager (void)
     ext2_panic ("can't create disk pager: %s", strerror (errno));
   upi->type = DISK;
   pager_bucket = ports_create_bucket ();
-  diskfs_start_disk_pager (upi, pager_bucket, MAY_CACHE, store->size,
-			   &disk_image);
+  diskfs_start_disk_pager (upi, pager_bucket, MAY_CACHE, 0,
+			   store->size, &disk_image);
+
 }
 
 /* Call this to create a FILE_DATA pager and return a send right.
@@ -817,7 +825,7 @@ diskfs_get_filemap (struct node *node, vm_prot_t prot)
 	  diskfs_nref_light (node);
 	  node->dn->pager =
 	    pager_create (upi, pager_bucket, MAY_CACHE,
-			  MEMORY_OBJECT_COPY_DELAY);
+			  MEMORY_OBJECT_COPY_DELAY, 0);
 	  if (node->dn->pager == 0)
 	    {
 	      diskfs_nrele_light (node);
diff --git a/fatfs/pager.c b/fatfs/pager.c
index f892c88f..8146e648 100644
--- a/fatfs/pager.c
+++ b/fatfs/pager.c
@@ -596,6 +596,13 @@ pager_unlock_page (struct user_pager_info *pager,
   return 0;
 }
 
+void
+pager_notify_evict (struct user_pager_info *pager,
+		    vm_offset_t page)
+{
+  assert (!"unrequested notification on eviction");
+}
+
 /* Grow the disk allocated to locked node NODE to be at least SIZE
    bytes, and set NODE->allocsize to the actual allocated size.  (If
    the allocated size is already SIZE bytes, do nothing.)  CRED
@@ -752,7 +759,7 @@ create_fat_pager (void)
   struct user_pager_info *upi = malloc (sizeof (struct user_pager_info));
   upi->type = FAT;
   pager_bucket = ports_create_bucket ();
-  diskfs_start_disk_pager (upi, pager_bucket, MAY_CACHE,
+  diskfs_start_disk_pager (upi, pager_bucket, MAY_CACHE, 0,
 			   bytes_per_sector * sectors_per_fat,
 			   &fat_image);
 }
@@ -794,7 +801,7 @@ diskfs_get_filemap (struct node *node, vm_prot_t prot)
           diskfs_nref_light (node);
           node->dn->pager =
             pager_create (upi, pager_bucket, MAY_CACHE,
-                          MEMORY_OBJECT_COPY_DELAY);
+                          MEMORY_OBJECT_COPY_DELAY, 0);
           if (node->dn->pager == 0)
             {
               diskfs_nrele_light (node);
diff --git a/isofs/pager.c b/isofs/pager.c
index f93e0c82..7a38ba33 100644
--- a/isofs/pager.c
+++ b/isofs/pager.c
@@ -94,6 +94,13 @@ pager_unlock_page (struct user_pager_info *pager,
   return EROFS;
 }
 
+void
+pager_notify_evict (struct user_pager_info *pager,
+		    vm_offset_t page)
+{
+  assert (!"unrequested notification on eviction");
+}
+
 /* Tell how big the file is. */
 error_t
 pager_report_extent (struct user_pager_info *pager,
@@ -137,7 +144,7 @@ create_disk_pager (void)
   upi->type = DISK;
   upi->np = 0;
   pager_bucket = ports_create_bucket ();
-  diskfs_start_disk_pager (upi, pager_bucket, 1, store->size, &disk_image);
+  diskfs_start_disk_pager (upi, pager_bucket, 1, 0, store->size, &disk_image);
   upi->p = diskfs_disk_pager;
 }
 
@@ -168,7 +175,8 @@ diskfs_get_filemap (struct node *np, vm_prot_t prot)
 	upi->type = FILE_DATA;
 	upi->np = np;
 	diskfs_nref_light (np);
-	upi->p = pager_create (upi, pager_bucket, 1, MEMORY_OBJECT_COPY_DELAY);
+	upi->p = pager_create (upi, pager_bucket, 1,
+			       MEMORY_OBJECT_COPY_DELAY, 0);
 	if (upi->p == 0)
 	  {
 	    diskfs_nrele_light (np);
diff --git a/libdiskfs/disk-pager.c b/libdiskfs/disk-pager.c
index 5795a281..8fe8f806 100644
--- a/libdiskfs/disk-pager.c
+++ b/libdiskfs/disk-pager.c
@@ -49,7 +49,8 @@ service_paging_requests (void *arg)
 
 void
 diskfs_start_disk_pager (struct user_pager_info *upi,
-			 struct port_bucket *pager_bucket, int may_cache,
+			 struct port_bucket *pager_bucket,
+			 int may_cache, int notify_on_evict,
 			 size_t size, void **image)
 {
   pthread_t thread;
@@ -68,7 +69,8 @@ diskfs_start_disk_pager (struct user_pager_info *upi,
 
   /* Create the pager.  */
   diskfs_disk_pager = pager_create (upi, pager_bucket,
-				    may_cache, MEMORY_OBJECT_COPY_NONE);
+				    may_cache, MEMORY_OBJECT_COPY_NONE,
+				    notify_on_evict);
   assert (diskfs_disk_pager);
 
   /* Get a port to the disk pager.  */
diff --git a/libdiskfs/diskfs-pager.h b/libdiskfs/diskfs-pager.h
index bd0a050c..a253069b 100644
--- a/libdiskfs/diskfs-pager.h
+++ b/libdiskfs/diskfs-pager.h
@@ -35,7 +35,8 @@ extern __thread struct disk_image_user *diskfs_exception_diu;
    mapped is returned in IMAGE.  INFO, PAGER_BUCKET, & MAY_CACHE are passed
    to `pager_create'.  */
 extern void diskfs_start_disk_pager (struct user_pager_info *info,
-				     struct port_bucket *pager_bucket, int may_cache,
+				     struct port_bucket *pager_bucket,
+				     int may_cache, int notify_on_evict,
 				     size_t size, void **image);
 
 extern struct pager *diskfs_disk_pager;
diff --git a/libpager/data-request.c b/libpager/data-request.c
index 4454facb..34b8b438 100644
--- a/libpager/data-request.c
+++ b/libpager/data-request.c
@@ -119,7 +119,8 @@ _pager_seqnos_memory_object_data_request (mach_port_t object,
     goto error_read;
 
   memory_object_data_supply (p->memobjcntl, offset, page, length, 1,
-			     write_lock ? VM_PROT_WRITE : VM_PROT_NONE, 0,
+			     write_lock ? VM_PROT_WRITE : VM_PROT_NONE,
+			     p->notify_on_evict ? 1 : 0,
 			     MACH_PORT_NULL);
   pthread_mutex_lock (&p->interlock);
   _pager_mark_object_error (p, offset, length, 0);
diff --git a/libpager/data-return.c b/libpager/data-return.c
index c70f0e8d..6a3b9037 100644
--- a/libpager/data-return.c
+++ b/libpager/data-return.c
@@ -39,6 +39,7 @@ _pager_do_write_request (mach_port_t object,
   struct pager *p;
   short *pm_entries;
   int npages, i;
+  char *notified;
   error_t *pagerrs;
   struct lock_request *lr;
   struct lock_list {struct lock_request *lr;
@@ -71,9 +72,6 @@ _pager_do_write_request (mach_port_t object,
       goto release_out;
     }
 
-  if (! dirty)
-    goto release_out;
-
   if (p->pager_state != NORMAL)
     {
       printf ("pager in wrong state for write\n");
@@ -83,6 +81,11 @@ _pager_do_write_request (mach_port_t object,
   npages = length / __vm_page_size;
   pagerrs = alloca (npages * sizeof (error_t));
 
+  notified = alloca (npages * (sizeof *notified));
+#ifndef NDEBUG
+  memset (notified, -1, npages * (sizeof *notified));
+#endif
+
   _pager_block_termination (p);	/* until we are done with the pagemap
 				   when the write completes. */
 
@@ -90,6 +93,24 @@ _pager_do_write_request (mach_port_t object,
 
   pm_entries = &p->pagemap[offset / __vm_page_size];
 
+  if (! dirty)
+    {
+      munmap ((void *) data, length);
+      if (!kcopy) {
+        /* Prepare notified array.  */
+        for (i = 0; i < npages; i++)
+          notified[i] = (p->notify_on_evict
+                         && ! (pm_entries[i] & PM_PAGEINWAIT));
+
+        _pager_release_seqno (p, seqno);
+        goto notify;
+      }
+      else {
+        _pager_allow_termination (p);
+        goto release_out;
+      }
+    }
+
   /* Make sure there are no other in-progress writes for any of these
      pages before we begin.  This imposes a little more serialization
      than we really have to require (because *all* future writes on
@@ -120,10 +141,6 @@ _pager_do_write_request (mach_port_t object,
     for (i = 0; i < npages; i++)
       pm_entries[i] |= PM_PAGINGOUT | PM_INIT;
 
-  if (!kcopy)
-    for (i = 0; i < npages; i++)
-      pm_entries[i] &= ~PM_INCORE;
-
   /* If this write occurs while a lock is pending, record
      it.  We have to keep this list because a lock request
      might come in while we do the I/O; in that case there
@@ -163,7 +180,10 @@ _pager_do_write_request (mach_port_t object,
   for (i = 0; i < npages; i++)
     {
       if (omitdata & (1 << i))
-	continue;
+	{
+	  notified[i] = 0;
+	  continue;
+	}
 
       if (pm_entries[i] & PM_WRITEWAIT)
 	wakeup = 1;
@@ -179,14 +199,22 @@ _pager_do_write_request (mach_port_t object,
 	pm_entries[i] |= PM_INVALID;
 
       if (pm_entries[i] & PM_PAGEINWAIT)
-	memory_object_data_supply (p->memobjcntl,
-				   offset + (vm_page_size * i),
-				   data + (vm_page_size * i),
-				   vm_page_size, 1,
-				   VM_PROT_NONE, 0, MACH_PORT_NULL);
+	{
+	  memory_object_data_supply (p->memobjcntl,
+				     offset + (vm_page_size * i),
+				     data + (vm_page_size * i),
+				     vm_page_size, 1,
+				     VM_PROT_NONE, 0, MACH_PORT_NULL);
+	  notified[i] = 0;
+	}
       else
-	munmap ((caddr_t) (data + (vm_page_size * i)),
-		vm_page_size);
+	{
+	  munmap ((void *) (data + (vm_page_size * i)),
+		  vm_page_size);
+	  notified[i] = (! kcopy && p->notify_on_evict);
+	  if (! kcopy)
+	    pm_entries[i] &= ~PM_INCORE;
+	}
 
       pm_entries[i] &= ~(PM_PAGINGOUT | PM_PAGEINWAIT | PM_WRITEWAIT);
     }
@@ -198,10 +226,29 @@ _pager_do_write_request (mach_port_t object,
   if (wakeup)
     pthread_cond_broadcast (&p->wakeup);
 
+ notify:
   _pager_allow_termination (p);
-
   pthread_mutex_unlock (&p->interlock);
 
+  for (i = 0; i < npages; i++)
+    {
+      assert (notified[i] == 0 || notified[i] == 1);
+      if (notified[i])
+	{
+	  short *pm_entry = &pm_entries[i];
+
+	  /* Do notify user.  */
+	  pager_notify_evict (p->upi, offset + (i * vm_page_size));
+
+	  /* Clear any error that is left.  Notification on eviction
+	     is used only to change association of page, so any
+	     error may no longer be valid.  */
+	  pthread_mutex_lock (&p->interlock);
+	  *pm_entry = SET_PM_ERROR (SET_PM_NEXTERROR (*pm_entry, 0), 0);
+	  pthread_mutex_unlock (&p->interlock);
+	}
+    }
+
   ports_port_deref (p);
   return 0;
 
diff --git a/libpager/pager-create.c b/libpager/pager-create.c
index 318c9f15..1aea6e96 100644
--- a/libpager/pager-create.c
+++ b/libpager/pager-create.c
@@ -22,7 +22,8 @@ struct pager *
 pager_create (struct user_pager_info *upi,
 	      struct port_bucket *bucket,
 	      boolean_t may_cache,
-	      memory_object_copy_strategy_t copy_strategy)
+	      memory_object_copy_strategy_t copy_strategy,
+	      boolean_t notify_on_evict)
 {
   struct pager *p;
 
@@ -38,6 +39,7 @@ pager_create (struct user_pager_info *upi,
   p->attribute_requests = 0;
   p->may_cache = may_cache;
   p->copy_strategy = copy_strategy;
+  p->notify_on_evict = notify_on_evict;
   p->memobjcntl = MACH_PORT_NULL;
   p->memobjname = MACH_PORT_NULL;
   p->seqno = -1;
diff --git a/libpager/pager.h b/libpager/pager.h
index 99fb3845..75ff108a 100644
--- a/libpager/pager.h
+++ b/libpager/pager.h
@@ -36,14 +36,17 @@ int pager_demuxer (mach_msg_header_t *inp,
    to receive requests.  U_PAGER will be provided to later calls to
    pager_find_address.  The pager will have one user reference
    created.  MAY_CACHE and COPY_STRATEGY are the original values of
-   those attributes as for memory_object_ready.  Users may create
-   references to pagers by use of the relevant ports library
-   functions.  On errors, return null and set errno.  */
+   those attributes as for memory_object_ready.  If NOTIFY_ON_EVICT is
+   non-zero, pager_notify_evict user callback will be called when page
+   is evicted.  Users may create references to pagers by use of the
+   relevant ports library functions.  On errors, return null and set
+   errno.  */
 struct pager *
 pager_create (struct user_pager_info *u_pager,
 	      struct port_bucket *bucket,
 	      boolean_t may_cache,
-	      memory_object_copy_strategy_t copy_strategy);
+	      memory_object_copy_strategy_t copy_strategy,
+	      boolean_t notify_on_evict);
 
 /* Return the user_pager_info struct associated with a pager. */
 struct user_pager_info *
@@ -172,6 +175,18 @@ error_t
 pager_unlock_page (struct user_pager_info *pager,
 		   vm_offset_t address);
 
+/* The user must define this function.  It is used when you want be
+   able to change association of pages to backing store.  To use it,
+   pass non-zero value in NOTIFY_ON_EVICT when pager is created with
+   pager_create.  You can change association of page only when
+   pager_notify_evict has been called and you haven't touched page
+   content after that.  Note there is a possibility that a page is
+   evicted, but user is not notified about that.  The user should be
+   able to handle this case.  */
+void
+pager_notify_evict (struct user_pager_info *pager,
+		    vm_offset_t page);
+
 /* The user must define this function.  It should report back (in
    *OFFSET and *SIZE the minimum valid address the pager will accept
    and the size of the object.   */
diff --git a/libpager/priv.h b/libpager/priv.h
index e6b2546b..7aa0fb44 100644
--- a/libpager/priv.h
+++ b/libpager/priv.h
@@ -49,6 +49,7 @@ struct pager
 
   boolean_t may_cache;
   memory_object_copy_strategy_t copy_strategy;
+  boolean_t notify_on_evict;
 
   /* Interface ports */
   memory_object_control_t memobjcntl;
diff --git a/storeio/pager.c b/storeio/pager.c
index cbae7eb6..7d787110 100644
--- a/storeio/pager.c
+++ b/storeio/pager.c
@@ -110,6 +110,13 @@ pager_unlock_page (struct user_pager_info *upi, vm_offset_t address)
     return 0;
 }
 
+void
+pager_notify_evict (struct user_pager_info *pager,
+		    vm_offset_t page)
+{
+  assert (!"unrequested notification on eviction");
+}
+
 /* The user must define this function.  It should report back (in
    *OFFSET and *SIZE the minimum valid address the pager will accept
    and the size of the object.   */
@@ -246,7 +253,7 @@ dev_get_memory_object (struct dev *dev, vm_prot_t prot, memory_object_t *memobj)
 	{
 	  dev->pager =
 	    pager_create ((struct user_pager_info *)dev, pager_port_bucket,
-			  1, MEMORY_OBJECT_COPY_DELAY);
+			  1, MEMORY_OBJECT_COPY_DELAY, 0);
 	  if (dev->pager == NULL)
 	    {
 	      pthread_mutex_unlock (&dev->pager_lock);
diff --git a/tmpfs/pager-stubs.c b/tmpfs/pager-stubs.c
index 25d70fe2..3cb264bb 100644
--- a/tmpfs/pager-stubs.c
+++ b/tmpfs/pager-stubs.c
@@ -57,6 +57,14 @@ pager_unlock_page (struct user_pager_info *pager,
   return EIEIO;
 }
 
+void
+pager_notify_evict (struct user_pager_info *pager,
+		    vm_offset_t page)
+{
+  abort();
+}
+
+
 /* The user must define this function.  It should report back (in
    *OFFSET and *SIZE the minimum valid address the pager will accept
    and the size of the object.   */
diff --git a/ufs/pager.c b/ufs/pager.c
index 1e3d140c..5d3e44ab 100644
--- a/ufs/pager.c
+++ b/ufs/pager.c
@@ -450,6 +450,13 @@ pager_unlock_page (struct user_pager_info *pager,
   return err;
 }
 
+void
+pager_notify_evict (struct user_pager_info *pager,
+		    vm_offset_t page)
+{
+  assert (!"unrequested notification on eviction");
+}
+
 /* Implement the pager_report_extent callback from the pager library.  See
    <hurd/pager.h> for the interface description. */
 inline error_t
@@ -502,7 +509,7 @@ create_disk_pager (void)
   upi->type = DISK;
   upi->np = 0;
   pager_bucket = ports_create_bucket ();
-  diskfs_start_disk_pager (upi, pager_bucket, MAY_CACHE, store->size,
+  diskfs_start_disk_pager (upi, pager_bucket, MAY_CACHE, 0, store->size,
 			   &disk_image);
   upi->p = diskfs_disk_pager;
 }
@@ -595,7 +602,7 @@ diskfs_get_filemap (struct node *np, vm_prot_t prot)
 	upi->unlocked_pagein_length = 0;
 	diskfs_nref_light (np);
 	upi->p = pager_create (upi, pager_bucket,
-			       MAY_CACHE, MEMORY_OBJECT_COPY_DELAY);
+			       MAY_CACHE, MEMORY_OBJECT_COPY_DELAY, 0);
 	if (upi->p == 0)
 	  {
 	    diskfs_nrele_light (np);