I did a cleanup pass on Andrew's code and rearranged a few minor things,
plus removed the vm_contig_[int]_pg_alloc() shim and just made the
original vm_contig_pg_alloc() take an mflags argument, and changed
the mflags type to an int.
It looks like nothing else in the system used the vm_contig_pg_alloc()
entry point anyway so I also made that static.
Please test.
-Matt
Index: vm/vm_contig.c
===================================================================
RCS file: /cvs/src/sys/vm/vm_contig.c,v
retrieving revision 1.12
diff -u -r1.12 vm_contig.c
--- vm/vm_contig.c 10 Nov 2004 20:19:51 -0000 1.12
+++ vm/vm_contig.c 22 Feb 2005 19:33:20 -0000
@@ -146,8 +146,8 @@
for (m = TAILQ_FIRST(&vm_page_queues[queue].pl); m != NULL; m = next) {
KASSERT(m->queue == queue,
- ("vm_contig_clean: page %p's queue is not %d", m, queue));
-
+ ("vm_contig_clean: page %p's queue is not %d",
+ m, queue));
next = TAILQ_NEXT(m, pageq);
if (vm_page_sleep_busy(m, TRUE, "vpctw0"))
@@ -170,15 +170,29 @@
return (TRUE);
}
}
-
if ((m->dirty == 0) && (m->busy == 0) && (m->hold_count == 0))
vm_page_cache(m);
}
-
return (FALSE);
}
/*
+ * vm_contig_pg_flush:
+ *
+ * Attempt to flush (count) pages from the given page queue. This may or
+ * may not succeed. Take up to <count> passes and delay 1/20 of a second
+ * between each pass.
+ */
+static void
+vm_contig_pg_flush(int queue, int count)
+{
+ while (count > 0) {
+ if (!vm_contig_pg_clean(queue))
+ break;
+ --count;
+ }
+}
+/*
* vm_contig_pg_alloc:
*
* Allocate contiguous pages from the VM. This function does not
@@ -187,15 +201,10 @@
*
* Malloc()'s data structures have been used for collection of
* statistics and for allocations of less than a page.
- *
*/
-int
-vm_contig_pg_alloc(
- unsigned long size,
- vm_paddr_t low,
- vm_paddr_t high,
- unsigned long alignment,
- unsigned long boundary)
+static int
+vm_contig_pg_alloc(unsigned long size, vm_paddr_t low, vm_paddr_t high,
+ unsigned long alignment, unsigned long boundary, int mflags)
{
int i, start, pass;
vm_offset_t phys;
@@ -212,13 +221,20 @@
panic("vm_contig_pg_alloc: boundary must be a power of 2");
start = 0;
- for (pass = 0; pass <= 1; pass++) {
- crit_enter();
-again:
+ crit_enter();
+
+ /*
+ * Three passes (0, 1, 2). Each pass scans the VM page list for
+ * free or cached pages. After each pass if the entire scan failed
+ * we attempt to flush inactive pages and reset the start index back
+ * to 0. For passes 1 and 2 we also attempt to flush active pages.
+ */
+ for (pass = 0; pass < 3; pass++) {
/*
- * Find first page in array that is free, within range, aligned, and
- * such that the boundary won't be crossed.
+ * Find first page in array that is free, within range,
+ * aligned, and such that the boundary won't be crossed.
*/
+again:
for (i = start; i < vmstats.v_page_count; i++) {
m = &pga[i];
phys = VM_PAGE_TO_PHYS(m);
@@ -244,13 +260,39 @@
if ((i == vmstats.v_page_count) ||
((VM_PAGE_TO_PHYS(&pga[i]) + size) > high)) {
-again1:
- if (vm_contig_pg_clean(PQ_INACTIVE))
- goto again1;
- if (vm_contig_pg_clean(PQ_ACTIVE))
- goto again1;
+ /*
+ * Best effort flush of all inactive pages.
+ * This is quite quick, for now stall all
+ * callers, even if they've specified M_NOWAIT.
+ */
+ vm_contig_pg_flush(PQ_INACTIVE,
+ vmstats.v_inactive_count);
+
+ crit_exit(); /* give interrupts a chance */
+ crit_enter();
+
+ /*
+ * Best effort flush of active pages.
+ *
+ * This is very, very slow.
+ * Only do this if the caller has agreed to M_WAITOK.
+ *
+ * If enough pages are flushed, we may succeed on
+ * next (final) pass, if not the caller, contigmalloc(),
+ * will fail in the index < 0 case.
+ */
+ if (pass > 0 && (mflags & M_WAITOK)) {
+ vm_contig_pg_flush (PQ_ACTIVE,
+ vmstats.v_active_count);
+ }
- crit_exit();
+ /*
+ * We're already too high in the address space
+ * to succeed, reset to 0 for the next iteration.
+ */
+ start = 0;
+ crit_exit(); /* give interrupts a chance */
+ crit_enter();
continue; /* next pass */
}
start = i;
@@ -423,13 +465,13 @@
int index;
void *rv;
- index = vm_contig_pg_alloc(size, low, high, alignment, boundary);
+ index = vm_contig_pg_alloc(size, low, high, alignment, boundary, flags);
if (index < 0) {
printf("contigmalloc_map: failed in index < 0 case!");
return NULL;
}
- rv = (void *) vm_contig_pg_kmap(index, size, map, flags);
+ rv = (void *)vm_contig_pg_kmap(index, size, map, flags);
if (!rv)
vm_contig_pg_free(index, size);
Index: vm/vm_page.h
===================================================================
RCS file: /cvs/src/sys/vm/vm_page.h,v
retrieving revision 1.16
diff -u -r1.16 vm_page.h
--- vm/vm_page.h 4 Oct 2004 09:05:26 -0000 1.16
+++ vm/vm_page.h 22 Feb 2005 19:32:17 -0000
@@ -418,7 +418,6 @@
vm_page_t vm_page_list_find(int basequeue, int index, boolean_t prefer_zero);
void vm_page_zero_invalid(vm_page_t m, boolean_t setvalid);
void vm_page_free_toq(vm_page_t m);
-int vm_contig_pg_alloc(u_long, vm_paddr_t, vm_paddr_t, u_long, u_long);
vm_offset_t vm_contig_pg_kmap(int, u_long, vm_map_t, int);
void vm_contig_pg_free(int, u_long);