take 2: request for review: backport of sx and rwlocks from 7.0 to

--SnV5plBeK2Ge1I9g
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline

Updated patch attached:

sizeof(struct sx) should be the same.
garbage removed from kern_sx.c.
alignment of "struct thread" set to constant rather than magic number.
uma_zalloc of "struct thread" fixed to use constant.



-- 
- Alfred Perlstein

--SnV5plBeK2Ge1I9g
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment;
	filename="netsmp_rwlock_freebsd6_09032007.diff"

Index: conf/NOTES
===================================================================
RCS file: /cvs/ncvs/src/sys/conf/NOTES,v
retrieving revision 1.1325.2.36
diff -c -r1.1325.2.36 NOTES
*** conf/NOTES	8 Jul 2007 15:30:28 -0000	1.1325.2.36
--- conf/NOTES	31 Aug 2007 00:39:59 -0000
***************
*** 189,200 ****
--- 189,214 ----
  # to disable it.
  options 	NO_ADAPTIVE_MUTEXES
  
+ # ADAPTIVE_RWLOCKS changes the behavior of reader/writer locks to spin
+ # if the thread that currently owns the rwlock is executing on another
+ # CPU.  This behaviour is enabled by default, so this option can be used
+ # to disable it.
+ options         NO_ADAPTIVE_RWLOCKS
+ 
+ 
  # ADAPTIVE_GIANT causes the Giant lock to also be made adaptive when
  # running without NO_ADAPTIVE_MUTEXES.  Normally, because Giant is assumed
  # to be held for extended periods, contention on Giant will cause a thread
  # to sleep rather than spinning.
  options 	ADAPTIVE_GIANT
  
+  
+ # ADAPTIVE_SX changes the behavior of sx locks to spin if the thread
+ # that currently owns the lock is executing on another CPU.  Note that
+ # in addition to enabling this option, individual sx locks must be
+ # initialized with the SX_ADAPTIVESPIN flag.
+ options   ADAPTIVE_SX
+ 
  # MUTEX_NOINLINE forces mutex operations to call functions to perform each
  # operation rather than inlining the simple cases.  This can be used to
  # shrink the size of the kernel text segment.  Note that this behavior is
***************
*** 207,212 ****
--- 221,240 ----
  # priority waiter.
  options 	MUTEX_WAKE_ALL
  
+ # RWLOCK_NOINLINE forces rwlock operations to call functions to perform each
+ # operation rather than inlining the simple cases.  This can be used to
+ # shrink the size of the kernel text segment.  Note that this behavior is
+ # already implied by the INVARIANT_SUPPORT, INVARIANTS, KTR, LOCK_PROFILING,
+ # and WITNESS options.
+ options         RWLOCK_NOINLINE
+ 
+ # SX_NOINLINE forces sx lock operations to call functions to perform each
+ # operation rather than inlining the simple cases.  This can be used to
+ # shrink the size of the kernel text segment.  Note that this behavior is
+ # already implied by the INVARIANT_SUPPORT, INVARIANTS, KTR, LOCK_PROFILING,
+ # and WITNESS options.
+ options    SX_NOINLINE
+ 
  # SMP Debugging Options:
  #
  # PREEMPTION allows the threads that are in the kernel to be preempted
Index: conf/files
===================================================================
RCS file: /cvs/ncvs/src/sys/conf/files,v
retrieving revision 1.1031.2.67
diff -c -r1.1031.2.67 files
*** conf/files	23 Aug 2007 22:30:14 -0000	1.1031.2.67
--- conf/files	31 Aug 2007 00:39:59 -0000
***************
*** 1312,1317 ****
--- 1312,1318 ----
  kern/kern_proc.c		standard
  kern/kern_prot.c		standard
  kern/kern_resource.c		standard
+ kern/kern_rwlock.c		standard
  kern/kern_sema.c		standard
  kern/kern_shutdown.c		standard
  kern/kern_sig.c			standard
Index: conf/options
===================================================================
RCS file: /cvs/ncvs/src/sys/conf/options,v
retrieving revision 1.510.2.21
diff -c -r1.510.2.21 options
*** conf/options	8 Jul 2007 15:30:28 -0000	1.510.2.21
--- conf/options	31 Aug 2007 00:39:59 -0000
***************
*** 60,66 ****
--- 60,68 ----
  
  # Miscellaneous options.
  ADAPTIVE_GIANT	opt_adaptive_mutexes.h
+ ADAPTIVE_SX
  NO_ADAPTIVE_MUTEXES	opt_adaptive_mutexes.h
+ NO_ADAPTIVE_RWLOCKS
  ALQ
  AUDIT		opt_global.h
  CODA_COMPAT_5	opt_coda.h
***************
*** 517,522 ****
--- 519,526 ----
  MSIZE			opt_global.h
  REGRESSION		opt_global.h
  RESTARTABLE_PANICS	opt_global.h
+ RWLOCK_NOINLINE		opt_global.h
+ SX_NOINLINE		opt_global.h
  VFS_BIO_DEBUG		opt_global.h
  
  # These are VM related options
Index: dev/acpica/acpi_ec.c
===================================================================
RCS file: /cvs/ncvs/src/sys/dev/acpica/acpi_ec.c,v
retrieving revision 1.65.2.2
diff -c -r1.65.2.2 acpi_ec.c
*** dev/acpica/acpi_ec.c	11 May 2006 17:41:00 -0000	1.65.2.2
--- dev/acpica/acpi_ec.c	31 Aug 2007 01:20:08 -0000
***************
*** 144,149 ****
--- 144,150 ----
  #include <sys/bus.h>
  #include <sys/malloc.h>
  #include <sys/module.h>
+ #include <sys/lock.h>
  #include <sys/sx.h>
  
  #include <machine/bus.h>
Index: kern/kern_ktrace.c
===================================================================
RCS file: /cvs/ncvs/src/sys/kern/kern_ktrace.c,v
retrieving revision 1.101.2.5
diff -c -r1.101.2.5 kern_ktrace.c
*** kern/kern_ktrace.c	6 Sep 2006 21:43:59 -0000	1.101.2.5
--- kern/kern_ktrace.c	31 Aug 2007 00:39:59 -0000
***************
*** 53,58 ****
--- 53,59 ----
  #include <sys/vnode.h>
  #include <sys/ktrace.h>
  #include <sys/sx.h>
+ #include <sys/condvar.h>
  #include <sys/sysctl.h>
  #include <sys/syslog.h>
  #include <sys/sysproto.h>
Index: kern/kern_rwlock.c
===================================================================
RCS file: kern/kern_rwlock.c
diff -N kern/kern_rwlock.c
*** /dev/null	1 Jan 1970 00:00:00 -0000
--- kern/kern_rwlock.c	31 Aug 2007 03:25:24 -0000
***************
*** 0 ****
--- 1,943 ----
+ /*-
+  * Copyright (c) 2006 John Baldwin <[e-mail]>
+  * All rights reserved.
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  * 3. Neither the name of the author nor the names of any co-contributors
+  *    may be used to endorse or promote products derived from this software
+  *    without specific prior written permission.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  */
+ 
+ /*
+  * Machine independent bits of reader/writer lock implementation.
+  */
+ 
+ #include <sys/cdefs.h>
+ __FBSDID("$FreeBSD: src/sys/kern/kern_rwlock.c,v 1.27 2007/06/26 21:31:56 attilio Exp $");
+ 
+ #include "opt_ddb.h"
+ 
+ #include <sys/param.h>
+ #include <sys/ktr.h>
+ #include <sys/lock.h>
+ #include <sys/mutex.h>
+ #include <sys/proc.h>
+ #include <sys/rwlock.h>
+ #include <sys/systm.h>
+ #include <sys/turnstile.h>
+ #include <sys/lock_profile.h>
+ #include <machine/cpu.h>
+ 
+ CTASSERT((RW_RECURSE & LO_CLASSFLAGS) == RW_RECURSE);
+ 
+ #if defined(SMP) && !defined(NO_ADAPTIVE_RWLOCKS)
+ #define	ADAPTIVE_RWLOCKS
+ #endif
+ 
+ #ifdef DDB
+ #include <ddb/ddb.h>
+ 
+ static void	db_show_rwlock(struct lock_object *lock);
+ #endif
+ 
+ struct lock_class lock_class_rw = {
+ 	.lc_name = "rw",
+ 	.lc_flags = LC_SLEEPLOCK | LC_RECURSABLE | LC_UPGRADABLE,
+ #ifdef DDB
+ 	.lc_ddb_show = db_show_rwlock,
+ #endif
+ };
+ 
+ /*
+  * Return a pointer to the owning thread if the lock is write-locked or
+  * NULL if the lock is unlocked or read-locked.
+  */
+ #define	rw_wowner(rw)							\
+ 	((rw)->rw_lock & RW_LOCK_READ ? NULL :				\
+ 	    (struct thread *)RW_OWNER((rw)->rw_lock))
+ 
+ /*
+  * Returns if a write owner is recursed.  Write ownership is not assured
+  * here and should be previously checked.
+  */
+ #define	rw_recursed(rw)		((rw)->rw_recurse != 0)
+ 
+ /*
+  * Return true if curthread helds the lock.
+  */
+ #define	rw_wlocked(rw)		(rw_wowner((rw)) == curthread)
+ 
+ /*
+  * Return a pointer to the owning thread for this lock who should receive
+  * any priority lent by threads that block on this lock.  Currently this
+  * is identical to rw_wowner().
+  */
+ #define	rw_owner(rw)		rw_wowner(rw)
+ 
+ #ifndef INVARIANTS
+ #define	_rw_assert(rw, what, file, line)
+ #endif
+ 
+ void
+ rw_init_flags(struct rwlock *rw, const char *name, int opts)
+ {
+ 	struct lock_object *lock;
+ 	int flags;
+ 
+ 	MPASS((opts & ~(RW_DUPOK | RW_NOPROFILE | RW_NOWITNESS | RW_QUIET |
+ 	    RW_RECURSE)) == 0);
+ 
+ 	flags = LO_UPGRADABLE | LO_RECURSABLE;
+ 	if (opts & RW_DUPOK)
+ 		flags |= LO_DUPOK;
+ 	if (!(opts & RW_NOWITNESS))
+ 		flags |= LO_WITNESS;
+ 	if (opts & RW_QUIET)
+ 		flags |= LO_QUIET;
+ 	flags |= opts & RW_RECURSE;
+ 
+ 	rw->rw_lock = RW_UNLOCKED;
+ 	rw->rw_recurse = 0;
+ 	lock = &rw->lock_object;
+ 	lock->lo_class = &lock_class_rw;
+ 	lock->lo_flags = flags;
+ 	lock->lo_name = lock->lo_type = name;
+ 	LOCK_LOG_INIT(lock, opts);
+ 	WITNESS_INIT(lock);
+ }
+ 
+ void
+ rw_destroy(struct rwlock *rw)
+ {
+ 
+ 	LOCK_LOG_DESTROY(&rw->lock_object, 0);
+ 	KASSERT(rw->rw_lock == RW_UNLOCKED, ("rw lock not unlocked"));
+ 	KASSERT(rw->rw_recurse == 0, ("rw lock still recursed"));
+ 	rw->rw_lock = RW_DESTROYED;
+ 	WITNESS_DESTROY(&rw->lock_object);
+ }
+ 
+ void
+ rw_sysinit(void *arg)
+ {
+ 	struct rw_args *args = arg;
+ 
+ 	rw_init(args->ra_rw, args->ra_desc);
+ }
+ 
+ int
+ rw_wowned(struct rwlock *rw)
+ {
+ 
+ 	return (rw_wowner(rw) == curthread);
+ }
+ 
+ void
+ _rw_wlock(struct rwlock *rw, const char *file, int line)
+ {
+ 
+ 	MPASS(curthread != NULL);
+ 	KASSERT(rw->rw_lock != RW_DESTROYED,
+ 	    ("rw_wlock() of destroyed rwlock @ %s:%d", file, line));
+ 	KASSERT(rw_wowner(rw) != curthread,
+ 	    ("%s (%s): wlock already held @ %s:%d", __func__,
+ 	    rw->lock_object.lo_name, file, line));
+ 	WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
+ 	    line);
+ 	__rw_wlock(rw, curthread, file, line);
+ 	LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line);
+ 	WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
+ 	curthread->td_locks++;
+ }
+ 
+ void
+ _rw_wunlock(struct rwlock *rw, const char *file, int line)
+ {
+ 
+ 	MPASS(curthread != NULL);
+ 	KASSERT(rw->rw_lock != RW_DESTROYED,
+ 	    ("rw_wunlock() of destroyed rwlock @ %s:%d", file, line));
+ 	_rw_assert(rw, RA_WLOCKED, file, line);
+ 	curthread->td_locks--;
+ 	WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line);
+ 	LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file,
+ 	    line);
+ 	if (!rw_recursed(rw))
+ 		lock_profile_release_lock(&rw->lock_object);
+ 	__rw_wunlock(rw, curthread, file, line);
+ }
+ 
+ void
+ _rw_rlock(struct rwlock *rw, const char *file, int line)
+ {
+ #ifdef ADAPTIVE_RWLOCKS
+ 	volatile struct thread *owner;
+ #endif
+ 	//uint64_t waittime = 0; /* XXX: notsup */
+ 	//int contested = 0; /* XXX: notsup */
+ 	uintptr_t x;
+ 
+ 	KASSERT(rw->rw_lock != RW_DESTROYED,
+ 	    ("rw_rlock() of destroyed rwlock @ %s:%d", file, line));
+ 	KASSERT(rw_wowner(rw) != curthread,
+ 	    ("%s (%s): wlock already held @ %s:%d", __func__,
+ 	    rw->lock_object.lo_name, file, line));
+ 	WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line);
+ 
+ 	/*
+ 	 * Note that we don't make any attempt to try to block read
+ 	 * locks once a writer has blocked on the lock.  The reason is
+ 	 * that we currently allow for read locks to recurse and we
+ 	 * don't keep track of all the holders of read locks.  Thus, if
+ 	 * we were to block readers once a writer blocked and a reader
+ 	 * tried to recurse on their reader lock after a writer had
+ 	 * blocked we would end up in a deadlock since the reader would
+ 	 * be blocked on the writer, and the writer would be blocked
+ 	 * waiting for the reader to release its original read lock.
+ 	 */
+ 	for (;;) {
+ 		/*
+ 		 * Handle the easy case.  If no other thread has a write
+ 		 * lock, then try to bump up the count of read locks.  Note
+ 		 * that we have to preserve the current state of the
+ 		 * RW_LOCK_WRITE_WAITERS flag.  If we fail to acquire a
+ 		 * read lock, then rw_lock must have changed, so restart
+ 		 * the loop.  Note that this handles the case of a
+ 		 * completely unlocked rwlock since such a lock is encoded
+ 		 * as a read lock with no waiters.
+ 		 */
+ 		x = rw->rw_lock;
+ 		if (x & RW_LOCK_READ) {
+ 
+ 			/*
+ 			 * The RW_LOCK_READ_WAITERS flag should only be set
+ 			 * if another thread currently holds a write lock,
+ 			 * and in that case RW_LOCK_READ should be clear.
+ 			 */
+ 			MPASS((x & RW_LOCK_READ_WAITERS) == 0);
+ 			if (atomic_cmpset_acq_ptr(&rw->rw_lock, x,
+ 			    x + RW_ONE_READER)) {
+ 				if (LOCK_LOG_TEST(&rw->lock_object, 0))
+ 					CTR4(KTR_LOCK,
+ 					    "%s: %p succeed %p -> %p", __func__,
+ 					    rw, (void *)x,
+ 					    (void *)(x + RW_ONE_READER));
+ 				if (RW_READERS(x) == 0)
+ 					lock_profile_obtain_lock_success(
+ 					    &rw->lock_object, contested, waittime,
+ 					    file, line);
+ 				break;
+ 			}
+ 			cpu_spinwait();
+ 			continue;
+ 		}
+ 		lock_profile_obtain_lock_failed(&rw->lock_object, &contested,
+ 		    &waittime);
+ 
+ 		/*
+ 		 * Okay, now it's the hard case.  Some other thread already
+ 		 * has a write lock, so acquire the turnstile lock so we can
+ 		 * begin the process of blocking.
+ 		 */
+ 		turnstile_lock(&rw->lock_object);
+ 
+ 		/*
+ 		 * The lock might have been released while we spun, so
+ 		 * recheck its state and restart the loop if there is no
+ 		 * longer a write lock.
+ 		 */
+ 		x = rw->rw_lock;
+ 		if (x & RW_LOCK_READ) {
+ 			turnstile_release(&rw->lock_object);
+ 			cpu_spinwait();
+ 			continue;
+ 		}
+ 
+ 		/*
+ 		 * Ok, it's still a write lock.  If the RW_LOCK_READ_WAITERS
+ 		 * flag is already set, then we can go ahead and block.  If
+ 		 * it is not set then try to set it.  If we fail to set it
+ 		 * drop the turnstile lock and restart the loop.
+ 		 */
+ 		if (!(x & RW_LOCK_READ_WAITERS)) {
+ 			if (!atomic_cmpset_ptr(&rw->rw_lock, x,
+ 			    x | RW_LOCK_READ_WAITERS)) {
+ 				turnstile_release(&rw->lock_object);
+ 				cpu_spinwait();
+ 				continue;
+ 			}
+ 			if (LOCK_LOG_TEST(&rw->lock_object, 0))
+ 				CTR2(KTR_LOCK, "%s: %p set read waiters flag",
+ 				    __func__, rw);
+ 		}
+ 
+ #ifdef ADAPTIVE_RWLOCKS
+ 		owner = (struct thread *)RW_OWNER(x);
+ 		/*
+ 		 * If the owner is running on another CPU, spin until
+ 		 * the owner stops running or the state of the lock
+ 		 * changes.
+ 		 */
+ 		if (TD_IS_RUNNING(owner)) {
+ 			turnstile_release(&rw->lock_object);
+ 			if (LOCK_LOG_TEST(&rw->lock_object, 0))
+ 				CTR3(KTR_LOCK, "%s: spinning on %p held by %p",
+ 				    __func__, rw, owner);
+ 			while ((struct thread*)RW_OWNER(rw->rw_lock)== owner &&
+ 			    TD_IS_RUNNING(owner))
+ 				cpu_spinwait();
+ 			continue;
+ 		}
+ #endif
+ 
+ 		/*
+ 		 * We were unable to acquire the lock and the read waiters
+ 		 * flag is set, so we must block on the turnstile.
+ 		 */
+ 		if (LOCK_LOG_TEST(&rw->lock_object, 0))
+ 			CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__,
+ 			    rw);
+ 		turnstile_wait_queue(&rw->lock_object, rw_owner(rw),
+ 		    TS_SHARED_QUEUE);
+ 		if (LOCK_LOG_TEST(&rw->lock_object, 0))
+ 			CTR2(KTR_LOCK, "%s: %p resuming from turnstile",
+ 			    __func__, rw);
+ 	}
+ 
+ 	/*
+ 	 * TODO: acquire "owner of record" here.  Here be turnstile dragons
+ 	 * however.  turnstiles don't like owners changing between calls to
+ 	 * turnstile_wait() currently.
+ 	 */
+ 
+ 	LOCK_LOG_LOCK("RLOCK", &rw->lock_object, 0, 0, file, line);
+ 	WITNESS_LOCK(&rw->lock_object, 0, file, line);
+ 	curthread->td_locks++;
+ }
+ 
+ void
+ _rw_runlock(struct rwlock *rw, const char *file, int line)
+ {
+ 	struct turnstile *ts;
+ 	uintptr_t x;
+ 
+ 	KASSERT(rw->rw_lock != RW_DESTROYED,
+ 	    ("rw_runlock() of destroyed rwlock @ %s:%d", file, line));
+ 	_rw_assert(rw, RA_RLOCKED, file, line);
+ 	curthread->td_locks--;
+ 	WITNESS_UNLOCK(&rw->lock_object, 0, file, line);
+ 	LOCK_LOG_LOCK("RUNLOCK", &rw->lock_object, 0, 0, file, line);
+ 
+ 	/* TODO: drop "owner of record" here. */
+ 
+ 	for (;;) {
+ 		/*
+ 		 * See if there is more than one read lock held.  If so,
+ 		 * just drop one and return.
+ 		 */
+ 		x = rw->rw_lock;
+ 		if (RW_READERS(x) > 1) {
+ 			if (atomic_cmpset_ptr(&rw->rw_lock, x,
+ 			    x - RW_ONE_READER)) {
+ 				if (LOCK_LOG_TEST(&rw->lock_object, 0))
+ 					CTR4(KTR_LOCK,
+ 					    "%s: %p succeeded %p -> %p",
+ 					    __func__, rw, (void *)x,
+ 					    (void *)(x - RW_ONE_READER));
+ 				break;
+ 			}
+ 			continue;
+ 		}
+ 
+ 
+ 		/*
+ 		 * We should never have read waiters while at least one
+ 		 * thread holds a read lock.  (See note above)
+ 		 */
+ 		KASSERT(!(x & RW_LOCK_READ_WAITERS),
+ 		    ("%s: waiting readers", __func__));
+ 
+ 		/*
+ 		 * If there aren't any waiters for a write lock, then try
+ 		 * to drop it quickly.
+ 		 */
+ 		if (!(x & RW_LOCK_WRITE_WAITERS)) {
+ 
+ 			/*
+ 			 * There shouldn't be any flags set and we should
+ 			 * be the only read lock.  If we fail to release
+ 			 * the single read lock, then another thread might
+ 			 * have just acquired a read lock, so go back up
+ 			 * to the multiple read locks case.
+ 			 */
+ 			MPASS(x == RW_READERS_LOCK(1));
+ 			if (atomic_cmpset_ptr(&rw->rw_lock, RW_READERS_LOCK(1),
+ 			    RW_UNLOCKED)) {
+ 				if (LOCK_LOG_TEST(&rw->lock_object, 0))
+ 					CTR2(KTR_LOCK, "%s: %p last succeeded",
+ 					    __func__, rw);
+ 				break;
+ 			}
+ 			continue;
+ 		}
+ 
+ 		/*
+ 		 * There should just be one reader with one or more
+ 		 * writers waiting.
+ 		 */
+ 		MPASS(x == (RW_READERS_LOCK(1) | RW_LOCK_WRITE_WAITERS));
+ 
+ 		/*
+ 		 * Ok, we know we have a waiting writer and we think we
+ 		 * are the last reader, so grab the turnstile lock.
+ 		 */
+ 		turnstile_lock(&rw->lock_object);
+ 
+ 		/*
+ 		 * Try to drop our lock leaving the lock in a unlocked
+ 		 * state.
+ 		 *
+ 		 * If you wanted to do explicit lock handoff you'd have to
+ 		 * do it here.  You'd also want to use turnstile_signal()
+ 		 * and you'd have to handle the race where a higher
+ 		 * priority thread blocks on the write lock before the
+ 		 * thread you wakeup actually runs and have the new thread
+ 		 * "steal" the lock.  For now it's a lot simpler to just
+ 		 * wakeup all of the waiters.
+ 		 *
+ 		 * As above, if we fail, then another thread might have
+ 		 * acquired a read lock, so drop the turnstile lock and
+ 		 * restart.
+ 		 */
+ 		if (!atomic_cmpset_ptr(&rw->rw_lock,
+ 		    RW_READERS_LOCK(1) | RW_LOCK_WRITE_WAITERS, RW_UNLOCKED)) {
+ 			turnstile_release(&rw->lock_object);
+ 			continue;
+ 		}
+ 		if (LOCK_LOG_TEST(&rw->lock_object, 0))
+ 			CTR2(KTR_LOCK, "%s: %p last succeeded with waiters",
+ 			    __func__, rw);
+ 
+ 		/*
+ 		 * Ok.  The lock is released and all that's left is to
+ 		 * wake up the waiters.  Note that the lock might not be
+ 		 * free anymore, but in that case the writers will just
+ 		 * block again if they run before the new lock holder(s)
+ 		 * release the lock.
+ 		 */
+ 		ts = turnstile_lookup(&rw->lock_object);
+ 		MPASS(ts != NULL);
+ 		turnstile_broadcast_queue(ts, TS_EXCLUSIVE_QUEUE);
+ 		turnstile_unpend_queue(ts, TS_SHARED_LOCK);
+ 		turnstile_release(&rw->lock_object);
+ 		break;
+ 	}
+ 	lock_profile_release_lock(&rw->lock_object);
+ }
+ 
+ /*
+  * This function is called when we are unable to obtain a write lock on the
+  * first try.  This means that at least one other thread holds either a
+  * read or write lock.
+  */
+ void
+ _rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
+ {
+ 	//struct turnstile *ts;
+ #ifdef ADAPTIVE_RWLOCKS
+ 	volatile struct thread *owner;
+ #endif
+ 	uintptr_t v;
+ 
+ 	if (rw_wlocked(rw)) {
+ 		KASSERT(rw->lock_object.lo_flags & RW_RECURSE,
+ 		    ("%s: recursing but non-recursive rw %s @ %s:%d\n",
+ 		    __func__, rw->lock_object.lo_name, file, line));
+ 		rw->rw_recurse++;
+ 		atomic_set_ptr(&rw->rw_lock, RW_LOCK_RECURSED);
+ 		if (LOCK_LOG_TEST(&rw->lock_object, 0))
+ 			CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw);
+ 		return;
+ 	}
+ 
+ 	if (LOCK_LOG_TEST(&rw->lock_object, 0))
+ 		CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__,
+ 		    rw->lock_object.lo_name, (void *)rw->rw_lock, file, line);
+ 
+ 	while (!_rw_write_lock(rw, tid)) {
+ 		turnstile_lock(&rw->lock_object);
+ 		v = rw->rw_lock;
+ 
+ 		/*
+ 		 * If the lock was released while spinning on the
+ 		 * turnstile chain lock, try again.
+ 		 */
+ 		if (v == RW_UNLOCKED) {
+ 			turnstile_release(&rw->lock_object);
+ 			cpu_spinwait();
+ 			continue;
+ 		}
+ 
+ 		/*
+ 		 * If the lock was released by a writer with both readers
+ 		 * and writers waiting and a reader hasn't woken up and
+ 		 * acquired the lock yet, rw_lock will be set to the
+ 		 * value RW_UNLOCKED | RW_LOCK_WRITE_WAITERS.  If we see
+ 		 * that value, try to acquire it once.  Note that we have
+ 		 * to preserve the RW_LOCK_WRITE_WAITERS flag as there are
+ 		 * other writers waiting still.  If we fail, restart the
+ 		 * loop.
+ 		 */
+ 		if (v == (RW_UNLOCKED | RW_LOCK_WRITE_WAITERS)) {
+ 			if (atomic_cmpset_acq_ptr(&rw->rw_lock,
+ 			    RW_UNLOCKED | RW_LOCK_WRITE_WAITERS,
+ 			    tid | RW_LOCK_WRITE_WAITERS)) {
+ 				turnstile_claim(&rw->lock_object);
+ 				CTR2(KTR_LOCK, "%s: %p claimed by new writer",
+ 				    __func__, rw);
+ 				break;
+ 			}
+ 			turnstile_release(&rw->lock_object);
+ 			cpu_spinwait();
+ 			continue;
+ 		}
+ 
+ 		/*
+ 		 * If the RW_LOCK_WRITE_WAITERS flag isn't set, then try to
+ 		 * set it.  If we fail to set it, then loop back and try
+ 		 * again.
+ 		 */
+ 		if (!(v & RW_LOCK_WRITE_WAITERS)) {
+ 			if (!atomic_cmpset_ptr(&rw->rw_lock, v,
+ 			    v | RW_LOCK_WRITE_WAITERS)) {
+ 				turnstile_release(&rw->lock_object);
+ 				cpu_spinwait();
+ 				continue;
+ 			}
+ 			if (LOCK_LOG_TEST(&rw->lock_object, 0))
+ 				CTR2(KTR_LOCK, "%s: %p set write waiters flag",
+ 				    __func__, rw);
+ 		}
+ 
+ #ifdef ADAPTIVE_RWLOCKS
+ 		/*
+ 		 * If the lock is write locked and the owner is
+ 		 * running on another CPU, spin until the owner stops
+ 		 * running or the state of the lock changes.
+ 		 */
+ 		owner = (struct thread *)RW_OWNER(v);
+ 		if (!(v & RW_LOCK_READ) && TD_IS_RUNNING(owner)) {
+ 			turnstile_release(&rw->lock_object);
+ 			if (LOCK_LOG_TEST(&rw->lock_object, 0))
+ 				CTR3(KTR_LOCK, "%s: spinning on %p held by %p",
+ 				    __func__, rw, owner);
+ 			while ((struct thread*)RW_OWNER(rw->rw_lock)== owner &&
+ 			    TD_IS_RUNNING(owner))
+ 				cpu_spinwait();
+ 			continue;
+ 		}
+ #endif
+ 
+ 		/*
+ 		 * We were unable to acquire the lock and the write waiters
+ 		 * flag is set, so we must block on the turnstile.
+ 		 */
+ 		if (LOCK_LOG_TEST(&rw->lock_object, 0))
+ 			CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__,
+ 			    rw);
+ 		turnstile_wait_queue(&rw->lock_object, rw_owner(rw),
+ 		    TS_EXCLUSIVE_QUEUE);
+ 		if (LOCK_LOG_TEST(&rw->lock_object, 0))
+ 			CTR2(KTR_LOCK, "%s: %p resuming from turnstile",
+ 			    __func__, rw);
+ 	}
+ }
+ 
+ /*
+  * This function is called if the first try at releasing a write lock failed.
+  * This means that one of the 2 waiter bits must be set indicating that at
+  * least one thread is waiting on this lock.
+  */
+ void
+ _rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line)
+ {
+ 	struct turnstile *ts;
+ 	uintptr_t v;
+ 	int queue;
+ 
+ 	if (rw_wlocked(rw) && rw_recursed(rw)) {
+ 		if ((--rw->rw_recurse) == 0)
+ 			atomic_clear_ptr(&rw->rw_lock, RW_LOCK_RECURSED);
+ 		if (LOCK_LOG_TEST(&rw->lock_object, 0))
+ 			CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw);
+ 		return;
+ 	}
+ 
+ 	KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS),
+ 	    ("%s: neither of the waiter flags are set", __func__));
+ 
+ 	if (LOCK_LOG_TEST(&rw->lock_object, 0))
+ 		CTR2(KTR_LOCK, "%s: %p contested", __func__, rw);
+ 
+ 	turnstile_lock(&rw->lock_object);
+ 	ts = turnstile_lookup(&rw->lock_object);
+ 
+ #ifdef ADAPTIVE_RWLOCKS
+ 	/*
+ 	 * There might not be a turnstile for this lock if all of
+ 	 * the waiters are adaptively spinning.  In that case, just
+ 	 * reset the lock to the unlocked state and return.
+ 	 */
+ 	if (ts == NULL) {
+ 		atomic_store_rel_ptr(&rw->rw_lock, RW_UNLOCKED);
+ 		if (LOCK_LOG_TEST(&rw->lock_object, 0))
+ 			CTR2(KTR_LOCK, "%s: %p no sleepers", __func__, rw);
+ 		turnstile_release(&rw->lock_object);
+ 		return;
+ 	}
+ #else
+ 	MPASS(ts != NULL);
+ #endif
+ 
+ 	/*
+ 	 * Use the same algo as sx locks for now.  Prefer waking up shared
+ 	 * waiters if we have any over writers.  This is probably not ideal.
+ 	 *
+ 	 * 'v' is the value we are going to write back to rw_lock.  If we
+ 	 * have waiters on both queues, we need to preserve the state of
+ 	 * the waiter flag for the queue we don't wake up.  For now this is
+ 	 * hardcoded for the algorithm mentioned above.
+ 	 *
+ 	 * In the case of both readers and writers waiting we wakeup the
+ 	 * readers but leave the RW_LOCK_WRITE_WAITERS flag set.  If a
+ 	 * new writer comes in before a reader it will claim the lock up
+ 	 * above.  There is probably a potential priority inversion in
+ 	 * there that could be worked around either by waking both queues
+ 	 * of waiters or doing some complicated lock handoff gymnastics.
+ 	 *
+ 	 * Note that in the ADAPTIVE_RWLOCKS case, if both flags are
+ 	 * set, there might not be any actual writers on the turnstile
+ 	 * as they might all be spinning.  In that case, we don't want
+ 	 * to preserve the RW_LOCK_WRITE_WAITERS flag as the turnstile
+ 	 * is going to go away once we wakeup all the readers.
+ 	 */
+ 	v = RW_UNLOCKED;
+ 	if (rw->rw_lock & RW_LOCK_READ_WAITERS) {
+ 		queue = TS_SHARED_QUEUE;
+ #ifdef ADAPTIVE_RWLOCKS
+ 		if (rw->rw_lock & RW_LOCK_WRITE_WAITERS &&
+ 		    !turnstile_empty_queue(ts, TS_EXCLUSIVE_QUEUE))
+ 			v |= RW_LOCK_WRITE_WAITERS;
+ #else
+ 		v |= (rw->rw_lock & RW_LOCK_WRITE_WAITERS);
+ #endif
+ 	} else
+ 		queue = TS_EXCLUSIVE_QUEUE;
+ 
+ #ifdef ADAPTIVE_RWLOCKS
+ 	/*
+ 	 * We have to make sure that we actually have waiters to
+ 	 * wakeup.  If they are all spinning, then we just need to
+ 	 * disown the turnstile and return.
+ 	 */
+ 	if (turnstile_empty_queue(ts, queue)) {
+ 		if (LOCK_LOG_TEST(&rw->lock_object, 0))
+ 			CTR2(KTR_LOCK, "%s: %p no sleepers 2", __func__, rw);
+ 		atomic_store_rel_ptr(&rw->rw_lock, v);
+ 		turnstile_disown(ts);
+ 		turnstile_release(&rw->lock_object);
+ 		return;
+ 	}
+ #endif
+ 
+ 	/* Wake up all waiters for the specific queue. */
+ 	if (LOCK_LOG_TEST(&rw->lock_object, 0))
+ 		CTR3(KTR_LOCK, "%s: %p waking up %s waiters", __func__, rw,
+ 		    queue == TS_SHARED_QUEUE ? "read" : "write");
+ 	turnstile_broadcast_queue(ts, queue);
+ 	atomic_store_rel_ptr(&rw->rw_lock, v);
+ 	turnstile_unpend_queue(ts, TS_EXCLUSIVE_LOCK);
+ 	turnstile_release(&rw->lock_object);
+ }
+ 
+ /*
+  * Attempt to do a non-blocking upgrade from a read lock to a write
+  * lock.  This will only succeed if this thread holds a single read
+  * lock.  Returns true if the upgrade succeeded and false otherwise.
+  */
+ int
+ _rw_try_upgrade(struct rwlock *rw, const char *file, int line)
+ {
+ 	uintptr_t v, tid;
+ 	int success;
+ 
+ 	KASSERT(rw->rw_lock != RW_DESTROYED,
+ 	    ("rw_try_upgrade() of destroyed rwlock @ %s:%d", file, line));
+ 	_rw_assert(rw, RA_RLOCKED, file, line);
+ 
+ 	/*
+ 	 * Attempt to switch from one reader to a writer.  If there
+ 	 * are any write waiters, then we will have to lock the
+ 	 * turnstile first to prevent races with another writer
+ 	 * calling turnstile_wait() before we have claimed this
+ 	 * turnstile.  So, do the simple case of no waiters first.
+ 	 */
+ 	tid = (uintptr_t)curthread;
+ 	if (!(rw->rw_lock & RW_LOCK_WRITE_WAITERS)) {
+ 		success = atomic_cmpset_ptr(&rw->rw_lock, RW_READERS_LOCK(1),
+ 		    tid);
+ 		goto out;
+ 	}
+ 
+ 	/*
+ 	 * Ok, we think we have write waiters, so lock the
+ 	 * turnstile.
+ 	 */
+ 	turnstile_lock(&rw->lock_object);
+ 
+ 	/*
+ 	 * Try to switch from one reader to a writer again.  This time
+ 	 * we honor the current state of the RW_LOCK_WRITE_WAITERS
+ 	 * flag.  If we obtain the lock with the flag set, then claim
+ 	 * ownership of the turnstile.  In the ADAPTIVE_RWLOCKS case
+ 	 * it is possible for there to not be an associated turnstile
+ 	 * even though there are waiters if all of the waiters are
+ 	 * spinning.
+ 	 */
+ 	v = rw->rw_lock & RW_LOCK_WRITE_WAITERS;
+ 	success = atomic_cmpset_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v,
+ 	    tid | v);
+ #ifdef ADAPTIVE_RWLOCKS
+ 	if (success && v && turnstile_lookup(&rw->lock_object) != NULL)
+ #else
+ 	if (success && v)
+ #endif
+ 		turnstile_claim(&rw->lock_object);
+ 	else
+ 		turnstile_release(&rw->lock_object);
+ out:
+ 	LOCK_LOG_TRY("WUPGRADE", &rw->lock_object, 0, success, file, line);
+ 	if (success)
+ 		WITNESS_UPGRADE(&rw->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
+ 		    file, line);
+ 	return (success);
+ }
+ 
+ /*
+  * Downgrade a write lock into a single read lock.
+  */
+ void
+ _rw_downgrade(struct rwlock *rw, const char *file, int line)
+ {
+ 	struct turnstile *ts;
+ 	uintptr_t tid, v;
+ 
+ 	KASSERT(rw->rw_lock != RW_DESTROYED,
+ 	    ("rw_downgrade() of destroyed rwlock @ %s:%d", file, line));
+ 	_rw_assert(rw, RA_WLOCKED | RA_NOTRECURSED, file, line);
+ #ifndef INVARIANTS
+ 	if (rw_recursed(rw))
+ 		panic("downgrade of a recursed lock");
+ #endif
+ 
+ 	WITNESS_DOWNGRADE(&rw->lock_object, 0, file, line);
+ 
+ 	/*
+ 	 * Convert from a writer to a single reader.  First we handle
+ 	 * the easy case with no waiters.  If there are any waiters, we
+ 	 * lock the turnstile, "disown" the lock, and awaken any read
+ 	 * waiters.
+ 	 */
+ 	tid = (uintptr_t)curthread;
+ 	if (atomic_cmpset_rel_ptr(&rw->rw_lock, tid, RW_READERS_LOCK(1)))
+ 		goto out;
+ 
+ 	/*
+ 	 * Ok, we think we have waiters, so lock the turnstile so we can
+ 	 * read the waiter flags without any races.
+ 	 */
+ 	turnstile_lock(&rw->lock_object);
+ 	v = rw->rw_lock;
+ 	MPASS(v & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS));
+ 
+ 	/*
+ 	 * Downgrade from a write lock while preserving
+ 	 * RW_LOCK_WRITE_WAITERS and give up ownership of the
+ 	 * turnstile.  If there are any read waiters, wake them up.
+ 	 *
+ 	 * For ADAPTIVE_RWLOCKS, we have to allow for the fact that
+ 	 * all of the read waiters might be spinning.  In that case,
+ 	 * act as if RW_LOCK_READ_WAITERS is not set.  Also, only
+ 	 * preserve the RW_LOCK_WRITE_WAITERS flag if at least one
+ 	 * writer is blocked on the turnstile.
+ 	 */
+ 	ts = turnstile_lookup(&rw->lock_object);
+ #ifdef ADAPTIVE_RWLOCKS
+ 	if (ts == NULL)
+ 		v &= ~(RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS);
+ 	else if (v & RW_LOCK_READ_WAITERS &&
+ 	    turnstile_empty_queue(ts, TS_SHARED_QUEUE))
+ 		v &= ~RW_LOCK_READ_WAITERS;
+ 	else if (v & RW_LOCK_WRITE_WAITERS &&
+ 	    turnstile_empty_queue(ts, TS_EXCLUSIVE_QUEUE))
+ 		v &= ~RW_LOCK_WRITE_WAITERS;
+ #else
+ 	MPASS(ts != NULL);
+ #endif
+ 	if (v & RW_LOCK_READ_WAITERS)
+ 		turnstile_broadcast_queue(ts, TS_SHARED_QUEUE);
+ 	atomic_store_rel_ptr(&rw->rw_lock, RW_READERS_LOCK(1) |
+ 	    (v & RW_LOCK_WRITE_WAITERS));
+ 	if (v & RW_LOCK_READ_WAITERS)
+ 		turnstile_unpend_queue(ts, TS_EXCLUSIVE_LOCK);
+ 	else if (ts)
+ 		turnstile_disown(ts);
+ 	turnstile_release(&rw->lock_object);
+ out:
+ 	LOCK_LOG_LOCK("WDOWNGRADE", &rw->lock_object, 0, 0, file, line);
+ }
+ 
+ #ifdef INVARIANT_SUPPORT
+ #ifndef INVARIANTS
+ #undef _rw_assert
+ #endif
+ 
+ /*
+  * In the non-WITNESS case, rw_assert() can only detect that at least
+  * *some* thread owns an rlock, but it cannot guarantee that *this*
+  * thread owns an rlock.
+  */
+ void
+ _rw_assert(struct rwlock *rw, int what, const char *file, int line)
+ {
+ 
+ 	if (panicstr != NULL)
+ 		return;
+ 	switch (what) {
+ 	case RA_LOCKED:
+ 	case RA_LOCKED | RA_RECURSED:
+ 	case RA_LOCKED | RA_NOTRECURSED:
+ 	case RA_RLOCKED:
+ #ifdef WITNESS
+ 		witness_assert(&rw->lock_object, what, file, line);
+ #else
+ 		/*
+ 		 * If some other thread has a write lock or we have one
+ 		 * and are asserting a read lock, fail.  Also, if no one
+ 		 * has a lock at all, fail.
+ 		 */
+ 		if (rw->rw_lock == RW_UNLOCKED ||
+ 		    (!(rw->rw_lock & RW_LOCK_READ) && (what == RA_RLOCKED ||
+ 		    rw_wowner(rw) != curthread)))
+ 			panic("Lock %s not %slocked @ %s:%d\n",
+ 			    rw->lock_object.lo_name, (what == RA_RLOCKED) ?
+ 			    "read " : "", file, line);
+ 
+ 		if (!(rw->rw_lock & RW_LOCK_READ)) {
+ 			if (rw_recursed(rw)) {
+ 				if (what & RA_NOTRECURSED)
+ 					panic("Lock %s recursed @ %s:%d\n",
+ 					    rw->lock_object.lo_name, file,
+ 					    line);
+ 			} else if (what & RA_RECURSED)
+ 				panic("Lock %s not recursed @ %s:%d\n",
+ 				    rw->lock_object.lo_name, file, line);
+ 		}
+ #endif
+ 		break;
+ 	case RA_WLOCKED:
+ 	case RA_WLOCKED | RA_RECURSED:
+ 	case RA_WLOCKED | RA_NOTRECURSED:
+ 		if (rw_wowner(rw) != curthread)
+ 			panic("Lock %s not exclusively locked @ %s:%d\n",
+ 			    rw->lock_object.lo_name, file, line);
+ 		if (rw_recursed(rw)) {
+ 			if (what & RA_NOTRECURSED)
+ 				panic("Lock %s recursed @ %s:%d\n",
+ 				    rw->lock_object.lo_name, file, line);
+ 		} else if (what & RA_RECURSED)
+ 			panic("Lock %s not recursed @ %s:%d\n",
+ 			    rw->lock_object.lo_name, file, line);
+ 		break;
+ 	case RA_UNLOCKED:
+ #ifdef WITNESS
+ 		witness_assert(&rw->lock_object, what, file, line);
+ #else
+ 		/*
+ 		 * If we hold a write lock fail.  We can't reliably check
+ 		 * to see if we hold a read lock or not.
+ 		 */
+ 		if (rw_wowner(rw) == curthread)
+ 			panic("Lock %s exclusively locked @ %s:%d\n",
+ 			    rw->lock_object.lo_name, file, line);
+ #endif
+ 		break;
+ 	default:
+ 		panic("Unknown rw lock assertion: %d @ %s:%d", what, file,
+ 		    line);
+ 	}
+ }
+ #endif /* INVARIANT_SUPPORT */
+ 
+ #ifdef DDB
+ void
+ db_show_rwlock(struct lock_object *lock)
+ {
+ 	struct rwlock *rw;
+ 	struct thread *td;
+ 
+ 	rw = (struct rwlock *)lock;
+ 
+ 	db_printf(" state: ");
+ 	if (rw->rw_lock == RW_UNLOCKED)
+ 		db_printf("UNLOCKED\n");
+ 	else if (rw->rw_lock == RW_DESTROYED) {
+ 		db_printf("DESTROYED\n");
+ 		return;
+ 	} else if (rw->rw_lock & RW_LOCK_READ)
+ 		db_printf("RLOCK: %ju locks\n",
+ 		    (uintmax_t)(RW_READERS(rw->rw_lock)));
+ 	else {
+ 		td = rw_wowner(rw);
+ 		db_printf("WLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
+ 		    td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm);
+ 		if (rw_recursed(rw))
+ 			db_printf(" recursed: %u\n", rw->rw_recurse);
+ 	}
+ 	db_printf(" waiters: ");
+ 	switch (rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)) {
+ 	case RW_LOCK_READ_WAITERS:
+ 		db_printf("readers\n");
+ 		break;
+ 	case RW_LOCK_WRITE_WAITERS:
+ 		db_printf("writers\n");
+ 		break;
+ 	case RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS:
+ 		db_printf("readers and writers\n");
+ 		break;
+ 	default:
+ 		db_printf("none\n");
+ 		break;
+ 	}
+ }
+ 
+ #endif
Index: kern/kern_sx.c
===================================================================
RCS file: /cvs/ncvs/src/sys/kern/kern_sx.c,v
retrieving revision 1.25.2.4
diff -c -r1.25.2.4 kern_sx.c
*** kern/kern_sx.c	17 Aug 2006 19:53:06 -0000	1.25.2.4
--- kern/kern_sx.c	31 Aug 2007 01:48:11 -0000
***************
*** 1,12 ****
  /*-
!  * Copyright (C) 2001 Jason Evans <[e-mail]>.  All rights reserved.
   *
   * Redistribution and use in source and binary forms, with or without
   * modification, are permitted provided that the following conditions
   * are met:
   * 1. Redistributions of source code must retain the above copyright
   *    notice(s), this list of conditions and the following disclaimer as
!  *    the first lines of this file unmodified other than the possible 
   *    addition of one or more copyright notices.
   * 2. Redistributions in binary form must reproduce the above copyright
   *    notice(s), this list of conditions and the following disclaimer in the
--- 1,14 ----
  /*-
!  * Copyright (c) 2007 Attilio Rao <[e-mail]>
!  * Copyright (c) 2001 Jason Evans <[e-mail]>
!  * All rights reserved.
   *
   * Redistribution and use in source and binary forms, with or without
   * modification, are permitted provided that the following conditions
   * are met:
   * 1. Redistributions of source code must retain the above copyright
   *    notice(s), this list of conditions and the following disclaimer as
!  *    the first lines of this file unmodified other than the possible
   *    addition of one or more copyright notices.
   * 2. Redistributions in binary form must reproduce the above copyright
   *    notice(s), this list of conditions and the following disclaimer in the
***************
*** 26,64 ****
   */
  
  /*
!  * Shared/exclusive locks.  This implementation assures deterministic lock
!  * granting behavior, so that slocks and xlocks are interleaved.
   *
   * Priority propagation will not generally raise the priority of lock holders,
   * so should not be relied upon in combination with sx locks.
   */
  
! #include <sys/cdefs.h>
! __FBSDID("$FreeBSD: src/sys/kern/kern_sx.c,v 1.25.2.4 2006/08/17 19:53:06 jhb Exp $");
! 
  #include "opt_ddb.h"
  
  #include <sys/param.h>
- #include <sys/systm.h>
  #include <sys/ktr.h>
- #include <sys/linker_set.h>
- #include <sys/condvar.h>
  #include <sys/lock.h>
  #include <sys/mutex.h>
  #include <sys/proc.h>
  #include <sys/sx.h>
  
  #ifdef DDB
  #include <ddb/ddb.h>
  
  static void	db_show_sx(struct lock_object *lock);
  #endif
  
  struct lock_class lock_class_sx = {
! 	"sx",
! 	LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE | LC_UPGRADABLE,
  #ifdef DDB
! 	db_show_sx
  #endif
  };
  
--- 28,117 ----
   */
  
  /*
!  * Shared/exclusive locks.  This implementation attempts to ensure
!  * deterministic lock granting behavior, so that slocks and xlocks are
!  * interleaved.
   *
   * Priority propagation will not generally raise the priority of lock holders,
   * so should not be relied upon in combination with sx locks.
   */
  
! #include "opt_adaptive_sx.h"
  #include "opt_ddb.h"
  
+ #include <sys/cdefs.h>
+ __FBSDID("$FreeBSD: src/sys/kern/kern_sx.c,v 1.54 2007/07/06 13:20:44 attilio Exp $");
+ 
  #include <sys/param.h>
  #include <sys/ktr.h>
  #include <sys/lock.h>
  #include <sys/mutex.h>
  #include <sys/proc.h>
+ #include <sys/sleepqueue.h>
  #include <sys/sx.h>
+ #include <sys/systm.h>
+ 
+ #ifdef ADAPTIVE_SX
+ #include <machine/cpu.h>
+ #endif
  
  #ifdef DDB
  #include <ddb/ddb.h>
+ #endif
+ 
+ #if !defined(SMP) && defined(ADAPTIVE_SX)
+ #error "You must have SMP to enable the ADAPTIVE_SX option"
+ #endif
+ 
+ CTASSERT(((SX_ADAPTIVESPIN | SX_RECURSE) & LO_CLASSFLAGS) ==
+     (SX_ADAPTIVESPIN | SX_RECURSE));
+ 
+ /* Handy macros for sleep queues. */
+ #define	SQ_EXCLUSIVE_QUEUE	0
+ #define	SQ_SHARED_QUEUE		1
  
+ /*
+  * Variations on DROP_GIANT()/PICKUP_GIANT() for use in this file.  We
+  * drop Giant anytime we have to sleep or if we adaptively spin.
+  */
+ #define	GIANT_DECLARE							\
+ 	int _giantcnt = 0;						\
+ 	WITNESS_SAVE_DECL(Giant)					\
+ 
+ #define	GIANT_SAVE() do {						\
+ 	if (mtx_owned(&Giant)) {					\
+ 		WITNESS_SAVE(&Giant.mtx_object, Giant);		\
+ 		while (mtx_owned(&Giant)) {				\
+ 			_giantcnt++;					\
+ 			mtx_unlock(&Giant);				\
+ 		}							\
+ 	}								\
+ } while (0)
+ 
+ #define GIANT_RESTORE() do {						\
+ 	if (_giantcnt > 0) {						\
+ 		mtx_assert(&Giant, MA_NOTOWNED);			\
+ 		while (_giantcnt--)					\
+ 			mtx_lock(&Giant);				\
+ 		WITNESS_RESTORE(&Giant.mtx_object, Giant);		\
+ 	}								\
+ } while (0)
+ 
+ /*
+  * Returns true if an exclusive lock is recursed.  It assumes
+  * curthread currently has an exclusive lock.
+  */
+ #define	sx_recursed(sx)		((sx)->sx_recurse != 0)
+ 
+ #ifdef DDB
  static void	db_show_sx(struct lock_object *lock);
  #endif
  
  struct lock_class lock_class_sx = {
! 	.lc_name = "sx",
! 	.lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE | LC_UPGRADABLE,
  #ifdef DDB
! 	.lc_ddb_show = db_show_sx,
  #endif
  };
  
***************
*** 75,317 ****
  }
  
  void
! sx_init(struct sx *sx, const char *description)
  {
  
! 	sx->sx_lock = mtx_pool_find(mtxpool_lockbuilder, sx);
! 	sx->sx_cnt = 0;
! 	cv_init(&sx->sx_shrd_cv, description);
! 	sx->sx_shrd_wcnt = 0;
! 	cv_init(&sx->sx_excl_cv, description);
! 	sx->sx_excl_wcnt = 0;
! 	sx->sx_xholder = NULL;
! 	lock_init(&sx->sx_object, &lock_class_sx, description, NULL,
! 	    LO_WITNESS | LO_RECURSABLE | LO_SLEEPABLE | LO_UPGRADABLE);
  }
  
  void
  sx_destroy(struct sx *sx)
  {
  
! 	KASSERT((sx->sx_cnt == 0 && sx->sx_shrd_wcnt == 0 && sx->sx_excl_wcnt ==
! 	    0), ("%s (%s): holders or waiters\n", __func__,
! 	    sx->sx_object.lo_name));
! 
! 	sx->sx_lock = NULL;
! 	cv_destroy(&sx->sx_shrd_cv);
! 	cv_destroy(&sx->sx_excl_cv);
! 
! 	lock_destroy(&sx->sx_object);
  }
  
! void
! _sx_slock(struct sx *sx, const char *file, int line)
  {
  
! 	mtx_lock(sx->sx_lock);
! 	KASSERT(sx->sx_xholder != curthread,
! 	    ("%s (%s): slock while xlock is held @ %s:%d\n", __func__,
! 	    sx->sx_object.lo_name, file, line));
! 	WITNESS_CHECKORDER(&sx->sx_object, LOP_NEWORDER, file, line);
! 
! 	/*
! 	 * Loop in case we lose the race for lock acquisition.
! 	 */
! 	while (sx->sx_cnt < 0) {
! 		sx->sx_shrd_wcnt++;
! 		cv_wait(&sx->sx_shrd_cv, sx->sx_lock);
! 		sx->sx_shrd_wcnt--;
  	}
  
! 	/* Acquire a shared lock. */
! 	sx->sx_cnt++;
! 
! 	LOCK_LOG_LOCK("SLOCK", &sx->sx_object, 0, 0, file, line);
! 	WITNESS_LOCK(&sx->sx_object, 0, file, line);
! 	curthread->td_locks++;
! 
! 	mtx_unlock(sx->sx_lock);
  }
  
  int
  _sx_try_slock(struct sx *sx, const char *file, int line)
  {
  
! 	mtx_lock(sx->sx_lock);
! 	if (sx->sx_cnt >= 0) {
! 		sx->sx_cnt++;
! 		LOCK_LOG_TRY("SLOCK", &sx->sx_object, 0, 1, file, line);
! 		WITNESS_LOCK(&sx->sx_object, LOP_TRYLOCK, file, line);
  		curthread->td_locks++;
- 		mtx_unlock(sx->sx_lock);
  		return (1);
- 	} else {
- 		LOCK_LOG_TRY("SLOCK", &sx->sx_object, 0, 0, file, line);
- 		mtx_unlock(sx->sx_lock);
- 		return (0);
  	}
  }
  
! void
! _sx_xlock(struct sx *sx, const char *file, int line)
  {
  
! 	mtx_lock(sx->sx_lock);
! 
! 	/*
! 	 * With sx locks, we're absolutely not permitted to recurse on
! 	 * xlocks, as it is fatal (deadlock). Normally, recursion is handled
! 	 * by WITNESS, but as it is not semantically correct to hold the
! 	 * xlock while in here, we consider it API abuse and put it under
! 	 * INVARIANTS.
! 	 */
! 	KASSERT(sx->sx_xholder != curthread,
! 	    ("%s (%s): xlock already held @ %s:%d", __func__,
! 	    sx->sx_object.lo_name, file, line));
! 	WITNESS_CHECKORDER(&sx->sx_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
  	    line);
! 
! 	/* Loop in case we lose the race for lock acquisition. */
! 	while (sx->sx_cnt != 0) {
! 		sx->sx_excl_wcnt++;
! 		cv_wait(&sx->sx_excl_cv, sx->sx_lock);
! 		sx->sx_excl_wcnt--;
  	}
  
! 	MPASS(sx->sx_cnt == 0);
! 
! 	/* Acquire an exclusive lock. */
! 	sx->sx_cnt--;
! 	sx->sx_xholder = curthread;
! 
! 	LOCK_LOG_LOCK("XLOCK", &sx->sx_object, 0, 0, file, line);
! 	WITNESS_LOCK(&sx->sx_object, LOP_EXCLUSIVE, file, line);
! 	curthread->td_locks++;
! 
! 	mtx_unlock(sx->sx_lock);
  }
  
  int
  _sx_try_xlock(struct sx *sx, const char *file, int line)
  {
  
! 	mtx_lock(sx->sx_lock);
! 	if (sx->sx_cnt == 0) {
! 		sx->sx_cnt--;
! 		sx->sx_xholder = curthread;
! 		LOCK_LOG_TRY("XLOCK", &sx->sx_object, 0, 1, file, line);
! 		WITNESS_LOCK(&sx->sx_object, LOP_EXCLUSIVE | LOP_TRYLOCK, file,
! 		    line);
  		curthread->td_locks++;
- 		mtx_unlock(sx->sx_lock);
- 		return (1);
- 	} else {
- 		LOCK_LOG_TRY("XLOCK", &sx->sx_object, 0, 0, file, line);
- 		mtx_unlock(sx->sx_lock);
- 		return (0);
  	}
  }
  
  void
  _sx_sunlock(struct sx *sx, const char *file, int line)
  {
  
! 	_sx_assert(sx, SX_SLOCKED, file, line);
! 	mtx_lock(sx->sx_lock);
  
  	curthread->td_locks--;
! 	WITNESS_UNLOCK(&sx->sx_object, 0, file, line);
  
! 	/* Release. */
! 	sx->sx_cnt--;
  
  	/*
! 	 * If we just released the last shared lock, wake any waiters up, giving
! 	 * exclusive lockers precedence.  In order to make sure that exclusive
! 	 * lockers won't be blocked forever, don't wake shared lock waiters if
! 	 * there are exclusive lock waiters.
  	 */
! 	if (sx->sx_excl_wcnt > 0) {
! 		if (sx->sx_cnt == 0)
! 			cv_signal(&sx->sx_excl_cv);
! 	} else if (sx->sx_shrd_wcnt > 0)
! 		cv_broadcast(&sx->sx_shrd_cv);
! 
! 	LOCK_LOG_LOCK("SUNLOCK", &sx->sx_object, 0, 0, file, line);
! 
! 	mtx_unlock(sx->sx_lock);
  }
  
  void
! _sx_xunlock(struct sx *sx, const char *file, int line)
  {
  
! 	_sx_assert(sx, SX_XLOCKED, file, line);
! 	mtx_lock(sx->sx_lock);
! 	MPASS(sx->sx_cnt == -1);
  
! 	curthread->td_locks--;
! 	WITNESS_UNLOCK(&sx->sx_object, LOP_EXCLUSIVE, file, line);
  
! 	/* Release. */
! 	sx->sx_cnt++;
! 	sx->sx_xholder = NULL;
  
  	/*
! 	 * Wake up waiters if there are any.  Give precedence to slock waiters.
  	 */
! 	if (sx->sx_shrd_wcnt > 0)
! 		cv_broadcast(&sx->sx_shrd_cv);
! 	else if (sx->sx_excl_wcnt > 0)
! 		cv_signal(&sx->sx_excl_cv);
  
! 	LOCK_LOG_LOCK("XUNLOCK", &sx->sx_object, 0, 0, file, line);
  
! 	mtx_unlock(sx->sx_lock);
  }
  
  int
! _sx_try_upgrade(struct sx *sx, const char *file, int line)
  {
  
! 	_sx_assert(sx, SX_SLOCKED, file, line);
! 	mtx_lock(sx->sx_lock);
  
! 	if (sx->sx_cnt == 1) {
! 		sx->sx_cnt = -1;
! 		sx->sx_xholder = curthread;
  
! 		LOCK_LOG_TRY("XUPGRADE", &sx->sx_object, 0, 1, file, line);
! 		WITNESS_UPGRADE(&sx->sx_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
! 		    file, line);
  
! 		mtx_unlock(sx->sx_lock);
! 		return (1);
! 	} else {
! 		LOCK_LOG_TRY("XUPGRADE", &sx->sx_object, 0, 0, file, line);
! 		mtx_unlock(sx->sx_lock);
! 		return (0);
  	}
  }
  
  void
! _sx_downgrade(struct sx *sx, const char *file, int line)
  {
  
! 	_sx_assert(sx, SX_XLOCKED, file, line);
! 	mtx_lock(sx->sx_lock);
! 	MPASS(sx->sx_cnt == -1);
  
! 	WITNESS_DOWNGRADE(&sx->sx_object, 0, file, line);
  
! 	sx->sx_cnt = 1;
! 	sx->sx_xholder = NULL;
!         if (sx->sx_shrd_wcnt > 0)
!                 cv_broadcast(&sx->sx_shrd_cv);
  
! 	LOCK_LOG_LOCK("XDOWNGRADE", &sx->sx_object, 0, 0, file, line);
  
! 	mtx_unlock(sx->sx_lock);
  }
  
  #ifdef INVARIANT_SUPPORT
--- 128,851 ----
  }
  
  void
! sx_init_flags(struct sx *sx, const char *description, int opts)
  {
+ 	struct lock_object *lock;
+ 	int flags;
  
! 	MPASS((opts & ~(SX_QUIET | SX_RECURSE | SX_NOWITNESS | SX_DUPOK |
! 	    SX_NOPROFILE | SX_ADAPTIVESPIN)) == 0);
! 
! 	bzero(sx, sizeof(*sx));
! 
! 	flags = LO_RECURSABLE | LO_SLEEPABLE | LO_UPGRADABLE;
! 	if (opts & SX_DUPOK)
! 		flags |= LO_DUPOK;
! 	if (!(opts & SX_NOWITNESS))
! 		flags |= LO_WITNESS;
! 	if (opts & SX_QUIET)
! 		flags |= LO_QUIET;
! 
! 	flags |= opts & (SX_ADAPTIVESPIN | SX_RECURSE);
! 	sx->sx_lock = SX_LOCK_UNLOCKED;
! 	sx->sx_recurse = 0;
! 	lock = &sx->lock_object;
! 	lock->lo_class = &lock_class_sx;
! 	lock->lo_flags = flags;
! 	lock->lo_name = lock->lo_type = description;
! 	LOCK_LOG_INIT(lock, opts);
! 	WITNESS_INIT(lock);
  }
  
  void
  sx_destroy(struct sx *sx)
  {
+ 	LOCK_LOG_DESTROY(&sx->lock_object, 0);
  
! 	KASSERT(sx->sx_lock == SX_LOCK_UNLOCKED, ("sx lock still held"));
! 	KASSERT(sx->sx_recurse == 0, ("sx lock still recursed"));
! 	sx->sx_lock = SX_LOCK_DESTROYED;
! 	WITNESS_DESTROY(&sx->lock_object);
  }
  
! int
! _sx_slock(struct sx *sx, int opts, const char *file, int line)
  {
+ 	int error = 0;
  
! 	MPASS(curthread != NULL);
! 	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
! 	    ("sx_slock() of destroyed sx @ %s:%d", file, line));
! 	WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER, file, line);
! 	error = __sx_slock(sx, opts, file, line);
! 	if (!error) {
! 		LOCK_LOG_LOCK("SLOCK", &sx->lock_object, 0, 0, file, line);
! 		WITNESS_LOCK(&sx->lock_object, 0, file, line);
! 		curthread->td_locks++;
  	}
  
! 	return (error);
  }
  
  int
  _sx_try_slock(struct sx *sx, const char *file, int line)
  {
+ 	uintptr_t x;
  
! 	x = sx->sx_lock;
! 	KASSERT(x != SX_LOCK_DESTROYED,
! 	    ("sx_try_slock() of destroyed sx @ %s:%d", file, line));
! 	if ((x & SX_LOCK_SHARED) && atomic_cmpset_acq_ptr(&sx->sx_lock, x,
! 	    x + SX_ONE_SHARER)) {
! 		LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 1, file, line);
! 		WITNESS_LOCK(&sx->lock_object, LOP_TRYLOCK, file, line);
  		curthread->td_locks++;
  		return (1);
  	}
+ 
+ 	LOCK_LOG_TRY("SLOCK", &sx->lock_object, 0, 0, file, line);
+ 	return (0);
  }
  
! int
! _sx_xlock(struct sx *sx, int opts, const char *file, int line)
  {
+ 	int error = 0;
  
! 	MPASS(curthread != NULL);
! 	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
! 	    ("sx_xlock() of destroyed sx @ %s:%d", file, line));
! 	WITNESS_CHECKORDER(&sx->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file,
  	    line);
! 	error = __sx_xlock(sx, curthread, opts, file, line);
! 	if (!error) {
! 		LOCK_LOG_LOCK("XLOCK", &sx->lock_object, 0, sx->sx_recurse,
! 		    file, line);
! 		WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line);
! 		curthread->td_locks++;
  	}
  
! 	return (error);
  }
  
  int
  _sx_try_xlock(struct sx *sx, const char *file, int line)
  {
+ 	int rval;
  
! 	MPASS(curthread != NULL);
! 	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
! 	    ("sx_try_xlock() of destroyed sx @ %s:%d", file, line));
! 
! 	if (sx_xlocked(sx) && (sx->lock_object.lo_flags & SX_RECURSE) != 0) {
! 		sx->sx_recurse++;
! 		atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
! 		rval = 1;
! 	} else
! 		rval = atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED,
! 		    (uintptr_t)curthread);
! 	LOCK_LOG_TRY("XLOCK", &sx->lock_object, 0, rval, file, line);
! 	if (rval) {
! 		WITNESS_LOCK(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
! 		    file, line);
  		curthread->td_locks++;
  	}
+ 
+ 	return (rval);
  }
  
  void
  _sx_sunlock(struct sx *sx, const char *file, int line)
  {
  
! 	MPASS(curthread != NULL);
! 	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
! 	    ("sx_sunlock() of destroyed sx @ %s:%d", file, line));
! 	_sx_assert(sx, SA_SLOCKED, file, line);
! 	curthread->td_locks--;
! 	WITNESS_UNLOCK(&sx->lock_object, 0, file, line);
! 	LOCK_LOG_LOCK("SUNLOCK", &sx->lock_object, 0, 0, file, line);
! #ifdef LOCK_PROFILING_SHARED
! 	if (SX_SHARERS(sx->sx_lock) == 1)
! 		lock_profile_release_lock(&sx->lock_object);
! #endif
! 	__sx_sunlock(sx, file, line);
! }
! 
! void
! _sx_xunlock(struct sx *sx, const char *file, int line)
! {
  
+ 	MPASS(curthread != NULL);
+ 	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
+ 	    ("sx_xunlock() of destroyed sx @ %s:%d", file, line));
+ 	_sx_assert(sx, SA_XLOCKED, file, line);
  	curthread->td_locks--;
! 	WITNESS_UNLOCK(&sx->lock_object, LOP_EXCLUSIVE, file, line);
! 	LOCK_LOG_LOCK("XUNLOCK", &sx->lock_object, 0, sx->sx_recurse, file,
! 	    line);
! 	if (!sx_recursed(sx))
! 		lock_profile_release_lock(&sx->lock_object);
! 	__sx_xunlock(sx, curthread, file, line);
! }
  
! /*
!  * Try to do a non-blocking upgrade from a shared lock to an exclusive lock.
!  * This will only succeed if this thread holds a single shared lock.
!  * Return 1 if if the upgrade succeed, 0 otherwise.
!  */
! int
! _sx_try_upgrade(struct sx *sx, const char *file, int line)
! {
! 	uintptr_t x;
! 	int success;
! 
! 	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
! 	    ("sx_try_upgrade() of destroyed sx @ %s:%d", file, line));
! 	_sx_assert(sx, SA_SLOCKED, file, line);
  
  	/*
! 	 * Try to switch from one shared lock to an exclusive lock.  We need
! 	 * to maintain the SX_LOCK_EXCLUSIVE_WAITERS flag if set so that
! 	 * we will wake up the exclusive waiters when we drop the lock.
  	 */
! 	x = sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS;
! 	success = atomic_cmpset_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) | x,
! 	    (uintptr_t)curthread | x);
! 	LOCK_LOG_TRY("XUPGRADE", &sx->lock_object, 0, success, file, line);
! 	if (success)
! 		WITNESS_UPGRADE(&sx->lock_object, LOP_EXCLUSIVE | LOP_TRYLOCK,
! 		    file, line);
! 	return (success);
  }
  
+ /*
+  * Downgrade an unrecursed exclusive lock into a single shared lock.
+  */
  void
! _sx_downgrade(struct sx *sx, const char *file, int line)
  {
+ 	uintptr_t x;
  
! 	KASSERT(sx->sx_lock != SX_LOCK_DESTROYED,
! 	    ("sx_downgrade() of destroyed sx @ %s:%d", file, line));
! 	_sx_assert(sx, SA_XLOCKED | SA_NOTRECURSED, file, line);
! #ifndef INVARIANTS
! 	if (sx_recursed(sx))
! 		panic("downgrade of a recursed lock");
! #endif
  
! 	WITNESS_DOWNGRADE(&sx->lock_object, 0, file, line);
  
! 	/*
! 	 * Try to switch from an exclusive lock with no shared waiters
! 	 * to one sharer with no shared waiters.  If there are
! 	 * exclusive waiters, we don't need to lock the sleep queue so
! 	 * long as we preserve the flag.  We do one quick try and if
! 	 * that fails we grab the sleepq lock to keep the flags from
! 	 * changing and do it the slow way.
! 	 *
! 	 * We have to lock the sleep queue if there are shared waiters
! 	 * so we can wake them up.
! 	 */
! 	x = sx->sx_lock;
! 	if (!(x & SX_LOCK_SHARED_WAITERS) &&
! 	    atomic_cmpset_rel_ptr(&sx->sx_lock, x, SX_SHARERS_LOCK(1) |
! 	    (x & SX_LOCK_EXCLUSIVE_WAITERS))) {
! 		LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line);
! 		return;
! 	}
  
  	/*
! 	 * Lock the sleep queue so we can read the waiters bits
! 	 * without any races and wakeup any shared waiters.
  	 */
! 	sleepq_lock(&sx->lock_object);
  
! 	/*
! 	 * Preserve SX_LOCK_EXCLUSIVE_WAITERS while downgraded to a single
! 	 * shared lock.  If there are any shared waiters, wake them up.
! 	 */
! 	x = sx->sx_lock;
! 	atomic_store_rel_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1) |
! 	    (x & SX_LOCK_EXCLUSIVE_WAITERS));
! 	if (x & SX_LOCK_SHARED_WAITERS)
! 		sleepq_broadcast_queue(&sx->lock_object, SLEEPQ_SX, -1,
! 		    SQ_SHARED_QUEUE);
! 	else
! 		sleepq_release(&sx->lock_object);
  
! 	LOCK_LOG_LOCK("XDOWNGRADE", &sx->lock_object, 0, 0, file, line);
  }
  
+ /*
+  * This function represents the so-called 'hard case' for sx_xlock
+  * operation.  All 'easy case' failures are redirected to this.  Note
+  * that ideally this would be a static function, but it needs to be
+  * accessible from at least sx.h.
+  */
  int
! _sx_xlock_hard(struct sx *sx, uintptr_t tid, int opts, const char *file,
!     int line)
  {
+ 	GIANT_DECLARE;
+ #ifdef ADAPTIVE_SX
+ 	volatile struct thread *owner;
+ #endif
+ 	/* uint64_t waittime = 0; */
+ 	uintptr_t x;
+ 	int /* contested = 0, */error = 0;
+ 
+ 	/* If we already hold an exclusive lock, then recurse. */
+ 	if (sx_xlocked(sx)) {
+ 		KASSERT((sx->lock_object.lo_flags & SX_RECURSE) != 0,
+ 	    ("_sx_xlock_hard: recursed on non-recursive sx %s @ %s:%d\n",
+ 		    sx->lock_object.lo_name, file, line));
+ 		sx->sx_recurse++;
+ 		atomic_set_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
+ 		if (LOCK_LOG_TEST(&sx->lock_object, 0))
+ 			CTR2(KTR_LOCK, "%s: %p recursing", __func__, sx);
+ 		return (0);
+ 	}
  
! 	if (LOCK_LOG_TEST(&sx->lock_object, 0))
! 		CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__,
! 		    sx->lock_object.lo_name, (void *)sx->sx_lock, file, line);
  
! 	while (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid)) {
! #ifdef ADAPTIVE_SX
! 		/*
! 		 * If the lock is write locked and the owner is
! 		 * running on another CPU, spin until the owner stops
! 		 * running or the state of the lock changes.
! 		 */
! 		x = sx->sx_lock;
! 		if (!(x & SX_LOCK_SHARED) &&
! 		    (sx->lock_object.lo_flags & SX_ADAPTIVESPIN)) {
! 			x = SX_OWNER(x);
! 			owner = (struct thread *)x;
! 			if (TD_IS_RUNNING(owner)) {
! 				if (LOCK_LOG_TEST(&sx->lock_object, 0))
! 					CTR3(KTR_LOCK,
! 					    "%s: spinning on %p held by %p",
! 					    __func__, sx, owner);
! 				GIANT_SAVE();
! 				lock_profile_obtain_lock_failed(
! 				    &sx->lock_object, &contested, &waittime);
! 				while (SX_OWNER(sx->sx_lock) == x &&
! 				    TD_IS_RUNNING(owner))
! 					cpu_spinwait();
! 				continue;
! 			}
! 		}
! #endif
  
! 		sleepq_lock(&sx->lock_object);
! 		x = sx->sx_lock;
  
! 		/*
! 		 * If the lock was released while spinning on the
! 		 * sleep queue chain lock, try again.
! 		 */
! 		if (x == SX_LOCK_UNLOCKED) {
! 			sleepq_release(&sx->lock_object);
! 			continue;
! 		}
! 
! #ifdef ADAPTIVE_SX
! 		/*
! 		 * The current lock owner might have started executing
! 		 * on another CPU (or the lock could have changed
! 		 * owners) while we were waiting on the sleep queue
! 		 * chain lock.  If so, drop the sleep queue lock and try
! 		 * again.
! 		 */
! 		if (!(x & SX_LOCK_SHARED) &&
! 		    (sx->lock_object.lo_flags & SX_ADAPTIVESPIN)) {
! 			owner = (struct thread *)SX_OWNER(x);
! 			if (TD_IS_RUNNING(owner)) {
! 				sleepq_release(&sx->lock_object);
! 				continue;
! 			}
! 		}
! #endif
! 
! 		/*
! 		 * If an exclusive lock was released with both shared
! 		 * and exclusive waiters and a shared waiter hasn't
! 		 * woken up and acquired the lock yet, sx_lock will be
! 		 * set to SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS.
! 		 * If we see that value, try to acquire it once.  Note
! 		 * that we have to preserve SX_LOCK_EXCLUSIVE_WAITERS
! 		 * as there are other exclusive waiters still.  If we
! 		 * fail, restart the loop.
! 		 */
! 		if (x == (SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS)) {
! 			if (atomic_cmpset_acq_ptr(&sx->sx_lock,
! 			    SX_LOCK_UNLOCKED | SX_LOCK_EXCLUSIVE_WAITERS,
! 			    tid | SX_LOCK_EXCLUSIVE_WAITERS)) {
! 				sleepq_release(&sx->lock_object);
! 				CTR2(KTR_LOCK, "%s: %p claimed by new writer",
! 				    __func__, sx);
! 				break;
! 			}
! 			sleepq_release(&sx->lock_object);
! 			continue;
! 		}
! 
! 		/*
! 		 * Try to set the SX_LOCK_EXCLUSIVE_WAITERS.  If we fail,
! 		 * than loop back and retry.
! 		 */
! 		if (!(x & SX_LOCK_EXCLUSIVE_WAITERS)) {
! 			if (!atomic_cmpset_ptr(&sx->sx_lock, x,
! 			    x | SX_LOCK_EXCLUSIVE_WAITERS)) {
! 				sleepq_release(&sx->lock_object);
! 				continue;
! 			}
! 			if (LOCK_LOG_TEST(&sx->lock_object, 0))
! 				CTR2(KTR_LOCK, "%s: %p set excl waiters flag",
! 				    __func__, sx);
! 		}
! 
! 		/*
! 		 * Since we have been unable to acquire the exclusive
! 		 * lock and the exclusive waiters flag is set, we have
! 		 * to sleep.
! 		 */
! #if 0
! 		if (LOCK_LOG_TEST(&sx->lock_object, 0))
! 			CTR2(KTR_LOCK, "%s: %p blocking on sleep queue",
! 			    __func__, sx);
! #endif
! 
! 		GIANT_SAVE();
! 		lock_profile_obtain_lock_failed(&sx->lock_object, &contested,
! 		    &waittime);
! 		sleepq_add_queue(&sx->lock_object, NULL, sx->lock_object.lo_name,
! 		    SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ?
! 		    SLEEPQ_INTERRUPTIBLE : 0), SQ_EXCLUSIVE_QUEUE);
! 		if (!(opts & SX_INTERRUPTIBLE))
! 			sleepq_wait(&sx->lock_object);
! 		else
! 			error = sleepq_wait_sig(&sx->lock_object);
! 
! 		if (error) {
! 			if (LOCK_LOG_TEST(&sx->lock_object, 0))
! 				CTR2(KTR_LOCK,
! 			"%s: interruptible sleep by %p suspended by signal",
! 				    __func__, sx);
! 			break;
! 		}
! 		if (LOCK_LOG_TEST(&sx->lock_object, 0))
! 			CTR2(KTR_LOCK, "%s: %p resuming from sleep queue",
! 			    __func__, sx);
  	}
+ 
+ 	GIANT_RESTORE();
+ 	if (!error)
+ 		lock_profile_obtain_lock_success(&sx->lock_object, contested,
+ 		    waittime, file, line);
+ 	return (error);
  }
  
+ /*
+  * This function represents the so-called 'hard case' for sx_xunlock
+  * operation.  All 'easy case' failures are redirected to this.  Note
+  * that ideally this would be a static function, but it needs to be
+  * accessible from at least sx.h.
+  */
  void
! _sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int line)
! {
! 	uintptr_t x;
! 	int queue;
! 
! 	MPASS(!(sx->sx_lock & SX_LOCK_SHARED));
! 
! 	/* If the lock is recursed, then unrecurse one level. */
! 	if (sx_xlocked(sx) && sx_recursed(sx)) {
! 		if ((--sx->sx_recurse) == 0)
! 			atomic_clear_ptr(&sx->sx_lock, SX_LOCK_RECURSED);
! 		if (LOCK_LOG_TEST(&sx->lock_object, 0))
! 			CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, sx);
! 		return;
! 	}
! 	MPASS(sx->sx_lock & (SX_LOCK_SHARED_WAITERS |
! 	    SX_LOCK_EXCLUSIVE_WAITERS));
! 	if (LOCK_LOG_TEST(&sx->lock_object, 0))
! 		CTR2(KTR_LOCK, "%s: %p contested", __func__, sx);
! 
! 	sleepq_lock(&sx->lock_object);
! 	x = SX_LOCK_UNLOCKED;
! 
! 	/*
! 	 * The wake up algorithm here is quite simple and probably not
! 	 * ideal.  It gives precedence to shared waiters if they are
! 	 * present.  For this condition, we have to preserve the
! 	 * state of the exclusive waiters flag.
! 	 */
! 	if (sx->sx_lock & SX_LOCK_SHARED_WAITERS) {
! 		queue = SQ_SHARED_QUEUE;
! 		x |= (sx->sx_lock & SX_LOCK_EXCLUSIVE_WAITERS);
! 	} else
! 		queue = SQ_EXCLUSIVE_QUEUE;
! 
! 	/* Wake up all the waiters for the specific queue. */
! 	if (LOCK_LOG_TEST(&sx->lock_object, 0))
! 		CTR3(KTR_LOCK, "%s: %p waking up all threads on %s queue",
! 		    __func__, sx, queue == SQ_SHARED_QUEUE ? "shared" :
! 		    "exclusive");
! 	atomic_store_rel_ptr(&sx->sx_lock, x);
! 	sleepq_broadcast_queue(&sx->lock_object, SLEEPQ_SX, -1, queue);
! }
! 
! /*
!  * This function represents the so-called 'hard case' for sx_slock
!  * operation.  All 'easy case' failures are redirected to this.  Note
!  * that ideally this would be a static function, but it needs to be
!  * accessible from at least sx.h.
!  */
! int
! _sx_slock_hard(struct sx *sx, int opts, const char *file, int line)
! {
! 	GIANT_DECLARE;
! #ifdef ADAPTIVE_SX
! 	volatile struct thread *owner;
! #endif
! #ifdef LOCK_PROFILING_SHARED
! 	uint64_t waittime = 0;
! 	int contested = 0;
! #endif
! 	uintptr_t x;
! 	int error = 0;
! 
! 	/*
! 	 * As with rwlocks, we don't make any attempt to try to block
! 	 * shared locks once there is an exclusive waiter.
! 	 */
! 	for (;;) {
! 		x = sx->sx_lock;
! 
! 		/*
! 		 * If no other thread has an exclusive lock then try to bump up
! 		 * the count of sharers.  Since we have to preserve the state
! 		 * of SX_LOCK_EXCLUSIVE_WAITERS, if we fail to acquire the
! 		 * shared lock loop back and retry.
! 		 */
! 		if (x & SX_LOCK_SHARED) {
! 			MPASS(!(x & SX_LOCK_SHARED_WAITERS));
! 			if (atomic_cmpset_acq_ptr(&sx->sx_lock, x,
! 			    x + SX_ONE_SHARER)) {
! #ifdef LOCK_PROFILING_SHARED
! 				if (SX_SHARERS(x) == 0)
! 					lock_profile_obtain_lock_success(
! 					    &sx->lock_object, contested,
! 					    waittime, file, line);
! #endif
! 				if (LOCK_LOG_TEST(&sx->lock_object, 0))
! 					CTR4(KTR_LOCK,
! 					    "%s: %p succeed %p -> %p", __func__,
! 					    sx, (void *)x,
! 					    (void *)(x + SX_ONE_SHARER));
! 				break;
! 			}
! 			continue;
! 		}
! 
! #ifdef ADAPTIVE_SX
! 		/*
! 		 * If the owner is running on another CPU, spin until
! 		 * the owner stops running or the state of the lock
! 		 * changes.
! 		 */
! 		else if (sx->lock_object.lo_flags & SX_ADAPTIVESPIN) {
! 			x = SX_OWNER(x);
! 			owner = (struct thread *)x;
! 			if (TD_IS_RUNNING(owner)) {
! 				if (LOCK_LOG_TEST(&sx->lock_object, 0))
! 					CTR3(KTR_LOCK,
! 					    "%s: spinning on %p held by %p",
! 					    __func__, sx, owner);
! 				GIANT_SAVE();
! #ifdef LOCK_PROFILING_SHARED
! 				lock_profile_obtain_lock_failed(
! 				    &sx->lock_object, &contested, &waittime);
! #endif
! 				while (SX_OWNER(sx->sx_lock) == x &&
! 				    TD_IS_RUNNING(owner))
! 					cpu_spinwait();
! 				continue;
! 			}
! 		}
! #endif
! 
! 		/*
! 		 * Some other thread already has an exclusive lock, so
! 		 * start the process of blocking.
! 		 */
! 		sleepq_lock(&sx->lock_object);
! 		x = sx->sx_lock;
! 
! 		/*
! 		 * The lock could have been released while we spun.
! 		 * In this case loop back and retry.
! 		 */
! 		if (x & SX_LOCK_SHARED) {
! 			sleepq_release(&sx->lock_object);
! 			continue;
! 		}
! 
! #ifdef ADAPTIVE_SX
! 		/*
! 		 * If the owner is running on another CPU, spin until
! 		 * the owner stops running or the state of the lock
! 		 * changes.
! 		 */
! 		if (!(x & SX_LOCK_SHARED) &&
! 		    (sx->lock_object.lo_flags & SX_ADAPTIVESPIN)) {
! 			owner = (struct thread *)SX_OWNER(x);
! 			if (TD_IS_RUNNING(owner)) {
! 				sleepq_release(&sx->lock_object);
! 				continue;
! 			}
! 		}
! #endif
! 
! 		/*
! 		 * Try to set the SX_LOCK_SHARED_WAITERS flag.  If we
! 		 * fail to set it drop the sleep queue lock and loop
! 		 * back.
! 		 */
! 		if (!(x & SX_LOCK_SHARED_WAITERS)) {
! 			if (!atomic_cmpset_ptr(&sx->sx_lock, x,
! 			    x | SX_LOCK_SHARED_WAITERS)) {
! 				sleepq_release(&sx->lock_object);
! 				continue;
! 			}
! 			if (LOCK_LOG_TEST(&sx->lock_object, 0))
! 				CTR2(KTR_LOCK, "%s: %p set shared waiters flag",
! 				    __func__, sx);
! 		}
! 
! 		/*
! 		 * Since we have been unable to acquire the shared lock,
! 		 * we have to sleep.
! 		 */
! 		if (LOCK_LOG_TEST(&sx->lock_object, 0))
! 			CTR2(KTR_LOCK, "%s: %p blocking on sleep queue",
! 			    __func__, sx);
! 
! 		GIANT_SAVE();
! #ifdef LOCK_PROFILING_SHARED
! 		lock_profile_obtain_lock_failed(&sx->lock_object, &contested,
! 		    &waittime);
! #endif
! 		sleepq_add_queue(&sx->lock_object, NULL, sx->lock_object.lo_name,
! 		    SLEEPQ_SX | ((opts & SX_INTERRUPTIBLE) ?
! 		    SLEEPQ_INTERRUPTIBLE : 0), SQ_SHARED_QUEUE);
! 		if (!(opts & SX_INTERRUPTIBLE))
! 			sleepq_wait(&sx->lock_object);
! 		else
! 			error = sleepq_wait_sig(&sx->lock_object);
! 
! 		if (error) {
! 			if (LOCK_LOG_TEST(&sx->lock_object, 0))
! 				CTR2(KTR_LOCK,
! 			"%s: interruptible sleep by %p suspended by signal",
! 				    __func__, sx);
! 			break;
! 		}
! 		if (LOCK_LOG_TEST(&sx->lock_object, 0))
! 			CTR2(KTR_LOCK, "%s: %p resuming from sleep queue",
! 			    __func__, sx);
! 	}
! 
! 	GIANT_RESTORE();
! 	return (error);
! }
! 
! /*
!  * This function represents the so-called 'hard case' for sx_sunlock
!  * operation.  All 'easy case' failures are redirected to this.  Note
!  * that ideally this would be a static function, but it needs to be
!  * accessible from at least sx.h.
!  */
! void
! _sx_sunlock_hard(struct sx *sx, const char *file, int line)
  {
+ 	uintptr_t x;
+ 
+ 	for (;;) {
+ 		x = sx->sx_lock;
+ 
+ 		/*
+ 		 * We should never have sharers while at least one thread
+ 		 * holds a shared lock.
+ 		 */
+ 		KASSERT(!(x & SX_LOCK_SHARED_WAITERS),
+ 		    ("%s: waiting sharers", __func__));
  
! 		/*
! 		 * See if there is more than one shared lock held.  If
! 		 * so, just drop one and return.
! 		 */
! 		if (SX_SHARERS(x) > 1) {
! 			if (atomic_cmpset_ptr(&sx->sx_lock, x,
! 			    x - SX_ONE_SHARER)) {
! 				if (LOCK_LOG_TEST(&sx->lock_object, 0))
! 					CTR4(KTR_LOCK,
! 					    "%s: %p succeeded %p -> %p",
! 					    __func__, sx, (void *)x,
! 					    (void *)(x - SX_ONE_SHARER));
! 				break;
! 			}
! 			continue;
! 		}
  
! 		/*
! 		 * If there aren't any waiters for an exclusive lock,
! 		 * then try to drop it quickly.
! 		 */
! 		if (!(x & SX_LOCK_EXCLUSIVE_WAITERS)) {
! 			MPASS(x == SX_SHARERS_LOCK(1));
! 			if (atomic_cmpset_ptr(&sx->sx_lock, SX_SHARERS_LOCK(1),
! 			    SX_LOCK_UNLOCKED)) {
! 				if (LOCK_LOG_TEST(&sx->lock_object, 0))
! 					CTR2(KTR_LOCK, "%s: %p last succeeded",
! 					    __func__, sx);
! 				break;
! 			}
! 			continue;
! 		}
  
! 		/*
! 		 * At this point, there should just be one sharer with
! 		 * exclusive waiters.
! 		 */
! 		MPASS(x == (SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS));
  
! 		sleepq_lock(&sx->lock_object);
  
! 		/*
! 		 * Wake up semantic here is quite simple:
! 		 * Just wake up all the exclusive waiters.
! 		 * Note that the state of the lock could have changed,
! 		 * so if it fails loop back and retry.
! 		 */
! 		if (!atomic_cmpset_ptr(&sx->sx_lock,
! 		    SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS,
! 		    SX_LOCK_UNLOCKED)) {
! 			sleepq_release(&sx->lock_object);
! 			continue;
! 		}
! 		if (LOCK_LOG_TEST(&sx->lock_object, 0))
! 			CTR2(KTR_LOCK, "%s: %p waking up all thread on"
! 			    "exclusive queue", __func__, sx);
! 		sleepq_broadcast_queue(&sx->lock_object, SLEEPQ_SX, -1,
! 		    SQ_EXCLUSIVE_QUEUE);
! 		break;
! 	}
  }
  
  #ifdef INVARIANT_SUPPORT
***************
*** 327,370 ****
  void
  _sx_assert(struct sx *sx, int what, const char *file, int line)
  {
  
  	if (panicstr != NULL)
  		return;
  	switch (what) {
! 	case SX_LOCKED:
! 	case SX_SLOCKED:
  #ifdef WITNESS
! 		witness_assert(&sx->sx_object, what, file, line);
  #else
! 		mtx_lock(sx->sx_lock);
! 		if (sx->sx_cnt <= 0 &&
! 		    (what == SX_SLOCKED || sx->sx_xholder != curthread))
  			panic("Lock %s not %slocked @ %s:%d\n",
! 			    sx->sx_object.lo_name, (what == SX_SLOCKED) ?
! 			    "share " : "", file, line);
! 		mtx_unlock(sx->sx_lock);
  #endif
  		break;
! 	case SX_XLOCKED:
! 		mtx_lock(sx->sx_lock);
! 		if (sx->sx_xholder != curthread)
  			panic("Lock %s not exclusively locked @ %s:%d\n",
! 			    sx->sx_object.lo_name, file, line);
! 		mtx_unlock(sx->sx_lock);
  		break;
! 	case SX_UNLOCKED:
  #ifdef WITNESS
! 		witness_assert(&sx->sx_object, what, file, line);
  #else
  		/*
! 		 * We are able to check only exclusive lock here,
! 		 * we cannot assert that *this* thread owns slock.
  		 */
! 		mtx_lock(sx->sx_lock);
! 		if (sx->sx_xholder == curthread)
  			panic("Lock %s exclusively locked @ %s:%d\n",
! 			    sx->sx_object.lo_name, file, line);
! 		mtx_unlock(sx->sx_lock);
  #endif
  		break;
  	default:
--- 861,936 ----
  void
  _sx_assert(struct sx *sx, int what, const char *file, int line)
  {
+ #ifndef WITNESS
+ 	int slocked = 0;
+ #endif
  
  	if (panicstr != NULL)
  		return;
  	switch (what) {
! 	case SA_SLOCKED:
! 	case SA_SLOCKED | SA_NOTRECURSED:
! 	case SA_SLOCKED | SA_RECURSED:
! #ifndef WITNESS
! 		slocked = 1;
! 		/* FALLTHROUGH */
! #endif
! 	case SA_LOCKED:
! 	case SA_LOCKED | SA_NOTRECURSED:
! 	case SA_LOCKED | SA_RECURSED:
  #ifdef WITNESS
! 		witness_assert(&sx->lock_object, what, file, line);
  #else
! 		/*
! 		 * If some other thread has an exclusive lock or we
! 		 * have one and are asserting a shared lock, fail.
! 		 * Also, if no one has a lock at all, fail.
! 		 */
! 		if (sx->sx_lock == SX_LOCK_UNLOCKED ||
! 		    (!(sx->sx_lock & SX_LOCK_SHARED) && (slocked ||
! 		    sx_xholder(sx) != curthread)))
  			panic("Lock %s not %slocked @ %s:%d\n",
! 			    sx->lock_object.lo_name, slocked ? "share " : "",
! 			    file, line);
! 
! 		if (!(sx->sx_lock & SX_LOCK_SHARED)) {
! 			if (sx_recursed(sx)) {
! 				if (what & SA_NOTRECURSED)
! 					panic("Lock %s recursed @ %s:%d\n",
! 					    sx->lock_object.lo_name, file,
! 					    line);
! 			} else if (what & SA_RECURSED)
! 				panic("Lock %s not recursed @ %s:%d\n",
! 				    sx->lock_object.lo_name, file, line);
! 		}
  #endif
  		break;
! 	case SA_XLOCKED:
! 	case SA_XLOCKED | SA_NOTRECURSED:
! 	case SA_XLOCKED | SA_RECURSED:
! 		if (sx_xholder(sx) != curthread)
  			panic("Lock %s not exclusively locked @ %s:%d\n",
! 			    sx->lock_object.lo_name, file, line);
! 		if (sx_recursed(sx)) {
! 			if (what & SA_NOTRECURSED)
! 				panic("Lock %s recursed @ %s:%d\n",
! 				    sx->lock_object.lo_name, file, line);
! 		} else if (what & SA_RECURSED)
! 			panic("Lock %s not recursed @ %s:%d\n",
! 			    sx->lock_object.lo_name, file, line);
  		break;
! 	case SA_UNLOCKED:
  #ifdef WITNESS
! 		witness_assert(&sx->lock_object, what, file, line);
  #else
  		/*
! 		 * If we hold an exclusve lock fail.  We can't
! 		 * reliably check to see if we hold a shared lock or
! 		 * not.
  		 */
! 		if (sx_xholder(sx) == curthread)
  			panic("Lock %s exclusively locked @ %s:%d\n",
! 			    sx->lock_object.lo_name, file, line);
  #endif
  		break;
  	default:
***************
*** 375,381 ****
  #endif	/* INVARIANT_SUPPORT */
  
  #ifdef DDB
! void
  db_show_sx(struct lock_object *lock)
  {
  	struct thread *td;
--- 941,947 ----
  #endif	/* INVARIANT_SUPPORT */
  
  #ifdef DDB
! static void
  db_show_sx(struct lock_object *lock)
  {
  	struct thread *td;
***************
*** 384,399 ****
  	sx = (struct sx *)lock;
  
  	db_printf(" state: ");
! 	if (sx->sx_cnt < 0) {
! 		td = sx->sx_xholder;
  		db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
  		    td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm);
! 	} else if (sx->sx_cnt > 0)
! 		db_printf("SLOCK: %d locks\n", sx->sx_cnt);
! 	else
! 		db_printf("UNLOCKED\n");
! 	db_printf(" waiters: %d shared, %d exclusive\n", sx->sx_shrd_wcnt,
! 	    sx->sx_excl_wcnt);
  }
  
  /*
--- 950,985 ----
  	sx = (struct sx *)lock;
  
  	db_printf(" state: ");
! 	if (sx->sx_lock == SX_LOCK_UNLOCKED)
! 		db_printf("UNLOCKED\n");
! 	else if (sx->sx_lock == SX_LOCK_DESTROYED) {
! 		db_printf("DESTROYED\n");
! 		return;
! 	} else if (sx->sx_lock & SX_LOCK_SHARED)
! 		db_printf("SLOCK: %ju\n", (uintmax_t)SX_SHARERS(sx->sx_lock));
! 	else {
! 		td = sx_xholder(sx);
  		db_printf("XLOCK: %p (tid %d, pid %d, \"%s\")\n", td,
  		    td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm);
! 		if (sx_recursed(sx))
! 			db_printf(" recursed: %d\n", sx->sx_recurse);
! 	}
! 
! 	db_printf(" waiters: ");
! 	switch(sx->sx_lock &
! 	    (SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS)) {
! 	case SX_LOCK_SHARED_WAITERS:
! 		db_printf("shared\n");
! 		break;
! 	case SX_LOCK_EXCLUSIVE_WAITERS:
! 		db_printf("exclusive\n");
! 		break;
! 	case SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS:
! 		db_printf("exclusive and shared\n");
! 		break;
! 	default:
! 		db_printf("none\n");
! 	}
  }
  
  /*
***************
*** 405,451 ****
  sx_chain(struct thread *td, struct thread **ownerp)
  {
  	struct sx *sx;
- 	struct cv *cv;
  
  	/*
! 	 * First, see if it looks like td is blocked on a condition
! 	 * variable.
  	 */
! 	cv = td->td_wchan;
! 	if (cv->cv_description != td->td_wmesg)
  		return (0);
  
- 	/*
- 	 * Ok, see if it looks like td is blocked on the exclusive
- 	 * condition variable.
- 	 */
- 	sx = (struct sx *)((char *)cv - offsetof(struct sx, sx_excl_cv));
- 	if (LOCK_CLASS(&sx->sx_object) == &lock_class_sx &&
- 	    sx->sx_excl_wcnt > 0)
- 		goto ok;
- 
- 	/*
- 	 * Second, see if it looks like td is blocked on the shared
- 	 * condition variable.
- 	 */
- 	sx = (struct sx *)((char *)cv - offsetof(struct sx, sx_shrd_cv));
- 	if (LOCK_CLASS(&sx->sx_object) == &lock_class_sx &&
- 	    sx->sx_shrd_wcnt > 0)
- 		goto ok;
- 
- 	/* Doesn't seem to be an sx lock. */
- 	return (0);
- 
- ok:
  	/* We think we have an sx lock, so output some details. */
  	db_printf("blocked on sx \"%s\" ", td->td_wmesg);
! 	if (sx->sx_cnt >= 0) {
! 		db_printf("SLOCK (count %d)\n", sx->sx_cnt);
! 		*ownerp = NULL;
! 	} else {
  		db_printf("XLOCK\n");
- 		*ownerp = sx->sx_xholder;
- 	}
  	return (1);
  }
  #endif
--- 991,1016 ----
  sx_chain(struct thread *td, struct thread **ownerp)
  {
  	struct sx *sx;
  
  	/*
! 	 * Check to see if this thread is blocked on an sx lock.
! 	 * First, we check the lock class.  If that is ok, then we
! 	 * compare the lock name against the wait message.
  	 */
! #define LOCK_CLASS(lo)	(lo)->lo_class
! 	sx = td->td_wchan;
! 	if (LOCK_CLASS(&sx->lock_object) != &lock_class_sx ||
! 	    sx->lock_object.lo_name != td->td_wmesg)
  		return (0);
  
  	/* We think we have an sx lock, so output some details. */
  	db_printf("blocked on sx \"%s\" ", td->td_wmesg);
! 	*ownerp = sx_xholder(sx);
! 	if (sx->sx_lock & SX_LOCK_SHARED)
! 		db_printf("SLOCK (count %ju)\n",
! 		    (uintmax_t)SX_SHARERS(sx->sx_lock));
! 	else
  		db_printf("XLOCK\n");
  	return (1);
  }
  #endif
Index: kern/kern_thread.c
===================================================================
RCS file: /cvs/ncvs/src/sys/kern/kern_thread.c,v
retrieving revision 1.216.2.6
diff -c -r1.216.2.6 kern_thread.c
*** kern/kern_thread.c	2 Sep 2006 17:29:57 -0000	1.216.2.6
--- kern/kern_thread.c	2 Sep 2007 21:56:36 -0000
***************
*** 305,311 ****
  
  	thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(),
  	    thread_ctor, thread_dtor, thread_init, thread_fini,
! 	    UMA_ALIGN_CACHE, 0);
  	ksegrp_zone = uma_zcreate("KSEGRP", sched_sizeof_ksegrp(),
  	    ksegrp_ctor, NULL, NULL, NULL,
  	    UMA_ALIGN_CACHE, 0);
--- 305,311 ----
  
  	thread_zone = uma_zcreate("THREAD", sched_sizeof_thread(),
  	    thread_ctor, thread_dtor, thread_init, thread_fini,
! 	    THREAD_ALIGN, 0);
  	ksegrp_zone = uma_zcreate("KSEGRP", sched_sizeof_ksegrp(),
  	    ksegrp_ctor, NULL, NULL, NULL,
  	    UMA_ALIGN_CACHE, 0);
Index: kern/subr_sleepqueue.c
===================================================================
RCS file: /cvs/ncvs/src/sys/kern/subr_sleepqueue.c,v
retrieving revision 1.18.2.4
diff -c -r1.18.2.4 subr_sleepqueue.c
*** kern/subr_sleepqueue.c	17 Aug 2006 19:53:06 -0000	1.18.2.4
--- kern/subr_sleepqueue.c	4 Sep 2007 01:28:11 -0000
***************
*** 82,87 ****
--- 82,93 ----
  #include <ddb/ddb.h>
  #endif
  
+ #include <vm/uma.h>
+ 
+ #ifdef DDB
+ #include <ddb/ddb.h>
+ #endif
+ 
  /*
   * Constants for the hash table of sleep queue chains.  These constants are
   * the same ones that 4BSD (and possibly earlier versions of BSD) used.
***************
*** 94,100 ****
  #define	SC_SHIFT	8
  #define	SC_HASH(wc)	(((uintptr_t)(wc) >> SC_SHIFT) & SC_MASK)
  #define	SC_LOOKUP(wc)	&sleepq_chains[SC_HASH(wc)]
! 
  /*
   * There two different lists of sleep queues.  Both lists are connected
   * via the sq_hash entries.  The first list is the sleep queue chain list
--- 100,106 ----
  #define	SC_SHIFT	8
  #define	SC_HASH(wc)	(((uintptr_t)(wc) >> SC_SHIFT) & SC_MASK)
  #define	SC_LOOKUP(wc)	&sleepq_chains[SC_HASH(wc)]
! #define NR_SLEEPQS      2
  /*
   * There two different lists of sleep queues.  Both lists are connected
   * via the sq_hash entries.  The first list is the sleep queue chain list
***************
*** 114,126 ****
   *  c - sleep queue chain lock
   */
  struct sleepqueue {
! 	TAILQ_HEAD(, thread) sq_blocked;	/* (c) Blocked threads. */
  	LIST_ENTRY(sleepqueue) sq_hash;		/* (c) Chain and free list. */
  	LIST_HEAD(, sleepqueue) sq_free;	/* (c) Free queues. */
  	void	*sq_wchan;			/* (c) Wait channel. */
  #ifdef INVARIANTS
  	int	sq_type;			/* (c) Queue type. */
! 	struct mtx *sq_lock;			/* (c) Associated lock. */
  #endif
  };
  
--- 120,132 ----
   *  c - sleep queue chain lock
   */
  struct sleepqueue {
! 	TAILQ_HEAD(, thread) sq_blocked[NR_SLEEPQS];	/* (c) Blocked threads. */
  	LIST_ENTRY(sleepqueue) sq_hash;		/* (c) Chain and free list. */
  	LIST_HEAD(, sleepqueue) sq_free;	/* (c) Free queues. */
  	void	*sq_wchan;			/* (c) Wait channel. */
  #ifdef INVARIANTS
  	int	sq_type;			/* (c) Queue type. */
! 	struct mtx *sq_lock;		/* (c) Associated lock. */
  #endif
  };
  
***************
*** 142,157 ****
      0, "maxmimum depth achieved of a single chain");
  #endif
  static struct sleepqueue_chain sleepq_chains[SC_TABLESIZE];
! 
! static MALLOC_DEFINE(M_SLEEPQUEUE, "sleep queues", "sleep queues");
  
  /*
   * Prototypes for non-exported routines.
   */
  static int	sleepq_check_timeout(void);
  static void	sleepq_switch(void *wchan);
  static void	sleepq_timeout(void *arg);
- static void	sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, int pri);
  
  /*
   * Early initialization of sleep queues that is called from the sleepinit()
--- 148,169 ----
      0, "maxmimum depth achieved of a single chain");
  #endif
  static struct sleepqueue_chain sleepq_chains[SC_TABLESIZE];
! static uma_zone_t sleepq_zone;
  
  /*
   * Prototypes for non-exported routines.
   */
+ static int	sleepq_catch_signals(void *wchan);
+ static int	sleepq_check_signals(void);
  static int	sleepq_check_timeout(void);
+ #ifdef INVARIANTS
+ static void	sleepq_dtor(void *mem, int size, void *arg);
+ #endif
+ static int	sleepq_init(void *mem, int size, int flags);
+ static void	sleepq_resume_thread(struct sleepqueue *sq, struct thread *td,
+ 		    int pri);
  static void	sleepq_switch(void *wchan);
  static void	sleepq_timeout(void *arg);
  
  /*
   * Early initialization of sleep queues that is called from the sleepinit()
***************
*** 182,202 ****
  		    NULL);
  #endif
  	}
  	thread0.td_sleepqueue = sleepq_alloc();
  }
  
  /*
!  * Malloc and initialize a new sleep queue for a new thread.
   */
  struct sleepqueue *
  sleepq_alloc(void)
  {
- 	struct sleepqueue *sq;
  
! 	sq = malloc(sizeof(struct sleepqueue), M_SLEEPQUEUE, M_WAITOK | M_ZERO);
! 	TAILQ_INIT(&sq->sq_blocked);
! 	LIST_INIT(&sq->sq_free);
! 	return (sq);
  }
  
  /*
--- 194,217 ----
  		    NULL);
  #endif
  	}
+ 	sleepq_zone = uma_zcreate("SLEEPQUEUE", sizeof(struct sleepqueue),
+ #ifdef INVARIANTS
+ 	    NULL, sleepq_dtor, sleepq_init, NULL, UMA_ALIGN_CACHE, 0);
+ #else
+ 	    NULL, NULL, sleepq_init, NULL, UMA_ALIGN_CACHE, 0);
+ #endif
+ 	
  	thread0.td_sleepqueue = sleepq_alloc();
  }
  
  /*
!  * Get a sleep queue for a new thread.
   */
  struct sleepqueue *
  sleepq_alloc(void)
  {
  
! 	return (uma_zalloc(sleepq_zone, M_WAITOK));
  }
  
  /*
***************
*** 206,214 ****
  sleepq_free(struct sleepqueue *sq)
  {
  
! 	MPASS(sq != NULL);
! 	MPASS(TAILQ_EMPTY(&sq->sq_blocked));
! 	free(sq, M_SLEEPQUEUE);
  }
  
  /*
--- 221,227 ----
  sleepq_free(struct sleepqueue *sq)
  {
  
! 	uma_zfree(sleepq_zone, sq);
  }
  
  /*
***************
*** 262,268 ****
   * woken up.
   */
  void
! sleepq_add(void *wchan, struct mtx *lock, const char *wmesg, int flags)
  {
  	struct sleepqueue_chain *sc;
  	struct sleepqueue *sq;
--- 275,282 ----
   * woken up.
   */
  void
! sleepq_add_queue(void *wchan, struct mtx *lock, const char *wmesg, int flags,
!     int queue)
  {
  	struct sleepqueue_chain *sc;
  	struct sleepqueue *sq;
***************
*** 273,282 ****
  	mtx_assert(&sc->sc_lock, MA_OWNED);
  	MPASS(td->td_sleepqueue != NULL);
  	MPASS(wchan != NULL);
  
  	/* If this thread is not allowed to sleep, die a horrible death. */
  	KASSERT(!(td->td_pflags & TDP_NOSLEEPING),
! 	    ("trying to sleep while sleeping is prohibited"));
  
  	/* Look up the sleep queue associated with the wait channel 'wchan'. */
  	sq = sleepq_lookup(wchan);
--- 287,297 ----
  	mtx_assert(&sc->sc_lock, MA_OWNED);
  	MPASS(td->td_sleepqueue != NULL);
  	MPASS(wchan != NULL);
+ 	MPASS((queue >= 0) && (queue < NR_SLEEPQS));
  
  	/* If this thread is not allowed to sleep, die a horrible death. */
  	KASSERT(!(td->td_pflags & TDP_NOSLEEPING),
! 	    ("Trying sleep, but thread marked as sleeping prohibited"));
  
  	/* Look up the sleep queue associated with the wait channel 'wchan'. */
  	sq = sleepq_lookup(wchan);
***************
*** 287,292 ****
--- 302,320 ----
  	 * into the sleep queue already in use by this wait channel.
  	 */
  	if (sq == NULL) {
+ #ifdef INVARIANTS
+ 		int i;
+ 
+ 		sq = td->td_sleepqueue;
+ 		for (i = 0; i < NR_SLEEPQS; i++)
+ 			KASSERT(TAILQ_EMPTY(&sq->sq_blocked[i]),
+ 				("thread's sleep queue %d is not empty", i));
+ 		KASSERT(LIST_EMPTY(&sq->sq_free),
+ 		    ("thread's sleep queue has a non-empty free list"));
+ 		KASSERT(sq->sq_wchan == NULL, ("stale sq_wchan pointer"));
+ 		sq->sq_lock = lock;
+ 		sq->sq_type = flags & SLEEPQ_TYPE;
+ #endif
  #ifdef SLEEPQUEUE_PROFILING
  		sc->sc_depth++;
  		if (sc->sc_depth > sc->sc_max_depth) {
***************
*** 297,321 ****
  #endif
  		sq = td->td_sleepqueue;
  		LIST_INSERT_HEAD(&sc->sc_queues, sq, sq_hash);
- 		KASSERT(TAILQ_EMPTY(&sq->sq_blocked),
- 		    ("thread's sleep queue has a non-empty queue"));
- 		KASSERT(LIST_EMPTY(&sq->sq_free),
- 		    ("thread's sleep queue has a non-empty free list"));
- 		KASSERT(sq->sq_wchan == NULL, ("stale sq_wchan pointer"));
  		sq->sq_wchan = wchan;
- #ifdef INVARIANTS
- 		sq->sq_lock = lock;
- 		sq->sq_type = flags & SLEEPQ_TYPE;
- #endif
  	} else {
  		MPASS(wchan == sq->sq_wchan);
  		MPASS(lock == sq->sq_lock);
  		MPASS((flags & SLEEPQ_TYPE) == sq->sq_type);
  		LIST_INSERT_HEAD(&sq->sq_free, td->td_sleepqueue, sq_hash);
  	}
! 	TAILQ_INSERT_TAIL(&sq->sq_blocked, td, td_slpq);
  	td->td_sleepqueue = NULL;
  	mtx_lock_spin(&sched_lock);
  	td->td_wchan = wchan;
  	td->td_wmesg = wmesg;
  	if (flags & SLEEPQ_INTERRUPTIBLE) {
--- 325,341 ----
  #endif
  		sq = td->td_sleepqueue;
  		LIST_INSERT_HEAD(&sc->sc_queues, sq, sq_hash);
  		sq->sq_wchan = wchan;
  	} else {
  		MPASS(wchan == sq->sq_wchan);
  		MPASS(lock == sq->sq_lock);
  		MPASS((flags & SLEEPQ_TYPE) == sq->sq_type);
  		LIST_INSERT_HEAD(&sq->sq_free, td->td_sleepqueue, sq_hash);
  	}
! 	TAILQ_INSERT_TAIL(&sq->sq_blocked[queue], td, td_slpq);
  	td->td_sleepqueue = NULL;
  	mtx_lock_spin(&sched_lock);
+ 	td->td_sqqueue = queue;
  	td->td_wchan = wchan;
  	td->td_wmesg = wmesg;
  	if (flags & SLEEPQ_INTERRUPTIBLE) {
***************
*** 606,617 ****
  	MPASS(td != NULL);
  	MPASS(sq->sq_wchan != NULL);
  	MPASS(td->td_wchan == sq->sq_wchan);
  	sc = SC_LOOKUP(sq->sq_wchan);
  	mtx_assert(&sc->sc_lock, MA_OWNED);
  	mtx_assert(&sched_lock, MA_OWNED);
  
  	/* Remove the thread from the queue. */
! 	TAILQ_REMOVE(&sq->sq_blocked, td, td_slpq);
  
  	/*
  	 * Get a sleep queue for this thread.  If this is the last waiter,
--- 626,638 ----
  	MPASS(td != NULL);
  	MPASS(sq->sq_wchan != NULL);
  	MPASS(td->td_wchan == sq->sq_wchan);
+ 	MPASS(td->td_sqqueue < NR_SLEEPQS && td->td_sqqueue >= 0);
  	sc = SC_LOOKUP(sq->sq_wchan);
  	mtx_assert(&sc->sc_lock, MA_OWNED);
  	mtx_assert(&sched_lock, MA_OWNED);
  
  	/* Remove the thread from the queue. */
! 	TAILQ_REMOVE(&sq->sq_blocked[td->td_sqqueue], td, td_slpq);
  
  	/*
  	 * Get a sleep queue for this thread.  If this is the last waiter,
***************
*** 652,668 ****
  	setrunnable(td);
  }
  
  /*
   * Find the highest priority thread sleeping on a wait channel and resume it.
   */
  void
! sleepq_signal(void *wchan, int flags, int pri)
  {
  	struct sleepqueue *sq;
  	struct thread *td, *besttd;
  
  	CTR2(KTR_PROC, "sleepq_signal(%p, %d)", wchan, flags);
  	KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
  	sq = sleepq_lookup(wchan);
  	if (sq == NULL) {
  		sleepq_release(wchan);
--- 673,723 ----
  	setrunnable(td);
  }
  
+ #ifdef INVARIANTS
+ /*
+  * UMA zone item deallocator.
+  */
+ static void
+ sleepq_dtor(void *mem, int size, void *arg)
+ {
+ 	struct sleepqueue *sq;
+ 	int i;
+ 
+ 	sq = mem;
+ 	for (i = 0; i < NR_SLEEPQS; i++)
+ 		MPASS(TAILQ_EMPTY(&sq->sq_blocked[i]));
+ }
+ #endif
+ 
+ /*
+  * UMA zone item initializer.
+  */
+ static int
+ sleepq_init(void *mem, int size, int flags)
+ {
+ 	struct sleepqueue *sq;
+ 	int i;
+ 
+ 	bzero(mem, size);
+ 	sq = mem;
+ 	for (i = 0; i < NR_SLEEPQS; i++)
+ 		TAILQ_INIT(&sq->sq_blocked[i]);
+ 	LIST_INIT(&sq->sq_free);
+ 	return (0);
+ }
+ 
  /*
   * Find the highest priority thread sleeping on a wait channel and resume it.
   */
  void
! sleepq_signal_queue(void *wchan, int flags, int pri, int queue)
  {
  	struct sleepqueue *sq;
  	struct thread *td, *besttd;
  
  	CTR2(KTR_PROC, "sleepq_signal(%p, %d)", wchan, flags);
  	KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
+ 	MPASS((queue >= 0) && (queue < NR_SLEEPQS));
  	sq = sleepq_lookup(wchan);
  	if (sq == NULL) {
  		sleepq_release(wchan);
***************
*** 678,684 ****
  	 * the tail of sleep queues.
  	 */
  	besttd = NULL;
! 	TAILQ_FOREACH(td, &sq->sq_blocked, td_slpq) {
  		if (besttd == NULL || td->td_priority < besttd->td_priority)
  			besttd = td;
  	}
--- 733,739 ----
  	 * the tail of sleep queues.
  	 */
  	besttd = NULL;
! 	TAILQ_FOREACH(td, &sq->sq_blocked[queue], td_slpq) {
  		if (besttd == NULL || td->td_priority < besttd->td_priority)
  			besttd = td;
  	}
***************
*** 693,704 ****
   * Resume all threads sleeping on a specified wait channel.
   */
  void
! sleepq_broadcast(void *wchan, int flags, int pri)
  {
  	struct sleepqueue *sq;
  
  	CTR2(KTR_PROC, "sleepq_broadcast(%p, %d)", wchan, flags);
  	KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
  	sq = sleepq_lookup(wchan);
  	if (sq == NULL) {
  		sleepq_release(wchan);
--- 748,760 ----
   * Resume all threads sleeping on a specified wait channel.
   */
  void
! sleepq_broadcast_queue(void *wchan, int flags, int pri, int queue)
  {
  	struct sleepqueue *sq;
  
  	CTR2(KTR_PROC, "sleepq_broadcast(%p, %d)", wchan, flags);
  	KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
+ 	MPASS((queue >= 0) && (queue < NR_SLEEPQS));
  	sq = sleepq_lookup(wchan);
  	if (sq == NULL) {
  		sleepq_release(wchan);
***************
*** 709,716 ****
  
  	/* Resume all blocked threads on the sleep queue. */
  	mtx_lock_spin(&sched_lock);
! 	while (!TAILQ_EMPTY(&sq->sq_blocked))
! 		sleepq_resume_thread(sq, TAILQ_FIRST(&sq->sq_blocked), pri);
  	mtx_unlock_spin(&sched_lock);
  	sleepq_release(wchan);
  }
--- 765,773 ----
  
  	/* Resume all blocked threads on the sleep queue. */
  	mtx_lock_spin(&sched_lock);
! 	while (!TAILQ_EMPTY(&sq->sq_blocked[queue]))
! 		sleepq_resume_thread(sq, TAILQ_FIRST(&sq->sq_blocked[queue]),
! 		    pri);
  	mtx_unlock_spin(&sched_lock);
  	sleepq_release(wchan);
  }
***************
*** 853,913 ****
  	mtx_lock_spin(&sched_lock);
  }
  
- #ifdef DDB
- DB_SHOW_COMMAND(sleepq, db_show_sleepqueue)
- {
- 	struct sleepqueue_chain *sc;
- 	struct sleepqueue *sq;
- #ifdef INVARIANTS
- 	struct lock_object *lock;
- #endif
- 	struct thread *td;
- 	void *wchan;
- 	int i;
- 
- 	if (!have_addr)
- 		return;
- 
- 	/*
- 	 * First, see if there is an active sleep queue for the wait channel
- 	 * indicated by the address.
- 	 */
- 	wchan = (void *)addr;
- 	sc = SC_LOOKUP(wchan);
- 	LIST_FOREACH(sq, &sc->sc_queues, sq_hash)
- 		if (sq->sq_wchan == wchan)
- 			goto found;
- 
- 	/*
- 	 * Second, see if there is an active sleep queue at the address
- 	 * indicated.
- 	 */
- 	for (i = 0; i < SC_TABLESIZE; i++)
- 		LIST_FOREACH(sq, &sleepq_chains[i].sc_queues, sq_hash) {
- 			if (sq == (struct sleepqueue *)addr)
- 				goto found;
- 		}
- 
- 	db_printf("Unable to locate a sleep queue via %p\n", (void *)addr);
- 	return;
- found:
- 	db_printf("Wait channel: %p\n", sq->sq_wchan);
- #ifdef INVARIANTS
- 	db_printf("Queue type: %d\n", sq->sq_type);
- 	if (sq->sq_lock) {
- 		lock = &sq->sq_lock->mtx_object;
- 		db_printf("Associated Interlock: %p - (%s) %s\n", lock,
- 		    LOCK_CLASS(lock)->lc_name, lock->lo_name);
- 	}
- #endif
- 	db_printf("Blocked threads:\n");
- 	if (TAILQ_EMPTY(&sq->sq_blocked))
- 		db_printf("\tempty\n");
- 	else
- 		TAILQ_FOREACH(td, &sq->sq_blocked, td_slpq) {
- 			db_printf("\t%p (tid %d, pid %d, \"%s\")\n", td,
- 			    td->td_tid, td->td_proc->p_pid,
- 			    td->td_proc->p_comm);
- 		}	
- }
- #endif
--- 910,912 ----
Index: kern/subr_turnstile.c
===================================================================
RCS file: /cvs/ncvs/src/sys/kern/subr_turnstile.c,v
retrieving revision 1.152.2.5
diff -c -r1.152.2.5 subr_turnstile.c
*** kern/subr_turnstile.c	23 Jan 2007 22:16:33 -0000	1.152.2.5
--- kern/subr_turnstile.c	31 Aug 2007 02:15:23 -0000
***************
*** 114,120 ****
   *  q - td_contested lock
   */
  struct turnstile {
! 	TAILQ_HEAD(, thread) ts_blocked;	/* (c + q) Blocked threads. */
  	TAILQ_HEAD(, thread) ts_pending;	/* (c) Pending threads. */
  	LIST_ENTRY(turnstile) ts_hash;		/* (c) Chain and free list. */
  	LIST_ENTRY(turnstile) ts_link;		/* (q) Contested locks. */
--- 114,121 ----
   *  q - td_contested lock
   */
  struct turnstile {
! 	/* struct mtx ts_lock;	*/		/* Spin lock for self. */
! 	TAILQ_HEAD(, thread) ts_blocked[2];	/* (c + q) Blocked threads. */
  	TAILQ_HEAD(, thread) ts_pending;	/* (c) Pending threads. */
  	LIST_ENTRY(turnstile) ts_hash;		/* (c) Chain and free list. */
  	LIST_ENTRY(turnstile) ts_link;		/* (q) Contested locks. */
***************
*** 143,148 ****
--- 144,155 ----
  static struct mtx td_contested_lock;
  static struct turnstile_chain turnstile_chains[TC_TABLESIZE];
  
+ /* XXX: stats, remove me */
+ static u_int turnstile_nullowners;
+ SYSCTL_UINT(_debug, OID_AUTO, turnstile_nullowners, CTLFLAG_RD,
+     &turnstile_nullowners, 0, "called with null owner on a shared queue");
+ 
+ 
  static MALLOC_DEFINE(M_TURNSTILE, "turnstiles", "turnstiles");
  
  /*
***************
*** 267,272 ****
--- 274,280 ----
  {
  	struct turnstile_chain *tc;
  	struct thread *td1, *td2;
+ 	int queue;
  
  	mtx_assert(&sched_lock, MA_OWNED);
  	MPASS(TD_ON_LOCK(td));
***************
*** 300,315 ****
  		 * Remove thread from blocked chain and determine where
  		 * it should be moved to.
  		 */
  		mtx_lock_spin(&td_contested_lock);
! 		TAILQ_REMOVE(&ts->ts_blocked, td, td_lockq);
! 		TAILQ_FOREACH(td1, &ts->ts_blocked, td_lockq) {
  			MPASS(td1->td_proc->p_magic == P_MAGIC);
  			if (td1->td_priority > td->td_priority)
  				break;
  		}
  
  		if (td1 == NULL)
! 			TAILQ_INSERT_TAIL(&ts->ts_blocked, td, td_lockq);
  		else
  			TAILQ_INSERT_BEFORE(td1, td, td_lockq);
  		mtx_unlock_spin(&td_contested_lock);
--- 308,325 ----
  		 * Remove thread from blocked chain and determine where
  		 * it should be moved to.
  		 */
+ 		queue = td->td_tsqueue;
+ 		MPASS(queue == TS_EXCLUSIVE_QUEUE || queue == TS_SHARED_QUEUE);
  		mtx_lock_spin(&td_contested_lock);
! 		TAILQ_REMOVE(&ts->ts_blocked[queue], td, td_lockq);
! 		TAILQ_FOREACH(td1, &ts->ts_blocked[queue], td_lockq) {
  			MPASS(td1->td_proc->p_magic == P_MAGIC);
  			if (td1->td_priority > td->td_priority)
  				break;
  		}
  
  		if (td1 == NULL)
! 			TAILQ_INSERT_TAIL(&ts->ts_blocked[queue], td, td_lockq);
  		else
  			TAILQ_INSERT_BEFORE(td1, td, td_lockq);
  		mtx_unlock_spin(&td_contested_lock);
***************
*** 412,418 ****
  	 * Note that we currently don't try to revoke lent priorities
  	 * when our priority goes up.
  	 */
! 	if (td == TAILQ_FIRST(&ts->ts_blocked) && td->td_priority < oldpri) {
  		mtx_unlock_spin(&tc->tc_lock);
  		critical_enter();
  		propagate_priority(td);
--- 422,431 ----
  	 * Note that we currently don't try to revoke lent priorities
  	 * when our priority goes up.
  	 */
! 	MPASS(td->td_tsqueue == TS_EXCLUSIVE_QUEUE ||
! 	    td->td_tsqueue == TS_SHARED_QUEUE);
! 	if (td == TAILQ_FIRST(&ts->ts_blocked[td->td_tsqueue]) &&
! 	    td->td_priority < oldpri) {
  		mtx_unlock_spin(&tc->tc_lock);
  		critical_enter();
  		propagate_priority(td);
***************
*** 429,436 ****
  {
  
  	mtx_assert(&td_contested_lock, MA_OWNED);
- 	MPASS(owner->td_proc->p_magic == P_MAGIC);
  	MPASS(ts->ts_owner == NULL);
  	ts->ts_owner = owner;
  	LIST_INSERT_HEAD(&owner->td_contested, ts, ts_link);
  }
--- 442,452 ----
  {
  
  	mtx_assert(&td_contested_lock, MA_OWNED);
  	MPASS(ts->ts_owner == NULL);
+ 	if (owner == NULL)
+ 		return;
+ 
+ 	MPASS(owner->td_proc->p_magic == P_MAGIC);
  	ts->ts_owner = owner;
  	LIST_INSERT_HEAD(&owner->td_contested, ts, ts_link);
  }
***************
*** 444,450 ****
  	struct turnstile *ts;
  
  	ts = malloc(sizeof(struct turnstile), M_TURNSTILE, M_WAITOK | M_ZERO);
! 	TAILQ_INIT(&ts->ts_blocked);
  	TAILQ_INIT(&ts->ts_pending);
  	LIST_INIT(&ts->ts_free);
  	return (ts);
--- 460,467 ----
  	struct turnstile *ts;
  
  	ts = malloc(sizeof(struct turnstile), M_TURNSTILE, M_WAITOK | M_ZERO);
! 	TAILQ_INIT(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]);
! 	TAILQ_INIT(&ts->ts_blocked[TS_SHARED_QUEUE]);
  	TAILQ_INIT(&ts->ts_pending);
  	LIST_INIT(&ts->ts_free);
  	return (ts);
***************
*** 458,464 ****
  {
  
  	MPASS(ts != NULL);
! 	MPASS(TAILQ_EMPTY(&ts->ts_blocked));
  	MPASS(TAILQ_EMPTY(&ts->ts_pending));
  	free(ts, M_TURNSTILE);
  }
--- 475,482 ----
  {
  
  	MPASS(ts != NULL);
! 	MPASS(TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]));
! 	MPASS(TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE]));
  	MPASS(TAILQ_EMPTY(&ts->ts_pending));
  	free(ts, M_TURNSTILE);
  }
***************
*** 507,512 ****
--- 525,546 ----
  }
  
  /*
+  * Return a pointer to the thread waiting on this turnstile with the
+  * most important priority or NULL if the turnstile has no waiters.
+  */
+ static struct thread *
+ turnstile_first_waiter(struct turnstile *ts)
+ {
+ 	struct thread *std, *xtd;
+ 
+ 	std = TAILQ_FIRST(&ts->ts_blocked[TS_SHARED_QUEUE]);
+ 	xtd = TAILQ_FIRST(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]);
+ 	if (xtd == NULL || (std != NULL && std->td_priority < xtd->td_priority))
+ 		return (std);
+ 	return (xtd);
+ }
+ 
+ /*
   * Take ownership of a turnstile and adjust the priority of the new
   * owner appropriately.
   */
***************
*** 527,533 ****
  	turnstile_setowner(ts, owner);
  	mtx_unlock_spin(&td_contested_lock);
  
! 	td = TAILQ_FIRST(&ts->ts_blocked);
  	MPASS(td != NULL);
  	MPASS(td->td_proc->p_magic == P_MAGIC);
  	mtx_unlock_spin(&tc->tc_lock);
--- 561,567 ----
  	turnstile_setowner(ts, owner);
  	mtx_unlock_spin(&td_contested_lock);
  
! 	td = turnstile_first_waiter(ts);
  	MPASS(td != NULL);
  	MPASS(td->td_proc->p_magic == P_MAGIC);
  	mtx_unlock_spin(&tc->tc_lock);
***************
*** 548,554 ****
   * turnstile chain locked and will return with it unlocked.
   */
  void
! turnstile_wait(struct lock_object *lock, struct thread *owner)
  {
  	struct turnstile_chain *tc;
  	struct turnstile *ts;
--- 582,588 ----
   * turnstile chain locked and will return with it unlocked.
   */
  void
! turnstile_wait_queue(struct lock_object *lock, struct thread *owner, int queue)
  {
  	struct turnstile_chain *tc;
  	struct turnstile *ts;
***************
*** 558,565 ****
  	tc = TC_LOOKUP(lock);
  	mtx_assert(&tc->tc_lock, MA_OWNED);
  	MPASS(td->td_turnstile != NULL);
! 	MPASS(owner != NULL);
! 	MPASS(owner->td_proc->p_magic == P_MAGIC);
  
  	/* Look up the turnstile associated with the lock 'lock'. */
  	ts = turnstile_lookup(lock);
--- 592,604 ----
  	tc = TC_LOOKUP(lock);
  	mtx_assert(&tc->tc_lock, MA_OWNED);
  	MPASS(td->td_turnstile != NULL);
! 	if (owner)
! 		MPASS(owner->td_proc->p_magic == P_MAGIC);
! 	/* XXX: stats, remove me */
! 	if (!owner && queue == TS_SHARED_QUEUE) {
! 		turnstile_nullowners++;
! 	}
! 	MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE);
  
  	/* Look up the turnstile associated with the lock 'lock'. */
  	ts = turnstile_lookup(lock);
***************
*** 582,606 ****
  		LIST_INSERT_HEAD(&tc->tc_turnstiles, ts, ts_hash);
  		KASSERT(TAILQ_EMPTY(&ts->ts_pending),
  		    ("thread's turnstile has pending threads"));
! 		KASSERT(TAILQ_EMPTY(&ts->ts_blocked),
! 		    ("thread's turnstile has a non-empty queue"));
  		KASSERT(LIST_EMPTY(&ts->ts_free),
  		    ("thread's turnstile has a non-empty free list"));
  		KASSERT(ts->ts_lockobj == NULL, ("stale ts_lockobj pointer"));
  		ts->ts_lockobj = lock;
  		mtx_lock_spin(&td_contested_lock);
! 		TAILQ_INSERT_TAIL(&ts->ts_blocked, td, td_lockq);
  		turnstile_setowner(ts, owner);
  		mtx_unlock_spin(&td_contested_lock);
  	} else {
! 		TAILQ_FOREACH(td1, &ts->ts_blocked, td_lockq)
  			if (td1->td_priority > td->td_priority)
  				break;
  		mtx_lock_spin(&td_contested_lock);
  		if (td1 != NULL)
  			TAILQ_INSERT_BEFORE(td1, td, td_lockq);
  		else
! 			TAILQ_INSERT_TAIL(&ts->ts_blocked, td, td_lockq);
  		mtx_unlock_spin(&td_contested_lock);
  		MPASS(td->td_turnstile != NULL);
  		LIST_INSERT_HEAD(&ts->ts_free, td->td_turnstile, ts_hash);
--- 621,647 ----
  		LIST_INSERT_HEAD(&tc->tc_turnstiles, ts, ts_hash);
  		KASSERT(TAILQ_EMPTY(&ts->ts_pending),
  		    ("thread's turnstile has pending threads"));
! 		KASSERT(TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]),
! 		    ("thread's turnstile has exclusive waiters"));
! 		KASSERT(TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE]),
! 		    ("thread's turnstile has shared waiters"));
  		KASSERT(LIST_EMPTY(&ts->ts_free),
  		    ("thread's turnstile has a non-empty free list"));
  		KASSERT(ts->ts_lockobj == NULL, ("stale ts_lockobj pointer"));
  		ts->ts_lockobj = lock;
  		mtx_lock_spin(&td_contested_lock);
! 		TAILQ_INSERT_TAIL(&ts->ts_blocked[queue], td, td_lockq);
  		turnstile_setowner(ts, owner);
  		mtx_unlock_spin(&td_contested_lock);
  	} else {
! 		TAILQ_FOREACH(td1, &ts->ts_blocked[queue], td_lockq)
  			if (td1->td_priority > td->td_priority)
  				break;
  		mtx_lock_spin(&td_contested_lock);
  		if (td1 != NULL)
  			TAILQ_INSERT_BEFORE(td1, td, td_lockq);
  		else
! 			TAILQ_INSERT_TAIL(&ts->ts_blocked[queue], td, td_lockq);
  		mtx_unlock_spin(&td_contested_lock);
  		MPASS(td->td_turnstile != NULL);
  		LIST_INSERT_HEAD(&ts->ts_free, td->td_turnstile, ts_hash);
***************
*** 664,670 ****
   * pending list.  This must be called with the turnstile chain locked.
   */
  int
! turnstile_signal(struct turnstile *ts)
  {
  	struct turnstile_chain *tc;
  	struct thread *td;
--- 705,711 ----
   * pending list.  This must be called with the turnstile chain locked.
   */
  int
! turnstile_signal_queue(struct turnstile *ts, int queue)
  {
  	struct turnstile_chain *tc;
  	struct thread *td;
***************
*** 675,689 ****
  	MPASS(ts->ts_owner == curthread);
  	tc = TC_LOOKUP(ts->ts_lockobj);
  	mtx_assert(&tc->tc_lock, MA_OWNED);
  
  	/*
  	 * Pick the highest priority thread blocked on this lock and
  	 * move it to the pending list.
  	 */
! 	td = TAILQ_FIRST(&ts->ts_blocked);
  	MPASS(td->td_proc->p_magic == P_MAGIC);
  	mtx_lock_spin(&td_contested_lock);
! 	TAILQ_REMOVE(&ts->ts_blocked, td, td_lockq);
  	mtx_unlock_spin(&td_contested_lock);
  	TAILQ_INSERT_TAIL(&ts->ts_pending, td, td_lockq);
  
--- 716,733 ----
  	MPASS(ts->ts_owner == curthread);
  	tc = TC_LOOKUP(ts->ts_lockobj);
  	mtx_assert(&tc->tc_lock, MA_OWNED);
+ 	MPASS(ts->ts_owner == curthread ||
+ 	    (queue == TS_EXCLUSIVE_QUEUE && ts->ts_owner == NULL));
+ 	MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE);
  
  	/*
  	 * Pick the highest priority thread blocked on this lock and
  	 * move it to the pending list.
  	 */
! 	td = TAILQ_FIRST(&ts->ts_blocked[queue]);
  	MPASS(td->td_proc->p_magic == P_MAGIC);
  	mtx_lock_spin(&td_contested_lock);
! 	TAILQ_REMOVE(&ts->ts_blocked[queue], td, td_lockq);
  	mtx_unlock_spin(&td_contested_lock);
  	TAILQ_INSERT_TAIL(&ts->ts_pending, td, td_lockq);
  
***************
*** 692,698 ****
  	 * give it to the about-to-be-woken thread.  Otherwise take a
  	 * turnstile from the free list and give it to the thread.
  	 */
! 	empty = TAILQ_EMPTY(&ts->ts_blocked);
  	if (empty) {
  		MPASS(LIST_EMPTY(&ts->ts_free));
  #ifdef TURNSTILE_PROFILING
--- 736,743 ----
  	 * give it to the about-to-be-woken thread.  Otherwise take a
  	 * turnstile from the free list and give it to the thread.
  	 */
! 	empty = TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]) &&
! 	    TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE]);
  	if (empty) {
  		MPASS(LIST_EMPTY(&ts->ts_free));
  #ifdef TURNSTILE_PROFILING
***************
*** 712,718 ****
   * the turnstile chain locked.
   */
  void
! turnstile_broadcast(struct turnstile *ts)
  {
  	struct turnstile_chain *tc;
  	struct turnstile *ts1;
--- 757,763 ----
   * the turnstile chain locked.
   */
  void
! turnstile_broadcast_queue(struct turnstile *ts, int queue)
  {
  	struct turnstile_chain *tc;
  	struct turnstile *ts1;
***************
*** 720,734 ****
  
  	MPASS(ts != NULL);
  	MPASS(curthread->td_proc->p_magic == P_MAGIC);
! 	MPASS(ts->ts_owner == curthread);
  	tc = TC_LOOKUP(ts->ts_lockobj);
  	mtx_assert(&tc->tc_lock, MA_OWNED);
  
  	/*
  	 * Transfer the blocked list to the pending list.
  	 */
  	mtx_lock_spin(&td_contested_lock);
! 	TAILQ_CONCAT(&ts->ts_pending, &ts->ts_blocked, td_lockq);
  	mtx_unlock_spin(&td_contested_lock);
  
  	/*
--- 765,781 ----
  
  	MPASS(ts != NULL);
  	MPASS(curthread->td_proc->p_magic == P_MAGIC);
! 	MPASS(ts->ts_owner == curthread ||
! 		    (queue == TS_EXCLUSIVE_QUEUE && ts->ts_owner == NULL));
  	tc = TC_LOOKUP(ts->ts_lockobj);
  	mtx_assert(&tc->tc_lock, MA_OWNED);
+ 	MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE);
  
  	/*
  	 * Transfer the blocked list to the pending list.
  	 */
  	mtx_lock_spin(&td_contested_lock);
! 	TAILQ_CONCAT(&ts->ts_pending, &ts->ts_blocked[queue], td_lockq);
  	mtx_unlock_spin(&td_contested_lock);
  
  	/*
***************
*** 756,770 ****
   * chain locked.
   */
  void
! turnstile_unpend(struct turnstile *ts)
  {
  	TAILQ_HEAD( ,thread) pending_threads;
  	struct turnstile_chain *tc;
  	struct thread *td;
  	u_char cp, pri;
  
  	MPASS(ts != NULL);
! 	MPASS(ts->ts_owner == curthread);
  	tc = TC_LOOKUP(ts->ts_lockobj);
  	mtx_assert(&tc->tc_lock, MA_OWNED);
  	MPASS(!TAILQ_EMPTY(&ts->ts_pending));
--- 803,819 ----
   * chain locked.
   */
  void
! turnstile_unpend_queue(struct turnstile *ts, int owner_type)
  {
  	TAILQ_HEAD( ,thread) pending_threads;
  	struct turnstile_chain *tc;
+ 	struct turnstile *nts;
  	struct thread *td;
  	u_char cp, pri;
  
  	MPASS(ts != NULL);
! 	MPASS(ts->ts_owner == curthread ||
! 	    (owner_type == TS_SHARED_LOCK && ts->ts_owner == NULL));
  	tc = TC_LOOKUP(ts->ts_lockobj);
  	mtx_assert(&tc->tc_lock, MA_OWNED);
  	MPASS(!TAILQ_EMPTY(&ts->ts_pending));
***************
*** 776,782 ****
  	TAILQ_INIT(&pending_threads);
  	TAILQ_CONCAT(&pending_threads, &ts->ts_pending, td_lockq);
  #ifdef INVARIANTS
! 	if (TAILQ_EMPTY(&ts->ts_blocked))
  		ts->ts_lockobj = NULL;
  #endif
  
--- 825,832 ----
  	TAILQ_INIT(&pending_threads);
  	TAILQ_CONCAT(&pending_threads, &ts->ts_pending, td_lockq);
  #ifdef INVARIANTS
! 	if (TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]) &&
! 	    TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE]))
  		ts->ts_lockobj = NULL;
  #endif
  
***************
*** 802,809 ****
  	pri = PRI_MAX;
  	mtx_lock_spin(&sched_lock);
  	mtx_lock_spin(&td_contested_lock);
! 	LIST_FOREACH(ts, &td->td_contested, ts_link) {
! 		cp = TAILQ_FIRST(&ts->ts_blocked)->td_priority;
  		if (cp < pri)
  			pri = cp;
  	}
--- 852,859 ----
  	pri = PRI_MAX;
  	mtx_lock_spin(&sched_lock);
  	mtx_lock_spin(&td_contested_lock);
! 	LIST_FOREACH(nts, &td->td_contested, ts_link) {
! 		cp = turnstile_first_waiter(nts)->td_priority;
  		if (cp < pri)
  			pri = cp;
  	}
***************
*** 837,855 ****
  }
  
  /*
   * Return the first thread in a turnstile.
   */
  struct thread *
! turnstile_head(struct turnstile *ts)
  {
  #ifdef INVARIANTS
  	struct turnstile_chain *tc;
  
  	MPASS(ts != NULL);
  	tc = TC_LOOKUP(ts->ts_lockobj);
  	mtx_assert(&tc->tc_lock, MA_OWNED);
  #endif
! 	return (TAILQ_FIRST(&ts->ts_blocked));
  }
  
  #ifdef DDB
--- 887,955 ----
  }
  
  /*
+  * Give up ownership of a turnstile.  This must be called with the
+  * turnstile chain locked.
+  */
+ void
+ turnstile_disown(struct turnstile *ts)
+ {
+ 	struct turnstile_chain *tc;
+ 	struct thread *td;
+ 	u_char cp, pri;
+ 
+ 	MPASS(ts != NULL);
+ 	MPASS(ts->ts_owner == curthread);
+ 	tc = TC_LOOKUP(ts->ts_lockobj);
+ 	mtx_assert(&tc->tc_lock, MA_OWNED);
+ 	MPASS(TAILQ_EMPTY(&ts->ts_pending));
+ 	MPASS(!TAILQ_EMPTY(&ts->ts_blocked[TS_EXCLUSIVE_QUEUE]) ||
+ 	    !TAILQ_EMPTY(&ts->ts_blocked[TS_SHARED_QUEUE]));
+ 
+ 	/*
+ 	 * Remove the turnstile from this thread's list of contested locks
+ 	 * since this thread doesn't own it anymore.  New threads will
+ 	 * not be blocking on the turnstile until it is claimed by a new
+ 	 * owner.
+ 	 */
+ 	mtx_lock_spin(&td_contested_lock);
+ 	ts->ts_owner = NULL;
+ 	LIST_REMOVE(ts, ts_link);
+ 	mtx_unlock_spin(&td_contested_lock);
+ 
+ 	/*
+ 	 * Adjust the priority of curthread based on other contested
+ 	 * locks it owns.  Don't lower the priority below the base
+ 	 * priority however.
+ 	 */
+ 	td = curthread;
+ 	pri = PRI_MAX;
+ 	mtx_lock_spin(&sched_lock);
+ 	mtx_lock_spin(&td_contested_lock);
+ 	LIST_FOREACH(ts, &td->td_contested, ts_link) {
+ 		cp = turnstile_first_waiter(ts)->td_priority;
+ 		if (cp < pri)
+ 			pri = cp;
+ 	}
+ 	mtx_unlock_spin(&td_contested_lock);
+ 	sched_unlend_prio(td, pri);
+ 	mtx_unlock_spin(&sched_lock);
+ }
+ 
+ /*
   * Return the first thread in a turnstile.
   */
  struct thread *
! turnstile_head_queue(struct turnstile *ts, int queue)
  {
  #ifdef INVARIANTS
  	struct turnstile_chain *tc;
  
  	MPASS(ts != NULL);
  	tc = TC_LOOKUP(ts->ts_lockobj);
+ 	MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE);
  	mtx_assert(&tc->tc_lock, MA_OWNED);
  #endif
! 	return (TAILQ_FIRST(&ts->ts_blocked[queue]));
  }
  
  #ifdef DDB
***************
*** 1146,1152 ****
   * Returns true if a turnstile is empty.
   */
  int
! turnstile_empty(struct turnstile *ts)
  {
  #ifdef INVARIANTS
  	struct turnstile_chain *tc;
--- 1246,1252 ----
   * Returns true if a turnstile is empty.
   */
  int
! turnstile_empty_queue(struct turnstile *ts, int queue)
  {
  #ifdef INVARIANTS
  	struct turnstile_chain *tc;
***************
*** 1154,1159 ****
  	MPASS(ts != NULL);
  	tc = TC_LOOKUP(ts->ts_lockobj);
  	mtx_assert(&tc->tc_lock, MA_OWNED);
  #endif
! 	return (TAILQ_EMPTY(&ts->ts_blocked));
  }
--- 1254,1260 ----
  	MPASS(ts != NULL);
  	tc = TC_LOOKUP(ts->ts_lockobj);
  	mtx_assert(&tc->tc_lock, MA_OWNED);
+ 	MPASS(queue == TS_SHARED_QUEUE || queue == TS_EXCLUSIVE_QUEUE);
  #endif
! 	return (TAILQ_EMPTY(&ts->ts_blocked[queue]));
  }
Index: netinet6/in6_src.c
===================================================================
RCS file: /cvs/ncvs/src/sys/netinet6/in6_src.c,v
retrieving revision 1.30.2.4
diff -c -r1.30.2.4 in6_src.c
*** netinet6/in6_src.c	25 Dec 2005 14:03:37 -0000	1.30.2.4
--- netinet6/in6_src.c	31 Aug 2007 01:23:38 -0000
***************
*** 76,81 ****
--- 76,82 ----
  #include <sys/errno.h>
  #include <sys/time.h>
  #include <sys/kernel.h>
+ #include <sys/lock.h>
  #include <sys/sx.h>
  
  #include <net/if.h>
Index: sys/_rwlock.h
===================================================================
RCS file: sys/_rwlock.h
diff -N sys/_rwlock.h
*** /dev/null	1 Jan 1970 00:00:00 -0000
--- sys/_rwlock.h	31 Aug 2007 00:39:59 -0000
***************
*** 0 ****
--- 1,44 ----
+ /*-
+  * Copyright (c) 2006 John Baldwin <[e-mail]>
+  * All rights reserved.
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  * 3. Neither the name of the author nor the names of any co-contributors
+  *    may be used to endorse or promote products derived from this software
+  *    without specific prior written permission.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  *
+  * $FreeBSD: src/sys/sys/_rwlock.h,v 1.4 2007/06/26 21:31:56 attilio Exp $
+  */
+ 
+ #ifndef _SYS__RWLOCK_H_
+ #define	_SYS__RWLOCK_H_
+ 
+ /*
+  * Reader/writer lock.
+  */
+ struct rwlock {
+ 	struct lock_object	lock_object;
+ 	volatile uintptr_t	rw_lock;
+ 	volatile unsigned	rw_recurse;
+ };
+ 
+ #endif /* !_SYS__RWLOCK_H_ */
Index: sys/_sx.h
===================================================================
RCS file: sys/_sx.h
diff -N sys/_sx.h
*** /dev/null	1 Jan 1970 00:00:00 -0000
--- sys/_sx.h	4 Sep 2007 01:20:03 -0000
***************
*** 0 ****
--- 1,59 ----
+ /*-
+  * Copyright (c) 2007 Attilio Rao <[e-mail]>
+  * All rights reserved.
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice(s), this list of conditions and the following disclaimer as
+  *    the first lines of this file unmodified other than the possible 
+  *    addition of one or more copyright notices.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice(s), this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
+  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+  * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
+  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+  * DAMAGE.
+  *
+  * $FreeBSD: src/sys/sys/_sx.h,v 1.1 2007/03/31 23:23:42 jhb Exp $
+  */
+ 
+ #ifndef	_SYS__SX_H_
+ #define	_SYS__SX_H_
+ 
+ #include <sys/condvar.h>
+ 
+ /*
+  * Shared/exclusive lock main structure definition.
+  *
+  * Note, to preserve compatibility we have extra fields from
+  * the previous implementation left over.
+  */
+ struct sx {
+ 	struct lock_object	lock_object;
+ 	/* was: struct mtx *sx_lock; */
+ 	volatile uintptr_t	sx_lock;
+ 	/* was: int sx_cnt; */
+ 	volatile unsigned	sx_recurse;
+ 	/*
+ 	 * The following fields are unused but kept to preserve
+ 	 * sizeof(struct sx) for 6.x compat.
+ 	 */
+ 	struct cv       sx_shrd_cv;	/* unused */
+ 	int             sx_shrd_wcnt;	/* unused */
+ 	struct cv       sx_excl_cv;	/* unused */
+ 	int             sx_excl_wcnt;	/* unused */
+ 	struct thread   *sx_xholder;	/* unused */
+ };
+ 
+ #endif	/* !_SYS__SX_H_ */
Index: sys/lock_profile.h
===================================================================
RCS file: sys/lock_profile.h
diff -N sys/lock_profile.h
*** /dev/null	1 Jan 1970 00:00:00 -0000
--- sys/lock_profile.h	31 Aug 2007 03:21:42 -0000
***************
*** 0 ****
--- 1,45 ----
+ /*-
+  * Copyright (c) 2007 Juniper Networks
+  * Author: Alfred Perlstein
+  * All rights reserved.
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice(s), this list of conditions and the following disclaimer as
+  *    the first lines of this file unmodified other than the possible 
+  *    addition of one or more copyright notices.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice(s), this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY
+  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+  * DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY
+  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+  * DAMAGE.
+  *
+  * $FreeBSD$
+  */
+ 
+ #ifndef	_LOCK_PROFILE_H_
+ #define	_LOCK_PROFILE_H_
+ 
+ /* temp stubs for lock profiling, not supported in 6.x (yet) */
+ 
+ #define lock_profile_obtain_lock_success(lock_object, a0, a1, file, line) \
+     do { ; } while (0)
+ #define lock_profile_release_lock(lock_object) \
+     do { ; } while (0)
+ #define lock_profile_obtain_lock_failed(lock_object, contested, waittime) \
+     do { ; } while (0)
+ 
+ #endif
+ 
Index: sys/proc.h
===================================================================
RCS file: /cvs/ncvs/src/sys/sys/proc.h,v
retrieving revision 1.432.2.10
diff -c -r1.432.2.10 proc.h
*** sys/proc.h	11 Jun 2007 11:27:04 -0000	1.432.2.10
--- sys/proc.h	2 Sep 2007 21:56:16 -0000
***************
*** 237,243 ****
--- 237,252 ----
   * When waiting to be run, threads are hung off the KSEGRP in priority order.
   * With N runnable and queued KSEs in the KSEGRP, the first N threads
   * are linked to them. Other threads are not yet assigned.
+  *
+  * We must force at least 16 byte alignment for "struct thread"
+  * because the rwlocks and sxlocks expect to use the bottom bits
+  * of the pointer for bookkeeping information.
+  *
+  * This causes problems for the thread0 data structure because it
+  * may not be properly aligned otherwise.
   */
+ #define THREAD_ALIGN	16
+ 
  struct thread {
  	struct proc	*td_proc;	/* (*) Associated process. */
  	struct ksegrp	*td_ksegrp;	/* (*) Associated KSEG. */
***************
*** 261,272 ****
--- 270,283 ----
  	int		td_inhibitors;	/* (j) Why can not run. */
  	int		td_pflags;	/* (k) Private thread (TDP_*) flags. */
  	int		td_dupfd;	/* (k) Ret value from fdopen. XXX */
+ 	int		td_sqqueue;	/* (t) Sleepqueue queue blocked on. */
  	void		*td_wchan;	/* (j) Sleep address. */
  	const char	*td_wmesg;	/* (j) Reason for sleep. */
  	u_char		td_lastcpu;	/* (j) Last cpu we were on. */
  	u_char		td_oncpu;	/* (j) Which cpu we are on. */
  	volatile u_char td_owepreempt;  /* (k*) Preempt on last critical_exit */
  	short		td_locks;	/* (k) Count of non-spin locks. */
+ 	u_char		td_tsqueue;	/* (t) Turnstile queue blocked on. */
  	struct turnstile *td_blocked;	/* (j) Lock process is blocked on. */
  	void		*td_ithd;	/* (n) Unused, kept to preserve ABI. */
  	const char	*td_lockname;	/* (j) Name of lock blocked on. */
***************
*** 324,330 ****
  	struct mdthread td_md;		/* (k) Any machine-dependent fields. */
  	struct td_sched	*td_sched;	/* (*) Scheduler-specific data. */
  	struct kaudit_record	*td_ar;	/* (k) Active audit record, if any. */
! };
  
  /*
   * Flags kept in td_flags:
--- 335,342 ----
  	struct mdthread td_md;		/* (k) Any machine-dependent fields. */
  	struct td_sched	*td_sched;	/* (*) Scheduler-specific data. */
  	struct kaudit_record	*td_ar;	/* (k) Active audit record, if any. */
! } __attribute__ ((aligned (THREAD_ALIGN)));
! 
  
  /*
   * Flags kept in td_flags:
Index: sys/rwlock.h
===================================================================
RCS file: sys/rwlock.h
diff -N sys/rwlock.h
*** /dev/null	1 Jan 1970 00:00:00 -0000
--- sys/rwlock.h	31 Aug 2007 03:26:18 -0000
***************
*** 0 ****
--- 1,223 ----
+ /*-
+  * Copyright (c) 2006 John Baldwin <[e-mail]>
+  * All rights reserved.
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  * 3. Neither the name of the author nor the names of any co-contributors
+  *    may be used to endorse or promote products derived from this software
+  *    without specific prior written permission.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  *
+  * $FreeBSD: src/sys/sys/rwlock.h,v 1.13 2007/06/26 21:31:56 attilio Exp $
+  */
+ 
+ #ifndef _SYS_RWLOCK_H_
+ #define _SYS_RWLOCK_H_
+ 
+ #include <sys/_lock.h>
+ #include <sys/_rwlock.h>
+ #include <sys/lock_profile.h>
+ 
+ #ifdef _KERNEL
+ #include <sys/pcpu.h>
+ #include <machine/atomic.h>
+ #endif
+ 
+ /*
+  * The rw_lock field consists of several fields.  The low bit indicates
+  * if the lock is locked with a read (shared) or write (exclusive) lock.
+  * A value of 0 indicates a write lock, and a value of 1 indicates a read
+  * lock.  Bit 1 is a boolean indicating if there are any threads waiting
+  * for a read lock.  Bit 2 is a boolean indicating if there are any threads
+  * waiting for a write lock.  The rest of the variable's definition is
+  * dependent on the value of the first bit.  For a write lock, it is a
+  * pointer to the thread holding the lock, similar to the mtx_lock field of
+  * mutexes.  For read locks, it is a count of read locks that are held.
+  *
+  * When the lock is not locked by any thread, it is encoded as a read lock
+  * with zero waiters.
+  *
+  * A note about memory barriers.  Write locks need to use the same memory
+  * barriers as mutexes: _acq when acquiring a write lock and _rel when
+  * releasing a write lock.  Read locks also need to use an _acq barrier when
+  * acquiring a read lock.  However, since read locks do not update any
+  * locked data (modulo bugs of course), no memory barrier is needed when
+  * releasing a read lock.
+  */
+ 
+ #define	RW_LOCK_READ		0x01
+ #define	RW_LOCK_READ_WAITERS	0x02
+ #define	RW_LOCK_WRITE_WAITERS	0x04
+ #define	RW_LOCK_RECURSED	0x08
+ #define	RW_LOCK_FLAGMASK						\
+ 	(RW_LOCK_READ | RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS |	\
+ 	RW_LOCK_RECURSED)
+ 
+ #define	RW_OWNER(x)		((x) & ~RW_LOCK_FLAGMASK)
+ #define	RW_READERS_SHIFT	4
+ #define	RW_READERS(x)		(RW_OWNER((x)) >> RW_READERS_SHIFT)
+ #define	RW_READERS_LOCK(x)	((x) << RW_READERS_SHIFT | RW_LOCK_READ)
+ #define	RW_ONE_READER		(1 << RW_READERS_SHIFT)
+ 
+ #define	RW_UNLOCKED		RW_READERS_LOCK(0)
+ #define	RW_DESTROYED		(RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)
+ 
+ #ifdef _KERNEL
+ 
+ /* Very simple operations on rw_lock. */
+ 
+ /* Try to obtain a write lock once. */
+ #define	_rw_write_lock(rw, tid)						\
+ 	atomic_cmpset_acq_ptr(&(rw)->rw_lock, RW_UNLOCKED, (tid))
+ 
+ /* Release a write lock quickly if there are no waiters. */
+ #define	_rw_write_unlock(rw, tid)					\
+ 	atomic_cmpset_rel_ptr(&(rw)->rw_lock, (tid), RW_UNLOCKED)
+ 
+ /*
+  * Full lock operations that are suitable to be inlined in non-debug
+  * kernels.  If the lock cannot be acquired or released trivially then
+  * the work is deferred to another function.
+  */
+ 
+ /* Acquire a write lock. */
+ #define	__rw_wlock(rw, tid, file, line) do {	\
+ 	uintptr_t _tid = (uintptr_t)(tid);				\
+ 	/* int contested = 0; XXX: notsup */                                             \
+         /*uint64_t waitstart = 0; XXX: notsup */                                         \
+ 						                        \
+ 	if (!_rw_write_lock((rw), _tid)) {				\
+ 		lock_profile_obtain_lock_failed(&(rw)->lock_object,	\
+ 		    &contested, &waitstart);				\
+ 		_rw_wlock_hard((rw), _tid, (file), (line));		\
+ 	}                                                               \
+ 	lock_profile_obtain_lock_success(&(rw)->lock_object, contested,	\
+ 	    waitstart, (file), (line));					\
+ } while (0)
+ 
+ /* Release a write lock. */
+ #define	__rw_wunlock(rw, tid, file, line) do {				\
+ 	uintptr_t _tid = (uintptr_t)(tid);				\
+ 									\
+ 	if (!_rw_write_unlock((rw), _tid))				\
+ 		_rw_wunlock_hard((rw), _tid, (file), (line));		\
+ } while (0)
+ 
+ /*
+  * Function prototypes.  Routines that start with _ are not part of the
+  * external API and should not be called directly.  Wrapper macros should
+  * be used instead.
+  */
+ 
+ #define	rw_init(rw, name)	rw_init_flags((rw), (name), 0)
+ void	rw_init_flags(struct rwlock *rw, const char *name, int opts);
+ void	rw_destroy(struct rwlock *rw);
+ void	rw_sysinit(void *arg);
+ int	rw_wowned(struct rwlock *rw);
+ void	_rw_wlock(struct rwlock *rw, const char *file, int line);
+ void	_rw_wunlock(struct rwlock *rw, const char *file, int line);
+ void	_rw_rlock(struct rwlock *rw, const char *file, int line);
+ void	_rw_runlock(struct rwlock *rw, const char *file, int line);
+ void	_rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file,
+ 	    int line);
+ void	_rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file,
+ 	    int line);
+ int	_rw_try_upgrade(struct rwlock *rw, const char *file, int line);
+ void	_rw_downgrade(struct rwlock *rw, const char *file, int line);
+ #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
+ void	_rw_assert(struct rwlock *rw, int what, const char *file, int line);
+ #endif
+ 
+ /*
+  * Public interface for lock operations.
+  *
+  * XXX: Missing try locks.
+  */
+ 
+ #ifndef LOCK_DEBUG
+ #error LOCK_DEBUG not defined, include <sys/lock.h> before <sys/rwlock.h>
+ #endif
+ #if LOCK_DEBUG > 0 || defined(RWLOCK_NOINLINE)
+ #define	rw_wlock(rw)		_rw_wlock((rw), LOCK_FILE, LOCK_LINE)
+ #define	rw_wunlock(rw)		_rw_wunlock((rw), LOCK_FILE, LOCK_LINE)
+ #else
+ #define	rw_wlock(rw)							\
+ 	__rw_wlock((rw), curthread, LOCK_FILE, LOCK_LINE)
+ #define	rw_wunlock(rw)							\
+ 	__rw_wunlock((rw), curthread, LOCK_FILE, LOCK_LINE)
+ #endif
+ #define	rw_rlock(rw)		_rw_rlock((rw), LOCK_FILE, LOCK_LINE)
+ #define	rw_runlock(rw)		_rw_runlock((rw), LOCK_FILE, LOCK_LINE)
+ #define	rw_try_upgrade(rw)	_rw_try_upgrade((rw), LOCK_FILE, LOCK_LINE)
+ #define	rw_downgrade(rw)	_rw_downgrade((rw), LOCK_FILE, LOCK_LINE)
+ #define	rw_sleep(chan, rw, pri, wmesg, timo)				\
+ 	_sleep((chan), &(rw)->lock_object, (pri), (wmesg), (timo))
+ 
+ #define	rw_initialized(rw)	lock_initalized(&(rw)->lock_object)
+ 
+ struct rw_args {
+ 	struct rwlock	*ra_rw;
+ 	const char 	*ra_desc;
+ };
+ 
+ #define	RW_SYSINIT(name, rw, desc)					\
+ 	static struct rw_args name##_args = {				\
+ 		(rw),							\
+ 		(desc),							\
+ 	};								\
+ 	SYSINIT(name##_rw_sysinit, SI_SUB_LOCK, SI_ORDER_MIDDLE,	\
+ 	    rw_sysinit, &name##_args);					\
+ 	SYSUNINIT(name##_rw_sysuninit, SI_SUB_LOCK, SI_ORDER_MIDDLE,	\
+ 	    rw_destroy, (rw))
+ 
+ /*
+  * Options passed to rw_init_flags().
+  */
+ #define	RW_DUPOK	0x01
+ #define	RW_NOPROFILE	0x02
+ #define	RW_NOWITNESS	0x04
+ #define	RW_QUIET	0x08
+ #define	RW_RECURSE	0x10
+ 
+ /*
+  * The INVARIANTS-enabled rw_assert() functionality.
+  *
+  * The constants need to be defined for INVARIANT_SUPPORT infrastructure
+  * support as _rw_assert() itself uses them and the latter implies that
+  * _rw_assert() must build.
+  */
+ #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
+ #define	RA_LOCKED		LA_LOCKED
+ #define	RA_RLOCKED		LA_SLOCKED
+ #define	RA_WLOCKED		LA_XLOCKED
+ #define	RA_UNLOCKED		LA_UNLOCKED
+ #define	RA_RECURSED		LA_RECURSED
+ #define	RA_NOTRECURSED		LA_NOTRECURSED
+ #endif
+ 
+ #ifdef INVARIANTS
+ #define	rw_assert(rw, what)	_rw_assert((rw), (what), LOCK_FILE, LOCK_LINE)
+ #else
+ #define	rw_assert(rw, what)
+ #endif
+ 
+ #endif /* _KERNEL */
+ #endif /* !_SYS_RWLOCK_H_ */
Index: sys/sleepqueue.h
===================================================================
RCS file: /cvs/ncvs/src/sys/sys/sleepqueue.h,v
retrieving revision 1.6.2.1
diff -c -r1.6.2.1 sleepqueue.h
*** sys/sleepqueue.h	27 Feb 2006 00:19:39 -0000	1.6.2.1
--- sys/sleepqueue.h	31 Aug 2007 01:47:52 -0000
***************
*** 26,40 ****
   * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   * SUCH DAMAGE.
   *
!  * $FreeBSD: src/sys/sys/sleepqueue.h,v 1.6.2.1 2006/02/27 00:19:39 davidxu Exp $
   */
  
  #ifndef _SYS_SLEEPQUEUE_H_
  #define _SYS_SLEEPQUEUE_H_
  
  /*
!  * Sleep queue interface.  Sleep/wakeup and condition variables use a sleep
!  * queue for the queue of threads blocked on a sleep channel.
   *
   * A thread calls sleepq_lock() to lock the sleep queue chain associated
   * with a given wait channel.  A thread can then call call sleepq_add() to
--- 26,41 ----
   * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   * SUCH DAMAGE.
   *
!  * $FreeBSD: src/sys/sys/sleepqueue.h,v 1.12 2007/03/31 23:23:42 jhb Exp $
   */
  
  #ifndef _SYS_SLEEPQUEUE_H_
  #define _SYS_SLEEPQUEUE_H_
  
  /*
!  * Sleep queue interface.  Sleep/wakeup, condition variables, and sx
!  * locks use a sleep queue for the queue of threads blocked on a sleep
!  * channel.
   *
   * A thread calls sleepq_lock() to lock the sleep queue chain associated
   * with a given wait channel.  A thread can then call call sleepq_add() to
***************
*** 84,107 ****
  #define	SLEEPQ_TYPE		0x0ff		/* Mask of sleep queue types. */
  #define	SLEEPQ_MSLEEP		0x00		/* Used by msleep/wakeup. */
  #define	SLEEPQ_CONDVAR		0x01		/* Used for a cv. */
  #define	SLEEPQ_INTERRUPTIBLE	0x100		/* Sleep is interruptible. */
  
  void	init_sleepqueues(void);
  void	sleepq_abort(struct thread *td, int intrval);
! void	sleepq_add(void *, struct mtx *, const char *, int);
  struct sleepqueue *sleepq_alloc(void);
! void	sleepq_broadcast(void *, int, int);
  void	sleepq_free(struct sleepqueue *);
  void	sleepq_lock(void *);
  struct sleepqueue *sleepq_lookup(void *);
  void	sleepq_release(void *);
  void	sleepq_remove(struct thread *, void *);
! void	sleepq_signal(void *, int, int);
  void	sleepq_set_timeout(void *wchan, int timo);
  int	sleepq_timedwait(void *wchan);
  int	sleepq_timedwait_sig(void *wchan);
  void	sleepq_wait(void *);
  int	sleepq_wait_sig(void *wchan);
  
  #endif	/* _KERNEL */
  #endif	/* !_SYS_SLEEPQUEUE_H_ */
--- 85,117 ----
  #define	SLEEPQ_TYPE		0x0ff		/* Mask of sleep queue types. */
  #define	SLEEPQ_MSLEEP		0x00		/* Used by msleep/wakeup. */
  #define	SLEEPQ_CONDVAR		0x01		/* Used for a cv. */
+ #define	SLEEPQ_SX		0x03		/* Used by an sx lock. */
  #define	SLEEPQ_INTERRUPTIBLE	0x100		/* Sleep is interruptible. */
  
  void	init_sleepqueues(void);
  void	sleepq_abort(struct thread *td, int intrval);
! void	sleepq_add_queue(void *, struct mtx *, const char *, int, int);
  struct sleepqueue *sleepq_alloc(void);
! void	sleepq_broadcast_queue(void *, int, int, int);
  void	sleepq_free(struct sleepqueue *);
  void	sleepq_lock(void *);
  struct sleepqueue *sleepq_lookup(void *);
  void	sleepq_release(void *);
  void	sleepq_remove(struct thread *, void *);
! void	sleepq_signal_queue(void *, int, int, int);
  void	sleepq_set_timeout(void *wchan, int timo);
  int	sleepq_timedwait(void *wchan);
  int	sleepq_timedwait_sig(void *wchan);
  void	sleepq_wait(void *);
  int	sleepq_wait_sig(void *wchan);
  
+ /* Preserve source compat with 6.x */
+ #define sleepq_add(wchan, lock, wmesg, flags)		    \
+     sleepq_add_queue(wchan, lock, wmesg, flags, 0)
+ #define sleepq_broadcast(wchan, flags, pri)		     \
+     sleepq_broadcast_queue(wchan, flags, pri, 0)
+ #define sleepq_signal(wchan, flags, pri)		    \
+     sleepq_signal_queue(wchan, flags, pri, 0)
+ 
  #endif	/* _KERNEL */
  #endif	/* !_SYS_SLEEPQUEUE_H_ */
Index: sys/sx.h
===================================================================
RCS file: /cvs/ncvs/src/sys/sys/sx.h,v
retrieving revision 1.21.2.5
diff -c -r1.21.2.5 sx.h
*** sys/sx.h	27 Aug 2007 13:45:35 -0000	1.21.2.5
--- sys/sx.h	31 Aug 2007 01:02:47 -0000
***************
*** 1,5 ****
  /*-
!  * Copyright (C) 2001 Jason Evans <[e-mail]>.  All rights reserved.
   *
   * Redistribution and use in source and binary forms, with or without
   * modification, are permitted provided that the following conditions
--- 1,7 ----
  /*-
!  * Copyright (c) 2007 Attilio Rao <[e-mail]>
!  * Copyright (c) 2001 Jason Evans <[e-mail]>
!  * All rights reserved.
   *
   * Redistribution and use in source and binary forms, with or without
   * modification, are permitted provided that the following conditions
***************
*** 24,62 ****
   * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
   * DAMAGE.
   *
!  * $FreeBSD: src/sys/sys/sx.h,v 1.21.2.5 2007/08/27 13:45:35 jhb Exp $
   */
  
  #ifndef	_SYS_SX_H_
  #define	_SYS_SX_H_
  
- #include <sys/queue.h>
  #include <sys/_lock.h>
! #include <sys/condvar.h>	/* XXX */
  
! struct sx {
! 	struct lock_object sx_object;	/* Common lock properties. */
! 	struct mtx	*sx_lock;	/* General protection lock. */
! 	int		sx_cnt;		/* -1: xlock, > 0: slock count. */
! 	struct cv	sx_shrd_cv;	/* slock waiters. */
! 	int		sx_shrd_wcnt;	/* Number of slock waiters. */
! 	struct cv	sx_excl_cv;	/* xlock waiters. */
! 	int		sx_excl_wcnt;	/* Number of xlock waiters. */
! 	struct thread	*sx_xholder;	/* Thread presently holding xlock. */
! };
  
  #ifdef _KERNEL
  void	sx_sysinit(void *arg);
! void	sx_init(struct sx *sx, const char *description);
  void	sx_destroy(struct sx *sx);
! void	_sx_slock(struct sx *sx, const char *file, int line);
! void	_sx_xlock(struct sx *sx, const char *file, int line);
  int	_sx_try_slock(struct sx *sx, const char *file, int line);
  int	_sx_try_xlock(struct sx *sx, const char *file, int line);
  void	_sx_sunlock(struct sx *sx, const char *file, int line);
  void	_sx_xunlock(struct sx *sx, const char *file, int line);
  int	_sx_try_upgrade(struct sx *sx, const char *file, int line);
  void	_sx_downgrade(struct sx *sx, const char *file, int line);
  #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
  void	_sx_assert(struct sx *sx, int what, const char *file, int line);
  #endif
--- 26,118 ----
   * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
   * DAMAGE.
   *
!  * $FreeBSD: src/sys/sys/sx.h,v 1.37 2007/07/06 13:20:44 attilio Exp $
   */
  
  #ifndef	_SYS_SX_H_
  #define	_SYS_SX_H_
  
  #include <sys/_lock.h>
! #include <sys/_sx.h>
! #include <sys/lock_profile.h>
  
! #ifdef	_KERNEL
! #include <machine/atomic.h>
! #endif
! 
! /*
!  * In general, the sx locks and rwlocks use very similar algorithms.
!  * The main difference in the implementations is how threads are
!  * blocked when a lock is unavailable.  For this, sx locks use sleep
!  * queues which do not support priority propagation, and rwlocks use
!  * turnstiles which do.
!  *
!  * The sx_lock field consists of several fields.  The low bit
!  * indicates if the lock is locked with a shared or exclusive lock.  A
!  * value of 0 indicates an exclusive lock, and a value of 1 indicates
!  * a shared lock.  Bit 1 is a boolean indicating if there are any
!  * threads waiting for a shared lock.  Bit 2 is a boolean indicating
!  * if there are any threads waiting for an exclusive lock.  Bit 3 is a
!  * boolean indicating if an exclusive lock is recursively held.  The
!  * rest of the variable's definition is dependent on the value of the
!  * first bit.  For an exclusive lock, it is a pointer to the thread
!  * holding the lock, similar to the mtx_lock field of mutexes.  For
!  * shared locks, it is a count of read locks that are held.
!  *
!  * When the lock is not locked by any thread, it is encoded as a
!  * shared lock with zero waiters.
!  *
!  * A note about memory barriers.  Exclusive locks need to use the same
!  * memory barriers as mutexes: _acq when acquiring an exclusive lock
!  * and _rel when releasing an exclusive lock.  On the other side,
!  * shared lock needs to use an _acq barrier when acquiring the lock
!  * but, since they don't update any locked data, no memory barrier is
!  * needed when releasing a shared lock.
!  */
! 
! #define	SX_LOCK_SHARED			0x01
! #define	SX_LOCK_SHARED_WAITERS		0x02
! #define	SX_LOCK_EXCLUSIVE_WAITERS	0x04
! #define	SX_LOCK_RECURSED		0x08
! #define	SX_LOCK_FLAGMASK						\
! 	(SX_LOCK_SHARED | SX_LOCK_SHARED_WAITERS |			\
! 	SX_LOCK_EXCLUSIVE_WAITERS | SX_LOCK_RECURSED)
! 
! #define	SX_OWNER(x)			((x) & ~SX_LOCK_FLAGMASK)
! #define	SX_SHARERS_SHIFT		4
! #define	SX_SHARERS(x)			(SX_OWNER(x) >> SX_SHARERS_SHIFT)
! #define	SX_SHARERS_LOCK(x)						\
! 	((x) << SX_SHARERS_SHIFT | SX_LOCK_SHARED)
! #define	SX_ONE_SHARER			(1 << SX_SHARERS_SHIFT)
! 
! #define	SX_LOCK_UNLOCKED		SX_SHARERS_LOCK(0)
! #define	SX_LOCK_DESTROYED						\
! 	(SX_LOCK_SHARED_WAITERS | SX_LOCK_EXCLUSIVE_WAITERS)
  
  #ifdef _KERNEL
+ 
+ /*
+  * Function prototipes.  Routines that start with an underscore are not part
+  * of the public interface and are wrappered with a macro.
+  */
  void	sx_sysinit(void *arg);
! #define	sx_init(sx, desc)	sx_init_flags((sx), (desc), 0)
! void	sx_init_flags(struct sx *sx, const char *description, int opts);
  void	sx_destroy(struct sx *sx);
! int	_sx_slock(struct sx *sx, int opts, const char *file, int line);
! int	_sx_xlock(struct sx *sx, int opts, const char *file, int line);
  int	_sx_try_slock(struct sx *sx, const char *file, int line);
  int	_sx_try_xlock(struct sx *sx, const char *file, int line);
  void	_sx_sunlock(struct sx *sx, const char *file, int line);
  void	_sx_xunlock(struct sx *sx, const char *file, int line);
  int	_sx_try_upgrade(struct sx *sx, const char *file, int line);
  void	_sx_downgrade(struct sx *sx, const char *file, int line);
+ int	_sx_xlock_hard(struct sx *sx, uintptr_t tid, int opts,
+ 	    const char *file, int line);
+ int	_sx_slock_hard(struct sx *sx, int opts, const char *file, int line);
+ void	_sx_xunlock_hard(struct sx *sx, uintptr_t tid, const char *file, int
+ 	    line);
+ void	_sx_sunlock_hard(struct sx *sx, const char *file, int line);
  #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
  void	_sx_assert(struct sx *sx, int what, const char *file, int line);
  #endif
***************
*** 79,93 ****
  	SYSUNINIT(name##_sx_sysuninit, SI_SUB_LOCK, SI_ORDER_MIDDLE,	\
  	    sx_destroy, (sxa))
  
! #define	sx_xlocked(sx)		((sx)->sx_cnt < 0 && (sx)->sx_xholder == curthread)
! #define	sx_slock(sx)		_sx_slock((sx), LOCK_FILE, LOCK_LINE)
! #define	sx_xlock(sx)		_sx_xlock((sx), LOCK_FILE, LOCK_LINE)
  #define	sx_try_slock(sx)	_sx_try_slock((sx), LOCK_FILE, LOCK_LINE)
  #define	sx_try_xlock(sx)	_sx_try_xlock((sx), LOCK_FILE, LOCK_LINE)
- #define	sx_sunlock(sx)		_sx_sunlock((sx), LOCK_FILE, LOCK_LINE)
- #define	sx_xunlock(sx)		_sx_xunlock((sx), LOCK_FILE, LOCK_LINE)
  #define	sx_try_upgrade(sx)	_sx_try_upgrade((sx), LOCK_FILE, LOCK_LINE)
  #define	sx_downgrade(sx)	_sx_downgrade((sx), LOCK_FILE, LOCK_LINE)
  #define	sx_unlock(sx) do {						\
  	if (sx_xlocked(sx))						\
  		sx_xunlock(sx);						\
--- 135,253 ----
  	SYSUNINIT(name##_sx_sysuninit, SI_SUB_LOCK, SI_ORDER_MIDDLE,	\
  	    sx_destroy, (sxa))
  
! /*
!  * Full lock operations that are suitable to be inlined in non-debug kernels.
!  * If the lock can't be acquired or released trivially then the work is
!  * deferred to 'tougher' functions.
!  */
! 
! /* Acquire an exclusive lock. */
! static __inline int
! __sx_xlock(struct sx *sx, struct thread *td, int opts, const char *file,
!     int line)
! {
! 	uintptr_t tid = (uintptr_t)td;
! 	int error = 0;
! 
! 	if (!atomic_cmpset_acq_ptr(&sx->sx_lock, SX_LOCK_UNLOCKED, tid))
! 		error = _sx_xlock_hard(sx, tid, opts, file, line);
! 	else
! 		lock_profile_obtain_lock_success(&sx->lock_object, 0, 0, file,
! 		    line);
! 
! 	return (error);
! }
! 
! /* Release an exclusive lock. */
! static __inline void
! __sx_xunlock(struct sx *sx, struct thread *td, const char *file, int line)
! {
! 	uintptr_t tid = (uintptr_t)td;
! 
! 	if (!atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_LOCK_UNLOCKED))
! 		_sx_xunlock_hard(sx, tid, file, line);
! }
! 
! /* Acquire a shared lock. */
! static __inline int
! __sx_slock(struct sx *sx, int opts, const char *file, int line)
! {
! 	uintptr_t x = sx->sx_lock;
! 	int error = 0;
! 
! 	if (!(x & SX_LOCK_SHARED) ||
! 	    !atomic_cmpset_acq_ptr(&sx->sx_lock, x, x + SX_ONE_SHARER))
! 		error = _sx_slock_hard(sx, opts, file, line);
! #ifdef LOCK_PROFILING_SHARED
! 	else if (SX_SHARERS(x) == 0)
! 		lock_profile_obtain_lock_success(&sx->lock_object, 0, 0, file,
! 		    line);
! #endif
! 
! 	return (error);
! }
! 
! /*
!  * Release a shared lock.  We can just drop a single shared lock so
!  * long as we aren't trying to drop the last shared lock when other
!  * threads are waiting for an exclusive lock.  This takes advantage of
!  * the fact that an unlocked lock is encoded as a shared lock with a
!  * count of 0.
!  */
! static __inline void
! __sx_sunlock(struct sx *sx, const char *file, int line)
! {
! 	uintptr_t x = sx->sx_lock;
! 
! 	if (x == (SX_SHARERS_LOCK(1) | SX_LOCK_EXCLUSIVE_WAITERS) ||
! 	    !atomic_cmpset_ptr(&sx->sx_lock, x, x - SX_ONE_SHARER))
! 		_sx_sunlock_hard(sx, file, line);
! }
! 
! /*
!  * Public interface for lock operations.
!  */
! #ifndef LOCK_DEBUG
! #error	"LOCK_DEBUG not defined, include <sys/lock.h> before <sys/sx.h>"
! #endif
! #if	(LOCK_DEBUG > 0) || defined(SX_NOINLINE)
! #define	sx_xlock(sx)		(void)_sx_xlock((sx), 0, LOCK_FILE, LOCK_LINE)
! #define	sx_xlock_sig(sx)						\
! 	_sx_xlock((sx), SX_INTERRUPTIBLE, LOCK_FILE, LOCK_LINE)
! #define	sx_xunlock(sx)		_sx_xunlock((sx), LOCK_FILE, LOCK_LINE)
! #define	sx_slock(sx)		(void)_sx_slock((sx), 0, LOCK_FILE, LOCK_LINE)
! #define	sx_slock_sig(sx)						\
! 	_sx_slock((sx), SX_INTERRUPTIBLE, LOCK_FILE, LOCK_LINE)
! #define	sx_sunlock(sx)		_sx_sunlock((sx), LOCK_FILE, LOCK_LINE)
! #else
! #define	sx_xlock(sx)							\
! 	(void)__sx_xlock((sx), curthread, 0, LOCK_FILE, LOCK_LINE)
! #define	sx_xlock_sig(sx)						\
! 	__sx_xlock((sx), curthread, SX_INTERRUPTIBLE, LOCK_FILE, LOCK_LINE)
! #define	sx_xunlock(sx)							\
! 	__sx_xunlock((sx), curthread, LOCK_FILE, LOCK_LINE)
! #define	sx_slock(sx)		(void)__sx_slock((sx), 0, LOCK_FILE, LOCK_LINE)
! #define	sx_slock_sig(sx)						\
! 	__sx_slock((sx), SX_INTERRUPTIBLE, LOCK_FILE, LOCK_LINE)
! #define	sx_sunlock(sx)		__sx_sunlock((sx), LOCK_FILE, LOCK_LINE)
! #endif	/* LOCK_DEBUG > 0 || SX_NOINLINE */
  #define	sx_try_slock(sx)	_sx_try_slock((sx), LOCK_FILE, LOCK_LINE)
  #define	sx_try_xlock(sx)	_sx_try_xlock((sx), LOCK_FILE, LOCK_LINE)
  #define	sx_try_upgrade(sx)	_sx_try_upgrade((sx), LOCK_FILE, LOCK_LINE)
  #define	sx_downgrade(sx)	_sx_downgrade((sx), LOCK_FILE, LOCK_LINE)
+ 
+ /*
+  * Return a pointer to the owning thread if the lock is exclusively
+  * locked.
+  */
+ #define	sx_xholder(sx)							\
+ 	((sx)->sx_lock & SX_LOCK_SHARED ? NULL :			\
+ 	(struct thread *)SX_OWNER((sx)->sx_lock))
+ 
+ #define	sx_xlocked(sx)							\
+ 	(((sx)->sx_lock & ~(SX_LOCK_FLAGMASK & ~SX_LOCK_SHARED)) ==	\
+ 	    (uintptr_t)curthread)
+ 
  #define	sx_unlock(sx) do {						\
  	if (sx_xlocked(sx))						\
  		sx_xunlock(sx);						\
***************
*** 95,111 ****
--- 255,293 ----
  		sx_sunlock(sx);						\
  } while (0)
  
+ #define	sx_sleep(chan, sx, pri, wmesg, timo)				\
+ 	_sleep((chan), &(sx)->lock_object, (pri), (wmesg), (timo))
+ 
+ /*
+  * Options passed to sx_init_flags().
+  */
+ #define	SX_DUPOK		0x01
+ #define	SX_NOPROFILE		0x02
+ #define	SX_NOWITNESS		0x04
+ #define	SX_QUIET		0x08
+ #define	SX_ADAPTIVESPIN		0x10
+ #define	SX_RECURSE		0x20
+ 
+ /*
+  * Options passed to sx_*lock_hard().
+  */
+ #define	SX_INTERRUPTIBLE	0x40
+ 
  #if defined(INVARIANTS) || defined(INVARIANT_SUPPORT)
  #define	SA_LOCKED		LA_LOCKED
  #define	SA_SLOCKED		LA_SLOCKED
  #define	SA_XLOCKED		LA_XLOCKED
  #define	SA_UNLOCKED		LA_UNLOCKED
+ #define	SA_RECURSED		LA_RECURSED
+ #define	SA_NOTRECURSED		LA_NOTRECURSED
  
  /* Backwards compatability. */
  #define	SX_LOCKED		LA_LOCKED
  #define	SX_SLOCKED		LA_SLOCKED
  #define	SX_XLOCKED		LA_XLOCKED
  #define	SX_UNLOCKED		LA_UNLOCKED
+ #define	SX_RECURSED		LA_RECURSED
+ #define	SX_NOTRECURSED		LA_NOTRECURSED
  #endif
  
  #ifdef INVARIANTS
Index: sys/turnstile.h
===================================================================
RCS file: /cvs/ncvs/src/sys/sys/turnstile.h,v
retrieving revision 1.7
diff -c -r1.7 turnstile.h
*** sys/turnstile.h	7 Jan 2005 02:29:24 -0000	1.7
--- sys/turnstile.h	31 Aug 2007 00:39:59 -0000
***************
*** 73,92 ****
  
  #ifdef _KERNEL
  
  void	init_turnstiles(void);
  void	turnstile_adjust(struct thread *, u_char);
  struct turnstile *turnstile_alloc(void);
! void	turnstile_broadcast(struct turnstile *);
  void	turnstile_claim(struct lock_object *);
! int	turnstile_empty(struct turnstile *);
  void	turnstile_free(struct turnstile *);
! struct thread *turnstile_head(struct turnstile *);
  void	turnstile_lock(struct lock_object *);
  struct turnstile *turnstile_lookup(struct lock_object *);
  void	turnstile_release(struct lock_object *);
! int	turnstile_signal(struct turnstile *);
! void	turnstile_unpend(struct turnstile *);
! void	turnstile_wait(struct lock_object *, struct thread *);
  
  #endif	/* _KERNEL */
  #endif	/* _SYS_TURNSTILE_H_ */
--- 73,115 ----
  
  #ifdef _KERNEL
  
+ /* Which queue to block on or which queue to wakeup one or more threads from. */
+ #define       TS_EXCLUSIVE_QUEUE      0
+ #define       TS_SHARED_QUEUE         1
+ 
+ /* The type of lock currently held. */
+ #define       TS_EXCLUSIVE_LOCK       TS_EXCLUSIVE_QUEUE
+ #define       TS_SHARED_LOCK          TS_SHARED_QUEUE
+ 
  void	init_turnstiles(void);
  void	turnstile_adjust(struct thread *, u_char);
  struct turnstile *turnstile_alloc(void);
! #define turnstile_wakeup(turnstile) turnstile_broadcast(turnstile)
! #define	turnstile_broadcast(turnstile) \
!     turnstile_broadcast_queue(turnstile, TS_EXCLUSIVE_QUEUE)
! void	turnstile_broadcast_queue(struct turnstile *, int);
  void	turnstile_claim(struct lock_object *);
! void	turnstile_disown(struct turnstile *);
! #define	turnstile_empty(turnstile)  \
!     turnstile_empty_queue(turnstile, TS_EXCLUSIVE_QUEUE);
! int	turnstile_empty_queue(struct turnstile *, int);
  void	turnstile_free(struct turnstile *);
! #define	turnstile_head(turnstile)   \
!     turnstile_head_queue(turnstile, TS_EXCLUSIVE_QUEUE)
! struct thread *turnstile_head_queue(struct turnstile *, int);
  void	turnstile_lock(struct lock_object *);
  struct turnstile *turnstile_lookup(struct lock_object *);
  void	turnstile_release(struct lock_object *);
! #define turnstile_signal(turnstile) \
!     turnstile_signal_queue(turnstile, TS_EXCLUSIVE_QUEUE)
! int	turnstile_signal_queue(struct turnstile *, int);
! struct turnstile *turnstile_trywait(struct lock_object *);
! #define	turnstile_unpend(turnstile) \
!     turnstile_unpend_queue(turnstile, TS_EXCLUSIVE_QUEUE);
! void	turnstile_unpend_queue(struct turnstile *, int);
! #define turnstile_wait(lock_object, thread) \
!     turnstile_wait_queue(lock_object, thread, TS_EXCLUSIVE_QUEUE) 
! void	turnstile_wait_queue(struct lock_object *, struct thread *, int);
  
  #endif	/* _KERNEL */
  #endif	/* _SYS_TURNSTILE_H_ */
Index: vm/vm_map.c
===================================================================
RCS file: /cvs/ncvs/src/sys/vm/vm_map.c,v
retrieving revision 1.366.2.4
diff -c -r1.366.2.4 vm_map.c
*** vm/vm_map.c	30 Aug 2007 02:32:04 -0000	1.366.2.4
--- vm/vm_map.c	31 Aug 2007 03:12:00 -0000
***************
*** 429,435 ****
  	if (map->system_map)
  		_mtx_lock_flags(&map->system_mtx, 0, file, line);
  	else
! 		_sx_xlock(&map->lock, file, line);
  	map->timestamp++;
  }
  
--- 429,435 ----
  	if (map->system_map)
  		_mtx_lock_flags(&map->system_mtx, 0, file, line);
  	else
! 		(void) _sx_xlock(&map->lock, 0, file, line);
  	map->timestamp++;
  }
  
***************
*** 450,456 ****
  	if (map->system_map)
  		_mtx_lock_flags(&map->system_mtx, 0, file, line);
  	else
! 		_sx_xlock(&map->lock, file, line);
  }
  
  void
--- 450,456 ----
  	if (map->system_map)
  		_mtx_lock_flags(&map->system_mtx, 0, file, line);
  	else
! 		(void) _sx_xlock(&map->lock, 0, file, line);
  }
  
  void

--SnV5plBeK2Ge1I9g
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: inline

_______________________________________________
[e-mail] mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-smp
To unsubscribe, send any mail to "[e-mail]"
--SnV5plBeK2Ge1I9g--