/*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
/*	  All Rights Reserved  	*/

/*	THIS IS UNPUBLISHED PROPRIETARY SOURCE CODE OF AT&T	*/
/*	The copyright notice above does not evidence any   	*/
/*	actual or intended publication of such source code.	*/

/*
 * +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 * 		PROPRIETARY NOTICE (Combined)
 *
 * This source code is unpublished proprietary information
 * constituting, or derived under license from AT&T's UNIX(r) System V.
 * In addition, portions of such source code were derived from Berkeley
 * 4.3 BSD under license from the Regents of the University of
 * California.
 *
 *
 *
 * 		Copyright Notice
 *
 * Notice of copyright on this source code product does not indicate
 * publication.
 *
 * 	Copyright (c) 1986-1990, 1995-1997 by Sun Microsystems, Inc.
 * 	(c) 1983, 1984, 1985, 1986, 1987, 1988, 1989  AT&T.
 *		All rights reserved.
 *
 */

/*
 * Copyright (c) 1998-1999 by Sun Microsystems, Inc.
 * All rights reserved.
 */

#pragma ident	"@(#)vm_as.c	1.123	99/08/31 SMI"

/*
 * VM - address spaces.
 */

#include <sys/types.h>
#include <sys/t_lock.h>
#include <sys/param.h>
#include <sys/errno.h>
#include <sys/systm.h>
#include <sys/mman.h>
#include <sys/sysmacros.h>
#include <sys/cpuvar.h>
#include <sys/sysinfo.h>
#include <sys/kmem.h>
#include <sys/vnode.h>
#include <sys/vmsystm.h>
#include <sys/cmn_err.h>
#include <sys/debug.h>
#include <sys/tnf_probe.h>
#include <sys/vtrace.h>
#include <sys/rce.h>

#include <vm/hat.h>
#include <vm/as.h>
#include <vm/seg.h>
#include <vm/seg_vn.h>
#include <vm/seg_dev.h>
#include <vm/seg_kmem.h>
#include <vm/seg_map.h>
#include <vm/seg_spt.h>
#include <vm/page.h>

clock_t deadlk_wait = 1; /* number of ticks to wait before retrying */

static struct kmem_cache *as_cache;
static struct kmem_cache *seg_skiplist_cache;

/*
 * The random number generator function is based on an algorithm
 * attributed to G. J. Mitchell and D. P. Moore (1958) and described
 * in Knuth's _The Art of Computer Programming_, Vol 2, _Seminumerical
 * Algorithms_, pages 26-29.  The seeds were generated by the random
 * number generator algorithm itself.
 */
static uint_t ssl_randtab[71] = {
	0x7d334262U, 0x6346d245U, 0x5f28efb9U, 0xc9bd5ee5U, 0x8380c56fU,
	0xaa24f1fbU, 0x8d890834U, 0x3a4b2c27U, 0x4a5b0aa7U, 0xed01b10eU,
	0x1a33b852U, 0x621a1076U, 0x427c69d2U, 0x28c7d209U, 0xdb7ce347U,
	0x0fd22580U, 0x87f39089U, 0x34c6455aU, 0x243e1a50U, 0x1de7a14eU,
	0x2bb5ba0fU, 0x0ad8db02U, 0xe58b99d2U, 0xc494c2bfU, 0xc8d6518aU,
	0x02fe313aU, 0x1e09edabU, 0x16948b04U, 0xdd79f72fU, 0xa8448c43U,
	0x28112533U, 0xf3f0a7e6U, 0xc3acdcd2U, 0xfd63bdcdU, 0xb97554f5U,
	0xc52b1f04U, 0x2b9d002dU, 0x1aa86589U, 0x2dda4fb4U, 0x3977d995U,
	0xf1b5ce9fU, 0x4c5b5a8dU, 0x6d5d6e8cU, 0x2279442aU, 0x54915c49U,
	0xd8f5e556U, 0x7e191817U, 0xcf11c7e4U, 0x7f0e7b14U, 0x5caffa77U,
	0xe7d34976U, 0xbdef82edU, 0xf29d3869U, 0x80746a85U, 0x872f279aU,
	0xf0c927f9U, 0x737803f3U, 0x3cd10937U, 0x98971885U, 0xe95a05ccU,
	0xc9650e2aU, 0x7cb4411fU, 0xa4991d38U, 0x37b0c27cU, 0xd9b22936U,
	0xb105b2a8U, 0x7eca15feU, 0x325b821aU, 0x8676f996U, 0x83845ab8U,
	0xf5c4b310U
};

static void as_setwatchprot(struct as *, struct seg *, caddr_t, size_t, uint_t);
static void as_clearwatchprot(struct as *, struct seg *, caddr_t, size_t);

static uint_t *ssl_n1p = &ssl_randtab[0];
static uint_t *ssl_n2p = &ssl_randtab[35];
static kmutex_t ssl_randlock;	/* serialize ssl_random */

static uint_t
ssl_random()
{
	uint_t r;

	mutex_enter(&ssl_randlock);
	*ssl_n1p++ += *ssl_n2p++;
	if (ssl_n1p >= &ssl_randtab[71])
		ssl_n1p = &ssl_randtab[0];
	if (ssl_n2p >= &ssl_randtab[71])
		ssl_n2p = &ssl_randtab[0];
	r = *ssl_n1p;
	mutex_exit(&ssl_randlock);
	return (r);
}

/*
 * skiplist level generator.  The probability of the new level being
 *	 > 0 is 1/SSL_BFACTOR,
 *	 > 1 is 1/SSL_BFACTOR**2,
 *	 > 2 is 1/SSL_BFACTOR**3, and so on.
 */
static uint_t
ssl_newlevel() {
	uint_t newlev = 1;
	uint_t r = ssl_random();

	while ((r & (SSL_BFACTOR - 1)) == 0 && newlev < SSL_NLEVELS) {
		r >>= SSL_LOG2BF;
		newlev++;
	}

	return (newlev - 1);
}

/*
 * Search a segment skiplist for addr. If a segment containing addr
 * exists, that segment is returned.  If no such segment exists, and
 * the list spans addresses greater than addr, then the first segment
 * whose base is greater than addr is returned.
 *
 * If the list does not span addresses greater than addr, and tail is
 * true, return the last element in the list.  Otherwise, return NULL.
 *
 * as->a_cache.spath points to the path recorded by a previous search
 * in this skiplist.  This information is needed for inserting and
 * deleting elements in the skiplist, and it is used as a cache for
 * the search.  If the segment to which the zeroth pointer in spath
 * refers is the one we seek, then we simply return it.  If the next
 * segment in the list is the one we seek, then we patch up the lower
 * values of spath and return it.  Otherwise, we search from the head
 * of the list, recording the search path in spath as we go.
 *
 * For a general description of skip lists, see "Skip Lists: A
 * Probabilistic Alternative to Balanced Trees" by William Pugh,
 * published in CACM Vol.33 No.6, June 1990
 */
static struct seg *
ssl_search(struct as *as, caddr_t addr, int tail)
{
	int i, hilev;
	struct seg *seg;
	seg_skiplist *ssl, *cssl;
	ssl_spath spath;

	ASSERT(AS_LOCK_HELD(as, &as->a_lock));

	mutex_enter(&as->a_contents);
	spath = *as->a_cache.spath;
	mutex_exit(&as->a_contents);
	cssl = spath.ssls[0];
	if (cssl != NULL && (seg = cssl->segs[0]) != NULL) {
		/* Are we looking for the encached segment? */
		struct seg *prev = seg->s_prev;
		if ((addr < seg->s_base + seg->s_size) &&
		    ((prev == NULL) || (addr >= prev->s_base + prev->s_size)))
			return (seg);

		/*
		 * ...Or are we looking for the immediate successor
		 * to the encached segment? (a common case)
		 */
		prev = seg;
		if ((seg = seg->s_next.skiplist->segs[0]) != NULL) {
			if ((addr >= prev->s_base + prev->s_size) &&
			    (addr < seg->s_base + seg->s_size)) {
				/* spath needs updating */
				ssl = prev->s_next.skiplist;
				for (i = 0;
				    i < SSL_NLEVELS &&
				    ssl->segs[i] != SSL_UNUSED;
				    i++)
					spath.ssls[i] = ssl;
				mutex_enter(&as->a_contents);
				*as->a_cache.spath = spath;
				mutex_exit(&as->a_contents);
				return (seg);
			}
		}
	}

	/* Cache didn't help; start search at head of skiplist */
	ssl = as->a_segs.skiplist;
	hilev = as->a_hilevel;
	for (i = hilev; i >= 0; i--) {
		seg = ssl->segs[i];
		while ((seg != NULL) && (seg->s_base + seg->s_size <= addr)) {
			ssl = seg->s_next.skiplist;
			seg = ssl->segs[i];
		}
		spath.ssls[i] = ssl;
	}

	mutex_enter(&as->a_contents);
	*as->a_cache.spath = spath;
	mutex_exit(&as->a_contents);

	if (tail && seg == NULL)
		return (as->a_tail);
	else
		return (seg);
}

/*
 * Verifying the segment lists is very time-consuming; it may not be
 * desirable always to define VERIFY_SEGLIST when DEBUG is set.
 */
#ifdef DEBUG
#define	VERIFY_SEGLIST
int do_as_verify = 0;
int do_ssl_verify = 0;
#endif

#ifdef VERIFY_SEGLIST
/*
 * verify that the skiplist is coherent.
 */
static void
ssl_verify(struct as *as)
{
	int i;
	struct seg *seg;
	ssl_spath spath;
	uint_t nsegs = 0;

	if (do_ssl_verify == 0)
		return;

	ASSERT(as->a_lrep == AS_LREP_SKIPLIST);

	spath = *as->a_cache.spath;

	for (seg = as->a_segs.skiplist->segs[0]; seg != NULL;
	    seg = seg->s_next.skiplist->segs[0]) {
		ASSERT(seg->s_as == as);

		for (i = 1; i < SSL_NLEVELS; i++)
			ASSERT(seg->s_next.skiplist->segs[i] == NULL ||
			    seg->s_next.skiplist->segs[i] == SSL_UNUSED ||
			    seg->s_next.skiplist->segs[i]->s_as == as);

		ASSERT(seg->s_prev == NULL || seg->s_prev->s_as == as);
		ASSERT(seg->s_prev == NULL ||
		    seg->s_prev->s_base < seg->s_base);
		ASSERT(seg->s_next.skiplist->segs[0] == NULL ||
		    seg->s_next.skiplist->segs[0]->s_base > seg->s_base);
		ASSERT(seg->s_next.skiplist->segs[0] != NULL ||
		    seg == as->a_tail);

		for (i = 0; i < SSL_NLEVELS; i++)
			if (spath.ssls[i] == seg->s_next.skiplist)
				spath.ssls[i] = NULL;
		nsegs++;
	}

	for (i = 0; i < SSL_NLEVELS; i++)
		ASSERT(spath.ssls[i] == NULL ||
		    spath.ssls[i] == as->a_segs.skiplist);

	ASSERT(as->a_nsegs == nsegs);
}
#endif /* VERIFY_SEGLIST */

/*
 * Insert a new element into a segment skiplist.  If a new high level
 * is selected, as->a_hilevel will be updated with the new value
 */
static int
ssl_insert(struct as *as, struct seg *newseg)
{
	int i, newlev, hilev;
	struct seg *seg;
	caddr_t addr, eaddr;
	ssl_spath *spath;

	ASSERT(AS_WRITE_HELD(as, &as->a_lock));

	addr = newseg->s_base;
	eaddr = addr + newseg->s_size;

again:
	seg = ssl_search(as, addr, 1);

	if (seg == NULL) {
		ASSERT(as->a_segs.skiplist->segs[0] == NULL &&
		    as->a_tail == NULL);
		as->a_tail = newseg;
		newseg->s_prev = NULL;
	} else {
		caddr_t base = seg->s_base;

		/*
		 * If top of seg is below the requested address, then
		 * the insertion point is at the end of the skiplist,
		 * and seg points to the tail of the list.  Otherwise,
		 * the insertion point is immediately before seg.
		 */
		if (base + seg->s_size <= addr) {
			newseg->s_prev = seg;
			as->a_tail = newseg;
		} else {
			if (addr >= base || eaddr > base) {
#ifdef __sparcv9cpu
				extern struct seg_ops segnf_ops;

				/*
				 * no-fault segs must disappear
				 * if overlaid.
				 * XXX need new segment type so
				 * we don't have to check s_ops
				 */
				if (seg->s_ops == &segnf_ops) {
					seg_unmap(seg);
					goto again;
				}
#endif
				return (-1);	/* overlapping segment */
			}
			newseg->s_prev = seg->s_prev;
			seg->s_prev = newseg;
		}
	}

	spath = as->a_cache.spath;
	newlev = ssl_newlevel();
	hilev = as->a_hilevel;
	if (newlev > hilev) {
		for (i = hilev + 1; i <= newlev; i++)
			spath->ssls[i] = as->a_segs.skiplist;
		as->a_hilevel = (uchar_t)newlev;
	}

	newseg->s_next.skiplist = kmem_cache_alloc(seg_skiplist_cache,
		KM_SLEEP);

	for (i = 0; i <= newlev; i++) {
		newseg->s_next.skiplist->segs[i] = spath->ssls[i]->segs[i];
		spath->ssls[i]->segs[i] = newseg;
	}

	for (i = newlev+1; i < SSL_NLEVELS; i++)	/* never deref these */
		newseg->s_next.skiplist->segs[i] = SSL_UNUSED;

	as->a_nsegs++;

#ifdef VERIFY_SEGLIST
	ssl_verify(as);
#endif
	return (0);
}

/*
 * remove the element containing addr from the skiplist, and return a
 * pointer to that element.
 */
static struct seg *
ssl_remove(struct as *as, caddr_t addr)
{
	int i, hilev;
	struct seg *seg;
	ssl_spath *spath;

	ASSERT(AS_WRITE_HELD(as, &as->a_lock));

	seg = ssl_search(as, addr, 0);

	if (seg == NULL || seg->s_base != addr)
		return (NULL);

	spath = as->a_cache.spath;
	hilev = as->a_hilevel;
	for (i = 0; i <= hilev; i++) {
		if (spath->ssls[i]->segs[i] != seg)
			break;

		spath->ssls[i]->segs[i] = seg->s_next.skiplist->segs[i];
	}

	if (seg->s_next.skiplist->segs[0] == NULL)
		as->a_tail = seg->s_prev;
	else
		seg->s_next.skiplist->segs[0]->s_prev = seg->s_prev;

	while (hilev > 0 && as->a_segs.skiplist->segs[hilev] == NULL)
		hilev--;
	as->a_hilevel = (uchar_t)hilev;

	kmem_cache_free(seg_skiplist_cache, seg->s_next.skiplist);
	seg->s_next.skiplist = NULL;

	as->a_nsegs--;

#ifdef VERIFY_SEGLIST
	ssl_verify(as);
#endif
	return (seg);
}

/*
 * change the simple linked list of segments into a skiplist
 */
static void
as_mutate_seglist(struct as *as)
{
	struct seg *car, *cdr;
#ifdef DEBUG
	uint_t nsegs = as->a_nsegs;
#endif

	cdr = as->a_tail;
	as->a_tail = NULL;

	/*
	 * seg_skiplist and ssl_spath are always the same size,
	 * so both can be allocated from seg_skiplist_cache.
	 */
	as->a_segs.skiplist = kmem_cache_alloc(seg_skiplist_cache, KM_SLEEP);
	bzero(as->a_segs.skiplist, sizeof (seg_skiplist));

	as->a_cache.spath = kmem_cache_alloc(seg_skiplist_cache, KM_SLEEP);
	bzero(as->a_cache.spath, sizeof (seg_skiplist));

	as->a_lrep = AS_LREP_SKIPLIST;
	as->a_nsegs = 0;

	/*
	 * Do the insertions from the top down, to take greatest
	 * advantage of the search path caching.
	 */
	while (cdr != NULL) {
		car = cdr;
		cdr = cdr->s_prev;
		(void) ssl_insert(as, car);
	}
#ifdef DEBUG
	/* make sure that we still have all the segments we started with */
	ASSERT(as->a_nsegs == nsegs);
#endif
}

/*
 * Search a segment linked list for addr. If a segment containing addr
 * exists, that segment is returned.  If no such segment exists, and
 * the list spans addresses greater than addr, then the first segment
 * whose base is greater than addr is returned; otherwise, NULL is
 * returned unless tail is true, in which case the last element of the
 * list is returned.
 */
struct seg *
as_findseg(struct as *as, caddr_t addr, int tail)
{
	struct seg *seg;
	caddr_t base;
	int forward;

	ASSERT(AS_LOCK_HELD(as, &as->a_lock));

	/* if the list has mutated into a skiplist, pass the buck */
	if (as->a_lrep == AS_LREP_SKIPLIST)
		return (ssl_search(as, addr, tail));

	if (as->a_segs.list == NULL)	/* address space has no segments */
		return (NULL);

	/* figure out the optimal place and direction to start searching */

	seg = as->a_cache.seglast;
	if (seg == NULL)
		seg = as->a_segs.list;

	forward = 0;
	if ((base = seg->s_base) <= addr) {
		caddr_t lastaddr;
		struct seg *hiseg;
		if (addr < (base + seg->s_size))
			return (seg);	/* seglast contained addr */

		hiseg = as->a_tail;	/* highest segment */
		if ((lastaddr = hiseg->s_base + hiseg->s_size) <= addr)
			return (tail? hiseg : NULL); /* addr is out of range */

		/* address is between seg and hiseg.  Which is closer? */
		if ((addr - base) > (lastaddr - addr)) {
			seg = hiseg;    /* start reverse search at tail */
		} else {
			seg = seg->s_next.list;
			forward++;	/* start forward search after seglast */
		}
	} else {
		struct seg *loseg = as->a_segs.list;	/* lowest segment */
		if (loseg->s_base > addr)
			return (loseg);	/* lowest seg is higher than addr */

		/* address is between seg and loseg.  Which is closer? */
		if ((addr - loseg->s_base) < (base - addr)) {
			seg = loseg;
			forward++;	/* start forward search at head */
		}
		/* otherwise, start reverse search at seglast */
	}

	if (forward) {
		while (seg->s_base + seg->s_size <= addr)
			seg = seg->s_next.list;
		if (seg->s_base <= addr)	/* addr is in seg */
			as->a_cache.seglast = seg;
		return (seg);
	} else {
		struct seg *prev;
		while (seg->s_base > addr) {
			prev = seg;
			seg = seg->s_prev;
			/*
			 * We know that the reverse search won't run
			 * off the beginning of the list, because of
			 * the check for (loseg->s_base > addr) above.
			 */
			ASSERT(seg != NULL);
		}
		if (seg->s_base + seg->s_size > addr) {
			as->a_cache.seglast = seg;
			return (seg);		/* addr is in seg */
		} else {
			return (prev);		/* addr is in no seg */
		}
	}
	/*NOTREACHED*/
}

#ifdef VERIFY_SEGLIST
/*
 * verify that the linked list is coherent
 */
static void
as_verify(struct as *as)
{
	struct seg *seg, *seglast;
	uint_t nsegs = 0;

	if (do_as_verify == 0)
		return;

	ASSERT(as->a_lrep == AS_LREP_LINKEDLIST);

	seglast = as->a_cache.seglast;

	for (seg = as->a_segs.list; seg != NULL; seg = seg->s_next.list) {
		ASSERT(seg->s_as == as);
		ASSERT(seg->s_prev == NULL || seg->s_prev->s_as == as);
		ASSERT(seg->s_prev == NULL ||
		    seg->s_prev->s_base < seg->s_base);
		ASSERT(seg->s_next.list == NULL ||
		    seg->s_next.list->s_base > seg->s_base);
		ASSERT(seg->s_next.list != NULL || seg == as->a_tail);
		if (seg == seglast)
			seglast = NULL;
		nsegs++;
	}
	ASSERT(seglast == NULL);
	ASSERT(as->a_nsegs == nsegs);
}
#endif /* VERIFY_SEGLIST */

int as_mutation_thresh = AS_MUTATION_THRESH;

/*
 * Add a new segment to the address space
 */
int
as_addseg(struct as *as, struct seg *newseg)
{
	struct seg *seg;
	caddr_t addr, eaddr;

	ASSERT(AS_WRITE_HELD(as, &as->a_lock));

	as->a_updatedir = 1;	/* inform /proc */

	if (as->a_nsegs >= as_mutation_thresh &&
	    as->a_lrep != AS_LREP_SKIPLIST && as != &kas)
		as_mutate_seglist(as);

	/* if the list has mutated into a skiplist, pass the buck */
	if (as->a_lrep == AS_LREP_SKIPLIST)
		return (ssl_insert(as, newseg));

	addr = newseg->s_base;
	eaddr = addr + newseg->s_size;

again:
	seg = as_findseg(as, newseg->s_base, 1);

	if (seg == NULL) {	/* list is empty */
		as->a_segs.list = as->a_tail = newseg;
		newseg->s_next.list = newseg->s_prev = NULL;
	} else {
		caddr_t base = seg->s_base;

		/*
		 * If top of seg is below the requested address, then
		 * the insertion point is at the end of the linked list,
		 * and seg points to the tail of the list.  Otherwise,
		 * the insertion point is immediately before seg.
		 */
		if (base + seg->s_size <= addr) {
			newseg->s_prev = seg;
			newseg->s_next.list = NULL;
			seg->s_next.list = as->a_tail = newseg;
		} else {
			if (addr >= base || eaddr > base) {
#ifdef __sparcv9cpu
				extern struct seg_ops segnf_ops;

				/*
				 * no-fault segs must disappear
				 * if overlaid.
				 * XXX need new segment type so
				 * we don't have to check s_ops
				 */
				if (seg->s_ops == &segnf_ops) {
					seg_unmap(seg);
					goto again;
				}
#endif
				return (-1);	/* overlapping segment */
			}
			newseg->s_next.list = seg;
			newseg->s_prev = seg->s_prev;
			seg->s_prev = newseg;
			if (as->a_segs.list == seg)
				/* newseg is at front */
				as->a_segs.list = newseg;
			else
				newseg->s_prev->s_next.list = newseg;
		}
	}
	as->a_cache.seglast = newseg;
	as->a_nsegs++;

#ifdef VERIFY_SEGLIST
	as_verify(as);
#endif
	return (0);
}

struct seg *
as_removeseg(struct as *as, caddr_t addr)
{
	struct seg *seg;

	ASSERT(AS_WRITE_HELD(as, &as->a_lock));

	as->a_updatedir = 1;	/* inform /proc */

	/* if the list has mutated into a skiplist, pass the buck */
	if (as->a_lrep == AS_LREP_SKIPLIST)
		return (ssl_remove(as, addr));

	seg = as_findseg(as, addr, 0);

	if (seg == NULL || seg->s_base != addr)
		return (NULL);

	as->a_cache.seglast = seg->s_next.list;

	if (seg == as->a_tail)
		as->a_tail = seg->s_prev;
	else
		seg->s_next.list->s_prev = seg->s_prev;

	if (seg == as->a_segs.list)
		as->a_segs.list = seg->s_next.list;
	else
		seg->s_prev->s_next.list = seg->s_next.list;

	as->a_nsegs--;
#ifdef VERIFY_SEGLIST
	as_verify(as);
#endif
	return (seg);
}

/*
 * Find a segment containing addr.
 */
struct seg *
as_segat(struct as *as, caddr_t addr)
{
	struct seg *seg;
	caddr_t base;

	ASSERT(AS_LOCK_HELD(as, &as->a_lock));

	seg = as_findseg(as, addr, 0);

	if ((seg != NULL) && ((base = seg->s_base) <= addr) &&
	    (addr < base + seg->s_size))
		return (seg);
	else
		return (NULL);
}

/*
 * Serialize all searches for holes in an address space to
 * prevent two or more threads from allocating the same virtual
 * address range.  The address space must not be "read/write"
 * locked by the caller since we may block.
 */
void
as_rangelock(struct as *as)
{
	mutex_enter(&as->a_contents);
	while (AS_ISCLAIMGAP(as))
		cv_wait(&as->a_cv, &as->a_contents);
	AS_SETCLAIMGAP(as);
	mutex_exit(&as->a_contents);
}

/*
 * Release hold on a_state & AS_CLAIMGAP and signal any other blocked threads.
 */
void
as_rangeunlock(struct as *as)
{
	mutex_enter(&as->a_contents);
	AS_CLRCLAIMGAP(as);
	cv_signal(&as->a_cv);
	mutex_exit(&as->a_contents);
}

/*ARGSUSED*/
static int
as_constructor(void *buf, void *cdrarg, int kmflags)
{
	struct as *as = buf;

	mutex_init(&as->a_contents, NULL, MUTEX_DEFAULT, NULL);
	cv_init(&as->a_cv, NULL, CV_DEFAULT, NULL);
	rw_init(&as->a_lock, NULL, RW_DEFAULT, NULL);
	return (0);
}

/*ARGSUSED1*/
static void
as_destructor(void *buf, void *cdrarg)
{
	struct as *as = buf;

	mutex_destroy(&as->a_contents);
	cv_destroy(&as->a_cv);
	rw_destroy(&as->a_lock);
}

void
as_init(void)
{
	as_cache = kmem_cache_create("as_cache", sizeof (struct as), 0,
		as_constructor, as_destructor, NULL, NULL, NULL, 0);
	seg_skiplist_cache = kmem_cache_create("seg_skiplist_cache",
		sizeof (seg_skiplist), 0, NULL, NULL, NULL, NULL, NULL, 0);
}

/*
 * Allocate and initialize an address space data structure.
 * We call hat_alloc to allow any machine dependent
 * information in the hat structure to be initialized.
 */
struct as *
as_alloc(void)
{
	struct as *as;

	as = kmem_cache_alloc(as_cache, KM_SLEEP);

	AS_CLRPGLCK(as);
	AS_CLRCLAIMGAP(as);
	AS_CLRUNMAPWAIT(as);
	as->a_vbits		= 0;
	as->a_hat		= NULL;
	as->a_hrm		= NULL;
	as->a_cache.seglast	= NULL;
	as->a_segs.list		= NULL;
	as->a_size		= 0;
	as->a_tail		= NULL;
	as->a_nsegs		= 0;
	as->a_lrep		= 0;
	as->a_hilevel		= 0;
	as->a_updatedir		= 0;
	as->a_objectdir		= NULL;
	as->a_sizedir		= 0;
	as->a_wpage		= NULL;
	as->a_nwpage		= 0;
	as->a_userlimit		= (caddr_t)USERLIMIT;

	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
	as->a_hat = hat_alloc(as);	/* create hat for default system mmu */
	AS_LOCK_EXIT(as, &as->a_lock);
	return (as);
}

/*
 * Free an address space data structure.
 * Need to free the hat first and then
 * all the segments on this as and finally
 * the space for the as struct itself.
 */
void
as_free(struct as *as)
{
	struct hat *hat = as->a_hat;

	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);
	hat_free_start(hat);
	while (AS_SEGP(as, as->a_segs) != NULL)
		seg_unmap(AS_SEGP(as, as->a_segs));

	hat_free_end(hat);

	/*
	 * SRM hook: provided for SRM accounting and control over process
	 * virtual memory usages.
	 * as_free() is entered when the entire address space, as, is being
	 * released. Even after the seg_unmap() calls in as_free(), as->a_size
	 * is still the size of the address space as it was at entry to
	 * as_free(), i.e. the sum of the s_size of each of the segments
	 * just unmapped from as.
	 * General notes: When memory is allocated in a non-atomic series of
	 * steps it is appropriate to use a SRM_LIMITMEMORY() call with
	 * LI_ENFORCE|LI_UPDATE flags to reserve the whole amount. Use
	 * LI_UPDATE to cancel the reservation in case of error or to adjust
	 * the reservation if the final allocation is different.
	 * SRM_LIMITMEMORY() calls are always made with as->a_lock held to
	 * ensure that the address space cannot change while it is running.
	 * There should be a SRM_LIMITMEMORY call in every place where
	 * as->a_size is changed.
	 */
	(void) SRM_LIMITMEMORY(as->a_size, as, LI_UPDATE | LI_FREE);

	if (as->a_lrep == AS_LREP_SKIPLIST) {
		kmem_cache_free(seg_skiplist_cache, as->a_segs.skiplist);
		kmem_cache_free(seg_skiplist_cache, as->a_cache.spath);
	}

	AS_LOCK_EXIT(as, &as->a_lock);

	/* /proc stuff */
	ASSERT(as->a_wpage == NULL && as->a_nwpage == 0);
	if (as->a_objectdir) {
		kmem_free(as->a_objectdir, as->a_sizedir * sizeof (vnode_t *));
		as->a_objectdir = NULL;
		as->a_sizedir = 0;
	}

	kmem_cache_free(as_cache, as);
}

int
as_dup(struct as *as, struct as **outas)
{
	struct as *newas;
	struct seg *seg, *newseg;
	int error;

	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);

	/*
	 * SRM hook: as_dup() is entered when the entire address space, as,
	 * is being duplicated to outas. During as_dup() (which usually
	 * happens as part of fork()) there is no need to worry about who
	 * owns the new memory - the new process will initially have the
	 * same SRM lnode uid as curproc - memory ownership totals will be
	 * adjusted when a process moves to a different lnode.
	 */
	if ((error = SRM_LIMITMEMORY(as->a_size, NULL,
	    LI_ENFORCE | LI_UPDATE | LI_ALLOC)) != 0) {
		AS_LOCK_EXIT(as, &as->a_lock);
		return (error);
	}

	as_clearwatch(as);
	newas = as_alloc();
	newas->a_userlimit = as->a_userlimit;
	AS_LOCK_ENTER(newas, &newas->a_lock, RW_WRITER);

	for (seg = AS_SEGP(as, as->a_segs); seg != NULL;
	    seg = AS_SEGP(as, seg->s_next)) {
		newseg = seg_alloc(newas, seg->s_base, seg->s_size);
		if (newseg == NULL) {
			/*
			 * SRM hook: release the amount reserved but not
			 * allocated, the as_free(newas) releases the rest.
			 */
			if (as->a_size > newas->a_size) {
				size_t a_sz;
				a_sz = as->a_size - newas->a_size;
				(void) SRM_LIMITMEMORY(a_sz, newas,
				    LI_UPDATE | LI_FREE);
			}

			AS_LOCK_EXIT(newas, &newas->a_lock);
			as_setwatch(as);
			AS_LOCK_EXIT(as, &as->a_lock);
			as_free(newas);
			return (-1);
		}
		if ((error = SEGOP_DUP(seg, newseg)) != 0) {
			/*
			 * SRM hook: release the amount reserved but not
			 * allocated, the as_free(newas) releases the rest.
			 */
			if (as->a_size > newas->a_size) {
				size_t a_sz;
				a_sz = as->a_size - newas->a_size;
				(void) SRM_LIMITMEMORY(a_sz, newas,
				    LI_UPDATE | LI_FREE);
			}

			/*
			 * We call seg_free() on the new seg
			 * because the segment is not set up
			 * completely; i.e. it has no ops.
			 */
			as_setwatch(as);
			AS_LOCK_EXIT(as, &as->a_lock);
			seg_free(newseg);
			AS_LOCK_EXIT(newas, &newas->a_lock);
			as_free(newas);
			return (error);
		}
		newas->a_size += seg->s_size;
	}

	error = hat_dup(as->a_hat, newas->a_hat, NULL, 0, HAT_DUP_ALL);
	AS_LOCK_EXIT(newas, &newas->a_lock);
	as_setwatch(as);
	AS_LOCK_EXIT(as, &as->a_lock);
	if (error != 0) {
		as_free(newas);
		return (error);
	}
	*outas = newas;
	return (0);
}

/*
 * Handle a ``fault'' at addr for size bytes.
 */
faultcode_t
as_fault(
	struct hat *hat,
	struct as *as,
	caddr_t addr,
	size_t size,
	enum fault_type type,
	enum seg_rw rw)
{
	struct seg *seg;
	caddr_t raddr;			/* rounded down addr */
	size_t rsize;			/* rounded up size */
	size_t ssize;
	faultcode_t res = 0;
	caddr_t addrsav;
	struct seg *segsav;
	int as_lock_held;
	klwp_t *lwp = ttolwp(curthread);

	ASSERT(as->a_hat == hat);
retry:
	/*
	 * Indicate that the lwp is not to be stopped while waiting
	 * for a pagefault.  This is to avoid deadlock while debugging
	 * a process via /proc over NFS (in particular).
	 */
	if (lwp != NULL)
		lwp->lwp_nostop++;

	/*
	 * same length must be used when we softlock and softunlock.
	 * We don't support softunlocking lengths less than
	 * the original length when there is largepage support.
	 * See seg_dev.c for more comments.
	 */
	switch (type) {

	case F_SOFTLOCK:
		CPU_STAT_ADD_K(cpu_vminfo.softlock, 1);
		break;

	case F_PROT:
		CPU_STAT_ADD_K(cpu_vminfo.prot_fault, 1);
		break;

	case F_INVAL:
		CPU_STAT_ENTER_K();
		CPU_STAT_ADDQ(CPU, cpu_vminfo.as_fault, 1);
		if (as == &kas)
			CPU_STAT_ADDQ(CPU, cpu_vminfo.kernel_asflt, 1);
		CPU_STAT_EXIT_K();
		break;
	}

	/* Kernel probe */
	TNF_PROBE_3(address_fault, "vm pagefault", /* CSTYLED */,
		tnf_opaque,	address,	addr,
		tnf_fault_type,	fault_type,	type,
		tnf_seg_access,	access,		rw);

	raddr = (caddr_t)((uintptr_t)addr & PAGEMASK);
	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
		(size_t)raddr;

	/*
	 * XXX -- Don't grab the as lock for segkmap. We should grab it for
	 * correctness, but then we could be stuck holding this lock for
	 * a LONG time if the fault needs to be resolved on a slow
	 * filesystem, and then no-one will be able to exec new commands,
	 * as exec'ing requires the write lock on the as.
	 */
	if (as == &kas && segkmap && segkmap->s_base <= raddr &&
	    raddr + size < segkmap->s_base + segkmap->s_size) {
		seg = segkmap;
		as_lock_held = 0;
	} else {
		AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
		seg = as_segat(as, raddr);
		if (seg == NULL) {
			AS_LOCK_EXIT(as, &as->a_lock);
			if (lwp != NULL)
				lwp->lwp_nostop--;
			return (FC_NOMAP);
		}
		as_lock_held = 1;
	}

	addrsav = raddr;
	segsav = seg;

	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
		if (raddr >= seg->s_base + seg->s_size) {
			seg = AS_SEGP(as, seg->s_next);  /* goto next seg */
			if (seg == NULL || raddr != seg->s_base) {
				res = FC_NOMAP;
				break;
			}
		}
		if (raddr + rsize > seg->s_base + seg->s_size)
			ssize = seg->s_base + seg->s_size - raddr;
		else
			ssize = rsize;
		res = SEGOP_FAULT(hat, seg, raddr, ssize, type, rw);
		if (res != 0)
			break;
	}

	/*
	 * If we were SOFTLOCKing and encountered a failure,
	 * we must SOFTUNLOCK the range we already did. (Maybe we
	 * should just panic if we are SOFTLOCKing or even SOFTUNLOCKing
	 * right here...)
	 */
	if (res != 0 && type == F_SOFTLOCK) {
		for (seg = segsav; addrsav < raddr; addrsav += ssize) {
			if (addrsav >= seg->s_base + seg->s_size)
				seg = AS_SEGP(as, seg->s_next);
			ASSERT(seg != NULL);
			/*
			 * Now call the fault routine again to perform the
			 * unlock using S_OTHER instead of the rw variable
			 * since we never got a chance to touch the pages.
			 */
			if (raddr > seg->s_base + seg->s_size)
				ssize = seg->s_base + seg->s_size - addrsav;
			else
				ssize = raddr - addrsav;
			(void) SEGOP_FAULT(hat, seg, addrsav, ssize,
			    F_SOFTUNLOCK, S_OTHER);
		}
	}
	if (as_lock_held)
		AS_LOCK_EXIT(as, &as->a_lock);
	if (lwp != NULL)
		lwp->lwp_nostop--;
	/*
	 * If the lower levels returned EDEADLK for a fault,
	 * It means that we should retry the fault.  Let's wait
	 * a bit also to let the deadlock causing condition clear.
	 * This is part of a gross hack to work around a design flaw
	 * in the ufs/sds logging code and should go away when the
	 * logging code is re-designed to fix the problem. See bug
	 * 4125102 for details of the problem.
	 */
	if (FC_ERRNO(res) == EDEADLK) {
		delay(deadlk_wait);
		res = 0;
		goto retry;
	}
	return (res);
}

/*
 * Asynchronous ``fault'' at addr for size bytes.
 */
faultcode_t
as_faulta(struct as *as, caddr_t addr, size_t size)
{
	struct seg *seg;
	caddr_t raddr;			/* rounded down addr */
	size_t rsize;			/* rounded up size */
	faultcode_t res = 0;
	klwp_t *lwp = ttolwp(curthread);

retry:
	/*
	 * Indicate that the lwp is not to be stopped while waiting
	 * for a pagefault.  This is to avoid deadlock while debugging
	 * a process via /proc over NFS (in particular).
	 */
	if (lwp != NULL)
		lwp->lwp_nostop++;

	raddr = (caddr_t)((uintptr_t)addr & PAGEMASK);
	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
		(size_t)raddr;

	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
	seg = as_segat(as, raddr);
	if (seg == NULL) {
		AS_LOCK_EXIT(as, &as->a_lock);
		if (lwp != NULL)
			lwp->lwp_nostop--;
		return (FC_NOMAP);
	}

	for (; rsize != 0; rsize -= PAGESIZE, raddr += PAGESIZE) {
		if (raddr >= seg->s_base + seg->s_size) {
			seg = AS_SEGP(as, seg->s_next);
			if (seg == NULL || addr != seg->s_base) {
				res = FC_NOMAP;
				break;
			}
		}
		res = SEGOP_FAULTA(seg, raddr);
		if (res != 0)
			break;
	}
	AS_LOCK_EXIT(as, &as->a_lock);
	if (lwp != NULL)
		lwp->lwp_nostop--;
	/*
	 * If the lower levels returned EDEADLK for a fault,
	 * It means that we should retry the fault.  Let's wait
	 * a bit also to let the deadlock causing condition clear.
	 * This is part of a gross hack to work around a design flaw
	 * in the ufs/sds logging code and should go away when the
	 * logging code is re-designed to fix the problem. See bug
	 * 4125102 for details of the problem.
	 */
	if (FC_ERRNO(res) == EDEADLK) {
		delay(deadlk_wait);
		res = 0;
		goto retry;
	}
	return (res);
}

/*
 * Set the virtual mapping for the interval from [addr : addr + size)
 * in address space `as' to have the specified protection.
 * It is ok for the range to cross over several segments,
 * as long as they are contiguous.
 */
int
as_setprot(struct as *as, caddr_t addr, size_t size, uint_t prot)
{
	struct seg *seg;
	size_t ssize;
	caddr_t raddr;			/* rounded down addr */
	size_t rsize;			/* rounded up size */
	int error = 0;

setprot_top:
	raddr = (caddr_t)((uintptr_t)addr & PAGEMASK);
	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
		(size_t)raddr;

	if (raddr + rsize < raddr)		/* check for wraparound */
		return (ENOMEM);

	/*
	 * This is ugly as sin...
	 * Normally, even though this is a modification operation,
	 * we only acquire the address space readers lock.
	 * The segment layer will acquire an exclusive lock of
	 * its own devising to actually perform the operation.
	 * However, if the address space has watchpoints present,
	 * we must acquire the writer lock on the address space for
	 * the benefit of as_clearwatchprot() and as_setwatchprot().
	 */
	AS_LOCK_ENTER(as, &as->a_lock, as->a_wpage? RW_WRITER : RW_READER);
	seg = as_segat(as, raddr);
	if (seg == NULL) {
		AS_LOCK_EXIT(as, &as->a_lock);
		return (ENOMEM);
	}

	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
		if (raddr >= seg->s_base + seg->s_size) {
			seg = AS_SEGP(as, seg->s_next);
			if (seg == NULL || raddr != seg->s_base) {
				error = ENOMEM;
				break;
			}
		}
		if ((raddr + rsize) > (seg->s_base + seg->s_size))
			ssize = seg->s_base + seg->s_size - raddr;
		else
			ssize = rsize;
		as_clearwatchprot(as, seg, raddr, ssize);
		error = SEGOP_SETPROT(seg, raddr, ssize, prot);
		as_setwatchprot(as, seg, raddr, ssize, prot);

		if (error == IE_NOMEM) {
			error = EAGAIN;
			break;
		}

		if (error == EAGAIN) {
			mutex_enter(&as->a_contents);
			AS_LOCK_EXIT(as, &as->a_lock);
			if (AS_ISUNMAPWAIT(as) == 0)
				cv_broadcast(&as->a_cv);
			AS_SETUNMAPWAIT(as);
			while (AS_ISUNMAPWAIT(as))
				cv_wait(&as->a_cv, &as->a_contents);
			mutex_exit(&as->a_contents);
			goto setprot_top;
		} else if (error != 0)
			break;
	}
	AS_LOCK_EXIT(as, &as->a_lock);
	return (error);
}

/*
 * Check to make sure that the interval [addr, addr + size)
 * in address space `as' has at least the specified protection.
 * It is ok for the range to cross over several segments, as long
 * as they are contiguous.
 */
int
as_checkprot(struct as *as, caddr_t addr, size_t size, uint_t prot)
{
	struct seg *seg;
	size_t ssize;
	caddr_t raddr;			/* rounded down addr */
	size_t rsize;			/* rounded up size */
	int error = 0;

	raddr = (caddr_t)((uintptr_t)addr & PAGEMASK);
	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
		(size_t)raddr;

	if (raddr + rsize < raddr)		/* check for wraparound */
		return (ENOMEM);

	/*
	 * This is ugly as sin...
	 * Normally, we only acquire the address space readers lock.
	 * However, if the address space has watchpoints present,
	 * we must acquire the writer lock on the address space for
	 * the benefit of as_clearwatchprot() and as_setwatchprot().
	 */
	AS_LOCK_ENTER(as, &as->a_lock, as->a_wpage? RW_WRITER : RW_READER);
	seg = as_segat(as, raddr);
	if (seg == NULL) {
		AS_LOCK_EXIT(as, &as->a_lock);
		return (ENOMEM);
	}

	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
		if (raddr >= seg->s_base + seg->s_size) {
			seg = AS_SEGP(as, seg->s_next);
			if (seg == NULL || raddr != seg->s_base) {
				error = ENOMEM;
				break;
			}
		}
		if ((raddr + rsize) > (seg->s_base + seg->s_size))
			ssize = seg->s_base + seg->s_size - raddr;
		else
			ssize = rsize;

		as_clearwatchprot(as, seg, raddr, ssize);
		error = SEGOP_CHECKPROT(seg, raddr, ssize, prot);
		as_setwatchprot(as, seg, raddr, ssize, prot);
		if (error != 0)
			break;
	}
	AS_LOCK_EXIT(as, &as->a_lock);
	return (error);
}

int
as_unmap(struct as *as, caddr_t addr, size_t size)
{
	struct seg *seg, *seg_next;
	caddr_t raddr, eaddr;
	size_t ssize;
	int err;
	size_t srm_size = 0;

top:
	raddr = (caddr_t)((uintptr_t)addr & PAGEMASK);
	eaddr = (caddr_t)(((uintptr_t)(addr + size) + PAGEOFFSET) & PAGEMASK);

	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);

	as->a_updatedir = 1;	/* inform /proc */

	/*
	 * Use as_findseg to find the first segment in the range, then
	 * step through the segments in order, following s_next.
	 */
	for (seg = as_findseg(as, raddr, 0); seg != NULL; seg = seg_next) {
		if (eaddr <= seg->s_base)
			break;		/* eaddr was in a gap; all done */

		/* this is implied by the test above */
		ASSERT(raddr < eaddr);

		if (raddr < seg->s_base)
			raddr = seg->s_base; 	/* raddr was in a gap */

		if (eaddr > (seg->s_base + seg->s_size))
			ssize = seg->s_base + seg->s_size - raddr;
		else
			ssize = eaddr - raddr;

		/*
		 * Save next segment pointer since seg can be
		 * destroyed during the segment unmap operation.
		 */
		seg_next = AS_SEGP(as, seg->s_next);

		as_clearwatchprot(as, seg, raddr, ssize);
		err = SEGOP_UNMAP(seg, raddr, ssize);
		if (err == EAGAIN) {
			/*
			 * Segment doesn't want to be unmapped now
			 * so wait; it'll wake us up when ready.
			 */
			as_setwatch(as);
			mutex_enter(&as->a_contents);
			AS_LOCK_EXIT(as, &as->a_lock);
			/*
			 * Anyone already asleep on this cv should
			 * be awakened because they are waiting for
			 * this condition to arise.
			 */
			if (AS_ISUNMAPWAIT(as) == 0)
				cv_broadcast(&as->a_cv);
			AS_SETUNMAPWAIT(as);
			while (AS_ISUNMAPWAIT(as))
				cv_wait(&as->a_cv, &as->a_contents);
			mutex_exit(&as->a_contents);
			goto top;
		} else if (err) {
			as_setwatch(as);
			AS_LOCK_EXIT(as, &as->a_lock);
			/*
			 * SRM hook: see comments in as_free() above.
			 */
			if (srm_size)
				(void) SRM_LIMITMEMORY(srm_size, as,
				    LI_UPDATE | LI_FREE);
			return (-1);
		}

		as->a_size -= ssize;
		/*
		 * SRM hook: just accumulate the change here inside the loop
		 * and then apply it at the error return above or the normal
		 * return below.
		 */
		srm_size += (size_t)ssize;
		raddr += ssize;
	}
	AS_LOCK_EXIT(as, &as->a_lock);
	/*
	 * SRM hook: as_unmap() is entered when size extra bytes have been
	 * requested to be unmapped at address addr in curproc. In the segment
	 * scan loop segments that overlap the target area are found, srm_size
	 * is the sum of the actual sizes freed from such segments.
	 */
	if (srm_size)
		(void) SRM_LIMITMEMORY(srm_size, as, LI_UPDATE | LI_FREE);
	return (0);
}

int
as_map(
	struct as *as,
	caddr_t addr,
	size_t size,
	int (*crfp)(),
	void *argsp)
{
	struct seg *seg;
	caddr_t raddr;			/* rounded down addr */
	size_t rsize;			/* rounded up size */
	int error;

	raddr = (caddr_t)((uintptr_t)addr & PAGEMASK);
	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
		(size_t)raddr;

	AS_LOCK_ENTER(as, &as->a_lock, RW_WRITER);

	/*
	 * check for wrap around
	 */
	if ((raddr + rsize < raddr) || (as->a_size > (ULONG_MAX - size))) {
		AS_LOCK_EXIT(as, &as->a_lock);
		return (ENOMEM);
	}

	as->a_updatedir = 1;	/* inform /proc */

	if (as != &kas &&
	    as->a_size + rsize > P_CURLIMIT(curproc, RLIMIT_VMEM)) {
		AS_LOCK_EXIT(as, &as->a_lock);
		return (ENOMEM);
	}

	/*
	 * SRM hook: as_map() is entered when size extra bytes have been
	 * requested for curproc. rsize is the rounded size that will
	 * actually be allocated. There are two error paths below where the
	 * rsize reservation is cancelled; the third error path is via
	 * as_unmap() which is expected to do the correct adjustment since
	 * the address space did actually reach the intended size before
	 * the as_ctl() error.
	 */
	if ((error = SRM_LIMITMEMORY((size_t)rsize, as,
	    LI_ENFORCE | LI_UPDATE | LI_ALLOC)) != 0) {
		AS_LOCK_EXIT(as, &as->a_lock);
		return (error);
	}

	seg = seg_alloc(as, addr, size);
	if (seg == NULL) {
		/*
		 * SRM hook: Undo the initial reservation.
		 */
		(void) SRM_LIMITMEMORY((size_t)rsize, as, LI_UPDATE | LI_FREE);

		AS_LOCK_EXIT(as, &as->a_lock);
		return (ENOMEM);
	}

	error = (*crfp)(seg, argsp);

	/*
	 * If some error occurred during the create function, destroy
	 * this segment.  Otherwise, if the address space is locked,
	 * establish memory locks for the new segment.
	 */
	if (error != 0) {
		seg_free(seg);
		/*
		 * SRM hook: On error undo the initial reservation.
		 */
		(void) SRM_LIMITMEMORY((size_t)rsize, as, LI_UPDATE | LI_FREE);
		AS_LOCK_EXIT(as, &as->a_lock);
	} else {
		/*
		 * Add size now so as_unmap will work if as_ctl fails.
		 */
		as->a_size += rsize;

		as_setwatch(as);
		mutex_enter(&as->a_contents);
		if (AS_ISPGLCK(as)) {
			mutex_exit(&as->a_contents);
			AS_LOCK_EXIT(as, &as->a_lock);
			error = as_ctl(as, addr, size, MC_LOCK, 0, 0, NULL, 0);
			if (error != 0)
				(void) as_unmap(as, addr, size);
		} else {
			mutex_exit(&as->a_contents);
			AS_LOCK_EXIT(as, &as->a_lock);
		}
	}
	return (error);
}

/*
 * Find a hole of at least size minlen within [base, base + len).
 * If flags specifies AH_HI, the hole will have the highest possible address
 * in the range. Otherwise, it will have the lowest possible address.
 * If flags specifies AH_CONTAIN, the hole will contain the address addr.
 * If an adequate hole is found, base and len are set to reflect the part of
 * the hole that is within range, and 0 is returned. Otherwise,
 * -1 is returned.
 * XXX This routine is not correct when base+len overflows caddr_t.
 */
/* VARARGS5 */
int
as_gap(struct as *as, size_t minlen, caddr_t *basep,
    size_t *lenp, uint_t flags, caddr_t addr)
{
	caddr_t lobound = *basep;
	caddr_t hibound = lobound + *lenp;
	struct seg *lseg, *hseg;
	caddr_t lo, hi;
	int forward;
	caddr_t save_base;
	size_t save_len;

	save_base = *basep;
	save_len = *lenp;
	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
	if (AS_SEGP(as, as->a_segs) == NULL) {
		if (valid_va_range(basep, lenp, minlen, flags & AH_DIR)) {
			AS_LOCK_EXIT(as, &as->a_lock);
			return (0);
		} else {
			AS_LOCK_EXIT(as, &as->a_lock);
			*basep = save_base;
			*lenp = save_len;
			return (-1);
		}
	}

	/*
	 * Set up to iterate over all the inter-segment holes in the given
	 * direction.  lseg is NULL for the lowest-addressed hole and hseg is
	 * NULL for the highest-addressed hole.  If moving backwards, we reset
	 * sseg to denote the highest-addressed segment.
	 */
	forward = (flags & AH_DIR) == AH_LO;
	if (forward) {
		hseg = as_findseg(as, lobound, 1);
		lseg = hseg->s_prev;
	} else {
		hseg = as_findseg(as, hibound, 1);
		if (hseg->s_base + hseg->s_size < hibound) {
			lseg = hseg;
			hseg = NULL;
		} else {
			lseg = hseg->s_prev;
		}
	}

	for (;;) {
		/*
		 * Set lo and hi to the hole's boundaries.  (We should really
		 * use MAXADDR in place of hibound in the expression below,
		 * but can't express it easily; using hibound in its place is
		 * harmless.)
		 */
		lo = (lseg == NULL) ? 0 : lseg->s_base + lseg->s_size;
		hi = (hseg == NULL) ? hibound : hseg->s_base;
		/*
		 * If the iteration has moved past the interval from lobound
		 * to hibound it's pointless to continue.
		 */
		if ((forward && lo > hibound) || (!forward && hi < lobound))
			break;
		else if (lo > hibound || hi < lobound)
			goto cont;
		/*
		 * Candidate hole lies at least partially within the allowable
		 * range.  Restrict it to fall completely within that range,
		 * i.e., to [max(lo, lobound), min(hi, hibound)].
		 */
		if (lo < lobound)
			lo = lobound;
		if (hi > hibound)
			hi = hibound;
		/*
		 * Verify that the candidate hole is big enough and meets
		 * hardware constraints.
		 */
		*basep = lo;
		*lenp = hi - lo;
		if (valid_va_range(basep, lenp, minlen,
		    forward ? AH_LO : AH_HI) &&
		    ((flags & AH_CONTAIN) == 0 ||
		    (*basep <= addr && *basep + *lenp > addr))) {
			AS_LOCK_EXIT(as, &as->a_lock);
			return (0);
		}

	cont:
		/*
		 * Move to the next hole.
		 */
		if (forward) {
			lseg = hseg;
			if (lseg == NULL)
				break;
			hseg = AS_SEGP(as, hseg->s_next);
		} else {
			hseg = lseg;
			if (hseg == NULL)
				break;
			lseg = lseg->s_prev;
		}
	}
	*basep = save_base;
	*lenp = save_len;
	AS_LOCK_EXIT(as, &as->a_lock);
	return (-1);
}

/*
 * Return the next range within [base, base + len) that is backed
 * with "real memory".  Skip holes and non-seg_vn segments.
 * We're lazy and only return one segment at a time.
 */
int
as_memory(struct as *as, caddr_t *basep, size_t *lenp)
{
	extern struct seg_ops segspt_shmops;	/* needs a header file */
	struct seg *seg;
	caddr_t addr, eaddr;
	caddr_t segend;

	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);

	addr = *basep;
	eaddr = addr + *lenp;

	if (seg = as_findseg(as, addr, 0)) {
		addr = MAX(seg->s_base, addr);
	}

	for (;;) {
		if (seg == NULL || addr >= eaddr || eaddr <= seg->s_base) {
			AS_LOCK_EXIT(as, &as->a_lock);
			return (EINVAL);
		}

		if (seg->s_ops == &segvn_ops) {
			segend = seg->s_base + seg->s_size;
			break;
		}

		/*
		 * We do ISM by looking into the private data
		 * to determine the real size of the segment.
		 */
		if (seg->s_ops == &segspt_shmops) {
			segend = seg->s_base + spt_realsize(seg);
			if (addr < segend)
				break;
		}

		seg = AS_SEGP(as, seg->s_next);

		if (seg != NULL)
			addr = seg->s_base;
	}

	*basep = addr;

	if (segend > eaddr)
		*lenp = eaddr - addr;
	else
		*lenp = segend - addr;

	AS_LOCK_EXIT(as, &as->a_lock);
	return (0);
}

/*
 * Swap the pages associated with the address space as out to
 * secondary storage, returning the number of bytes actually
 * swapped.
 *
 * The value returned is intended to correlate well with the process's
 * memory requirements.  Its usefulness for this purpose depends on
 * how well the segment-level routines do at returning accurate
 * information.
 */
size_t
as_swapout(struct as *as)
{
	struct seg *seg;
	size_t swpcnt = 0;

	/*
	 * Kernel-only processes have given up their address
	 * spaces.  Of course, we shouldn't be attempting to
	 * swap out such processes in the first place...
	 */
	if (as == NULL)
		return (0);

	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);

	/*
	 * Free all mapping resources associated with the address
	 * space.  The segment-level swapout routines capitalize
	 * on this unmapping by scavanging pages that have become
	 * unmapped here.
	 */
	hat_swapout(as->a_hat);

	/*
	 * Call the swapout routines of all segments in the address
	 * space to do the actual work, accumulating the amount of
	 * space reclaimed.
	 */
	for (seg = AS_SEGP(as, as->a_segs); seg != NULL;
	    seg = AS_SEGP(as, seg->s_next)) {
		struct seg_ops *ov = seg->s_ops;

		/*
		 * We have to check to see if the seg has
		 * an ops vector because the seg may have
		 * been in the middle of being set up when
		 * the process was picked for swapout.
		 */
		if ((ov != NULL) && (ov->swapout != NULL))
			swpcnt += SEGOP_SWAPOUT(seg);
	}
	AS_LOCK_EXIT(as, &as->a_lock);
	return (swpcnt);
}

/*
 * Determine whether data from the mappings in interval [addr, addr + size)
 * are in the primary memory (core) cache.
 */
int
as_incore(struct as *as, caddr_t addr,
    size_t size, char *vec, size_t *sizep)
{
	struct seg *seg;
	size_t ssize;
	caddr_t raddr;		/* rounded down addr */
	size_t rsize;		/* rounded up size */
	size_t isize;			/* iteration size */
	int error = 0;		/* result, assume success */

	*sizep = 0;
	raddr = (caddr_t)((uintptr_t)addr & PAGEMASK);
	rsize = ((((size_t)addr + size) + PAGEOFFSET) & PAGEMASK) -
		(size_t)raddr;

	if (raddr + rsize < raddr)		/* check for wraparound */
		return (ENOMEM);

	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
	seg = as_segat(as, raddr);
	if (seg == NULL) {
		AS_LOCK_EXIT(as, &as->a_lock);
		return (-1);
	}

	for (; rsize != 0; rsize -= ssize, raddr += ssize) {
		if (raddr >= seg->s_base + seg->s_size) {
			seg = AS_SEGP(as, seg->s_next);
			if (seg == NULL || raddr != seg->s_base) {
				error = -1;
				break;
			}
		}
		if ((raddr + rsize) > (seg->s_base + seg->s_size))
			ssize = seg->s_base + seg->s_size - raddr;
		else
			ssize = rsize;
		*sizep += isize = SEGOP_INCORE(seg, raddr, ssize, vec);
		if (isize != ssize) {
			error = -1;
			break;
		}
		vec += btopr(ssize);
	}
	AS_LOCK_EXIT(as, &as->a_lock);
	return (error);
}

/*
 * Cache control operations over the interval [addr, addr + size) in
 * address space "as".
 */
int
as_ctl(struct as *as, caddr_t addr, size_t size, int func, int attr,
    uintptr_t arg, ulong_t *lock_map, size_t pos)
{
	struct seg *seg;	/* working segment */
	caddr_t raddr;		/* rounded down addr */
	size_t rsize;		/* rounded up size */
	size_t ssize;		/* size of seg */
	int error = 0;			/* result */

retry:
	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);

	/*
	 * If these are address space lock/unlock operations, loop over
	 * all segments in the address space, as appropriate.
	 */
	if (func == MC_LOCKAS) {
		if (arg & MCL_FUTURE) {
			mutex_enter(&as->a_contents);
			AS_SETPGLCK(as);
			mutex_exit(&as->a_contents);
		}
		if ((arg & MCL_CURRENT) == 0) {
			AS_LOCK_EXIT(as, &as->a_lock);
			return (0);
		}

		for (seg = AS_SEGP(as, as->a_segs); seg != NULL;
		    seg = AS_SEGP(as, seg->s_next)) {
			error = SEGOP_LOCKOP(seg, seg->s_base,
			    seg->s_size, attr, MC_LOCK, lock_map, pos);
			if (error != 0)
				break;
			pos += seg_pages(seg);
		}

		AS_LOCK_EXIT(as, &as->a_lock);
		goto lockerr;
	} else if (func == MC_UNLOCKAS) {
		mutex_enter(&as->a_contents);
		AS_CLRPGLCK(as);
		mutex_exit(&as->a_contents);

		for (seg = AS_SEGP(as, as->a_segs); seg != NULL;
		    seg = AS_SEGP(as, seg->s_next)) {
			error = SEGOP_LOCKOP(seg, seg->s_base,
			    seg->s_size, attr, MC_UNLOCK, NULL, 0);
			if (error != 0)
				break;
		}

		AS_LOCK_EXIT(as, &as->a_lock);
		goto lockerr;
	}

	/*
	 * Normalize addresses and sizes.
	 */
	raddr = (caddr_t)((uintptr_t)addr & PAGEMASK);
	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
		(size_t)raddr;

	if (raddr + rsize < raddr) {		/* check for wraparound */
		AS_LOCK_EXIT(as, &as->a_lock);
		return (ENOMEM);
	}

	/*
	 * Get initial segment.
	 */
	if ((seg = as_segat(as, raddr)) == NULL) {
		AS_LOCK_EXIT(as, &as->a_lock);
		return (ENOMEM);
	}

	/*
	 * Loop over all segments.  If a hole in the address range is
	 * discovered, then fail.  For each segment, perform the appropriate
	 * control operation.
	 */
	while (rsize != 0) {

		/*
		 * Make sure there's no hole, calculate the portion
		 * of the next segment to be operated over.
		 */
		if (raddr >= seg->s_base + seg->s_size) {
			seg = AS_SEGP(as, seg->s_next);
			if (seg == NULL || raddr != seg->s_base) {
				AS_LOCK_EXIT(as, &as->a_lock);
				return (ENOMEM);
			}
		}
		if ((raddr + rsize) > (seg->s_base + seg->s_size))
			ssize = seg->s_base + seg->s_size - raddr;
		else
			ssize = rsize;

		/*
		 * Dispatch on specific function.
		 */
		switch (func) {

		/*
		 * Synchronize cached data from mappings with backing
		 * objects.
		 */
		case MC_SYNC:
			if (error = SEGOP_SYNC(seg, raddr, ssize,
			    attr, (uint_t)arg)) {
				AS_LOCK_EXIT(as, &as->a_lock);
				return (error);
			}
			break;

		/*
		 * Lock pages in memory.
		 */
		case MC_LOCK:
			if (error = SEGOP_LOCKOP(seg, raddr, ssize,
			    attr, func, lock_map, pos)) {
				AS_LOCK_EXIT(as, &as->a_lock);
				goto lockerr;
			}
			break;

		/*
		 * Unlock mapped pages.
		 */
		case MC_UNLOCK:
			(void) SEGOP_LOCKOP(seg, raddr, ssize, attr, func,
			    (ulong_t *)NULL, (size_t)NULL);
			break;

		/*
		 * Store VM advise for mapped pages in segment layer.
		 */
		case MC_ADVISE:
			if (error = SEGOP_ADVISE(seg, raddr, ssize,
			    (uint_t)arg)) {
				AS_LOCK_EXIT(as, &as->a_lock);
				return (error);
			}
			break;

		/*
		 * Can't happen.
		 */
		default:
			cmn_err(CE_PANIC, "as_ctl: bad operation %d", func);
			/* NOTREACHED */
		}

		rsize -= ssize;
		raddr += ssize;
	}
	AS_LOCK_EXIT(as, &as->a_lock);
	return (0);
lockerr:
	/*
	 * If the lower levels returned EDEADLK for a segment lockop,
	 * it means that we should retry the operation.  Let's wait
	 * a bit also to let the deadlock causing condition clear.
	 * This is part of a gross hack to work around a design flaw
	 * in the ufs/sds logging code and should go away when the
	 * logging code is re-designed to fix the problem. See bug
	 * 4125102 for details of the problem.
	 */
	if (error == EDEADLK) {
		delay(deadlk_wait);
		error = 0;
		goto retry;
	}
	return (error);
}

/*
 * Special code for exec to move the stack segment from its interim
 * place in the old address to the right place in the new address space.
 */
/*ARGSUSED*/
int
as_exec(struct as *oas, caddr_t ostka, size_t stksz,
    struct as *nas, caddr_t nstka, uint_t hatflag)
{
	struct seg *stkseg;

	AS_LOCK_ENTER(oas, &oas->a_lock, RW_WRITER);
	ASSERT(oas->a_wpage == NULL && oas->a_nwpage == 0);
	stkseg = as_removeseg(oas, ostka);
	ASSERT(stkseg != NULL);
	ASSERT(stkseg->s_base == ostka && stkseg->s_size == stksz);
	stkseg->s_as = nas;
	stkseg->s_base = nstka;

	/*
	 * It's ok to lock the address space we are about to exec to.
	 */
	AS_LOCK_ENTER(nas, &nas->a_lock, RW_WRITER);
	ASSERT(nas->a_wpage == NULL && nas->a_nwpage == 0);
	nas->a_size += stkseg->s_size;
	oas->a_size -= stkseg->s_size;
	(void) as_addseg(nas, stkseg);
	AS_LOCK_EXIT(nas, &nas->a_lock);
	AS_LOCK_EXIT(oas, &oas->a_lock);
	return (0);
}

static int
f_decode(faultcode_t fault_err)
{
	int error = 0;

	switch (FC_CODE(fault_err)) {
	case FC_OBJERR:
		error = FC_ERRNO(fault_err);
		break;
	case FC_PROT:
		error = EACCES;
		break;
	default:
		error = EFAULT;
		break;
	}
	return (error);
}

/*
 * lock pages in a given address space. Return shadow list. If
 * the list is NULL, the MMU mapping is also locked.
 */
int
as_pagelock(struct as *as, struct page ***ppp, caddr_t addr,
    size_t size, enum seg_rw rw)
{
	size_t rsize;
	caddr_t base;
	caddr_t raddr;
	faultcode_t fault_err;
	struct seg *seg;
	int res;

	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_START,
	    "as_pagelock_start: addr %x size %d", addr, size);

	/*
	 * if the request crosses two segments let
	 * as_fault handle it.
	 */
	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
	seg = as_findseg(as, addr, 0);
	if ((seg == NULL) || ((base = seg->s_base) > addr) ||
	    (addr + size) > base + seg->s_size) {
		AS_LOCK_EXIT(as, &as->a_lock);
		goto slow;
	}

	raddr = (caddr_t)((uintptr_t)addr & PAGEMASK);
	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
		(size_t)raddr;

	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_START,
	    "seg_lock_1_start: raddr %x rsize %d", raddr, rsize);

	/*
	 * try to lock pages and pass back shadow list
	 */
	res = SEGOP_PAGELOCK(seg, raddr, rsize, ppp, L_PAGELOCK, rw);

	TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_END, "seg_lock_1_end");
	AS_LOCK_EXIT(as, &as->a_lock);
	if (res == 0) {
		return (0);
	} else if (res == ENOTSUP) {
		goto slow;
	}

	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_FAULT_START,
	    "as_fault_start: addr %x size %d", addr, size);

	/*
	 * we might get here because of some COW fault or non
	 * existing page. Let as_fault deal with it. Just load
	 * the page, don't lock the MMU mapping.
	 */
	fault_err = as_fault(as->a_hat, as, addr, size, F_INVAL, rw);
	if (fault_err != 0) {
		return (f_decode(fault_err));
	}

	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_START,
	    "seg_lock_2_start: raddr %x rsize %d", raddr, rsize);

	/*
	 * try fast path again
	 */
	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
	res = SEGOP_PAGELOCK(seg, raddr, rsize, ppp, L_PAGELOCK, rw);
	AS_LOCK_EXIT(as, &as->a_lock);

	TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_SEG_LOCK_END, "seg_lock_2_end");
	if (res == 0) {
		return (0);
	}
slow:
	/*
	 * load the page and lock the MMU mapping.
	 */
	fault_err = as_fault(as->a_hat, as, addr, size, F_SOFTLOCK, rw);
	if (fault_err != 0) {
		return (f_decode(fault_err));
	}
	*ppp = NULL;

	TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_LOCK_END, "as_pagelock_end");
	return (0);
}

/*
 * unlock pages in a given address range
 */
void
as_pageunlock(struct as *as, struct page **pp, caddr_t addr, size_t size,
    enum seg_rw rw)
{
	struct seg *seg;
	size_t rsize;
	caddr_t raddr;

	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_START,
	    "as_pageunlock_start: addr %x size %d", addr, size);

	/*
	 * if the shadow list is NULL, as_pagelock was
	 * falling back to as_fault
	 */
	if (pp == NULL) {
		(void) as_fault(as->a_hat, as, addr, size, F_SOFTUNLOCK, rw);
		return;
	}
	raddr = (caddr_t)((uintptr_t)addr & PAGEMASK);
	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
		(size_t)raddr;
	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
	seg = as_findseg(as, addr, 0);
	ASSERT(seg);
	TRACE_2(TR_FAC_PHYSIO, TR_PHYSIO_SEG_UNLOCK_START,
	    "seg_unlock_start: raddr %x rsize %d", raddr, rsize);
	SEGOP_PAGELOCK(seg, raddr, rsize, &pp, L_PAGEUNLOCK, rw);
	AS_LOCK_EXIT(as, &as->a_lock);
	TRACE_0(TR_FAC_PHYSIO, TR_PHYSIO_AS_UNLOCK_END, "as_pageunlock_end");
}

/*
 * reclaim cached pages in a given address range
 */
void
as_pagereclaim(struct as *as, struct page **pp, caddr_t addr,
    size_t size, enum seg_rw rw)
{
	struct seg *seg;
	size_t rsize;
	caddr_t raddr;

	ASSERT(AS_READ_HELD(as, &as->a_lock));
	ASSERT(pp != NULL);

	raddr = (caddr_t)((uintptr_t)addr & PAGEMASK);
	rsize = (((size_t)(addr + size) + PAGEOFFSET) & PAGEMASK) -
		(size_t)raddr;
	seg = as_findseg(as, addr, 0);
	ASSERT(seg);
	SEGOP_PAGELOCK(seg, raddr, rsize, &pp, L_PAGERECLAIM, rw);
}

#define	MAXPAGEFLIP	4
#define	MAXPAGEFLIPSIZ	MAXPAGEFLIP*PAGESIZE

/*
 * This function tries to flip pages behind (as_to, addr_to) and
 * (kas, kaddr) of "size" bytes.
 * It serves as a fast path for "copyout". The only difference is
 * that "kaddr" will lose its original data content and get back what
 * was behind (as_to, addr_to).
 *
 * Since it tries to emulate copyout semantic, the destination has to
 * be user-writable.
 * On exit, the number of bytes that failed to flip is returned in
 * "size". The caller should use copyout to complete them.
 *
 * Error return:
 * FC_NOMAP: no valid user mapping loaded.
 * FC_PROT: protection violation encountered.
 * FC_OBJERR: page busy.
 * FC_NOSUPPORT: operation not supported on this platform.
 *
 * Assumption:
 * Caller makes sure it can give away the pages behind kaddr.
 */
faultcode_t
as_pageflip(struct as *as_to, caddr_t addr_to, caddr_t kaddr, size_t *sizep)
{
	struct	page *pp_to[MAXPAGEFLIP], *pp_from[MAXPAGEFLIP];
	ssize_t	i;
	size_t	total = *sizep;
	faultcode_t	res;
	extern	void page_flip();

	ASSERT(as_to == curproc->p_as);
	/* Otherwise we have to hold the as lock */

	if (as_to->a_wpage)	/* watchpoints */
		return (FC_NOSUPPORT);

	if (total > MAXPAGEFLIPSIZ)
		*sizep = MAXPAGEFLIPSIZ;

	/*
	 * XXX - Will this work on every segment type?
	 * (E.g. it will break any segment driver that caches
	 * mappings.)
	 *
	 * If we are not sure, we can always check the segment type.
	 */
	res = hat_pageflip(as_to->a_hat, addr_to, kaddr, sizep,
	    &pp_to[0], &pp_from[0]);
	if (total > MAXPAGEFLIPSIZ) {
		i = MAXPAGEFLIP - (*sizep >> PAGESHIFT);
		*sizep += total - MAXPAGEFLIPSIZ;
	} else {
		i = (total - *sizep) >> PAGESHIFT;
	}
	while (--i >= 0) {
		ASSERT(PAGE_EXCL(pp_to[i]));
		ASSERT(PAGE_EXCL(pp_from[i]));
		(void) page_flip(pp_to[i], pp_from[i]);
	}

	return (res);
}

/*
 * Support functions for the watchpoint facility.
 * They are here to support the watchpoint functionality.
 * On a system without watchpoint support, they would do nothing.
 */

/*
 * Setup all of the uninitialized watched pages that we can.
 */
void
as_setwatch(struct as *as)
{
	struct watched_page *pwp;
	struct seg *seg;
	caddr_t vaddr;
	uint_t prot;

	if ((pwp = as->a_wpage) == NULL)
		return;

	ASSERT(AS_WRITE_HELD(as, &as->a_lock));

	do {
		vaddr = pwp->wp_vaddr;
		if (pwp->wp_oprot != 0 ||	/* already set up */
		    (seg = as_segat(as, vaddr)) == NULL ||
		    SEGOP_GETPROT(seg, vaddr, 0, &prot) != 0)
			continue;

		pwp->wp_oprot = prot;
		if (pwp->wp_read)
			prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
		if (pwp->wp_write)
			prot &= ~PROT_WRITE;
		if (pwp->wp_exec)
			prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
		if (!(pwp->wp_flags & WP_NOWATCH) && prot != pwp->wp_oprot)
			(void) SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot);
		pwp->wp_prot = prot;
	} while ((pwp = pwp->wp_forw) != as->a_wpage);
}

/*
 * Clear all of the watched pages in the address space.
 */
void
as_clearwatch(struct as *as)
{
	struct watched_page *pwp;
	struct seg *seg;
	caddr_t vaddr;
	uint_t prot;

	if ((pwp = as->a_wpage) == NULL)
		return;

	ASSERT(AS_WRITE_HELD(as, &as->a_lock));

	do {
		vaddr = pwp->wp_vaddr;
		if (pwp->wp_oprot == 0 ||	/* not set up */
		    (seg = as_segat(as, vaddr)) == NULL)
			continue;

		if ((prot = pwp->wp_oprot) != pwp->wp_prot)
			(void) SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot);
		pwp->wp_oprot = 0;
		pwp->wp_prot = 0;
	} while ((pwp = pwp->wp_forw) != as->a_wpage);
}

/*
 * Force a new setup for all the watched pages in the range.
 */
static void
as_setwatchprot(struct as *as, struct seg *seg, caddr_t addr, size_t size,
    uint_t prot)
{
	struct watched_page *pwp;
	caddr_t eaddr = addr + size;
	caddr_t vaddr;

	if ((pwp = as->a_wpage) == NULL)
		return;

	ASSERT(AS_WRITE_HELD(as, &as->a_lock));

	do {
		vaddr = pwp->wp_vaddr;
		if (eaddr <= vaddr)
			break;
		if (addr >= vaddr + PAGESIZE)
			continue;

		pwp->wp_oprot = prot;
		if (pwp->wp_read)
			prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
		if (pwp->wp_write)
			prot &= ~PROT_WRITE;
		if (pwp->wp_exec)
			prot &= ~(PROT_READ|PROT_WRITE|PROT_EXEC);
		if (!(pwp->wp_flags & WP_NOWATCH) && prot != pwp->wp_oprot)
			(void) SEGOP_SETPROT(seg, vaddr, PAGESIZE, prot);
		pwp->wp_prot = prot;
	} while ((pwp = pwp->wp_forw) != as->a_wpage);
}

/*
 * Clear all of the watched pages in the range.
 */
static void
as_clearwatchprot(struct as *as, struct seg *seg, caddr_t addr, size_t size)
{
	caddr_t eaddr = addr + size;
	struct watched_page *pwp;
	uint_t prot;

	if ((pwp = as->a_wpage) == NULL)
		return;

	ASSERT(AS_WRITE_HELD(as, &as->a_lock));

	do {
		if (eaddr <= pwp->wp_vaddr)
			break;
		if (addr >= pwp->wp_vaddr + PAGESIZE)
			continue;
		if ((prot = pwp->wp_oprot) != 0) {
			if (prot != pwp->wp_prot)
				(void) SEGOP_SETPROT(seg, pwp->wp_vaddr,
					PAGESIZE, prot);
			pwp->wp_oprot = 0;
			pwp->wp_prot = 0;
		}
	} while ((pwp = pwp->wp_forw) != as->a_wpage);
}

void
as_signal_proc(struct as *as, k_siginfo_t *siginfo)
{
	struct proc *p;

	mutex_enter(&pidlock);
	for (p = practive; p; p = p->p_next) {
		if (p->p_as == as) {
			mutex_enter(&p->p_lock);
			if (p->p_as == as)
				sigaddq(p, NULL, siginfo, KM_NOSLEEP);
			mutex_exit(&p->p_lock);
		}
	}
	mutex_exit(&pidlock);
}

/*
 * return memory object ID
 */
int
as_getmemid(struct as *as, caddr_t addr, memid_t *memidp)
{
	struct seg	*seg;
	int		sts;

	AS_LOCK_ENTER(as, &as->a_lock, RW_READER);
	seg = as_segat(as, addr);
	if (seg == NULL) {
		AS_LOCK_EXIT(as, &as->a_lock);
		return (EFAULT);
	}
	/*
	 * catch old drivers which may not support getmemid
	 */
	if (seg->s_ops->getmemid == NULL) {
		AS_LOCK_EXIT(as, &as->a_lock);
		return (ENODEV);
	}

	sts = SEGOP_GETMEMID(seg, addr, memidp);

	AS_LOCK_EXIT(as, &as->a_lock);
	return (sts);
}
