/*
 * Copyright 1991-1998 by Open Software Foundation, Inc. 
 *              All Rights Reserved 
 *  
 * Permission to use, copy, modify, and distribute this software and 
 * its documentation for any purpose and without fee is hereby granted, 
 * provided that the above copyright notice appears in all copies and 
 * that both the copyright notice and this permission notice appear in 
 * supporting documentation. 
 *  
 * OSF DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE 
 * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 
 * FOR A PARTICULAR PURPOSE. 
 *  
 * IN NO EVENT SHALL OSF BE LIABLE FOR ANY SPECIAL, INDIRECT, OR 
 * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 
 * LOSS OF USE, DATA OR PROFITS, WHETHER IN ACTION OF CONTRACT, 
 * NEGLIGENCE, OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION 
 * WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 
 */
/*
 * MkLinux
 */
/* CMU_HIST */
/*
 * Revision 2.1.2.1.2.2  92/05/26  11:19:19  jeffreyh
 * 		Since some code tests locks not using simple_lock/simple_unlock
 * 	routines (intel pmap TLB routines) but just testing the lock
 * 	word, used xchgl instead of bts (bit set).
 * 	[92/05/18            bernadat]
 * 
 * Revision 2.1.2.1.2.1  92/02/18  18:46:21  jeffreyh
 * 	Moved from cbus/i386_lock.s and adapted for MACH_LOCK_MON
 * 	[91/12/23            bernadat]
 * 
 * Revision 2.1.4.1  91/09/26  04:27:29  bernadat
 * 	Created
 * 	[91/06/27  05:02:08  bernadat]
 * 
 */
/* CMU_ENDHIST */
/* 
 * Mach Operating System
 * Copyright (c) 1989 Carnegie-Mellon University
 * All rights reserved.  The CMU software License Agreement specifies
 * the terms and conditions for use and redistribution.
 */

#include <cpus.h>
#include <mach_rt.h>
#include <platforms.h>
#include <mach_ldebug.h>
#include <i386/asm.h>
#include <assym.s>
#include <kern/etap_options.h>


/*
 *	When performance isn't the only concern, it's
 *	nice to build stack frames...
 */
#define	BUILD_STACK_FRAMES   ((MACH_LDEBUG || ETAP_LOCK_TRACE) && MACH_KDB)

#if	BUILD_STACK_FRAMES

#define	L_PC		4(%ebp)
#define	L_ARG0		8(%ebp)
#define	L_ARG1		12(%ebp)

#define SWT_HI          -4(%ebp)
#define SWT_LO          -8(%ebp)
#define MISSED          -12(%ebp)

#else   /* BUILD_STACK_FRAMES */

#undef	FRAME
#undef	EMARF
#define	FRAME
#define	EMARF
#define	L_PC		(%esp)
#define	L_ARG0		4(%esp)
#define	L_ARG1		8(%esp)

#endif   /* BUILD_STACK_FRAMES */


#define	M_ILK		(%edx)
#define	M_LOCKED	1(%edx)
#define	M_WAITERS	2(%edx)
#if	MACH_LDEBUG
#define	M_TYPE		4(%edx)
#define	M_PC		8(%edx)
#define	M_THREAD	12(%edx)
#endif	/* MACH_LDEBUG */

#include <i386/AT386/mp/mp.h>
#if	(NCPUS > 1)
#define	CX(addr,reg)	addr(,reg,4)
#else
#define	CPU_NUMBER(reg)
#define	CX(addr,reg)	addr
#endif	/* (NCPUS > 1) */

#if	MACH_LDEBUG
/*
 *  Routines for general lock debugging.
 */
#define	S_TYPE		4(%edx)
#define	S_PC		8(%edx)
#define	S_THREAD	12(%edx)
#define	S_DURATIONH	16(%edx)
#define	S_DURATIONL	20(%edx)

/* 
 * Checks for expected lock types and calls "panic" on
 * mismatch.  Detects calls to Mutex functions with
 * type simplelock and vice versa.
 */
#define	CHECK_MUTEX_TYPE()					\
	cmpl	$MUTEX_TAG, M_TYPE			;	\
	je	1f					;	\
	pushl	$2f					;	\
	call	EXT(panic)				;	\
	hlt						;	\
	.data						;	\
2:	String	"not a mutex!"				;	\
	.text						;	\
1:

#define	CHECK_SIMPLE_LOCK_TYPE()				\
	cmpl	$SIMPLE_LOCK_TAG,S_TYPE 		;	\
	je	1f					;	\
	pushl	$2f					;	\
	call	EXT(panic)				;	\
	hlt						;	\
	.data						;	\
2:	String	"not a simple lock!"			;	\
	.text						;	\
1:

/*
 * If one or more simplelocks are currently held by a thread,
 * an attempt to acquire a mutex will cause this check to fail
 * (since a mutex lock may context switch, holding a simplelock
 * is not a good thing).
 */
#if	MACH_RT
#define CHECK_PREEMPTION_LEVEL()				\
	movl	$CPD_PREEMPTION_LEVEL,%eax		;	\
	cmpl	$0,%gs:(%eax)				;	\
	je	1f					;	\
	pushl	$2f					;	\
	call	EXT(panic)				;	\
	hlt						;	\
	.data						;	\
2:	String	"preemption_level != 0!"		;	\
	.text						;	\
1:
#else	/* MACH_RT */
#define	CHECK_PREEMPTION_LEVEL()
#endif	/* MACH_RT */

#define	CHECK_NO_SIMPLELOCKS()					\
	movl	$CPD_SIMPLE_LOCK_COUNT,%eax		;	\
	cmpl	$0,%gs:(%eax)				;	\
	je	1f					;	\
	pushl	$2f					;	\
	call	EXT(panic)				;	\
	hlt						;	\
	.data						;	\
2:	String	"simple_locks_held!"			;	\
	.text						;	\
1:

/* 
 * Verifies return to the correct thread in "unlock" situations.
 */
#define	CHECK_THREAD(thd)					\
	movl	$CPD_ACTIVE_THREAD,%eax			;	\
	movl	%gs:(%eax),%ecx				;	\
	testl	%ecx,%ecx				;	\
	je	1f					;	\
	cmpl	%ecx,thd				;	\
	je	1f					;	\
	pushl	$2f					;	\
	call	EXT(panic)				;	\
	hlt						;	\
	.data						;	\
2:	String	"wrong thread!"				;	\
	.text						;	\
1:

#define	CHECK_MYLOCK(thd)					\
	movl	$CPD_ACTIVE_THREAD,%eax			;	\
	movl	%gs:(%eax),%ecx				;	\
	testl	%ecx,%ecx				;	\
	je	1f					;	\
	cmpl	%ecx,thd				;	\
	jne	1f					;	\
	pushl	$2f					;	\
	call	EXT(panic)				;	\
	hlt						;	\
	.data						;	\
2:	String	"mylock attempt!"			;	\
	.text						;	\
1:

#define	METER_SIMPLE_LOCK_LOCK(reg)				\
	pushl	reg					;	\
	call	EXT(meter_simple_lock)			;	\
	popl	reg

#define	METER_SIMPLE_LOCK_UNLOCK(reg)				\
	pushl	reg					;	\
	call	EXT(meter_simple_unlock)		;	\
	popl	reg

#else	/* MACH_LDEBUG */
#define	CHECK_MUTEX_TYPE()
#define	CHECK_SIMPLE_LOCK_TYPE
#define	CHECK_THREAD(thd)
#define CHECK_PREEMPTION_LEVEL()
#define	CHECK_NO_SIMPLELOCKS()
#define	CHECK_MYLOCK(thd)
#define	METER_SIMPLE_LOCK_LOCK(reg)
#define	METER_SIMPLE_LOCK_UNLOCK(reg)
#endif	/* MACH_LDEBUG */


/*
 *	void hw_lock_init(hw_lock_t)
 *
 *	Initialize a hardware lock.
 */
ENTRY(hw_lock_init)
	FRAME
	movl	L_ARG0,%edx		/ fetch lock pointer
	xorl	%eax,%eax
	movb	%al,0(%edx)		/ clear the lock
	EMARF
	ret

/*
 *	void hw_lock_lock(hw_lock_t)
 *
 *	Acquire lock, spinning until it becomes available.
 *	MACH_RT:  also return with preemption disabled.
 */
ENTRY(hw_lock_lock)
	FRAME
	movl	L_ARG0,%edx		/ fetch lock pointer

1:	DISABLE_PREEMPTION(%eax)
	movb	$1,%cl
	xchgb	0(%edx),%cl		/ try to acquire the HW lock
	testb	%cl,%cl			/ success?
	jne	3f

	EMARF				/ if yes, then nothing left to do
	ret

3:	ENABLE_PREEMPTION(%eax)		/ no reason we can't be preemptable now

	movb	$1,%cl
2:	testb	%cl,0(%edx)		/ spin checking lock value in cache
	jne	2b			/ non-zero means locked, keep spinning
	jmp	1b			/ zero means unlocked, try to grab it

/*
 *	void hw_lock_unlock(hw_lock_t)
 *
 *	Unconditionally release lock.
 *	MACH_RT:  release preemption level.
 */
ENTRY(hw_lock_unlock)
	FRAME
	movl	L_ARG0,%edx		/ fetch lock pointer
	xorl	%eax,%eax
	xchgb	0(%edx),%al		/ clear the lock... a mov instruction
					/ ...might be cheaper and less paranoid
	ENABLE_PREEMPTION(%eax)
	EMARF
	ret

/*
 *	unsigned int hw_lock_try(hw_lock_t)
 *	MACH_RT:  returns with preemption disabled on success.
 */
ENTRY(hw_lock_try)
	FRAME
	movl	L_ARG0,%edx		/ fetch lock pointer

	DISABLE_PREEMPTION(%eax)
	movb	$1,%cl
	xchgb	0(%edx),%cl		/ try to acquire the HW lock
	testb	%cl,%cl			/ success?
	jne	1f			/ if yes, let the caller know

	movl	$1,%eax			/ success
	EMARF
	ret

1:	ENABLE_PREEMPTION(%eax)		/ failure:  release preemption...
	xorl	%eax,%eax		/ ...and return failure
	EMARF
	ret	

/*
 *	unsigned int hw_lock_held(hw_lock_t)
 *	MACH_RT:  doesn't change preemption state.
 *	N.B.  Racy, of course.
 */
ENTRY(hw_lock_held)
	FRAME
	movl	L_ARG0,%edx		/ fetch lock pointer

	movb	$1,%cl
	testb	%cl,0(%edx)		/ check lock value
	jne	1f			/ non-zero means locked
	xorl	%eax,%eax		/ tell caller:  lock wasn't locked
	EMARF
	ret	

1:	movl	$1,%eax			/ tell caller:  lock was locked
	EMARF
	ret
	


#if	0


ENTRY(_usimple_lock_init)
	FRAME
	movl	L_ARG0,%edx		/ fetch lock pointer
	xorl	%eax,%eax
	movb	%al,USL_INTERLOCK(%edx)	/ unlock the HW lock
	EMARF
	ret

ENTRY(_simple_lock)
	FRAME
	movl	L_ARG0,%edx		/ fetch lock pointer

	CHECK_SIMPLE_LOCK_TYPE()

	DISABLE_PREEMPTION(%eax)

sl_get_hw:
	movb	$1,%cl
	xchgb	USL_INTERLOCK(%edx),%cl	/ try to acquire the HW lock
	testb	%cl,%cl			/ did we succeed?

#if	MACH_LDEBUG
	je	5f
	CHECK_MYLOCK(S_THREAD)
	jmp	sl_get_hw
5:
#else	/* MACH_LDEBUG */
	jne	sl_get_hw		/ no, try again
#endif	/* MACH_LDEBUG */

#if	MACH_LDEBUG
	movl	L_PC,%ecx
	movl	%ecx,S_PC
	movl	$CPD_ACTIVE_THREAD,%eax
	movl	%gs:(%eax),%ecx
	movl	%ecx,S_THREAD
	incl	CX(EXT(simple_lock_count),%eax)
#if 0
	METER_SIMPLE_LOCK_LOCK(%edx)
#endif
#if	NCPUS == 1
	pushf
	pushl	%edx
	cli
	call	EXT(lock_stack_push)
	popl	%edx
	popfl
#endif	/* NCPUS == 1 */
#endif	/* MACH_LDEBUG */

	EMARF
	ret

ENTRY(_simple_lock_try)
	FRAME
	movl	L_ARG0,%edx		/ fetch lock pointer

	CHECK_SIMPLE_LOCK_TYPE()

	DISABLE_PREEMPTION(%eax)

	movb	$1,%cl
	xchgb	USL_INTERLOCK(%edx),%cl	/ try to acquire the HW lock
	testb	%cl,%cl			/ did we succeed?
	jne	1f			/ no, return failure

#if	MACH_LDEBUG
	movl	L_PC,%ecx
	movl	%ecx,S_PC
	movl	$CPD_ACTIVE_THREAD,%eax
	movl	%gs:(%eax),%ecx
	movl	%ecx,S_THREAD
	incl	CX(EXT(simple_lock_count),%eax)
#if 0
	METER_SIMPLE_LOCK_LOCK(%edx)
#endif
#if	NCPUS == 1
	pushf
	pushl	%edx
	cli
	call	EXT(lock_stack_push)
	popl	%edx
	popfl
#endif	/* NCPUS == 1 */
#endif	/* MACH_LDEBUG */

	movl	$1,%eax			/ return success

	EMARF
	ret

1:
	ENABLE_PREEMPTION(%eax)

	xorl	%eax,%eax		/ and return failure

	EMARF
	ret

ENTRY(_simple_unlock)
	FRAME
	movl	L_ARG0,%edx		/ fetch lock pointer

	CHECK_SIMPLE_LOCK_TYPE()
	CHECK_THREAD(S_THREAD)

#if	MACH_LDEBUG
	xorl	%eax,%eax
	movl	%eax,S_THREAD		/ disown thread
	MP_DISABLE_PREEMPTION(%eax)
	CPU_NUMBER(%eax)
	decl	CX(EXT(simple_lock_count),%eax)
	MP_ENABLE_PREEMPTION(%eax)
#if 0
	METER_SIMPLE_LOCK_UNLOCK(%edx)
#endif
#if	NCPUS == 1
	pushf
	pushl	%edx
	cli
	call	EXT(lock_stack_pop)
	popl	%edx
	popfl
#endif	/* NCPUS == 1 */
#endif	/* MACH_LDEBUG */

	xorb	%cl,%cl
	xchgb	USL_INTERLOCK(%edx),%cl	/ unlock the HW lock

	ENABLE_PREEMPTION(%eax)

	EMARF
	ret

#endif	/* 0 */


ENTRY(mutex_init)
	FRAME
	movl	L_ARG0,%edx		/ fetch lock pointer
	xorl	%eax,%eax
	movb	%al,M_ILK		/ clear interlock
	movb	%al,M_LOCKED		/ clear locked flag
	movw	%ax,M_WAITERS		/ init waiter count

#if	MACH_LDEBUG
	movl	$MUTEX_TAG,M_TYPE	/ set lock type
	movl	%eax,M_PC		/ init caller pc
	movl	%eax,M_THREAD		/ and owning thread
#endif
#if	ETAP_LOCK_TRACE
	movl	L_ARG1,%ecx		/ fetch event type
	pushl	%ecx			/ push event type
	pushl	%edx			/ push mutex address
	call	EXT(etap_mutex_init)	/ init ETAP data
	addl	$8,%esp
#endif	/* ETAP_LOCK_TRACE */

	EMARF
	ret

ENTRY(_mutex_lock)
	FRAME

#if	ETAP_LOCK_TRACE
	subl	$12,%esp		/ make room for locals
	movl	$0,SWT_HI		/ set wait time to zero (HI)
	movl	$0,SWT_LO		/ set wait time to zero (LO)
	movl	$0,MISSED		/ clear local miss marker
#endif	/* ETAP_LOCK_TRACE */

	movl	L_ARG0,%edx		/ fetch lock pointer

	CHECK_MUTEX_TYPE()
	CHECK_NO_SIMPLELOCKS()
	CHECK_PREEMPTION_LEVEL()

ml_retry:
	DISABLE_PREEMPTION(%eax)

ml_get_hw:
	movb	$1,%cl
	xchgb	%cl,M_ILK
	testb	%cl,%cl			/ did we succeed?
	jne	ml_get_hw		/ no, try again

/
/ Beware of a race between this code path and the inline ASM fast-path locking
/ sequence which attempts to lock a mutex by directly setting the locked flag
/ 

	movb	$1,%cl
	xchgb	%cl,M_LOCKED		/ try to set locked flag
	testb	%cl,%cl			/ is the mutex locked?
	jne	ml_fail			/ yes, we lose

#if	MACH_LDEBUG
	movl	L_PC,%ecx
	movl	%ecx,M_PC
	movl	$CPD_ACTIVE_THREAD,%eax
	movl	%gs:(%eax),%ecx
	movl	%ecx,M_THREAD
	testl	%ecx,%ecx
	je	3f
	incl	TH_MUTEX_COUNT(%ecx)
3:
#endif

	xorb	%cl,%cl
	xchgb	%cl,M_ILK

	ENABLE_PREEMPTION(%eax)

#if	ETAP_LOCK_TRACE
	movl	L_PC,%eax		/ fetch pc
	pushl	SWT_LO			/ push wait time (low)
	pushl	SWT_HI			/ push wait time (high)
	pushl	%eax			/ push pc
	pushl	%edx			/ push mutex address
	call	EXT(etap_mutex_hold)	/ collect hold timestamp
	addl	$16+12,%esp		/ clean up stack, adjusting for locals
#endif	/* ETAP_LOCK_TRACE */

	EMARF
	ret

ml_fail:
#if	ETAP_LOCK_TRACE
	cmp	$0,MISSED		/ did we already take a wait timestamp?
	jne	ml_block		/ yup. carry-on
	pushl	%edx			/ push mutex address
	call	EXT(etap_mutex_miss)	/ get wait timestamp
	movl	%eax,SWT_HI		/ set wait time (high word)
	movl	%edx,SWT_LO		/ set wait time (low word)
	popl	%edx			/ clean up stack
	movl	$1,MISSED		/ mark wait timestamp as taken
#endif	/* ETAP_LOCK_TRACE */

ml_block:
	CHECK_MYLOCK(M_THREAD)
	pushl	%edx			/ push mutex address
	call	EXT(mutex_lock_wait)	/ wait for the lock
	addl	$4,%esp
	movl	L_ARG0,%edx		/ refetch lock pointer
	jmp	ml_retry		/ and try again

ENTRY(_mutex_try)
	FRAME

#if	ETAP_LOCK_TRACE
	subl	$8,%esp			/ make room for locals
	movl	$0,SWT_HI		/ set wait time to zero (HI)
	movl	$0,SWT_LO		/ set wait time to zero (LO)
#endif	/* ETAP_LOCK_TRACE */

	movl	L_ARG0,%edx		/ fetch lock pointer

	CHECK_MUTEX_TYPE()
	CHECK_NO_SIMPLELOCKS()

	xorl	%eax,%eax
	movb	$1,%al			/ locked value for mutex
	xchgb	%al,M_LOCKED		/ swap locked values
	xorb	$1,%al			/ generate return value

#if	MACH_LDEBUG || ETAP_LOCK_TRACE
	testl	%eax,%eax		/ did we succeed?
	je	2f			/ no, skip
#endif

#if	MACH_LDEBUG
	movl	L_PC,%ecx
	movl	%ecx,M_PC
	movl	$CPD_ACTIVE_THREAD,%ecx
	movl	%gs:(%ecx),%ecx
	movl	%ecx,M_THREAD
	testl	%ecx,%ecx
	je	1f
	incl	TH_MUTEX_COUNT(%ecx)
1:
#endif

#if	ETAP_LOCK_TRACE
	movl	L_PC,%eax		/ fetch pc
	pushl	SWT_LO			/ push wait time (low)
	pushl	SWT_HI			/ push wait time (high)
	pushl	%eax			/ push pc
	pushl	%edx			/ push mutex address
	call	EXT(etap_mutex_hold)	/ get start hold timestamp
	addl	$16,%esp		/ clean up stack, adjusting for locals
	movl	$1,%eax			/ put back successful return value
#endif	/* ETAP_LOCK_TRACE */

#if	MACH_LDEBUG || ETAP_LOCK_TRACE
2:
#if	ETAP_LOCK_TRACE
	addl	$8,%esp			/ pop stack claimed on entry
#endif
#endif

	EMARF
	ret

ENTRY(mutex_unlock)
	FRAME
	movl	L_ARG0,%edx		/ fetch lock pointer

#if	ETAP_LOCK_TRACE
	pushl	%edx			/ push mutex address
	call	EXT(etap_mutex_unlock)	/ collect ETAP data
	popl	%edx			/ restore mutex address
#endif	/* ETAP_LOCK_TRACE */

	CHECK_MUTEX_TYPE()
	CHECK_THREAD(M_THREAD)

	DISABLE_PREEMPTION(%eax)

mu_get_hw:
	movb	$1,%cl
	xchgb	%cl,M_ILK
	testb	%cl,%cl			/ did we succeed?
	jne	mu_get_hw		/ no, try again

	cmpw	$0,M_WAITERS		/ are there any waiters?
	jne	mu_wakeup		/ yes, more work to do

mu_doit:
#if	MACH_LDEBUG
	xorl	%eax,%eax
	movl	%eax,M_THREAD		/ disown thread
	movl	$CPD_ACTIVE_THREAD,%eax
	movl	%gs:(%eax),%ecx
	testl	%ecx,%ecx
	je	0f
	decl	TH_MUTEX_COUNT(%ecx)
0:
#endif

	xorb	%cl,%cl
	xchgb	%cl,M_LOCKED		/ unlock the mutex

	xorb	%cl,%cl
	xchgb	%cl,M_ILK

	ENABLE_PREEMPTION(%eax)

	EMARF
	ret

mu_wakeup:
	pushl	%edx			/ push mutex address
	call	EXT(mutex_unlock_wakeup)/ yes, wake a thread
	addl	$4,%esp
	movl	L_ARG0,%edx		/ refetch lock pointer
	jmp	mu_doit

ENTRY(interlock_unlock)
	FRAME
	movl	L_ARG0,%edx

	xorb	%cl,%cl
	xchgb	%cl,M_ILK

	ENABLE_PREEMPTION(%eax)

	EMARF
	ret

	
ENTRY(_disable_preemption)
#if	MACH_RT
	_DISABLE_PREEMPTION(%eax)
#endif	/* MACH_RT */
	ret

ENTRY(_enable_preemption)
#if	MACH_RT
#if	MACH_ASSERT
	movl	$CPD_PREEMPTION_LEVEL,%eax
	cmpl	$0,%gs:(%eax)
	jg	1f
	pushl	$2f
	call	EXT(panic)
	hlt
	.data
2:	String	"_enable_preemption: preemption_level <= 0!"
	.text
1:
#endif	/* MACH_ASSERT */
	_ENABLE_PREEMPTION(%eax)
#endif	/* MACH_RT */
	ret

ENTRY(_enable_preemption_no_check)
#if	MACH_RT
#if	MACH_ASSERT
	movl	$CPD_PREEMPTION_LEVEL,%eax
	cmpl	$0,%gs:(%eax)
	jg	1f
	pushl	$2f
	call	EXT(panic)
	hlt
	.data
2:	String	"_enable_preemption_no_check: preemption_level <= 0!"
	.text
1:
#endif	/* MACH_ASSERT */
	_ENABLE_PREEMPTION_NO_CHECK(%eax)
#endif	/* MACH_RT */
	ret
	
	
ENTRY(_mp_disable_preemption)
#if	MACH_RT && NCPUS > 1
	_DISABLE_PREEMPTION(%eax)
#endif	/* MACH_RT && NCPUS > 1*/
	ret

ENTRY(_mp_enable_preemption)
#if	MACH_RT && NCPUS > 1
#if	MACH_ASSERT
	movl	$CPD_PREEMPTION_LEVEL,%eax
	cmpl	$0,%gs:(%eax)
	jg	1f
	pushl	$2f
	call	EXT(panic)
	hlt
	.data
2:	String	"_mp_enable_preemption: preemption_level <= 0!"
	.text
1:
#endif	/* MACH_ASSERT */
	_ENABLE_PREEMPTION(%eax)
#endif	/* MACH_RT && NCPUS > 1 */
	ret

ENTRY(_mp_enable_preemption_no_check)
#if	MACH_RT && NCPUS > 1
#if	MACH_ASSERT
	movl	$CPD_PREEMPTION_LEVEL,%eax
	cmpl	$0,%gs:(%eax)
	jg	1f
	pushl	$2f
	call	EXT(panic)
	hlt
	.data
2:	String	"_mp_enable_preemption_no_check: preemption_level <= 0!"
	.text
1:
#endif	/* MACH_ASSERT */
	_ENABLE_PREEMPTION_NO_CHECK(%eax)
#endif	/* MACH_RT && NCPUS > 1 */
	ret
	
	
ENTRY(i_bit_set)
	movl	S_ARG0,%edx
	movl	S_ARG1,%eax
	lock
	bts	%edx,(%eax)
	ret

ENTRY(i_bit_clear)
	movl	S_ARG0,%edx
	movl	S_ARG1,%eax
	lock
	btr	%edx,(%eax)
	ret

ENTRY(bit_lock)
	movl	S_ARG0,%ecx
	movl	S_ARG1,%eax
1:
	lock
	bts	%ecx,(%eax)
	jb	1b
	ret

ENTRY(bit_lock_try)
	movl	S_ARG0,%ecx
	movl	S_ARG1,%eax
	lock
	bts	%ecx,(%eax)
	jb	bit_lock_failed
	ret			/ %eax better not be null !
bit_lock_failed:
	xorl	%eax,%eax
	ret

ENTRY(bit_unlock)
	movl	S_ARG0,%ecx
	movl	S_ARG1,%eax
	lock
	btr	%ecx,(%eax)
	ret
