Update ARM atomic operations (tas, cas, casp, xinc/xdec, ainc/adec)
to work correctly on multiprocessors (armv7 and later).  This entails
changing SWPW to LDREX/STREX, and adding DMB barriers.

Also add emulation of DMB to the kw (armv5) kernel, so the new library
routines can still be used with older ARM systems.

Reference: /n/sources/patch/armv7-atomic
Date: Tue Apr 19 13:14:13 GMT 2016
Signed-off-by: miller@hamnavoe.com

--- /sys/src/libc/arm/atom.s	Tue Apr 19 13:07:30 2016
+++ /sys/src/libc/arm/atom.s	Sun May 29 13:19:21 2016
@@ -1,3 +1,4 @@
+#define DMB	MCR 15, 0, R0, C7, C10, 5
 #define	CLREX		WORD	$0xf57ff01f
 #define	LDREX(a,r)	WORD	$(0xe<<28|0x01900f9f | (a)<<16 | (r)<<12)
 /* `The order of operands is from left to right in dataflow order' - asm man */
@@ -18,11 +19,14 @@
 	STREX(2,0,4)	/*	STREX	0(R0),R2,R4	*/
 	CMP.S	$0, R4
 	BNE	spincas
+	MOVW	$0, R0
+	DMB
 	MOVW	$1, R0
 	RET
 fail:
 	CLREX
 	MOVW	$0, R0
+	DMB
 	RET
 
 TEXT _xinc(SB), $0	/* void	_xinc(long *); */
@@ -33,6 +37,8 @@
 	STREX(3,0,4)	/*	STREX	0(R0),R3,R4	*/
 	CMP.S	$0, R4
 	BNE	spinainc
+	MOVW	$0, R0
+	DMB
 	MOVW	R3, R0
 	RET
 
@@ -44,6 +50,8 @@
 	STREX(3,0,4)	/*	STREX	0(R0),R3,R4	*/
 	CMP.S	$0, R4
 	BNE	spinadec
+	MOVW	$0, R0
+	DMB
 	MOVW	R3, R0
 	RET
 
@@ -53,6 +61,8 @@
 
 TEXT storecond(SB), $0	/* int storecond(long *, long); */
 	MOVW	ov+4(FP), R3
-	STREX(3,0,0)	/*	STREX	0(R0),R3,R0	*/
-	RSB	$1, R0
+	STREX(3,0,4)	/*	STREX	0(R0),R3,R4	*/
+	MOVW	$0, R0
+	DMB
+	RSB	$1, R4, R0
 	RET
--- /sys/src/libc/arm/tas.s	Tue Apr 19 13:07:35 2016
+++ /sys/src/libc/arm/tas.s	Tue Apr 19 13:07:32 2016
@@ -1,5 +1,29 @@
-TEXT	_tas(SB), 1, $-4
-	MOVW	R0,R1
-	MOVW	$1,R0
-	SWPW	R0,(R1)		/* fix: deprecated in armv7 */
+#define DMB	MCR 15, 0, R0, C7, C10, 5
+#define	STREX(f,tp,r) WORD $(0xe<<28|0x01800f90 | (tp)<<16 | (r)<<12 | (f)<<0)
+#define	LDREX(fp,t)   WORD $(0xe<<28|0x01900f9f | (fp)<<16 | (t)<<12)
+#define	CLREX	WORD	$0xf57ff01f
+
+TEXT	_tas(SB), $-4			/* _tas(ulong *) */
+	/* returns old (R0) after modifying (R0) */
+	MOVW	R0,R5
+	MOVW	$0, R0
+	DMB
+
+	MOVW	$1,R2		/* new value of (R0) */
+tas1:
+	LDREX(5,1)		/* LDREX 0(R5),R1 */
+	CMP.S	$0, R1		/* old value non-zero (lock taken)? */
+	BNE	lockbusy	/* we lose */
+	STREX(2,5,4)		/* STREX R2,(R5),R4 */
+	CMP.S	$0, R4
+	BNE	tas1		/* strex failed? try again */
+	MOVW	$0, R0
+	DMB
+	MOVW	R1, R0
+	RET
+lockbusy:
+	CLREX
+	MOVW	$0, R0
+	DMB
+	MOVW	R1, R0
 	RET
--- /sys/src/ape/lib/ap/arm/atom.s	Tue Apr 19 13:07:39 2016
+++ /sys/src/ape/lib/ap/arm/atom.s	Sun May 29 13:19:19 2016
@@ -1,3 +1,4 @@
+#define DMB	MCR 15, 0, R0, C7, C10, 5
 #define	CLREX		WORD	$0xf57ff01f
 #define	LDREX(a,r)	WORD	$(0xe<<28|0x01900f9f | (a)<<16 | (r)<<12)
 /* `The order of operands is from left to right in dataflow order' - asm man */
@@ -18,11 +19,14 @@
 	STREX(2,0,4)	/*	STREX	0(R0),R2,R4	*/
 	CMP.S	$0, R4
 	BNE	spincas
+	MOVW	$0, R0
+	DMB
 	MOVW	$1, R0
 	RET
 fail:
 	CLREX
 	MOVW	$0, R0
+	DMB
 	RET
 
 TEXT _xinc(SB), $0	/* void	_xinc(long *); */
@@ -33,6 +37,8 @@
 	STREX(3,0,4)	/*	STREX	0(R0),R3,R4	*/
 	CMP.S	$0, R4
 	BNE	spinainc
+	MOVW	$0, R0
+	DMB
 	MOVW	R3, R0
 	RET
 
@@ -44,6 +50,8 @@
 	STREX(3,0,4)	/*	STREX	0(R0),R3,R4	*/
 	CMP.S	$0, R4
 	BNE	spinadec
+	MOVW	$0, R0
+	DMB
 	MOVW	R3, R0
 	RET
 
@@ -53,6 +61,8 @@
 
 TEXT storecond(SB), $0	/* int storecond(long *, long); */
 	MOVW	ov+4(FP), R3
-	STREX(3,0,0)	/*	STREX	0(R0),R3,R0	*/
-	RSB	$1, R0
+	STREX(3,0,4)	/*	STREX	0(R0),R3,R4	*/
+	MOVW	$0, R0
+	DMB
+	RSB	$1, R4, R0
 	RET
--- /sys/src/ape/lib/ap/arm/tas.s	Tue Apr 19 13:07:43 2016
+++ /sys/src/ape/lib/ap/arm/tas.s	Tue Apr 19 13:07:41 2016
@@ -1,5 +1,29 @@
-TEXT	tas(SB), $-4
-	MOVW	R0,R1
-	MOVW	$1,R0
-	SWPW	R0,(R1)
+#define DMB	MCR 15, 0, R0, C7, C10, 5
+#define	STREX(f,tp,r) WORD $(0xe<<28|0x01800f90 | (tp)<<16 | (r)<<12 | (f)<<0)
+#define	LDREX(fp,t)   WORD $(0xe<<28|0x01900f9f | (fp)<<16 | (t)<<12)
+#define	CLREX	WORD	$0xf57ff01f
+
+TEXT	tas(SB), $-4			/* tas(ulong *) */
+	/* returns old (R0) after modifying (R0) */
+	MOVW	R0,R5
+	MOVW	$0, R0
+	DMB
+
+	MOVW	$1,R2		/* new value of (R0) */
+tas1:
+	LDREX(5,1)		/* LDREX 0(R5),R1 */
+	CMP.S	$0, R1		/* old value non-zero (lock taken)? */
+	BNE	lockbusy	/* we lose */
+	STREX(2,5,4)		/* STREX R2,(R5),R4 */
+	CMP.S	$0, R4
+	BNE	tas1		/* strex failed? try again */
+	MOVW	$0, R0
+	DMB
+	MOVW	R1, R0
+	RET
+lockbusy:
+	CLREX
+	MOVW	$0, R0
+	DMB
+	MOVW	R1, R0
 	RET
--- /sys/src/9/kw/fpiarm.c	Tue Apr 19 13:07:48 2016
+++ /sys/src/9/kw/fpiarm.c	Tue Apr 19 13:07:45 2016
@@ -468,6 +468,14 @@
 	spllo();
 }
 
+void
+dmb(ulong pc, ulong op, Ureg *ur)
+{
+	USED(pc);
+	USED(op);
+	USED(ur);
+}
+
 int ldrexvalid;
 
 void
@@ -533,6 +541,7 @@
 	{ 0x01800f90, 0x0ff00ff0, strex },
 	{ 0xf57ff01f, 0xffffffff, clrex },
 	{ 0x0ed00100, 0x0ef08100, casemu },
+	{ 0xee070fba, 0xffffffff, dmb },
 	{ 0x00000000, 0x00000000, nil }
 };