Text file src/internal/runtime/atomic/atomic_arm.s

     1  // Copyright 2015 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  #include "go_asm.h"
     6  #include "textflag.h"
     7  #include "funcdata.h"
     8  
     9  // bool armcas(int32 *val, int32 old, int32 new)
    10  // Atomically:
    11  //	if(*val == old){
    12  //		*val = new;
    13  //		return 1;
    14  //	}else
    15  //		return 0;
    16  //
    17  // To implement ·cas in sys_$GOOS_arm.s
    18  // using the native instructions, use:
    19  //
    20  //	TEXT ·cas(SB),NOSPLIT,$0
    21  //		B	·armcas(SB)
    22  //
    23  TEXT ·armcas(SB),NOSPLIT,$0-13
    24  	MOVW	ptr+0(FP), R1
    25  	MOVW	old+4(FP), R2
    26  	MOVW	new+8(FP), R3
    27  casl:
    28  	LDREX	(R1), R0
    29  	CMP	R0, R2
    30  	BNE	casfail
    31  
    32  #ifndef GOARM_7
    33  	MOVB	internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
    34  	CMP	$0, R11
    35  	BEQ	2(PC)
    36  #endif
    37  	DMB	MB_ISHST
    38  
    39  	STREX	R3, (R1), R0
    40  	CMP	$0, R0
    41  	BNE	casl
    42  	MOVW	$1, R0
    43  
    44  #ifndef GOARM_7
    45  	CMP	$0, R11
    46  	BEQ	2(PC)
    47  #endif
    48  	DMB	MB_ISH
    49  
    50  	MOVB	R0, ret+12(FP)
    51  	RET
    52  casfail:
    53  	MOVW	$0, R0
    54  	MOVB	R0, ret+12(FP)
    55  	RET
    56  
    57  // stubs
    58  
    59  TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$0-8
    60  	B	·Load(SB)
    61  
    62  TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-8
    63  	B	·Load(SB)
    64  
    65  TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-8
    66  	B 	·Load(SB)
    67  
    68  TEXT ·Casint32(SB),NOSPLIT,$0-13
    69  	B	·Cas(SB)
    70  
    71  TEXT ·Casint64(SB),NOSPLIT,$-4-21
    72  	B	·Cas64(SB)
    73  
    74  TEXT ·Casuintptr(SB),NOSPLIT,$0-13
    75  	B	·Cas(SB)
    76  
    77  TEXT ·Casp1(SB),NOSPLIT,$0-13
    78  	B	·Cas(SB)
    79  
    80  TEXT ·CasRel(SB),NOSPLIT,$0-13
    81  	B	·Cas(SB)
    82  
    83  TEXT ·Loadint32(SB),NOSPLIT,$0-8
    84  	B	·Load(SB)
    85  
    86  TEXT ·Loadint64(SB),NOSPLIT,$-4-12
    87  	B	·Load64(SB)
    88  
    89  TEXT ·Loaduintptr(SB),NOSPLIT,$0-8
    90  	B	·Load(SB)
    91  
    92  TEXT ·Loaduint(SB),NOSPLIT,$0-8
    93  	B	·Load(SB)
    94  
    95  TEXT ·Storeint32(SB),NOSPLIT,$0-8
    96  	B	·Store(SB)
    97  
    98  TEXT ·Storeint64(SB),NOSPLIT,$0-12
    99  	B	·Store64(SB)
   100  
   101  TEXT ·Storeuintptr(SB),NOSPLIT,$0-8
   102  	B	·Store(SB)
   103  
   104  TEXT ·StorepNoWB(SB),NOSPLIT,$0-8
   105  	B	·Store(SB)
   106  
   107  TEXT ·StoreRel(SB),NOSPLIT,$0-8
   108  	B	·Store(SB)
   109  
   110  TEXT ·StoreReluintptr(SB),NOSPLIT,$0-8
   111  	B	·Store(SB)
   112  
   113  TEXT ·Xaddint32(SB),NOSPLIT,$0-12
   114  	B	·Xadd(SB)
   115  
   116  TEXT ·Xaddint64(SB),NOSPLIT,$-4-20
   117  	B	·Xadd64(SB)
   118  
   119  TEXT ·Xadduintptr(SB),NOSPLIT,$0-12
   120  	B	·Xadd(SB)
   121  
   122  TEXT ·Xchgint32(SB),NOSPLIT,$0-12
   123  	B	·Xchg(SB)
   124  
   125  TEXT ·Xchgint64(SB),NOSPLIT,$-4-20
   126  	B	·Xchg64(SB)
   127  
   128  // 64-bit atomics
   129  // The native ARM implementations use LDREXD/STREXD, which are
   130  // available on ARMv6k or later. We use them only on ARMv7.
   131  // On older ARM, we use Go implementations which simulate 64-bit
   132  // atomics with locks.
   133  TEXT armCas64<>(SB),NOSPLIT,$0-21
   134  	// addr is already in R1
   135  	MOVW	old_lo+4(FP), R2
   136  	MOVW	old_hi+8(FP), R3
   137  	MOVW	new_lo+12(FP), R4
   138  	MOVW	new_hi+16(FP), R5
   139  cas64loop:
   140  	LDREXD	(R1), R6	// loads R6 and R7
   141  	CMP	R2, R6
   142  	BNE	cas64fail
   143  	CMP	R3, R7
   144  	BNE	cas64fail
   145  
   146  	DMB	MB_ISHST
   147  
   148  	STREXD	R4, (R1), R0	// stores R4 and R5
   149  	CMP	$0, R0
   150  	BNE	cas64loop
   151  	MOVW	$1, R0
   152  
   153  	DMB	MB_ISH
   154  
   155  	MOVBU	R0, swapped+20(FP)
   156  	RET
   157  cas64fail:
   158  	MOVW	$0, R0
   159  	MOVBU	R0, swapped+20(FP)
   160  	RET
   161  
   162  TEXT armXadd64<>(SB),NOSPLIT,$0-20
   163  	// addr is already in R1
   164  	MOVW	delta_lo+4(FP), R2
   165  	MOVW	delta_hi+8(FP), R3
   166  
   167  add64loop:
   168  	LDREXD	(R1), R4	// loads R4 and R5
   169  	ADD.S	R2, R4
   170  	ADC	R3, R5
   171  
   172  	DMB	MB_ISHST
   173  
   174  	STREXD	R4, (R1), R0	// stores R4 and R5
   175  	CMP	$0, R0
   176  	BNE	add64loop
   177  
   178  	DMB	MB_ISH
   179  
   180  	MOVW	R4, new_lo+12(FP)
   181  	MOVW	R5, new_hi+16(FP)
   182  	RET
   183  
   184  TEXT armXchg64<>(SB),NOSPLIT,$0-20
   185  	// addr is already in R1
   186  	MOVW	new_lo+4(FP), R2
   187  	MOVW	new_hi+8(FP), R3
   188  
   189  swap64loop:
   190  	LDREXD	(R1), R4	// loads R4 and R5
   191  
   192  	DMB	MB_ISHST
   193  
   194  	STREXD	R2, (R1), R0	// stores R2 and R3
   195  	CMP	$0, R0
   196  	BNE	swap64loop
   197  
   198  	DMB	MB_ISH
   199  
   200  	MOVW	R4, old_lo+12(FP)
   201  	MOVW	R5, old_hi+16(FP)
   202  	RET
   203  
   204  TEXT armLoad64<>(SB),NOSPLIT,$0-12
   205  	// addr is already in R1
   206  
   207  	LDREXD	(R1), R2	// loads R2 and R3
   208  	DMB	MB_ISH
   209  
   210  	MOVW	R2, val_lo+4(FP)
   211  	MOVW	R3, val_hi+8(FP)
   212  	RET
   213  
   214  TEXT armStore64<>(SB),NOSPLIT,$0-12
   215  	// addr is already in R1
   216  	MOVW	val_lo+4(FP), R2
   217  	MOVW	val_hi+8(FP), R3
   218  
   219  store64loop:
   220  	LDREXD	(R1), R4	// loads R4 and R5
   221  
   222  	DMB	MB_ISHST
   223  
   224  	STREXD	R2, (R1), R0	// stores R2 and R3
   225  	CMP	$0, R0
   226  	BNE	store64loop
   227  
   228  	DMB	MB_ISH
   229  	RET
   230  
   231  // The following functions all panic if their address argument isn't
   232  // 8-byte aligned. Since we're calling back into Go code to do this,
   233  // we have to cooperate with stack unwinding. In the normal case, the
   234  // functions tail-call into the appropriate implementation, which
   235  // means they must not open a frame. Hence, when they go down the
   236  // panic path, at that point they push the LR to create a real frame
   237  // (they don't need to pop it because panic won't return; however, we
   238  // do need to set the SP delta back).
   239  
   240  // Check if R1 is 8-byte aligned, panic if not.
   241  // Clobbers R2.
   242  #define CHECK_ALIGN \
   243  	AND.S	$7, R1, R2 \
   244  	BEQ 	4(PC) \
   245  	MOVW.W	R14, -4(R13) /* prepare a real frame */ \
   246  	BL	·panicUnaligned(SB) \
   247  	ADD	$4, R13 /* compensate SP delta */
   248  
   249  TEXT ·Cas64(SB),NOSPLIT,$-4-21
   250  	NO_LOCAL_POINTERS
   251  	MOVW	addr+0(FP), R1
   252  	CHECK_ALIGN
   253  
   254  #ifndef GOARM_7
   255  	MOVB	internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
   256  	CMP	$1, R11
   257  	BEQ	2(PC)
   258  	JMP	·goCas64(SB)
   259  #endif
   260  	JMP	armCas64<>(SB)
   261  
   262  TEXT ·Xadd64(SB),NOSPLIT,$-4-20
   263  	NO_LOCAL_POINTERS
   264  	MOVW	addr+0(FP), R1
   265  	CHECK_ALIGN
   266  
   267  #ifndef GOARM_7
   268  	MOVB	internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
   269  	CMP	$1, R11
   270  	BEQ	2(PC)
   271  	JMP	·goXadd64(SB)
   272  #endif
   273  	JMP	armXadd64<>(SB)
   274  
   275  TEXT ·Xchg64(SB),NOSPLIT,$-4-20
   276  	NO_LOCAL_POINTERS
   277  	MOVW	addr+0(FP), R1
   278  	CHECK_ALIGN
   279  
   280  #ifndef GOARM_7
   281  	MOVB	internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
   282  	CMP	$1, R11
   283  	BEQ	2(PC)
   284  	JMP	·goXchg64(SB)
   285  #endif
   286  	JMP	armXchg64<>(SB)
   287  
   288  TEXT ·Load64(SB),NOSPLIT,$-4-12
   289  	NO_LOCAL_POINTERS
   290  	MOVW	addr+0(FP), R1
   291  	CHECK_ALIGN
   292  
   293  #ifndef GOARM_7
   294  	MOVB	internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
   295  	CMP	$1, R11
   296  	BEQ	2(PC)
   297  	JMP	·goLoad64(SB)
   298  #endif
   299  	JMP	armLoad64<>(SB)
   300  
   301  TEXT ·Store64(SB),NOSPLIT,$-4-12
   302  	NO_LOCAL_POINTERS
   303  	MOVW	addr+0(FP), R1
   304  	CHECK_ALIGN
   305  
   306  #ifndef GOARM_7
   307  	MOVB	internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
   308  	CMP	$1, R11
   309  	BEQ	2(PC)
   310  	JMP	·goStore64(SB)
   311  #endif
   312  	JMP	armStore64<>(SB)
   313  

View as plain text