1881 lines
77 KiB
LLVM
1881 lines
77 KiB
LLVM
|
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
|
||
|
|
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX900 %s
|
||
|
|
; RUN: llc -mtriple=amdgcn -mcpu=gfx908 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX908 %s
|
||
|
|
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX90A %s
|
||
|
|
; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX10 %s
|
||
|
|
; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX11 %s
|
||
|
|
|
||
|
|
define amdgpu_kernel void @global_atomic_fadd_ret_f32(ptr addrspace(1) %ptr) #0 {
|
||
|
|
; GFX900-LABEL: global_atomic_fadd_ret_f32:
|
||
|
|
; GFX900: ; %bb.0:
|
||
|
|
; GFX900-NEXT: s_mov_b64 s[4:5], exec
|
||
|
|
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
|
||
|
|
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
|
||
|
|
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX900-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GFX900-NEXT: s_and_saveexec_b64 s[2:3], vcc
|
||
|
|
; GFX900-NEXT: s_cbranch_execz .LBB0_4
|
||
|
|
; GFX900-NEXT: ; %bb.1:
|
||
|
|
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX900-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
|
||
|
|
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
|
||
|
|
; GFX900-NEXT: s_mov_b64 s[4:5], 0
|
||
|
|
; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v1
|
||
|
|
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX900-NEXT: s_load_dword s6, s[0:1], 0x0
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v1, s6
|
||
|
|
; GFX900-NEXT: .LBB0_2: ; %atomicrmw.start
|
||
|
|
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v5, v1
|
||
|
|
; GFX900-NEXT: v_add_f32_e32 v4, v5, v2
|
||
|
|
; GFX900-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
|
||
|
|
; GFX900-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX900-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
|
||
|
|
; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
||
|
|
; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
||
|
|
; GFX900-NEXT: s_cbranch_execnz .LBB0_2
|
||
|
|
; GFX900-NEXT: ; %bb.3: ; %Flow
|
||
|
|
; GFX900-NEXT: s_or_b64 exec, exec, s[4:5]
|
||
|
|
; GFX900-NEXT: .LBB0_4: ; %Flow1
|
||
|
|
; GFX900-NEXT: s_or_b64 exec, exec, s[2:3]
|
||
|
|
; GFX900-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GFX900-NEXT: v_mad_f32 v0, v0, 4.0, s0
|
||
|
|
; GFX900-NEXT: global_store_dword v[0:1], v0, off
|
||
|
|
; GFX900-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX908-LABEL: global_atomic_fadd_ret_f32:
|
||
|
|
; GFX908: ; %bb.0:
|
||
|
|
; GFX908-NEXT: s_mov_b64 s[4:5], exec
|
||
|
|
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
|
||
|
|
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
|
||
|
|
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX908-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GFX908-NEXT: s_and_saveexec_b64 s[2:3], vcc
|
||
|
|
; GFX908-NEXT: s_cbranch_execz .LBB0_4
|
||
|
|
; GFX908-NEXT: ; %bb.1:
|
||
|
|
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX908-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
|
||
|
|
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
|
||
|
|
; GFX908-NEXT: s_mov_b64 s[4:5], 0
|
||
|
|
; GFX908-NEXT: v_mul_f32_e32 v2, 4.0, v1
|
||
|
|
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX908-NEXT: s_load_dword s6, s[0:1], 0x0
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v1, s6
|
||
|
|
; GFX908-NEXT: .LBB0_2: ; %atomicrmw.start
|
||
|
|
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v5, v1
|
||
|
|
; GFX908-NEXT: v_add_f32_e32 v4, v5, v2
|
||
|
|
; GFX908-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
|
||
|
|
; GFX908-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX908-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
|
||
|
|
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
||
|
|
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
||
|
|
; GFX908-NEXT: s_cbranch_execnz .LBB0_2
|
||
|
|
; GFX908-NEXT: ; %bb.3: ; %Flow
|
||
|
|
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
|
||
|
|
; GFX908-NEXT: .LBB0_4: ; %Flow1
|
||
|
|
; GFX908-NEXT: s_or_b64 exec, exec, s[2:3]
|
||
|
|
; GFX908-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GFX908-NEXT: v_mad_f32 v0, v0, 4.0, s0
|
||
|
|
; GFX908-NEXT: global_store_dword v[0:1], v0, off
|
||
|
|
; GFX908-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX90A-LABEL: global_atomic_fadd_ret_f32:
|
||
|
|
; GFX90A: ; %bb.0:
|
||
|
|
; GFX90A-NEXT: s_mov_b64 s[4:5], exec
|
||
|
|
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
|
||
|
|
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
|
||
|
|
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX90A-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
|
||
|
|
; GFX90A-NEXT: s_cbranch_execz .LBB0_4
|
||
|
|
; GFX90A-NEXT: ; %bb.1:
|
||
|
|
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX90A-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
|
||
|
|
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
|
||
|
|
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
|
||
|
|
; GFX90A-NEXT: v_mul_f32_e32 v2, 4.0, v1
|
||
|
|
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX90A-NEXT: s_load_dword s6, s[0:1], 0x0
|
||
|
|
; GFX90A-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX90A-NEXT: v_mov_b32_e32 v1, s6
|
||
|
|
; GFX90A-NEXT: .LBB0_2: ; %atomicrmw.start
|
||
|
|
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX90A-NEXT: v_mov_b32_e32 v5, v1
|
||
|
|
; GFX90A-NEXT: v_add_f32_e32 v4, v5, v2
|
||
|
|
; GFX90A-NEXT: buffer_wbl2
|
||
|
|
; GFX90A-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
|
||
|
|
; GFX90A-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX90A-NEXT: buffer_invl2
|
||
|
|
; GFX90A-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
|
||
|
|
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
||
|
|
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
||
|
|
; GFX90A-NEXT: s_cbranch_execnz .LBB0_2
|
||
|
|
; GFX90A-NEXT: ; %bb.3: ; %Flow
|
||
|
|
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
|
||
|
|
; GFX90A-NEXT: .LBB0_4: ; %Flow1
|
||
|
|
; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3]
|
||
|
|
; GFX90A-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GFX90A-NEXT: v_mad_f32 v0, v0, 4.0, s0
|
||
|
|
; GFX90A-NEXT: global_store_dword v[0:1], v0, off
|
||
|
|
; GFX90A-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX10-LABEL: global_atomic_fadd_ret_f32:
|
||
|
|
; GFX10: ; %bb.0:
|
||
|
|
; GFX10-NEXT: s_mov_b32 s4, exec_lo
|
||
|
|
; GFX10-NEXT: s_mov_b32 s3, 0
|
||
|
|
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
|
||
|
|
; GFX10-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
|
||
|
|
; GFX10-NEXT: s_and_saveexec_b32 s2, vcc_lo
|
||
|
|
; GFX10-NEXT: s_cbranch_execz .LBB0_4
|
||
|
|
; GFX10-NEXT: ; %bb.1:
|
||
|
|
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX10-NEXT: s_bcnt1_i32_b32 s4, s4
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
|
||
|
|
; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v1
|
||
|
|
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX10-NEXT: s_load_dword s5, s[0:1], 0x0
|
||
|
|
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v1, s5
|
||
|
|
; GFX10-NEXT: .LBB0_2: ; %atomicrmw.start
|
||
|
|
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v5, v1
|
||
|
|
; GFX10-NEXT: v_add_f32_e32 v4, v5, v2
|
||
|
|
; GFX10-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
|
||
|
|
; GFX10-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX10-NEXT: buffer_gl0_inv
|
||
|
|
; GFX10-NEXT: buffer_gl1_inv
|
||
|
|
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
|
||
|
|
; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3
|
||
|
|
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
|
||
|
|
; GFX10-NEXT: s_cbranch_execnz .LBB0_2
|
||
|
|
; GFX10-NEXT: ; %bb.3: ; %Flow
|
||
|
|
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
|
||
|
|
; GFX10-NEXT: .LBB0_4: ; %Flow1
|
||
|
|
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s2
|
||
|
|
; GFX10-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GFX10-NEXT: v_mad_f32 v0, v0, 4.0, s0
|
||
|
|
; GFX10-NEXT: global_store_dword v[0:1], v0, off
|
||
|
|
; GFX10-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX11-LABEL: global_atomic_fadd_ret_f32:
|
||
|
|
; GFX11: ; %bb.0:
|
||
|
|
; GFX11-NEXT: s_mov_b32 s4, exec_lo
|
||
|
|
; GFX11-NEXT: s_mov_b32 s3, 0
|
||
|
|
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
|
||
|
|
; GFX11-NEXT: s_mov_b32 s2, exec_lo
|
||
|
|
; GFX11-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
|
||
|
|
; GFX11-NEXT: s_cbranch_execz .LBB0_4
|
||
|
|
; GFX11-NEXT: ; %bb.1:
|
||
|
|
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX11-NEXT: s_bcnt1_i32_b32 s4, s4
|
||
|
|
; GFX11-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
|
||
|
|
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX11-NEXT: s_load_b32 s5, s[0:1], 0x0
|
||
|
|
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX11-NEXT: v_dual_mul_f32 v2, 4.0, v1 :: v_dual_mov_b32 v1, s5
|
||
|
|
; GFX11-NEXT: .LBB0_2: ; %atomicrmw.start
|
||
|
|
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX11-NEXT: v_mov_b32_e32 v5, v1
|
||
|
|
; GFX11-NEXT: v_add_f32_e32 v4, v5, v2
|
||
|
|
; GFX11-NEXT: global_atomic_cmpswap_b32 v1, v3, v[4:5], s[0:1] glc
|
||
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX11-NEXT: buffer_gl0_inv
|
||
|
|
; GFX11-NEXT: buffer_gl1_inv
|
||
|
|
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
|
||
|
|
; GFX11-NEXT: s_or_b32 s3, vcc_lo, s3
|
||
|
|
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
|
||
|
|
; GFX11-NEXT: s_cbranch_execnz .LBB0_2
|
||
|
|
; GFX11-NEXT: ; %bb.3: ; %Flow
|
||
|
|
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3
|
||
|
|
; GFX11-NEXT: .LBB0_4: ; %Flow1
|
||
|
|
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s2
|
||
|
|
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GFX11-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GFX11-NEXT: v_mul_f32_e32 v0, 4.0, v0
|
||
|
|
; GFX11-NEXT: v_add_f32_e32 v0, s0, v0
|
||
|
|
; GFX11-NEXT: global_store_b32 v[0:1], v0, off
|
||
|
|
; GFX11-NEXT: s_nop 0
|
||
|
|
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
|
||
|
|
; GFX11-NEXT: s_endpgm
|
||
|
|
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 seq_cst
|
||
|
|
store float %result, ptr addrspace(1) undef
|
||
|
|
ret void
|
||
|
|
}
|
||
|
|
|
||
|
|
define amdgpu_kernel void @global_atomic_fadd_ret_f32_ieee(ptr addrspace(1) %ptr) #2 {
|
||
|
|
; GFX900-LABEL: global_atomic_fadd_ret_f32_ieee:
|
||
|
|
; GFX900: ; %bb.0:
|
||
|
|
; GFX900-NEXT: s_mov_b64 s[4:5], exec
|
||
|
|
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
|
||
|
|
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
|
||
|
|
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX900-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GFX900-NEXT: s_and_saveexec_b64 s[2:3], vcc
|
||
|
|
; GFX900-NEXT: s_cbranch_execz .LBB1_4
|
||
|
|
; GFX900-NEXT: ; %bb.1:
|
||
|
|
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX900-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
|
||
|
|
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
|
||
|
|
; GFX900-NEXT: s_mov_b64 s[4:5], 0
|
||
|
|
; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v1
|
||
|
|
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX900-NEXT: s_load_dword s6, s[0:1], 0x0
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v1, s6
|
||
|
|
; GFX900-NEXT: .LBB1_2: ; %atomicrmw.start
|
||
|
|
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v5, v1
|
||
|
|
; GFX900-NEXT: v_add_f32_e32 v4, v5, v2
|
||
|
|
; GFX900-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
|
||
|
|
; GFX900-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX900-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
|
||
|
|
; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
||
|
|
; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
||
|
|
; GFX900-NEXT: s_cbranch_execnz .LBB1_2
|
||
|
|
; GFX900-NEXT: ; %bb.3: ; %Flow
|
||
|
|
; GFX900-NEXT: s_or_b64 exec, exec, s[4:5]
|
||
|
|
; GFX900-NEXT: .LBB1_4: ; %Flow1
|
||
|
|
; GFX900-NEXT: s_or_b64 exec, exec, s[2:3]
|
||
|
|
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GFX900-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GFX900-NEXT: v_mul_f32_e32 v0, 4.0, v0
|
||
|
|
; GFX900-NEXT: v_add_f32_e32 v0, s0, v0
|
||
|
|
; GFX900-NEXT: global_store_dword v[0:1], v0, off
|
||
|
|
; GFX900-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX908-LABEL: global_atomic_fadd_ret_f32_ieee:
|
||
|
|
; GFX908: ; %bb.0:
|
||
|
|
; GFX908-NEXT: s_mov_b64 s[4:5], exec
|
||
|
|
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
|
||
|
|
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
|
||
|
|
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX908-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GFX908-NEXT: s_and_saveexec_b64 s[2:3], vcc
|
||
|
|
; GFX908-NEXT: s_cbranch_execz .LBB1_4
|
||
|
|
; GFX908-NEXT: ; %bb.1:
|
||
|
|
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX908-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
|
||
|
|
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
|
||
|
|
; GFX908-NEXT: s_mov_b64 s[4:5], 0
|
||
|
|
; GFX908-NEXT: v_mul_f32_e32 v2, 4.0, v1
|
||
|
|
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX908-NEXT: s_load_dword s6, s[0:1], 0x0
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v1, s6
|
||
|
|
; GFX908-NEXT: .LBB1_2: ; %atomicrmw.start
|
||
|
|
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v5, v1
|
||
|
|
; GFX908-NEXT: v_add_f32_e32 v4, v5, v2
|
||
|
|
; GFX908-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
|
||
|
|
; GFX908-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX908-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
|
||
|
|
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
||
|
|
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
||
|
|
; GFX908-NEXT: s_cbranch_execnz .LBB1_2
|
||
|
|
; GFX908-NEXT: ; %bb.3: ; %Flow
|
||
|
|
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
|
||
|
|
; GFX908-NEXT: .LBB1_4: ; %Flow1
|
||
|
|
; GFX908-NEXT: s_or_b64 exec, exec, s[2:3]
|
||
|
|
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GFX908-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GFX908-NEXT: v_mul_f32_e32 v0, 4.0, v0
|
||
|
|
; GFX908-NEXT: v_add_f32_e32 v0, s0, v0
|
||
|
|
; GFX908-NEXT: global_store_dword v[0:1], v0, off
|
||
|
|
; GFX908-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX90A-LABEL: global_atomic_fadd_ret_f32_ieee:
|
||
|
|
; GFX90A: ; %bb.0:
|
||
|
|
; GFX90A-NEXT: s_mov_b64 s[4:5], exec
|
||
|
|
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
|
||
|
|
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
|
||
|
|
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX90A-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
|
||
|
|
; GFX90A-NEXT: s_cbranch_execz .LBB1_2
|
||
|
|
; GFX90A-NEXT: ; %bb.1:
|
||
|
|
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX90A-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
|
||
|
|
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v2, s4
|
||
|
|
; GFX90A-NEXT: v_mov_b32_e32 v1, 0
|
||
|
|
; GFX90A-NEXT: v_mul_f32_e32 v2, 4.0, v2
|
||
|
|
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX90A-NEXT: global_atomic_add_f32 v1, v1, v2, s[0:1] glc
|
||
|
|
; GFX90A-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX90A-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX90A-NEXT: .LBB1_2:
|
||
|
|
; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3]
|
||
|
|
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GFX90A-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GFX90A-NEXT: v_mul_f32_e32 v0, 4.0, v0
|
||
|
|
; GFX90A-NEXT: v_add_f32_e32 v0, s0, v0
|
||
|
|
; GFX90A-NEXT: global_store_dword v[0:1], v0, off
|
||
|
|
; GFX90A-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX10-LABEL: global_atomic_fadd_ret_f32_ieee:
|
||
|
|
; GFX10: ; %bb.0:
|
||
|
|
; GFX10-NEXT: s_mov_b32 s4, exec_lo
|
||
|
|
; GFX10-NEXT: s_mov_b32 s3, 0
|
||
|
|
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
|
||
|
|
; GFX10-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
|
||
|
|
; GFX10-NEXT: s_and_saveexec_b32 s2, vcc_lo
|
||
|
|
; GFX10-NEXT: s_cbranch_execz .LBB1_4
|
||
|
|
; GFX10-NEXT: ; %bb.1:
|
||
|
|
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX10-NEXT: s_bcnt1_i32_b32 s4, s4
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
|
||
|
|
; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v1
|
||
|
|
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX10-NEXT: s_load_dword s5, s[0:1], 0x0
|
||
|
|
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v1, s5
|
||
|
|
; GFX10-NEXT: .LBB1_2: ; %atomicrmw.start
|
||
|
|
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v5, v1
|
||
|
|
; GFX10-NEXT: v_add_f32_e32 v4, v5, v2
|
||
|
|
; GFX10-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
|
||
|
|
; GFX10-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX10-NEXT: buffer_gl0_inv
|
||
|
|
; GFX10-NEXT: buffer_gl1_inv
|
||
|
|
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
|
||
|
|
; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3
|
||
|
|
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
|
||
|
|
; GFX10-NEXT: s_cbranch_execnz .LBB1_2
|
||
|
|
; GFX10-NEXT: ; %bb.3: ; %Flow
|
||
|
|
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
|
||
|
|
; GFX10-NEXT: .LBB1_4: ; %Flow1
|
||
|
|
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s2
|
||
|
|
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GFX10-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GFX10-NEXT: v_mul_f32_e32 v0, 4.0, v0
|
||
|
|
; GFX10-NEXT: v_add_f32_e32 v0, s0, v0
|
||
|
|
; GFX10-NEXT: global_store_dword v[0:1], v0, off
|
||
|
|
; GFX10-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX11-LABEL: global_atomic_fadd_ret_f32_ieee:
|
||
|
|
; GFX11: ; %bb.0:
|
||
|
|
; GFX11-NEXT: s_mov_b32 s3, exec_lo
|
||
|
|
; GFX11-NEXT: s_mov_b32 s2, exec_lo
|
||
|
|
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
|
||
|
|
; GFX11-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
|
||
|
|
; GFX11-NEXT: s_cbranch_execz .LBB1_2
|
||
|
|
; GFX11-NEXT: ; %bb.1:
|
||
|
|
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX11-NEXT: s_bcnt1_i32_b32 s3, s3
|
||
|
|
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
|
||
|
|
; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mul_f32 v1, 4.0, v1
|
||
|
|
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX11-NEXT: global_atomic_add_f32 v1, v2, v1, s[0:1] glc
|
||
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX11-NEXT: buffer_gl0_inv
|
||
|
|
; GFX11-NEXT: buffer_gl1_inv
|
||
|
|
; GFX11-NEXT: .LBB1_2:
|
||
|
|
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s2
|
||
|
|
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GFX11-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GFX11-NEXT: v_mul_f32_e32 v0, 4.0, v0
|
||
|
|
; GFX11-NEXT: v_add_f32_e32 v0, s0, v0
|
||
|
|
; GFX11-NEXT: global_store_b32 v[0:1], v0, off
|
||
|
|
; GFX11-NEXT: s_nop 0
|
||
|
|
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
|
||
|
|
; GFX11-NEXT: s_endpgm
|
||
|
|
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
|
||
|
|
store float %result, ptr addrspace(1) undef
|
||
|
|
ret void
|
||
|
|
}
|
||
|
|
|
||
|
|
define amdgpu_kernel void @global_atomic_fadd_noret_f32(ptr addrspace(1) %ptr) #0 {
|
||
|
|
; GFX900-LABEL: global_atomic_fadd_noret_f32:
|
||
|
|
; GFX900: ; %bb.0:
|
||
|
|
; GFX900-NEXT: s_mov_b64 s[2:3], exec
|
||
|
|
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
|
||
|
|
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
|
||
|
|
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX900-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
||
|
|
; GFX900-NEXT: s_cbranch_execz .LBB2_3
|
||
|
|
; GFX900-NEXT: ; %bb.1:
|
||
|
|
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX900-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
|
||
|
|
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
|
||
|
|
; GFX900-NEXT: s_mov_b64 s[2:3], 0
|
||
|
|
; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v0
|
||
|
|
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v1, s4
|
||
|
|
; GFX900-NEXT: .LBB2_2: ; %atomicrmw.start
|
||
|
|
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX900-NEXT: v_add_f32_e32 v0, v1, v2
|
||
|
|
; GFX900-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
|
||
|
|
; GFX900-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX900-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
|
||
|
|
; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v1, v0
|
||
|
|
; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
||
|
|
; GFX900-NEXT: s_cbranch_execnz .LBB2_2
|
||
|
|
; GFX900-NEXT: .LBB2_3:
|
||
|
|
; GFX900-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX908-LABEL: global_atomic_fadd_noret_f32:
|
||
|
|
; GFX908: ; %bb.0:
|
||
|
|
; GFX908-NEXT: s_mov_b64 s[2:3], exec
|
||
|
|
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
|
||
|
|
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
|
||
|
|
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
||
|
|
; GFX908-NEXT: s_cbranch_execz .LBB2_2
|
||
|
|
; GFX908-NEXT: ; %bb.1:
|
||
|
|
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX908-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
|
||
|
|
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v0, 0
|
||
|
|
; GFX908-NEXT: v_mul_f32_e32 v1, 4.0, v1
|
||
|
|
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX908-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
|
||
|
|
; GFX908-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX908-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX908-NEXT: .LBB2_2:
|
||
|
|
; GFX908-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX90A-LABEL: global_atomic_fadd_noret_f32:
|
||
|
|
; GFX90A: ; %bb.0:
|
||
|
|
; GFX90A-NEXT: s_mov_b64 s[2:3], exec
|
||
|
|
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
|
||
|
|
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
|
||
|
|
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
||
|
|
; GFX90A-NEXT: s_cbranch_execz .LBB2_2
|
||
|
|
; GFX90A-NEXT: ; %bb.1:
|
||
|
|
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
|
||
|
|
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
|
||
|
|
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
|
||
|
|
; GFX90A-NEXT: v_mul_f32_e32 v1, 4.0, v1
|
||
|
|
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX90A-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
|
||
|
|
; GFX90A-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX90A-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX90A-NEXT: .LBB2_2:
|
||
|
|
; GFX90A-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX10-LABEL: global_atomic_fadd_noret_f32:
|
||
|
|
; GFX10: ; %bb.0:
|
||
|
|
; GFX10-NEXT: s_mov_b32 s3, exec_lo
|
||
|
|
; GFX10-NEXT: s_mov_b32 s2, 0
|
||
|
|
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
|
||
|
|
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
|
||
|
|
; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo
|
||
|
|
; GFX10-NEXT: s_cbranch_execz .LBB2_3
|
||
|
|
; GFX10-NEXT: ; %bb.1:
|
||
|
|
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX10-NEXT: s_bcnt1_i32_b32 s3, s3
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
|
||
|
|
; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v0
|
||
|
|
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX10-NEXT: s_load_dword s4, s[0:1], 0x0
|
||
|
|
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v1, s4
|
||
|
|
; GFX10-NEXT: .LBB2_2: ; %atomicrmw.start
|
||
|
|
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX10-NEXT: v_add_f32_e32 v0, v1, v2
|
||
|
|
; GFX10-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
|
||
|
|
; GFX10-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX10-NEXT: buffer_gl0_inv
|
||
|
|
; GFX10-NEXT: buffer_gl1_inv
|
||
|
|
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v1, v0
|
||
|
|
; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2
|
||
|
|
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
|
||
|
|
; GFX10-NEXT: s_cbranch_execnz .LBB2_2
|
||
|
|
; GFX10-NEXT: .LBB2_3:
|
||
|
|
; GFX10-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX11-LABEL: global_atomic_fadd_noret_f32:
|
||
|
|
; GFX11: ; %bb.0:
|
||
|
|
; GFX11-NEXT: s_mov_b32 s2, exec_lo
|
||
|
|
; GFX11-NEXT: s_mov_b32 s3, exec_lo
|
||
|
|
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
|
||
|
|
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
|
||
|
|
; GFX11-NEXT: s_cbranch_execz .LBB2_2
|
||
|
|
; GFX11-NEXT: ; %bb.1:
|
||
|
|
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX11-NEXT: s_bcnt1_i32_b32 s2, s2
|
||
|
|
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
|
||
|
|
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
|
||
|
|
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX11-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
|
||
|
|
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
|
||
|
|
; GFX11-NEXT: buffer_gl0_inv
|
||
|
|
; GFX11-NEXT: buffer_gl1_inv
|
||
|
|
; GFX11-NEXT: .LBB2_2:
|
||
|
|
; GFX11-NEXT: s_endpgm
|
||
|
|
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
|
||
|
|
ret void
|
||
|
|
}
|
||
|
|
|
||
|
|
define amdgpu_kernel void @global_atomic_fadd_noret_f32_ieee(ptr addrspace(1) %ptr) #2 {
|
||
|
|
; GFX900-LABEL: global_atomic_fadd_noret_f32_ieee:
|
||
|
|
; GFX900: ; %bb.0:
|
||
|
|
; GFX900-NEXT: s_mov_b64 s[2:3], exec
|
||
|
|
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
|
||
|
|
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
|
||
|
|
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX900-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
||
|
|
; GFX900-NEXT: s_cbranch_execz .LBB3_3
|
||
|
|
; GFX900-NEXT: ; %bb.1:
|
||
|
|
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX900-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
|
||
|
|
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
|
||
|
|
; GFX900-NEXT: s_mov_b64 s[2:3], 0
|
||
|
|
; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v0
|
||
|
|
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v1, s4
|
||
|
|
; GFX900-NEXT: .LBB3_2: ; %atomicrmw.start
|
||
|
|
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX900-NEXT: v_add_f32_e32 v0, v1, v2
|
||
|
|
; GFX900-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
|
||
|
|
; GFX900-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX900-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
|
||
|
|
; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v1, v0
|
||
|
|
; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
||
|
|
; GFX900-NEXT: s_cbranch_execnz .LBB3_2
|
||
|
|
; GFX900-NEXT: .LBB3_3:
|
||
|
|
; GFX900-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX908-LABEL: global_atomic_fadd_noret_f32_ieee:
|
||
|
|
; GFX908: ; %bb.0:
|
||
|
|
; GFX908-NEXT: s_mov_b64 s[2:3], exec
|
||
|
|
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
|
||
|
|
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
|
||
|
|
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
||
|
|
; GFX908-NEXT: s_cbranch_execz .LBB3_2
|
||
|
|
; GFX908-NEXT: ; %bb.1:
|
||
|
|
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX908-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
|
||
|
|
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v0, 0
|
||
|
|
; GFX908-NEXT: v_mul_f32_e32 v1, 4.0, v1
|
||
|
|
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX908-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
|
||
|
|
; GFX908-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX908-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX908-NEXT: .LBB3_2:
|
||
|
|
; GFX908-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX90A-LABEL: global_atomic_fadd_noret_f32_ieee:
|
||
|
|
; GFX90A: ; %bb.0:
|
||
|
|
; GFX90A-NEXT: s_mov_b64 s[2:3], exec
|
||
|
|
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
|
||
|
|
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
|
||
|
|
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
||
|
|
; GFX90A-NEXT: s_cbranch_execz .LBB3_2
|
||
|
|
; GFX90A-NEXT: ; %bb.1:
|
||
|
|
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
|
||
|
|
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
|
||
|
|
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
|
||
|
|
; GFX90A-NEXT: v_mul_f32_e32 v1, 4.0, v1
|
||
|
|
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX90A-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
|
||
|
|
; GFX90A-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX90A-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX90A-NEXT: .LBB3_2:
|
||
|
|
; GFX90A-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX10-LABEL: global_atomic_fadd_noret_f32_ieee:
|
||
|
|
; GFX10: ; %bb.0:
|
||
|
|
; GFX10-NEXT: s_mov_b32 s3, exec_lo
|
||
|
|
; GFX10-NEXT: s_mov_b32 s2, 0
|
||
|
|
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
|
||
|
|
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
|
||
|
|
; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo
|
||
|
|
; GFX10-NEXT: s_cbranch_execz .LBB3_3
|
||
|
|
; GFX10-NEXT: ; %bb.1:
|
||
|
|
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX10-NEXT: s_bcnt1_i32_b32 s3, s3
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
|
||
|
|
; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v0
|
||
|
|
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX10-NEXT: s_load_dword s4, s[0:1], 0x0
|
||
|
|
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v1, s4
|
||
|
|
; GFX10-NEXT: .LBB3_2: ; %atomicrmw.start
|
||
|
|
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX10-NEXT: v_add_f32_e32 v0, v1, v2
|
||
|
|
; GFX10-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
|
||
|
|
; GFX10-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX10-NEXT: buffer_gl0_inv
|
||
|
|
; GFX10-NEXT: buffer_gl1_inv
|
||
|
|
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v1, v0
|
||
|
|
; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2
|
||
|
|
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
|
||
|
|
; GFX10-NEXT: s_cbranch_execnz .LBB3_2
|
||
|
|
; GFX10-NEXT: .LBB3_3:
|
||
|
|
; GFX10-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX11-LABEL: global_atomic_fadd_noret_f32_ieee:
|
||
|
|
; GFX11: ; %bb.0:
|
||
|
|
; GFX11-NEXT: s_mov_b32 s2, exec_lo
|
||
|
|
; GFX11-NEXT: s_mov_b32 s3, exec_lo
|
||
|
|
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
|
||
|
|
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
|
||
|
|
; GFX11-NEXT: s_cbranch_execz .LBB3_2
|
||
|
|
; GFX11-NEXT: ; %bb.1:
|
||
|
|
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX11-NEXT: s_bcnt1_i32_b32 s2, s2
|
||
|
|
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
|
||
|
|
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
|
||
|
|
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX11-NEXT: global_atomic_add_f32 v1, v0, s[0:1]
|
||
|
|
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
|
||
|
|
; GFX11-NEXT: buffer_gl0_inv
|
||
|
|
; GFX11-NEXT: buffer_gl1_inv
|
||
|
|
; GFX11-NEXT: .LBB3_2:
|
||
|
|
; GFX11-NEXT: s_endpgm
|
||
|
|
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
|
||
|
|
ret void
|
||
|
|
}
|
||
|
|
|
||
|
|
define amdgpu_kernel void @global_atomic_fadd_ret_f32_agent(ptr addrspace(1) %ptr) #0 {
|
||
|
|
; GFX900-LABEL: global_atomic_fadd_ret_f32_agent:
|
||
|
|
; GFX900: ; %bb.0:
|
||
|
|
; GFX900-NEXT: s_mov_b64 s[4:5], exec
|
||
|
|
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
|
||
|
|
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
|
||
|
|
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX900-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GFX900-NEXT: s_and_saveexec_b64 s[2:3], vcc
|
||
|
|
; GFX900-NEXT: s_cbranch_execz .LBB4_4
|
||
|
|
; GFX900-NEXT: ; %bb.1:
|
||
|
|
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX900-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
|
||
|
|
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
|
||
|
|
; GFX900-NEXT: s_mov_b64 s[4:5], 0
|
||
|
|
; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v1
|
||
|
|
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX900-NEXT: s_load_dword s6, s[0:1], 0x0
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v1, s6
|
||
|
|
; GFX900-NEXT: .LBB4_2: ; %atomicrmw.start
|
||
|
|
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v5, v1
|
||
|
|
; GFX900-NEXT: v_add_f32_e32 v4, v5, v2
|
||
|
|
; GFX900-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
|
||
|
|
; GFX900-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX900-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
|
||
|
|
; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
||
|
|
; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
||
|
|
; GFX900-NEXT: s_cbranch_execnz .LBB4_2
|
||
|
|
; GFX900-NEXT: ; %bb.3: ; %Flow
|
||
|
|
; GFX900-NEXT: s_or_b64 exec, exec, s[4:5]
|
||
|
|
; GFX900-NEXT: .LBB4_4: ; %Flow1
|
||
|
|
; GFX900-NEXT: s_or_b64 exec, exec, s[2:3]
|
||
|
|
; GFX900-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GFX900-NEXT: v_mad_f32 v0, v0, 4.0, s0
|
||
|
|
; GFX900-NEXT: global_store_dword v[0:1], v0, off
|
||
|
|
; GFX900-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX908-LABEL: global_atomic_fadd_ret_f32_agent:
|
||
|
|
; GFX908: ; %bb.0:
|
||
|
|
; GFX908-NEXT: s_mov_b64 s[4:5], exec
|
||
|
|
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
|
||
|
|
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
|
||
|
|
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX908-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GFX908-NEXT: s_and_saveexec_b64 s[2:3], vcc
|
||
|
|
; GFX908-NEXT: s_cbranch_execz .LBB4_4
|
||
|
|
; GFX908-NEXT: ; %bb.1:
|
||
|
|
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX908-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
|
||
|
|
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
|
||
|
|
; GFX908-NEXT: s_mov_b64 s[4:5], 0
|
||
|
|
; GFX908-NEXT: v_mul_f32_e32 v2, 4.0, v1
|
||
|
|
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX908-NEXT: s_load_dword s6, s[0:1], 0x0
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v1, s6
|
||
|
|
; GFX908-NEXT: .LBB4_2: ; %atomicrmw.start
|
||
|
|
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v5, v1
|
||
|
|
; GFX908-NEXT: v_add_f32_e32 v4, v5, v2
|
||
|
|
; GFX908-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
|
||
|
|
; GFX908-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX908-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
|
||
|
|
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
||
|
|
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
||
|
|
; GFX908-NEXT: s_cbranch_execnz .LBB4_2
|
||
|
|
; GFX908-NEXT: ; %bb.3: ; %Flow
|
||
|
|
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
|
||
|
|
; GFX908-NEXT: .LBB4_4: ; %Flow1
|
||
|
|
; GFX908-NEXT: s_or_b64 exec, exec, s[2:3]
|
||
|
|
; GFX908-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GFX908-NEXT: v_mad_f32 v0, v0, 4.0, s0
|
||
|
|
; GFX908-NEXT: global_store_dword v[0:1], v0, off
|
||
|
|
; GFX908-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX90A-LABEL: global_atomic_fadd_ret_f32_agent:
|
||
|
|
; GFX90A: ; %bb.0:
|
||
|
|
; GFX90A-NEXT: s_mov_b64 s[4:5], exec
|
||
|
|
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
|
||
|
|
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
|
||
|
|
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX90A-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
|
||
|
|
; GFX90A-NEXT: s_cbranch_execz .LBB4_2
|
||
|
|
; GFX90A-NEXT: ; %bb.1:
|
||
|
|
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX90A-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
|
||
|
|
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v2, s4
|
||
|
|
; GFX90A-NEXT: v_mov_b32_e32 v1, 0
|
||
|
|
; GFX90A-NEXT: v_mul_f32_e32 v2, 4.0, v2
|
||
|
|
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX90A-NEXT: global_atomic_add_f32 v1, v1, v2, s[0:1] glc
|
||
|
|
; GFX90A-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX90A-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX90A-NEXT: .LBB4_2:
|
||
|
|
; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3]
|
||
|
|
; GFX90A-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GFX90A-NEXT: v_mad_f32 v0, v0, 4.0, s0
|
||
|
|
; GFX90A-NEXT: global_store_dword v[0:1], v0, off
|
||
|
|
; GFX90A-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX10-LABEL: global_atomic_fadd_ret_f32_agent:
|
||
|
|
; GFX10: ; %bb.0:
|
||
|
|
; GFX10-NEXT: s_mov_b32 s4, exec_lo
|
||
|
|
; GFX10-NEXT: s_mov_b32 s3, 0
|
||
|
|
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
|
||
|
|
; GFX10-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
|
||
|
|
; GFX10-NEXT: s_and_saveexec_b32 s2, vcc_lo
|
||
|
|
; GFX10-NEXT: s_cbranch_execz .LBB4_4
|
||
|
|
; GFX10-NEXT: ; %bb.1:
|
||
|
|
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX10-NEXT: s_bcnt1_i32_b32 s4, s4
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
|
||
|
|
; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v1
|
||
|
|
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX10-NEXT: s_load_dword s5, s[0:1], 0x0
|
||
|
|
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v1, s5
|
||
|
|
; GFX10-NEXT: .LBB4_2: ; %atomicrmw.start
|
||
|
|
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v5, v1
|
||
|
|
; GFX10-NEXT: v_add_f32_e32 v4, v5, v2
|
||
|
|
; GFX10-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
|
||
|
|
; GFX10-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX10-NEXT: buffer_gl0_inv
|
||
|
|
; GFX10-NEXT: buffer_gl1_inv
|
||
|
|
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
|
||
|
|
; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3
|
||
|
|
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
|
||
|
|
; GFX10-NEXT: s_cbranch_execnz .LBB4_2
|
||
|
|
; GFX10-NEXT: ; %bb.3: ; %Flow
|
||
|
|
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
|
||
|
|
; GFX10-NEXT: .LBB4_4: ; %Flow1
|
||
|
|
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s2
|
||
|
|
; GFX10-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GFX10-NEXT: v_mad_f32 v0, v0, 4.0, s0
|
||
|
|
; GFX10-NEXT: global_store_dword v[0:1], v0, off
|
||
|
|
; GFX10-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX11-LABEL: global_atomic_fadd_ret_f32_agent:
|
||
|
|
; GFX11: ; %bb.0:
|
||
|
|
; GFX11-NEXT: s_mov_b32 s3, exec_lo
|
||
|
|
; GFX11-NEXT: s_mov_b32 s2, exec_lo
|
||
|
|
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
|
||
|
|
; GFX11-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
|
||
|
|
; GFX11-NEXT: s_cbranch_execz .LBB4_2
|
||
|
|
; GFX11-NEXT: ; %bb.1:
|
||
|
|
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX11-NEXT: s_bcnt1_i32_b32 s3, s3
|
||
|
|
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
|
||
|
|
; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mul_f32 v1, 4.0, v1
|
||
|
|
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX11-NEXT: global_atomic_add_f32 v1, v2, v1, s[0:1] glc
|
||
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX11-NEXT: buffer_gl0_inv
|
||
|
|
; GFX11-NEXT: buffer_gl1_inv
|
||
|
|
; GFX11-NEXT: .LBB4_2:
|
||
|
|
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s2
|
||
|
|
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GFX11-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GFX11-NEXT: v_mul_f32_e32 v0, 4.0, v0
|
||
|
|
; GFX11-NEXT: v_add_f32_e32 v0, s0, v0
|
||
|
|
; GFX11-NEXT: global_store_b32 v[0:1], v0, off
|
||
|
|
; GFX11-NEXT: s_nop 0
|
||
|
|
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
|
||
|
|
; GFX11-NEXT: s_endpgm
|
||
|
|
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
|
||
|
|
store float %result, ptr addrspace(1) undef
|
||
|
|
ret void
|
||
|
|
}
|
||
|
|
|
||
|
|
define amdgpu_kernel void @global_atomic_fadd_ret_f32_system(ptr addrspace(1) %ptr) #0 {
|
||
|
|
; GFX900-LABEL: global_atomic_fadd_ret_f32_system:
|
||
|
|
; GFX900: ; %bb.0:
|
||
|
|
; GFX900-NEXT: s_mov_b64 s[4:5], exec
|
||
|
|
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
|
||
|
|
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
|
||
|
|
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX900-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GFX900-NEXT: s_and_saveexec_b64 s[2:3], vcc
|
||
|
|
; GFX900-NEXT: s_cbranch_execz .LBB5_4
|
||
|
|
; GFX900-NEXT: ; %bb.1:
|
||
|
|
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX900-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
|
||
|
|
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
|
||
|
|
; GFX900-NEXT: s_mov_b64 s[4:5], 0
|
||
|
|
; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v1
|
||
|
|
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX900-NEXT: s_load_dword s6, s[0:1], 0x0
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v1, s6
|
||
|
|
; GFX900-NEXT: .LBB5_2: ; %atomicrmw.start
|
||
|
|
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v5, v1
|
||
|
|
; GFX900-NEXT: v_add_f32_e32 v4, v5, v2
|
||
|
|
; GFX900-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
|
||
|
|
; GFX900-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX900-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
|
||
|
|
; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
||
|
|
; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
||
|
|
; GFX900-NEXT: s_cbranch_execnz .LBB5_2
|
||
|
|
; GFX900-NEXT: ; %bb.3: ; %Flow
|
||
|
|
; GFX900-NEXT: s_or_b64 exec, exec, s[4:5]
|
||
|
|
; GFX900-NEXT: .LBB5_4: ; %Flow1
|
||
|
|
; GFX900-NEXT: s_or_b64 exec, exec, s[2:3]
|
||
|
|
; GFX900-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GFX900-NEXT: v_mad_f32 v0, v0, 4.0, s0
|
||
|
|
; GFX900-NEXT: global_store_dword v[0:1], v0, off
|
||
|
|
; GFX900-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX908-LABEL: global_atomic_fadd_ret_f32_system:
|
||
|
|
; GFX908: ; %bb.0:
|
||
|
|
; GFX908-NEXT: s_mov_b64 s[4:5], exec
|
||
|
|
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
|
||
|
|
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
|
||
|
|
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX908-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GFX908-NEXT: s_and_saveexec_b64 s[2:3], vcc
|
||
|
|
; GFX908-NEXT: s_cbranch_execz .LBB5_4
|
||
|
|
; GFX908-NEXT: ; %bb.1:
|
||
|
|
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX908-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
|
||
|
|
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
|
||
|
|
; GFX908-NEXT: s_mov_b64 s[4:5], 0
|
||
|
|
; GFX908-NEXT: v_mul_f32_e32 v2, 4.0, v1
|
||
|
|
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX908-NEXT: s_load_dword s6, s[0:1], 0x0
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v1, s6
|
||
|
|
; GFX908-NEXT: .LBB5_2: ; %atomicrmw.start
|
||
|
|
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v5, v1
|
||
|
|
; GFX908-NEXT: v_add_f32_e32 v4, v5, v2
|
||
|
|
; GFX908-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
|
||
|
|
; GFX908-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX908-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
|
||
|
|
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
||
|
|
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
||
|
|
; GFX908-NEXT: s_cbranch_execnz .LBB5_2
|
||
|
|
; GFX908-NEXT: ; %bb.3: ; %Flow
|
||
|
|
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
|
||
|
|
; GFX908-NEXT: .LBB5_4: ; %Flow1
|
||
|
|
; GFX908-NEXT: s_or_b64 exec, exec, s[2:3]
|
||
|
|
; GFX908-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GFX908-NEXT: v_mad_f32 v0, v0, 4.0, s0
|
||
|
|
; GFX908-NEXT: global_store_dword v[0:1], v0, off
|
||
|
|
; GFX908-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX90A-LABEL: global_atomic_fadd_ret_f32_system:
|
||
|
|
; GFX90A: ; %bb.0:
|
||
|
|
; GFX90A-NEXT: s_mov_b64 s[4:5], exec
|
||
|
|
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
|
||
|
|
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
|
||
|
|
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX90A-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
|
||
|
|
; GFX90A-NEXT: s_cbranch_execz .LBB5_4
|
||
|
|
; GFX90A-NEXT: ; %bb.1:
|
||
|
|
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX90A-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
|
||
|
|
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
|
||
|
|
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
|
||
|
|
; GFX90A-NEXT: v_mul_f32_e32 v2, 4.0, v1
|
||
|
|
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX90A-NEXT: s_load_dword s6, s[0:1], 0x0
|
||
|
|
; GFX90A-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX90A-NEXT: v_mov_b32_e32 v1, s6
|
||
|
|
; GFX90A-NEXT: .LBB5_2: ; %atomicrmw.start
|
||
|
|
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX90A-NEXT: v_mov_b32_e32 v5, v1
|
||
|
|
; GFX90A-NEXT: v_add_f32_e32 v4, v5, v2
|
||
|
|
; GFX90A-NEXT: buffer_wbl2
|
||
|
|
; GFX90A-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
|
||
|
|
; GFX90A-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX90A-NEXT: buffer_invl2
|
||
|
|
; GFX90A-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
|
||
|
|
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
||
|
|
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
||
|
|
; GFX90A-NEXT: s_cbranch_execnz .LBB5_2
|
||
|
|
; GFX90A-NEXT: ; %bb.3: ; %Flow
|
||
|
|
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
|
||
|
|
; GFX90A-NEXT: .LBB5_4: ; %Flow1
|
||
|
|
; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3]
|
||
|
|
; GFX90A-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GFX90A-NEXT: v_mad_f32 v0, v0, 4.0, s0
|
||
|
|
; GFX90A-NEXT: global_store_dword v[0:1], v0, off
|
||
|
|
; GFX90A-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX10-LABEL: global_atomic_fadd_ret_f32_system:
|
||
|
|
; GFX10: ; %bb.0:
|
||
|
|
; GFX10-NEXT: s_mov_b32 s4, exec_lo
|
||
|
|
; GFX10-NEXT: s_mov_b32 s3, 0
|
||
|
|
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
|
||
|
|
; GFX10-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
|
||
|
|
; GFX10-NEXT: s_and_saveexec_b32 s2, vcc_lo
|
||
|
|
; GFX10-NEXT: s_cbranch_execz .LBB5_4
|
||
|
|
; GFX10-NEXT: ; %bb.1:
|
||
|
|
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX10-NEXT: s_bcnt1_i32_b32 s4, s4
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
|
||
|
|
; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v1
|
||
|
|
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX10-NEXT: s_load_dword s5, s[0:1], 0x0
|
||
|
|
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v1, s5
|
||
|
|
; GFX10-NEXT: .LBB5_2: ; %atomicrmw.start
|
||
|
|
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v5, v1
|
||
|
|
; GFX10-NEXT: v_add_f32_e32 v4, v5, v2
|
||
|
|
; GFX10-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
|
||
|
|
; GFX10-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX10-NEXT: buffer_gl0_inv
|
||
|
|
; GFX10-NEXT: buffer_gl1_inv
|
||
|
|
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
|
||
|
|
; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3
|
||
|
|
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
|
||
|
|
; GFX10-NEXT: s_cbranch_execnz .LBB5_2
|
||
|
|
; GFX10-NEXT: ; %bb.3: ; %Flow
|
||
|
|
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
|
||
|
|
; GFX10-NEXT: .LBB5_4: ; %Flow1
|
||
|
|
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s2
|
||
|
|
; GFX10-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GFX10-NEXT: v_mad_f32 v0, v0, 4.0, s0
|
||
|
|
; GFX10-NEXT: global_store_dword v[0:1], v0, off
|
||
|
|
; GFX10-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX11-LABEL: global_atomic_fadd_ret_f32_system:
|
||
|
|
; GFX11: ; %bb.0:
|
||
|
|
; GFX11-NEXT: s_mov_b32 s4, exec_lo
|
||
|
|
; GFX11-NEXT: s_mov_b32 s3, 0
|
||
|
|
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
|
||
|
|
; GFX11-NEXT: s_mov_b32 s2, exec_lo
|
||
|
|
; GFX11-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
|
||
|
|
; GFX11-NEXT: s_cbranch_execz .LBB5_4
|
||
|
|
; GFX11-NEXT: ; %bb.1:
|
||
|
|
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX11-NEXT: s_bcnt1_i32_b32 s4, s4
|
||
|
|
; GFX11-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s4
|
||
|
|
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX11-NEXT: s_load_b32 s5, s[0:1], 0x0
|
||
|
|
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX11-NEXT: v_dual_mul_f32 v2, 4.0, v1 :: v_dual_mov_b32 v1, s5
|
||
|
|
; GFX11-NEXT: .LBB5_2: ; %atomicrmw.start
|
||
|
|
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX11-NEXT: v_mov_b32_e32 v5, v1
|
||
|
|
; GFX11-NEXT: v_add_f32_e32 v4, v5, v2
|
||
|
|
; GFX11-NEXT: global_atomic_cmpswap_b32 v1, v3, v[4:5], s[0:1] glc
|
||
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX11-NEXT: buffer_gl0_inv
|
||
|
|
; GFX11-NEXT: buffer_gl1_inv
|
||
|
|
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
|
||
|
|
; GFX11-NEXT: s_or_b32 s3, vcc_lo, s3
|
||
|
|
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
|
||
|
|
; GFX11-NEXT: s_cbranch_execnz .LBB5_2
|
||
|
|
; GFX11-NEXT: ; %bb.3: ; %Flow
|
||
|
|
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3
|
||
|
|
; GFX11-NEXT: .LBB5_4: ; %Flow1
|
||
|
|
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s2
|
||
|
|
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GFX11-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GFX11-NEXT: v_mul_f32_e32 v0, 4.0, v0
|
||
|
|
; GFX11-NEXT: v_add_f32_e32 v0, s0, v0
|
||
|
|
; GFX11-NEXT: global_store_b32 v[0:1], v0, off
|
||
|
|
; GFX11-NEXT: s_nop 0
|
||
|
|
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
|
||
|
|
; GFX11-NEXT: s_endpgm
|
||
|
|
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("one-as") seq_cst
|
||
|
|
store float %result, ptr addrspace(1) undef
|
||
|
|
ret void
|
||
|
|
}
|
||
|
|
|
||
|
|
define amdgpu_kernel void @global_atomic_fadd_ret_f32_wrong_subtarget(ptr addrspace(1) %ptr) #1 {
|
||
|
|
; GCN-LABEL: global_atomic_fadd_ret_f32_wrong_subtarget:
|
||
|
|
; GCN: ; %bb.0:
|
||
|
|
; GCN-NEXT: s_mov_b64 s[4:5], exec
|
||
|
|
; GCN-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
|
||
|
|
; GCN-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
|
||
|
|
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GCN-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GCN-NEXT: s_and_saveexec_b64 s[2:3], vcc
|
||
|
|
; GCN-NEXT: s_cbranch_execz .LBB6_4
|
||
|
|
; GCN-NEXT: ; %bb.1:
|
||
|
|
; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GCN-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
|
||
|
|
; GCN-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
|
||
|
|
; GCN-NEXT: s_mov_b64 s[4:5], 0
|
||
|
|
; GCN-NEXT: v_mul_f32_e32 v2, 4.0, v1
|
||
|
|
; GCN-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GCN-NEXT: s_load_dword s6, s[0:1], 0x0
|
||
|
|
; GCN-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GCN-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GCN-NEXT: v_mov_b32_e32 v1, s6
|
||
|
|
; GCN-NEXT: .LBB6_2: ; %atomicrmw.start
|
||
|
|
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GCN-NEXT: v_mov_b32_e32 v5, v1
|
||
|
|
; GCN-NEXT: v_add_f32_e32 v4, v5, v2
|
||
|
|
; GCN-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
|
||
|
|
; GCN-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GCN-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
|
||
|
|
; GCN-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
||
|
|
; GCN-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
||
|
|
; GCN-NEXT: s_cbranch_execnz .LBB6_2
|
||
|
|
; GCN-NEXT: ; %bb.3: ; %Flow
|
||
|
|
; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
|
||
|
|
; GCN-NEXT: .LBB6_4: ; %Flow1
|
||
|
|
; GCN-NEXT: s_or_b64 exec, exec, s[2:3]
|
||
|
|
; GCN-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GCN-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GCN-NEXT: v_mad_f32 v0, v0, 4.0, s0
|
||
|
|
; GCN-NEXT: global_store_dword v[0:1], v0, off
|
||
|
|
; GCN-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX11-LABEL: global_atomic_fadd_ret_f32_wrong_subtarget:
|
||
|
|
; GFX11: ; %bb.0:
|
||
|
|
; GFX11-NEXT: s_mov_b64 s[4:5], exec
|
||
|
|
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s4, 0
|
||
|
|
; GFX11-NEXT: v_mbcnt_hi_u32_b32 v0, s5, v0
|
||
|
|
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX11-NEXT: ; implicit-def: $vgpr1
|
||
|
|
; GFX11-NEXT: s_and_saveexec_b64 s[2:3], vcc
|
||
|
|
; GFX11-NEXT: s_cbranch_execz .LBB6_4
|
||
|
|
; GFX11-NEXT: ; %bb.1:
|
||
|
|
; GFX11-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX11-NEXT: s_bcnt1_i32_b64 s7, s[4:5]
|
||
|
|
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s7
|
||
|
|
; GFX11-NEXT: s_mov_b64 s[4:5], 0
|
||
|
|
; GFX11-NEXT: v_mul_f32_e32 v2, 4.0, v1
|
||
|
|
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX11-NEXT: s_load_dword s6, s[0:1], 0x0
|
||
|
|
; GFX11-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX11-NEXT: v_mov_b32_e32 v1, s6
|
||
|
|
; GFX11-NEXT: .LBB6_2: ; %atomicrmw.start
|
||
|
|
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX11-NEXT: v_mov_b32_e32 v5, v1
|
||
|
|
; GFX11-NEXT: v_add_f32_e32 v4, v5, v2
|
||
|
|
; GFX11-NEXT: global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
|
||
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX11-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
|
||
|
|
; GFX11-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
|
||
|
|
; GFX11-NEXT: s_andn2_b64 exec, exec, s[4:5]
|
||
|
|
; GFX11-NEXT: s_cbranch_execnz .LBB6_2
|
||
|
|
; GFX11-NEXT: ; %bb.3: ; %Flow
|
||
|
|
; GFX11-NEXT: s_or_b64 exec, exec, s[4:5]
|
||
|
|
; GFX11-NEXT: .LBB6_4: ; %Flow1
|
||
|
|
; GFX11-NEXT: s_or_b64 exec, exec, s[2:3]
|
||
|
|
; GFX11-NEXT: v_readfirstlane_b32 s0, v1
|
||
|
|
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
|
||
|
|
; GFX11-NEXT: v_mad_f32 v0, v0, 4.0, s0
|
||
|
|
; GFX11-NEXT: global_store_dword v[0:1], v0, off
|
||
|
|
; GFX11-NEXT: s_endpgm
|
||
|
|
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
|
||
|
|
store float %result, ptr addrspace(1) undef
|
||
|
|
ret void
|
||
|
|
}
|
||
|
|
|
||
|
|
define amdgpu_kernel void @global_atomic_fadd_noret_f32_wrong_subtarget(ptr addrspace(1) %ptr) #1 {
|
||
|
|
; GCN-LABEL: global_atomic_fadd_noret_f32_wrong_subtarget:
|
||
|
|
; GCN: ; %bb.0:
|
||
|
|
; GCN-NEXT: s_mov_b64 s[2:3], exec
|
||
|
|
; GCN-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
|
||
|
|
; GCN-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
|
||
|
|
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
||
|
|
; GCN-NEXT: s_cbranch_execz .LBB7_2
|
||
|
|
; GCN-NEXT: ; %bb.1:
|
||
|
|
; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GCN-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
|
||
|
|
; GCN-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
|
||
|
|
; GCN-NEXT: v_mov_b32_e32 v0, 0
|
||
|
|
; GCN-NEXT: v_mul_f32_e32 v1, 4.0, v1
|
||
|
|
; GCN-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GCN-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
|
||
|
|
; GCN-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GCN-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GCN-NEXT: .LBB7_2:
|
||
|
|
; GCN-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX11-LABEL: global_atomic_fadd_noret_f32_wrong_subtarget:
|
||
|
|
; GFX11: ; %bb.0:
|
||
|
|
; GFX11-NEXT: s_mov_b64 s[2:3], exec
|
||
|
|
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
|
||
|
|
; GFX11-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
|
||
|
|
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX11-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
||
|
|
; GFX11-NEXT: s_cbranch_execz .LBB7_2
|
||
|
|
; GFX11-NEXT: ; %bb.1:
|
||
|
|
; GFX11-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX11-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
|
||
|
|
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
|
||
|
|
; GFX11-NEXT: v_mov_b32_e32 v0, 0
|
||
|
|
; GFX11-NEXT: v_mul_f32_e32 v1, 4.0, v1
|
||
|
|
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX11-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
|
||
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX11-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX11-NEXT: .LBB7_2:
|
||
|
|
; GFX11-NEXT: s_endpgm
|
||
|
|
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
|
||
|
|
ret void
|
||
|
|
}
|
||
|
|
|
||
|
|
define amdgpu_kernel void @global_atomic_fadd_noret_f32_safe(ptr addrspace(1) %ptr) {
|
||
|
|
; GFX900-LABEL: global_atomic_fadd_noret_f32_safe:
|
||
|
|
; GFX900: ; %bb.0:
|
||
|
|
; GFX900-NEXT: s_mov_b64 s[2:3], exec
|
||
|
|
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
|
||
|
|
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
|
||
|
|
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX900-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
||
|
|
; GFX900-NEXT: s_cbranch_execz .LBB8_3
|
||
|
|
; GFX900-NEXT: ; %bb.1:
|
||
|
|
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX900-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
|
||
|
|
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
|
||
|
|
; GFX900-NEXT: s_mov_b64 s[2:3], 0
|
||
|
|
; GFX900-NEXT: v_mul_f32_e32 v2, 4.0, v0
|
||
|
|
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v1, s4
|
||
|
|
; GFX900-NEXT: .LBB8_2: ; %atomicrmw.start
|
||
|
|
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX900-NEXT: v_add_f32_e32 v0, v1, v2
|
||
|
|
; GFX900-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
|
||
|
|
; GFX900-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX900-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
|
||
|
|
; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v1, v0
|
||
|
|
; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
||
|
|
; GFX900-NEXT: s_cbranch_execnz .LBB8_2
|
||
|
|
; GFX900-NEXT: .LBB8_3:
|
||
|
|
; GFX900-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX908-LABEL: global_atomic_fadd_noret_f32_safe:
|
||
|
|
; GFX908: ; %bb.0:
|
||
|
|
; GFX908-NEXT: s_mov_b64 s[2:3], exec
|
||
|
|
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
|
||
|
|
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
|
||
|
|
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
||
|
|
; GFX908-NEXT: s_cbranch_execz .LBB8_3
|
||
|
|
; GFX908-NEXT: ; %bb.1:
|
||
|
|
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX908-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
|
||
|
|
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
|
||
|
|
; GFX908-NEXT: s_mov_b64 s[2:3], 0
|
||
|
|
; GFX908-NEXT: v_mul_f32_e32 v2, 4.0, v0
|
||
|
|
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX908-NEXT: s_load_dword s4, s[0:1], 0x0
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v1, s4
|
||
|
|
; GFX908-NEXT: .LBB8_2: ; %atomicrmw.start
|
||
|
|
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX908-NEXT: v_add_f32_e32 v0, v1, v2
|
||
|
|
; GFX908-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
|
||
|
|
; GFX908-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX908-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
|
||
|
|
; GFX908-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v1, v0
|
||
|
|
; GFX908-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
||
|
|
; GFX908-NEXT: s_cbranch_execnz .LBB8_2
|
||
|
|
; GFX908-NEXT: .LBB8_3:
|
||
|
|
; GFX908-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX90A-LABEL: global_atomic_fadd_noret_f32_safe:
|
||
|
|
; GFX90A: ; %bb.0:
|
||
|
|
; GFX90A-NEXT: s_mov_b64 s[2:3], exec
|
||
|
|
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
|
||
|
|
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
|
||
|
|
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
||
|
|
; GFX90A-NEXT: s_cbranch_execz .LBB8_3
|
||
|
|
; GFX90A-NEXT: ; %bb.1:
|
||
|
|
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX90A-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
|
||
|
|
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, s5
|
||
|
|
; GFX90A-NEXT: s_mov_b64 s[2:3], 0
|
||
|
|
; GFX90A-NEXT: v_mul_f32_e32 v2, 4.0, v0
|
||
|
|
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX90A-NEXT: s_load_dword s4, s[0:1], 0x0
|
||
|
|
; GFX90A-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX90A-NEXT: v_mov_b32_e32 v1, s4
|
||
|
|
; GFX90A-NEXT: .LBB8_2: ; %atomicrmw.start
|
||
|
|
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX90A-NEXT: v_add_f32_e32 v0, v1, v2
|
||
|
|
; GFX90A-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
|
||
|
|
; GFX90A-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX90A-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
|
||
|
|
; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
||
|
|
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
|
||
|
|
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
||
|
|
; GFX90A-NEXT: s_cbranch_execnz .LBB8_2
|
||
|
|
; GFX90A-NEXT: .LBB8_3:
|
||
|
|
; GFX90A-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX10-LABEL: global_atomic_fadd_noret_f32_safe:
|
||
|
|
; GFX10: ; %bb.0:
|
||
|
|
; GFX10-NEXT: s_mov_b32 s3, exec_lo
|
||
|
|
; GFX10-NEXT: s_mov_b32 s2, 0
|
||
|
|
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
|
||
|
|
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
|
||
|
|
; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo
|
||
|
|
; GFX10-NEXT: s_cbranch_execz .LBB8_3
|
||
|
|
; GFX10-NEXT: ; %bb.1:
|
||
|
|
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX10-NEXT: s_bcnt1_i32_b32 s3, s3
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
|
||
|
|
; GFX10-NEXT: v_mul_f32_e32 v2, 4.0, v0
|
||
|
|
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX10-NEXT: s_load_dword s4, s[0:1], 0x0
|
||
|
|
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v1, s4
|
||
|
|
; GFX10-NEXT: .LBB8_2: ; %atomicrmw.start
|
||
|
|
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX10-NEXT: v_add_f32_e32 v0, v1, v2
|
||
|
|
; GFX10-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
|
||
|
|
; GFX10-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX10-NEXT: buffer_gl0_inv
|
||
|
|
; GFX10-NEXT: buffer_gl1_inv
|
||
|
|
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v1, v0
|
||
|
|
; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2
|
||
|
|
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
|
||
|
|
; GFX10-NEXT: s_cbranch_execnz .LBB8_2
|
||
|
|
; GFX10-NEXT: .LBB8_3:
|
||
|
|
; GFX10-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX11-LABEL: global_atomic_fadd_noret_f32_safe:
|
||
|
|
; GFX11: ; %bb.0:
|
||
|
|
; GFX11-NEXT: s_mov_b32 s3, exec_lo
|
||
|
|
; GFX11-NEXT: s_mov_b32 s2, 0
|
||
|
|
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
|
||
|
|
; GFX11-NEXT: s_mov_b32 s4, exec_lo
|
||
|
|
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
|
||
|
|
; GFX11-NEXT: s_cbranch_execz .LBB8_3
|
||
|
|
; GFX11-NEXT: ; %bb.1:
|
||
|
|
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX11-NEXT: s_bcnt1_i32_b32 s3, s3
|
||
|
|
; GFX11-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
|
||
|
|
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX11-NEXT: s_load_b32 s4, s[0:1], 0x0
|
||
|
|
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX11-NEXT: v_dual_mul_f32 v2, 4.0, v0 :: v_dual_mov_b32 v1, s4
|
||
|
|
; GFX11-NEXT: .LBB8_2: ; %atomicrmw.start
|
||
|
|
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX11-NEXT: v_add_f32_e32 v0, v1, v2
|
||
|
|
; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
|
||
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX11-NEXT: buffer_gl0_inv
|
||
|
|
; GFX11-NEXT: buffer_gl1_inv
|
||
|
|
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
|
||
|
|
; GFX11-NEXT: v_mov_b32_e32 v1, v0
|
||
|
|
; GFX11-NEXT: s_or_b32 s2, vcc_lo, s2
|
||
|
|
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
|
||
|
|
; GFX11-NEXT: s_cbranch_execnz .LBB8_2
|
||
|
|
; GFX11-NEXT: .LBB8_3:
|
||
|
|
; GFX11-NEXT: s_endpgm
|
||
|
|
%result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
|
||
|
|
ret void
|
||
|
|
}
|
||
|
|
|
||
|
|
define amdgpu_kernel void @infer_as_before_atomic(ptr addrspace(4) %arg) #0 {
|
||
|
|
; GFX900-LABEL: infer_as_before_atomic:
|
||
|
|
; GFX900: ; %bb.0:
|
||
|
|
; GFX900-NEXT: s_mov_b64 s[2:3], exec
|
||
|
|
; GFX900-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
|
||
|
|
; GFX900-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
|
||
|
|
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX900-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
||
|
|
; GFX900-NEXT: s_cbranch_execz .LBB9_3
|
||
|
|
; GFX900-NEXT: ; %bb.1:
|
||
|
|
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX900-NEXT: s_bcnt1_i32_b64 s5, s[2:3]
|
||
|
|
; GFX900-NEXT: s_mov_b64 s[2:3], 0
|
||
|
|
; GFX900-NEXT: v_cvt_f32_ubyte0_e32 v2, s5
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
|
||
|
|
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX900-NEXT: s_load_dword s4, s[0:1], 0x0
|
||
|
|
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v1, s4
|
||
|
|
; GFX900-NEXT: .LBB9_2: ; %atomicrmw.start
|
||
|
|
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX900-NEXT: v_add_f32_e32 v0, v1, v2
|
||
|
|
; GFX900-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
|
||
|
|
; GFX900-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
|
||
|
|
; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v1, v0
|
||
|
|
; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3]
|
||
|
|
; GFX900-NEXT: s_cbranch_execnz .LBB9_2
|
||
|
|
; GFX900-NEXT: .LBB9_3:
|
||
|
|
; GFX900-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX908-LABEL: infer_as_before_atomic:
|
||
|
|
; GFX908: ; %bb.0:
|
||
|
|
; GFX908-NEXT: s_mov_b64 s[2:3], exec
|
||
|
|
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
|
||
|
|
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
|
||
|
|
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
||
|
|
; GFX908-NEXT: s_cbranch_execz .LBB9_2
|
||
|
|
; GFX908-NEXT: ; %bb.1:
|
||
|
|
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX908-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v0, 0
|
||
|
|
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
|
||
|
|
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
|
||
|
|
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX908-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
|
||
|
|
; GFX908-NEXT: .LBB9_2:
|
||
|
|
; GFX908-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX90A-LABEL: infer_as_before_atomic:
|
||
|
|
; GFX90A: ; %bb.0:
|
||
|
|
; GFX90A-NEXT: s_mov_b64 s[2:3], exec
|
||
|
|
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
|
||
|
|
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
|
||
|
|
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
|
||
|
|
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
|
||
|
|
; GFX90A-NEXT: s_cbranch_execz .LBB9_2
|
||
|
|
; GFX90A-NEXT: ; %bb.1:
|
||
|
|
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
|
||
|
|
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
|
||
|
|
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
|
||
|
|
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
|
||
|
|
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX90A-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
|
||
|
|
; GFX90A-NEXT: .LBB9_2:
|
||
|
|
; GFX90A-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX10-LABEL: infer_as_before_atomic:
|
||
|
|
; GFX10: ; %bb.0:
|
||
|
|
; GFX10-NEXT: s_mov_b32 s3, exec_lo
|
||
|
|
; GFX10-NEXT: s_mov_b32 s2, 0
|
||
|
|
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s3, 0
|
||
|
|
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
|
||
|
|
; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo
|
||
|
|
; GFX10-NEXT: s_cbranch_execz .LBB9_3
|
||
|
|
; GFX10-NEXT: ; %bb.1:
|
||
|
|
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX10-NEXT: s_bcnt1_i32_b32 s3, s3
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v3, 0
|
||
|
|
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
|
||
|
|
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
|
||
|
|
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX10-NEXT: s_load_dword s4, s[0:1], 0x0
|
||
|
|
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v1, s4
|
||
|
|
; GFX10-NEXT: .LBB9_2: ; %atomicrmw.start
|
||
|
|
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX10-NEXT: v_add_f32_e32 v0, v1, v2
|
||
|
|
; GFX10-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
|
||
|
|
; GFX10-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v1, v0
|
||
|
|
; GFX10-NEXT: s_or_b32 s2, vcc_lo, s2
|
||
|
|
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
|
||
|
|
; GFX10-NEXT: s_cbranch_execnz .LBB9_2
|
||
|
|
; GFX10-NEXT: .LBB9_3:
|
||
|
|
; GFX10-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX11-LABEL: infer_as_before_atomic:
|
||
|
|
; GFX11: ; %bb.0:
|
||
|
|
; GFX11-NEXT: s_mov_b32 s2, exec_lo
|
||
|
|
; GFX11-NEXT: s_mov_b32 s3, exec_lo
|
||
|
|
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
|
||
|
|
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
|
||
|
|
; GFX11-NEXT: s_cbranch_execz .LBB9_2
|
||
|
|
; GFX11-NEXT: ; %bb.1:
|
||
|
|
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x24
|
||
|
|
; GFX11-NEXT: s_bcnt1_i32_b32 s2, s2
|
||
|
|
; GFX11-NEXT: v_mov_b32_e32 v0, 0
|
||
|
|
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
|
||
|
|
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
|
||
|
|
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX11-NEXT: global_atomic_add_f32 v0, v1, s[0:1]
|
||
|
|
; GFX11-NEXT: .LBB9_2:
|
||
|
|
; GFX11-NEXT: s_nop 0
|
||
|
|
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
|
||
|
|
; GFX11-NEXT: s_endpgm
|
||
|
|
%load = load ptr, ptr addrspace(4) %arg
|
||
|
|
%v = atomicrmw fadd ptr %load, float 1.0 syncscope("agent-one-as") monotonic, align 4
|
||
|
|
ret void
|
||
|
|
}
|
||
|
|
|
||
|
|
define amdgpu_kernel void @global_atomic_fadd_ret_bf16_agent(ptr addrspace(1) %ptr) #0 {
|
||
|
|
; GFX900-LABEL: global_atomic_fadd_ret_bf16_agent:
|
||
|
|
; GFX900: ; %bb.0:
|
||
|
|
; GFX900-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
|
||
|
|
; GFX900-NEXT: s_mov_b64 s[0:1], 0
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v0, 0
|
||
|
|
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX900-NEXT: s_and_b32 s2, s4, -4
|
||
|
|
; GFX900-NEXT: s_mov_b32 s3, s5
|
||
|
|
; GFX900-NEXT: s_load_dword s6, s[2:3], 0x0
|
||
|
|
; GFX900-NEXT: s_and_b32 s4, s4, 3
|
||
|
|
; GFX900-NEXT: s_lshl_b32 s4, s4, 3
|
||
|
|
; GFX900-NEXT: s_lshl_b32 s5, 0xffff, s4
|
||
|
|
; GFX900-NEXT: s_not_b32 s5, s5
|
||
|
|
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v1, s6
|
||
|
|
; GFX900-NEXT: .LBB10_1: ; %atomicrmw.start
|
||
|
|
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v2, v1
|
||
|
|
; GFX900-NEXT: v_lshrrev_b32_sdwa v1, s4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
|
||
|
|
; GFX900-NEXT: v_add_f32_e32 v1, 4.0, v1
|
||
|
|
; GFX900-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
|
||
|
|
; GFX900-NEXT: v_and_or_b32 v1, v2, s5, v1
|
||
|
|
; GFX900-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[2:3] glc
|
||
|
|
; GFX900-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX900-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
|
||
|
|
; GFX900-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
||
|
|
; GFX900-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
||
|
|
; GFX900-NEXT: s_cbranch_execnz .LBB10_1
|
||
|
|
; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end
|
||
|
|
; GFX900-NEXT: s_or_b64 exec, exec, s[0:1]
|
||
|
|
; GFX900-NEXT: v_lshrrev_b32_e32 v0, s4, v1
|
||
|
|
; GFX900-NEXT: global_store_short v[0:1], v0, off
|
||
|
|
; GFX900-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX908-LABEL: global_atomic_fadd_ret_bf16_agent:
|
||
|
|
; GFX908: ; %bb.0:
|
||
|
|
; GFX908-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
|
||
|
|
; GFX908-NEXT: s_mov_b64 s[0:1], 0
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v0, 0
|
||
|
|
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX908-NEXT: s_and_b32 s2, s4, -4
|
||
|
|
; GFX908-NEXT: s_mov_b32 s3, s5
|
||
|
|
; GFX908-NEXT: s_load_dword s6, s[2:3], 0x0
|
||
|
|
; GFX908-NEXT: s_and_b32 s4, s4, 3
|
||
|
|
; GFX908-NEXT: s_lshl_b32 s4, s4, 3
|
||
|
|
; GFX908-NEXT: s_lshl_b32 s5, 0xffff, s4
|
||
|
|
; GFX908-NEXT: s_not_b32 s5, s5
|
||
|
|
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v1, s6
|
||
|
|
; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
|
||
|
|
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v2, v1
|
||
|
|
; GFX908-NEXT: v_lshrrev_b32_sdwa v1, s4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
|
||
|
|
; GFX908-NEXT: v_add_f32_e32 v1, 4.0, v1
|
||
|
|
; GFX908-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
|
||
|
|
; GFX908-NEXT: v_and_or_b32 v1, v2, s5, v1
|
||
|
|
; GFX908-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[2:3] glc
|
||
|
|
; GFX908-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX908-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
|
||
|
|
; GFX908-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
||
|
|
; GFX908-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
||
|
|
; GFX908-NEXT: s_cbranch_execnz .LBB10_1
|
||
|
|
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
|
||
|
|
; GFX908-NEXT: s_or_b64 exec, exec, s[0:1]
|
||
|
|
; GFX908-NEXT: v_lshrrev_b32_e32 v0, s4, v1
|
||
|
|
; GFX908-NEXT: global_store_short v[0:1], v0, off
|
||
|
|
; GFX908-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX90A-LABEL: global_atomic_fadd_ret_bf16_agent:
|
||
|
|
; GFX90A: ; %bb.0:
|
||
|
|
; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
|
||
|
|
; GFX90A-NEXT: s_mov_b64 s[0:1], 0
|
||
|
|
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
|
||
|
|
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX90A-NEXT: s_and_b32 s2, s4, -4
|
||
|
|
; GFX90A-NEXT: s_mov_b32 s3, s5
|
||
|
|
; GFX90A-NEXT: s_load_dword s6, s[2:3], 0x0
|
||
|
|
; GFX90A-NEXT: s_and_b32 s4, s4, 3
|
||
|
|
; GFX90A-NEXT: s_lshl_b32 s4, s4, 3
|
||
|
|
; GFX90A-NEXT: s_lshl_b32 s5, 0xffff, s4
|
||
|
|
; GFX90A-NEXT: s_not_b32 s5, s5
|
||
|
|
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX90A-NEXT: v_mov_b32_e32 v1, s6
|
||
|
|
; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
|
||
|
|
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX90A-NEXT: v_mov_b32_e32 v3, v1
|
||
|
|
; GFX90A-NEXT: v_lshrrev_b32_sdwa v1, s4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
|
||
|
|
; GFX90A-NEXT: v_add_f32_e32 v1, 4.0, v1
|
||
|
|
; GFX90A-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
|
||
|
|
; GFX90A-NEXT: v_and_or_b32 v2, v3, s5, v1
|
||
|
|
; GFX90A-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[2:3] glc
|
||
|
|
; GFX90A-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX90A-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
|
||
|
|
; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
||
|
|
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
||
|
|
; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
|
||
|
|
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
|
||
|
|
; GFX90A-NEXT: s_or_b64 exec, exec, s[0:1]
|
||
|
|
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s4, v1
|
||
|
|
; GFX90A-NEXT: global_store_short v[0:1], v0, off
|
||
|
|
; GFX90A-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX10-LABEL: global_atomic_fadd_ret_bf16_agent:
|
||
|
|
; GFX10: ; %bb.0:
|
||
|
|
; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v0, 0
|
||
|
|
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX10-NEXT: s_and_b32 s0, s2, -4
|
||
|
|
; GFX10-NEXT: s_mov_b32 s1, s3
|
||
|
|
; GFX10-NEXT: s_and_b32 s2, s2, 3
|
||
|
|
; GFX10-NEXT: s_load_dword s3, s[0:1], 0x0
|
||
|
|
; GFX10-NEXT: s_lshl_b32 s2, s2, 3
|
||
|
|
; GFX10-NEXT: s_lshl_b32 s4, 0xffff, s2
|
||
|
|
; GFX10-NEXT: s_not_b32 s4, s4
|
||
|
|
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v1, s3
|
||
|
|
; GFX10-NEXT: s_mov_b32 s3, 0
|
||
|
|
; GFX10-NEXT: .LBB10_1: ; %atomicrmw.start
|
||
|
|
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v2, v1
|
||
|
|
; GFX10-NEXT: v_lshrrev_b32_sdwa v1, s2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
|
||
|
|
; GFX10-NEXT: v_add_f32_e32 v1, 4.0, v1
|
||
|
|
; GFX10-NEXT: v_lshlrev_b32_sdwa v1, s2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
|
||
|
|
; GFX10-NEXT: v_and_or_b32 v1, v2, s4, v1
|
||
|
|
; GFX10-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
|
||
|
|
; GFX10-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX10-NEXT: buffer_gl0_inv
|
||
|
|
; GFX10-NEXT: buffer_gl1_inv
|
||
|
|
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
|
||
|
|
; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3
|
||
|
|
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
|
||
|
|
; GFX10-NEXT: s_cbranch_execnz .LBB10_1
|
||
|
|
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
|
||
|
|
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
|
||
|
|
; GFX10-NEXT: v_lshrrev_b32_e32 v0, s2, v1
|
||
|
|
; GFX10-NEXT: global_store_short v[0:1], v0, off
|
||
|
|
; GFX10-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX11-LABEL: global_atomic_fadd_ret_bf16_agent:
|
||
|
|
; GFX11: ; %bb.0:
|
||
|
|
; GFX11-NEXT: s_load_b64 s[2:3], s[0:1], 0x24
|
||
|
|
; GFX11-NEXT: v_mov_b32_e32 v0, 0
|
||
|
|
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX11-NEXT: s_and_b32 s0, s2, -4
|
||
|
|
; GFX11-NEXT: s_mov_b32 s1, s3
|
||
|
|
; GFX11-NEXT: s_and_b32 s2, s2, 3
|
||
|
|
; GFX11-NEXT: s_load_b32 s3, s[0:1], 0x0
|
||
|
|
; GFX11-NEXT: s_lshl_b32 s2, s2, 3
|
||
|
|
; GFX11-NEXT: s_lshl_b32 s4, 0xffff, s2
|
||
|
|
; GFX11-NEXT: s_not_b32 s4, s4
|
||
|
|
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX11-NEXT: v_mov_b32_e32 v1, s3
|
||
|
|
; GFX11-NEXT: s_mov_b32 s3, 0
|
||
|
|
; GFX11-NEXT: .LBB10_1: ; %atomicrmw.start
|
||
|
|
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX11-NEXT: v_mov_b32_e32 v2, v1
|
||
|
|
; GFX11-NEXT: v_lshrrev_b32_e32 v1, s2, v2
|
||
|
|
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
|
||
|
|
; GFX11-NEXT: v_add_f32_e32 v1, 4.0, v1
|
||
|
|
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v1
|
||
|
|
; GFX11-NEXT: v_lshlrev_b32_e32 v1, s2, v1
|
||
|
|
; GFX11-NEXT: v_and_or_b32 v1, v2, s4, v1
|
||
|
|
; GFX11-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] glc
|
||
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX11-NEXT: buffer_gl0_inv
|
||
|
|
; GFX11-NEXT: buffer_gl1_inv
|
||
|
|
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
|
||
|
|
; GFX11-NEXT: s_or_b32 s3, vcc_lo, s3
|
||
|
|
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
|
||
|
|
; GFX11-NEXT: s_cbranch_execnz .LBB10_1
|
||
|
|
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
|
||
|
|
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3
|
||
|
|
; GFX11-NEXT: v_lshrrev_b32_e32 v0, s2, v1
|
||
|
|
; GFX11-NEXT: global_store_b16 v[0:1], v0, off
|
||
|
|
; GFX11-NEXT: s_nop 0
|
||
|
|
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
|
||
|
|
; GFX11-NEXT: s_endpgm
|
||
|
|
%result = atomicrmw fadd ptr addrspace(1) %ptr, bfloat 4.0 syncscope("agent") seq_cst
|
||
|
|
store bfloat %result, ptr addrspace(1) undef
|
||
|
|
ret void
|
||
|
|
}
|
||
|
|
|
||
|
|
define amdgpu_kernel void @global_atomic_fadd_ret_bf16_system(ptr addrspace(1) %ptr) #0 {
|
||
|
|
; GFX900-LABEL: global_atomic_fadd_ret_bf16_system:
|
||
|
|
; GFX900: ; %bb.0:
|
||
|
|
; GFX900-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
|
||
|
|
; GFX900-NEXT: s_mov_b64 s[0:1], 0
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v0, 0
|
||
|
|
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX900-NEXT: s_and_b32 s2, s4, -4
|
||
|
|
; GFX900-NEXT: s_mov_b32 s3, s5
|
||
|
|
; GFX900-NEXT: s_load_dword s6, s[2:3], 0x0
|
||
|
|
; GFX900-NEXT: s_and_b32 s4, s4, 3
|
||
|
|
; GFX900-NEXT: s_lshl_b32 s4, s4, 3
|
||
|
|
; GFX900-NEXT: s_lshl_b32 s5, 0xffff, s4
|
||
|
|
; GFX900-NEXT: s_not_b32 s5, s5
|
||
|
|
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v1, s6
|
||
|
|
; GFX900-NEXT: .LBB11_1: ; %atomicrmw.start
|
||
|
|
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX900-NEXT: v_mov_b32_e32 v2, v1
|
||
|
|
; GFX900-NEXT: v_lshrrev_b32_sdwa v1, s4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
|
||
|
|
; GFX900-NEXT: v_add_f32_e32 v1, 4.0, v1
|
||
|
|
; GFX900-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
|
||
|
|
; GFX900-NEXT: v_and_or_b32 v1, v2, s5, v1
|
||
|
|
; GFX900-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[2:3] glc
|
||
|
|
; GFX900-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX900-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX900-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
|
||
|
|
; GFX900-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
||
|
|
; GFX900-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
||
|
|
; GFX900-NEXT: s_cbranch_execnz .LBB11_1
|
||
|
|
; GFX900-NEXT: ; %bb.2: ; %atomicrmw.end
|
||
|
|
; GFX900-NEXT: s_or_b64 exec, exec, s[0:1]
|
||
|
|
; GFX900-NEXT: v_lshrrev_b32_e32 v0, s4, v1
|
||
|
|
; GFX900-NEXT: global_store_short v[0:1], v0, off
|
||
|
|
; GFX900-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX908-LABEL: global_atomic_fadd_ret_bf16_system:
|
||
|
|
; GFX908: ; %bb.0:
|
||
|
|
; GFX908-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
|
||
|
|
; GFX908-NEXT: s_mov_b64 s[0:1], 0
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v0, 0
|
||
|
|
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX908-NEXT: s_and_b32 s2, s4, -4
|
||
|
|
; GFX908-NEXT: s_mov_b32 s3, s5
|
||
|
|
; GFX908-NEXT: s_load_dword s6, s[2:3], 0x0
|
||
|
|
; GFX908-NEXT: s_and_b32 s4, s4, 3
|
||
|
|
; GFX908-NEXT: s_lshl_b32 s4, s4, 3
|
||
|
|
; GFX908-NEXT: s_lshl_b32 s5, 0xffff, s4
|
||
|
|
; GFX908-NEXT: s_not_b32 s5, s5
|
||
|
|
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v1, s6
|
||
|
|
; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
|
||
|
|
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX908-NEXT: v_mov_b32_e32 v2, v1
|
||
|
|
; GFX908-NEXT: v_lshrrev_b32_sdwa v1, s4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
|
||
|
|
; GFX908-NEXT: v_add_f32_e32 v1, 4.0, v1
|
||
|
|
; GFX908-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
|
||
|
|
; GFX908-NEXT: v_and_or_b32 v1, v2, s5, v1
|
||
|
|
; GFX908-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[2:3] glc
|
||
|
|
; GFX908-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX908-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
|
||
|
|
; GFX908-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
||
|
|
; GFX908-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
||
|
|
; GFX908-NEXT: s_cbranch_execnz .LBB11_1
|
||
|
|
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
|
||
|
|
; GFX908-NEXT: s_or_b64 exec, exec, s[0:1]
|
||
|
|
; GFX908-NEXT: v_lshrrev_b32_e32 v0, s4, v1
|
||
|
|
; GFX908-NEXT: global_store_short v[0:1], v0, off
|
||
|
|
; GFX908-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX90A-LABEL: global_atomic_fadd_ret_bf16_system:
|
||
|
|
; GFX90A: ; %bb.0:
|
||
|
|
; GFX90A-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
|
||
|
|
; GFX90A-NEXT: s_mov_b64 s[0:1], 0
|
||
|
|
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
|
||
|
|
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX90A-NEXT: s_and_b32 s2, s4, -4
|
||
|
|
; GFX90A-NEXT: s_mov_b32 s3, s5
|
||
|
|
; GFX90A-NEXT: s_load_dword s6, s[2:3], 0x0
|
||
|
|
; GFX90A-NEXT: s_and_b32 s4, s4, 3
|
||
|
|
; GFX90A-NEXT: s_lshl_b32 s4, s4, 3
|
||
|
|
; GFX90A-NEXT: s_lshl_b32 s5, 0xffff, s4
|
||
|
|
; GFX90A-NEXT: s_not_b32 s5, s5
|
||
|
|
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX90A-NEXT: v_mov_b32_e32 v1, s6
|
||
|
|
; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
|
||
|
|
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX90A-NEXT: v_mov_b32_e32 v3, v1
|
||
|
|
; GFX90A-NEXT: v_lshrrev_b32_sdwa v1, s4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
|
||
|
|
; GFX90A-NEXT: v_add_f32_e32 v1, 4.0, v1
|
||
|
|
; GFX90A-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
|
||
|
|
; GFX90A-NEXT: v_and_or_b32 v2, v3, s5, v1
|
||
|
|
; GFX90A-NEXT: buffer_wbl2
|
||
|
|
; GFX90A-NEXT: global_atomic_cmpswap v1, v0, v[2:3], s[2:3] glc
|
||
|
|
; GFX90A-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX90A-NEXT: buffer_invl2
|
||
|
|
; GFX90A-NEXT: buffer_wbinvl1_vol
|
||
|
|
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
|
||
|
|
; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
|
||
|
|
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
|
||
|
|
; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
|
||
|
|
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
|
||
|
|
; GFX90A-NEXT: s_or_b64 exec, exec, s[0:1]
|
||
|
|
; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s4, v1
|
||
|
|
; GFX90A-NEXT: global_store_short v[0:1], v0, off
|
||
|
|
; GFX90A-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX10-LABEL: global_atomic_fadd_ret_bf16_system:
|
||
|
|
; GFX10: ; %bb.0:
|
||
|
|
; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v0, 0
|
||
|
|
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX10-NEXT: s_and_b32 s0, s2, -4
|
||
|
|
; GFX10-NEXT: s_mov_b32 s1, s3
|
||
|
|
; GFX10-NEXT: s_and_b32 s2, s2, 3
|
||
|
|
; GFX10-NEXT: s_load_dword s3, s[0:1], 0x0
|
||
|
|
; GFX10-NEXT: s_lshl_b32 s2, s2, 3
|
||
|
|
; GFX10-NEXT: s_lshl_b32 s4, 0xffff, s2
|
||
|
|
; GFX10-NEXT: s_not_b32 s4, s4
|
||
|
|
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v1, s3
|
||
|
|
; GFX10-NEXT: s_mov_b32 s3, 0
|
||
|
|
; GFX10-NEXT: .LBB11_1: ; %atomicrmw.start
|
||
|
|
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX10-NEXT: v_mov_b32_e32 v2, v1
|
||
|
|
; GFX10-NEXT: v_lshrrev_b32_sdwa v1, s2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
|
||
|
|
; GFX10-NEXT: v_add_f32_e32 v1, 4.0, v1
|
||
|
|
; GFX10-NEXT: v_lshlrev_b32_sdwa v1, s2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
|
||
|
|
; GFX10-NEXT: v_and_or_b32 v1, v2, s4, v1
|
||
|
|
; GFX10-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
|
||
|
|
; GFX10-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX10-NEXT: buffer_gl0_inv
|
||
|
|
; GFX10-NEXT: buffer_gl1_inv
|
||
|
|
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
|
||
|
|
; GFX10-NEXT: s_or_b32 s3, vcc_lo, s3
|
||
|
|
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
|
||
|
|
; GFX10-NEXT: s_cbranch_execnz .LBB11_1
|
||
|
|
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
|
||
|
|
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
|
||
|
|
; GFX10-NEXT: v_lshrrev_b32_e32 v0, s2, v1
|
||
|
|
; GFX10-NEXT: global_store_short v[0:1], v0, off
|
||
|
|
; GFX10-NEXT: s_endpgm
|
||
|
|
;
|
||
|
|
; GFX11-LABEL: global_atomic_fadd_ret_bf16_system:
|
||
|
|
; GFX11: ; %bb.0:
|
||
|
|
; GFX11-NEXT: s_load_b64 s[2:3], s[0:1], 0x24
|
||
|
|
; GFX11-NEXT: v_mov_b32_e32 v0, 0
|
||
|
|
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX11-NEXT: s_and_b32 s0, s2, -4
|
||
|
|
; GFX11-NEXT: s_mov_b32 s1, s3
|
||
|
|
; GFX11-NEXT: s_and_b32 s2, s2, 3
|
||
|
|
; GFX11-NEXT: s_load_b32 s3, s[0:1], 0x0
|
||
|
|
; GFX11-NEXT: s_lshl_b32 s2, s2, 3
|
||
|
|
; GFX11-NEXT: s_lshl_b32 s4, 0xffff, s2
|
||
|
|
; GFX11-NEXT: s_not_b32 s4, s4
|
||
|
|
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
|
||
|
|
; GFX11-NEXT: v_mov_b32_e32 v1, s3
|
||
|
|
; GFX11-NEXT: s_mov_b32 s3, 0
|
||
|
|
; GFX11-NEXT: .LBB11_1: ; %atomicrmw.start
|
||
|
|
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
|
||
|
|
; GFX11-NEXT: v_mov_b32_e32 v2, v1
|
||
|
|
; GFX11-NEXT: v_lshrrev_b32_e32 v1, s2, v2
|
||
|
|
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
|
||
|
|
; GFX11-NEXT: v_add_f32_e32 v1, 4.0, v1
|
||
|
|
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v1
|
||
|
|
; GFX11-NEXT: v_lshlrev_b32_e32 v1, s2, v1
|
||
|
|
; GFX11-NEXT: v_and_or_b32 v1, v2, s4, v1
|
||
|
|
; GFX11-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] glc
|
||
|
|
; GFX11-NEXT: s_waitcnt vmcnt(0)
|
||
|
|
; GFX11-NEXT: buffer_gl0_inv
|
||
|
|
; GFX11-NEXT: buffer_gl1_inv
|
||
|
|
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
|
||
|
|
; GFX11-NEXT: s_or_b32 s3, vcc_lo, s3
|
||
|
|
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
|
||
|
|
; GFX11-NEXT: s_cbranch_execnz .LBB11_1
|
||
|
|
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
|
||
|
|
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3
|
||
|
|
; GFX11-NEXT: v_lshrrev_b32_e32 v0, s2, v1
|
||
|
|
; GFX11-NEXT: global_store_b16 v[0:1], v0, off
|
||
|
|
; GFX11-NEXT: s_nop 0
|
||
|
|
; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
|
||
|
|
; GFX11-NEXT: s_endpgm
|
||
|
|
%result = atomicrmw fadd ptr addrspace(1) %ptr, bfloat 4.0 syncscope("one-as") seq_cst
|
||
|
|
store bfloat %result, ptr addrspace(1) undef
|
||
|
|
ret void
|
||
|
|
}
|
||
|
|
|
||
|
|
attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" }
|
||
|
|
attributes #1 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "target-cpu"="gfx803" "target-features"="+atomic-fadd-no-rtn-insts" "amdgpu-unsafe-fp-atomics"="true" }
|
||
|
|
attributes #2 = { "amdgpu-unsafe-fp-atomics"="true" }
|