; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX900 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx908 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX908 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX90A %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX10 %s
; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX11 %s

define amdgpu_kernel void @global_atomic_fadd_ret_f32(ptr addrspace(1) %ptr) #0 {
; GFX900-LABEL: global_atomic_fadd_ret_f32:
; GFX900:       ; %bb.0:
; GFX900-NEXT:    s_mov_b64 s[4:5], exec
; GFX900-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX900-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT:    ; implicit-def: $vgpr1
; GFX900-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX900-NEXT:    s_cbranch_execz .LBB0_4
; GFX900-NEXT:  ; %bb.1:
; GFX900-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT:    s_bcnt1_i32_b64 s7, s[4:5]
; GFX900-NEXT:    v_cvt_f32_ubyte0_e32 v1, s7
; GFX900-NEXT:    s_mov_b64 s[4:5], 0
; GFX900-NEXT:    v_mul_f32_e32 v2, 4.0, v1
; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
; GFX900-NEXT:    s_load_dword s6, s[0:1], 0x0
; GFX900-NEXT:    v_mov_b32_e32 v3, 0
; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
; GFX900-NEXT:    v_mov_b32_e32 v1, s6
; GFX900-NEXT:  .LBB0_2: ; %atomicrmw.start
; GFX900-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT:    v_mov_b32_e32 v5, v1
; GFX900-NEXT:    v_add_f32_e32 v4, v5, v2
; GFX900-NEXT:    global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX900-NEXT:    s_waitcnt vmcnt(0)
; GFX900-NEXT:    buffer_wbinvl1_vol
; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v5
; GFX900-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GFX900-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GFX900-NEXT:    s_cbranch_execnz .LBB0_2
; GFX900-NEXT:  ; %bb.3: ; %Flow
; GFX900-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX900-NEXT:  .LBB0_4: ; %Flow1
; GFX900-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX900-NEXT:    v_readfirstlane_b32 s0, v1
; GFX900-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GFX900-NEXT:    v_mad_f32 v0, v0, 4.0, s0
; GFX900-NEXT:    global_store_dword v[0:1], v0, off
; GFX900-NEXT:    s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_ret_f32:
; GFX908:       ; %bb.0:
; GFX908-NEXT:    s_mov_b64 s[4:5], exec
; GFX908-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX908-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT:    ; implicit-def: $vgpr1
; GFX908-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX908-NEXT:    s_cbranch_execz .LBB0_4
; GFX908-NEXT:  ; %bb.1:
; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT:    s_bcnt1_i32_b64 s7, s[4:5]
; GFX908-NEXT:    v_cvt_f32_ubyte0_e32 v1, s7
; GFX908-NEXT:    s_mov_b64 s[4:5], 0
; GFX908-NEXT:    v_mul_f32_e32 v2, 4.0, v1
; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
; GFX908-NEXT:    s_load_dword s6, s[0:1], 0x0
; GFX908-NEXT:    v_mov_b32_e32 v3, 0
; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
; GFX908-NEXT:    v_mov_b32_e32 v1, s6
; GFX908-NEXT:  .LBB0_2: ; %atomicrmw.start
; GFX908-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT:    v_mov_b32_e32 v5, v1
; GFX908-NEXT:    v_add_f32_e32 v4, v5, v2
; GFX908-NEXT:    global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX908-NEXT:    s_waitcnt vmcnt(0)
; GFX908-NEXT:    buffer_wbinvl1_vol
; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v5
; GFX908-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT:    s_cbranch_execnz .LBB0_2
; GFX908-NEXT:  ; %bb.3: ; %Flow
; GFX908-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT:  .LBB0_4: ; %Flow1
; GFX908-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX908-NEXT:    v_readfirstlane_b32 s0, v1
; GFX908-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GFX908-NEXT:    v_mad_f32 v0, v0, 4.0, s0
; GFX908-NEXT:    global_store_dword v[0:1], v0, off
; GFX908-NEXT:    s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_ret_f32:
; GFX90A:       ; %bb.0:
; GFX90A-NEXT:    s_mov_b64 s[4:5], exec
; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT:    ; implicit-def: $vgpr1
; GFX90A-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT:    s_cbranch_execz .LBB0_4
; GFX90A-NEXT:  ; %bb.1:
; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT:    s_bcnt1_i32_b64 s7, s[4:5]
; GFX90A-NEXT:    v_cvt_f32_ubyte0_e32 v1, s7
; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
; GFX90A-NEXT:    v_mul_f32_e32 v2, 4.0, v1
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    s_load_dword s6, s[0:1], 0x0
; GFX90A-NEXT:    v_mov_b32_e32 v3, 0
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_mov_b32_e32 v1, s6
; GFX90A-NEXT:  .LBB0_2: ; %atomicrmw.start
; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT:    v_mov_b32_e32 v5, v1
; GFX90A-NEXT:    v_add_f32_e32 v4, v5, v2
; GFX90A-NEXT:    buffer_wbl2
; GFX90A-NEXT:    global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    buffer_invl2
; GFX90A-NEXT:    buffer_wbinvl1_vol
; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v5
; GFX90A-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT:    s_cbranch_execnz .LBB0_2
; GFX90A-NEXT:  ; %bb.3: ; %Flow
; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT:  .LBB0_4: ; %Flow1
; GFX90A-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX90A-NEXT:    v_readfirstlane_b32 s0, v1
; GFX90A-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GFX90A-NEXT:    v_mad_f32 v0, v0, 4.0, s0
; GFX90A-NEXT:    global_store_dword v[0:1], v0, off
; GFX90A-NEXT:    s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_ret_f32:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_mov_b32 s4, exec_lo
; GFX10-NEXT:    s_mov_b32 s3, 0
; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX10-NEXT:    ; implicit-def: $vgpr1
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT:    s_and_saveexec_b32 s2, vcc_lo
; GFX10-NEXT:    s_cbranch_execz .LBB0_4
; GFX10-NEXT:  ; %bb.1:
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT:    s_bcnt1_i32_b32 s4, s4
; GFX10-NEXT:    v_mov_b32_e32 v3, 0
; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v1, s4
; GFX10-NEXT:    v_mul_f32_e32 v2, 4.0, v1
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_load_dword s5, s[0:1], 0x0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_mov_b32_e32 v1, s5
; GFX10-NEXT:  .LBB0_2: ; %atomicrmw.start
; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT:    v_mov_b32_e32 v5, v1
; GFX10-NEXT:    v_add_f32_e32 v4, v5, v2
; GFX10-NEXT:    global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v5
; GFX10-NEXT:    s_or_b32 s3, vcc_lo, s3
; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-NEXT:    s_cbranch_execnz .LBB0_2
; GFX10-NEXT:  ; %bb.3: ; %Flow
; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s3
; GFX10-NEXT:  .LBB0_4: ; %Flow1
; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s2
; GFX10-NEXT:    v_readfirstlane_b32 s0, v1
; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GFX10-NEXT:    v_mad_f32 v0, v0, 4.0, s0
; GFX10-NEXT:    global_store_dword v[0:1], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_ret_f32:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_mov_b32 s4, exec_lo
; GFX11-NEXT:    s_mov_b32 s3, 0
; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX11-NEXT:    s_mov_b32 s2, exec_lo
; GFX11-NEXT:    ; implicit-def: $vgpr1
; GFX11-NEXT:    v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT:    s_cbranch_execz .LBB0_4
; GFX11-NEXT:  ; %bb.1:
; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT:    s_bcnt1_i32_b32 s4, s4
; GFX11-NEXT:    v_mov_b32_e32 v3, 0
; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v1, s4
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_load_b32 s5, s[0:1], 0x0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_dual_mul_f32 v2, 4.0, v1 :: v_dual_mov_b32 v1, s5
; GFX11-NEXT:  .LBB0_2: ; %atomicrmw.start
; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT:    v_mov_b32_e32 v5, v1
; GFX11-NEXT:    v_add_f32_e32 v4, v5, v2
; GFX11-NEXT:    global_atomic_cmpswap_b32 v1, v3, v[4:5], s[0:1] glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v5
; GFX11-NEXT:    s_or_b32 s3, vcc_lo, s3
; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s3
; GFX11-NEXT:    s_cbranch_execnz .LBB0_2
; GFX11-NEXT:  ; %bb.3: ; %Flow
; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s3
; GFX11-NEXT:  .LBB0_4: ; %Flow1
; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s2
; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GFX11-NEXT:    v_readfirstlane_b32 s0, v1
; GFX11-NEXT:    v_mul_f32_e32 v0, 4.0, v0
; GFX11-NEXT:    v_add_f32_e32 v0, s0, v0
; GFX11-NEXT:    global_store_b32 v[0:1], v0, off
; GFX11-NEXT:    s_nop 0
; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT:    s_endpgm
  %result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 seq_cst
  store float %result, ptr addrspace(1) undef
  ret void
}

define amdgpu_kernel void @global_atomic_fadd_ret_f32_ieee(ptr addrspace(1) %ptr) #2 {
; GFX900-LABEL: global_atomic_fadd_ret_f32_ieee:
; GFX900:       ; %bb.0:
; GFX900-NEXT:    s_mov_b64 s[4:5], exec
; GFX900-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX900-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT:    ; implicit-def: $vgpr1
; GFX900-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX900-NEXT:    s_cbranch_execz .LBB1_4
; GFX900-NEXT:  ; %bb.1:
; GFX900-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT:    s_bcnt1_i32_b64 s7, s[4:5]
; GFX900-NEXT:    v_cvt_f32_ubyte0_e32 v1, s7
; GFX900-NEXT:    s_mov_b64 s[4:5], 0
; GFX900-NEXT:    v_mul_f32_e32 v2, 4.0, v1
; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
; GFX900-NEXT:    s_load_dword s6, s[0:1], 0x0
; GFX900-NEXT:    v_mov_b32_e32 v3, 0
; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
; GFX900-NEXT:    v_mov_b32_e32 v1, s6
; GFX900-NEXT:  .LBB1_2: ; %atomicrmw.start
; GFX900-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT:    v_mov_b32_e32 v5, v1
; GFX900-NEXT:    v_add_f32_e32 v4, v5, v2
; GFX900-NEXT:    global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX900-NEXT:    s_waitcnt vmcnt(0)
; GFX900-NEXT:    buffer_wbinvl1_vol
; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v5
; GFX900-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GFX900-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GFX900-NEXT:    s_cbranch_execnz .LBB1_2
; GFX900-NEXT:  ; %bb.3: ; %Flow
; GFX900-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX900-NEXT:  .LBB1_4: ; %Flow1
; GFX900-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX900-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GFX900-NEXT:    v_readfirstlane_b32 s0, v1
; GFX900-NEXT:    v_mul_f32_e32 v0, 4.0, v0
; GFX900-NEXT:    v_add_f32_e32 v0, s0, v0
; GFX900-NEXT:    global_store_dword v[0:1], v0, off
; GFX900-NEXT:    s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_ret_f32_ieee:
; GFX908:       ; %bb.0:
; GFX908-NEXT:    s_mov_b64 s[4:5], exec
; GFX908-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX908-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT:    ; implicit-def: $vgpr1
; GFX908-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX908-NEXT:    s_cbranch_execz .LBB1_4
; GFX908-NEXT:  ; %bb.1:
; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT:    s_bcnt1_i32_b64 s7, s[4:5]
; GFX908-NEXT:    v_cvt_f32_ubyte0_e32 v1, s7
; GFX908-NEXT:    s_mov_b64 s[4:5], 0
; GFX908-NEXT:    v_mul_f32_e32 v2, 4.0, v1
; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
; GFX908-NEXT:    s_load_dword s6, s[0:1], 0x0
; GFX908-NEXT:    v_mov_b32_e32 v3, 0
; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
; GFX908-NEXT:    v_mov_b32_e32 v1, s6
; GFX908-NEXT:  .LBB1_2: ; %atomicrmw.start
; GFX908-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT:    v_mov_b32_e32 v5, v1
; GFX908-NEXT:    v_add_f32_e32 v4, v5, v2
; GFX908-NEXT:    global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX908-NEXT:    s_waitcnt vmcnt(0)
; GFX908-NEXT:    buffer_wbinvl1_vol
; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v5
; GFX908-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT:    s_cbranch_execnz .LBB1_2
; GFX908-NEXT:  ; %bb.3: ; %Flow
; GFX908-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT:  .LBB1_4: ; %Flow1
; GFX908-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX908-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GFX908-NEXT:    v_readfirstlane_b32 s0, v1
; GFX908-NEXT:    v_mul_f32_e32 v0, 4.0, v0
; GFX908-NEXT:    v_add_f32_e32 v0, s0, v0
; GFX908-NEXT:    global_store_dword v[0:1], v0, off
; GFX908-NEXT:    s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_ret_f32_ieee:
; GFX90A:       ; %bb.0:
; GFX90A-NEXT:    s_mov_b64 s[4:5], exec
; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT:    ; implicit-def: $vgpr1
; GFX90A-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT:    s_cbranch_execz .LBB1_2
; GFX90A-NEXT:  ; %bb.1:
; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
; GFX90A-NEXT:    v_cvt_f32_ubyte0_e32 v2, s4
; GFX90A-NEXT:    v_mov_b32_e32 v1, 0
; GFX90A-NEXT:    v_mul_f32_e32 v2, 4.0, v2
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    global_atomic_add_f32 v1, v1, v2, s[0:1] glc
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    buffer_wbinvl1_vol
; GFX90A-NEXT:  .LBB1_2:
; GFX90A-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX90A-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GFX90A-NEXT:    v_readfirstlane_b32 s0, v1
; GFX90A-NEXT:    v_mul_f32_e32 v0, 4.0, v0
; GFX90A-NEXT:    v_add_f32_e32 v0, s0, v0
; GFX90A-NEXT:    global_store_dword v[0:1], v0, off
; GFX90A-NEXT:    s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_ret_f32_ieee:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_mov_b32 s4, exec_lo
; GFX10-NEXT:    s_mov_b32 s3, 0
; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX10-NEXT:    ; implicit-def: $vgpr1
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT:    s_and_saveexec_b32 s2, vcc_lo
; GFX10-NEXT:    s_cbranch_execz .LBB1_4
; GFX10-NEXT:  ; %bb.1:
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT:    s_bcnt1_i32_b32 s4, s4
; GFX10-NEXT:    v_mov_b32_e32 v3, 0
; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v1, s4
; GFX10-NEXT:    v_mul_f32_e32 v2, 4.0, v1
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_load_dword s5, s[0:1], 0x0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_mov_b32_e32 v1, s5
; GFX10-NEXT:  .LBB1_2: ; %atomicrmw.start
; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT:    v_mov_b32_e32 v5, v1
; GFX10-NEXT:    v_add_f32_e32 v4, v5, v2
; GFX10-NEXT:    global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v5
; GFX10-NEXT:    s_or_b32 s3, vcc_lo, s3
; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-NEXT:    s_cbranch_execnz .LBB1_2
; GFX10-NEXT:  ; %bb.3: ; %Flow
; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s3
; GFX10-NEXT:  .LBB1_4: ; %Flow1
; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s2
; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GFX10-NEXT:    v_readfirstlane_b32 s0, v1
; GFX10-NEXT:    v_mul_f32_e32 v0, 4.0, v0
; GFX10-NEXT:    v_add_f32_e32 v0, s0, v0
; GFX10-NEXT:    global_store_dword v[0:1], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_ret_f32_ieee:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_mov_b32 s3, exec_lo
; GFX11-NEXT:    s_mov_b32 s2, exec_lo
; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX11-NEXT:    ; implicit-def: $vgpr1
; GFX11-NEXT:    v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT:    s_cbranch_execz .LBB1_2
; GFX11-NEXT:  ; %bb.1:
; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT:    s_bcnt1_i32_b32 s3, s3
; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v1, s3
; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mul_f32 v1, 4.0, v1
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    global_atomic_add_f32 v1, v2, v1, s[0:1] glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:  .LBB1_2:
; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s2
; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GFX11-NEXT:    v_readfirstlane_b32 s0, v1
; GFX11-NEXT:    v_mul_f32_e32 v0, 4.0, v0
; GFX11-NEXT:    v_add_f32_e32 v0, s0, v0
; GFX11-NEXT:    global_store_b32 v[0:1], v0, off
; GFX11-NEXT:    s_nop 0
; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT:    s_endpgm
  %result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
  store float %result, ptr addrspace(1) undef
  ret void
}

define amdgpu_kernel void @global_atomic_fadd_noret_f32(ptr addrspace(1) %ptr) #0 {
; GFX900-LABEL: global_atomic_fadd_noret_f32:
; GFX900:       ; %bb.0:
; GFX900-NEXT:    s_mov_b64 s[2:3], exec
; GFX900-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX900-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GFX900-NEXT:    s_cbranch_execz .LBB2_3
; GFX900-NEXT:  ; %bb.1:
; GFX900-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT:    s_bcnt1_i32_b64 s5, s[2:3]
; GFX900-NEXT:    v_cvt_f32_ubyte0_e32 v0, s5
; GFX900-NEXT:    s_mov_b64 s[2:3], 0
; GFX900-NEXT:    v_mul_f32_e32 v2, 4.0, v0
; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
; GFX900-NEXT:    s_load_dword s4, s[0:1], 0x0
; GFX900-NEXT:    v_mov_b32_e32 v3, 0
; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
; GFX900-NEXT:    v_mov_b32_e32 v1, s4
; GFX900-NEXT:  .LBB2_2: ; %atomicrmw.start
; GFX900-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT:    v_add_f32_e32 v0, v1, v2
; GFX900-NEXT:    global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX900-NEXT:    s_waitcnt vmcnt(0)
; GFX900-NEXT:    buffer_wbinvl1_vol
; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GFX900-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
; GFX900-NEXT:    v_mov_b32_e32 v1, v0
; GFX900-NEXT:    s_andn2_b64 exec, exec, s[2:3]
; GFX900-NEXT:    s_cbranch_execnz .LBB2_2
; GFX900-NEXT:  .LBB2_3:
; GFX900-NEXT:    s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_noret_f32:
; GFX908:       ; %bb.0:
; GFX908-NEXT:    s_mov_b64 s[2:3], exec
; GFX908-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX908-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT:    s_cbranch_execz .LBB2_2
; GFX908-NEXT:  ; %bb.1:
; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
; GFX908-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
; GFX908-NEXT:    v_mov_b32_e32 v0, 0
; GFX908-NEXT:    v_mul_f32_e32 v1, 4.0, v1
; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
; GFX908-NEXT:    global_atomic_add_f32 v0, v1, s[0:1]
; GFX908-NEXT:    s_waitcnt vmcnt(0)
; GFX908-NEXT:    buffer_wbinvl1_vol
; GFX908-NEXT:  .LBB2_2:
; GFX908-NEXT:    s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_noret_f32:
; GFX90A:       ; %bb.0:
; GFX90A-NEXT:    s_mov_b64 s[2:3], exec
; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GFX90A-NEXT:    s_cbranch_execz .LBB2_2
; GFX90A-NEXT:  ; %bb.1:
; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
; GFX90A-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
; GFX90A-NEXT:    v_mul_f32_e32 v1, 4.0, v1
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    global_atomic_add_f32 v0, v1, s[0:1]
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    buffer_wbinvl1_vol
; GFX90A-NEXT:  .LBB2_2:
; GFX90A-NEXT:    s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_noret_f32:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_mov_b32 s3, exec_lo
; GFX10-NEXT:    s_mov_b32 s2, 0
; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT:    s_and_saveexec_b32 s4, vcc_lo
; GFX10-NEXT:    s_cbranch_execz .LBB2_3
; GFX10-NEXT:  ; %bb.1:
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT:    s_bcnt1_i32_b32 s3, s3
; GFX10-NEXT:    v_mov_b32_e32 v3, 0
; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v0, s3
; GFX10-NEXT:    v_mul_f32_e32 v2, 4.0, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_load_dword s4, s[0:1], 0x0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_mov_b32_e32 v1, s4
; GFX10-NEXT:  .LBB2_2: ; %atomicrmw.start
; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT:    v_add_f32_e32 v0, v1, v2
; GFX10-NEXT:    global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX10-NEXT:    v_mov_b32_e32 v1, v0
; GFX10-NEXT:    s_or_b32 s2, vcc_lo, s2
; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
; GFX10-NEXT:    s_cbranch_execnz .LBB2_2
; GFX10-NEXT:  .LBB2_3:
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_noret_f32:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_mov_b32 s2, exec_lo
; GFX11-NEXT:    s_mov_b32 s3, exec_lo
; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11-NEXT:    v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT:    s_cbranch_execz .LBB2_2
; GFX11-NEXT:  ; %bb.1:
; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT:    s_bcnt1_i32_b32 s2, s2
; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v0, s2
; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    global_atomic_add_f32 v1, v0, s[0:1]
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:  .LBB2_2:
; GFX11-NEXT:    s_endpgm
  %result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @global_atomic_fadd_noret_f32_ieee(ptr addrspace(1) %ptr) #2 {
; GFX900-LABEL: global_atomic_fadd_noret_f32_ieee:
; GFX900:       ; %bb.0:
; GFX900-NEXT:    s_mov_b64 s[2:3], exec
; GFX900-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX900-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GFX900-NEXT:    s_cbranch_execz .LBB3_3
; GFX900-NEXT:  ; %bb.1:
; GFX900-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT:    s_bcnt1_i32_b64 s5, s[2:3]
; GFX900-NEXT:    v_cvt_f32_ubyte0_e32 v0, s5
; GFX900-NEXT:    s_mov_b64 s[2:3], 0
; GFX900-NEXT:    v_mul_f32_e32 v2, 4.0, v0
; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
; GFX900-NEXT:    s_load_dword s4, s[0:1], 0x0
; GFX900-NEXT:    v_mov_b32_e32 v3, 0
; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
; GFX900-NEXT:    v_mov_b32_e32 v1, s4
; GFX900-NEXT:  .LBB3_2: ; %atomicrmw.start
; GFX900-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT:    v_add_f32_e32 v0, v1, v2
; GFX900-NEXT:    global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX900-NEXT:    s_waitcnt vmcnt(0)
; GFX900-NEXT:    buffer_wbinvl1_vol
; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GFX900-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
; GFX900-NEXT:    v_mov_b32_e32 v1, v0
; GFX900-NEXT:    s_andn2_b64 exec, exec, s[2:3]
; GFX900-NEXT:    s_cbranch_execnz .LBB3_2
; GFX900-NEXT:  .LBB3_3:
; GFX900-NEXT:    s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_noret_f32_ieee:
; GFX908:       ; %bb.0:
; GFX908-NEXT:    s_mov_b64 s[2:3], exec
; GFX908-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX908-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT:    s_cbranch_execz .LBB3_2
; GFX908-NEXT:  ; %bb.1:
; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
; GFX908-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
; GFX908-NEXT:    v_mov_b32_e32 v0, 0
; GFX908-NEXT:    v_mul_f32_e32 v1, 4.0, v1
; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
; GFX908-NEXT:    global_atomic_add_f32 v0, v1, s[0:1]
; GFX908-NEXT:    s_waitcnt vmcnt(0)
; GFX908-NEXT:    buffer_wbinvl1_vol
; GFX908-NEXT:  .LBB3_2:
; GFX908-NEXT:    s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_noret_f32_ieee:
; GFX90A:       ; %bb.0:
; GFX90A-NEXT:    s_mov_b64 s[2:3], exec
; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GFX90A-NEXT:    s_cbranch_execz .LBB3_2
; GFX90A-NEXT:  ; %bb.1:
; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
; GFX90A-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
; GFX90A-NEXT:    v_mul_f32_e32 v1, 4.0, v1
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    global_atomic_add_f32 v0, v1, s[0:1]
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    buffer_wbinvl1_vol
; GFX90A-NEXT:  .LBB3_2:
; GFX90A-NEXT:    s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_noret_f32_ieee:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_mov_b32 s3, exec_lo
; GFX10-NEXT:    s_mov_b32 s2, 0
; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT:    s_and_saveexec_b32 s4, vcc_lo
; GFX10-NEXT:    s_cbranch_execz .LBB3_3
; GFX10-NEXT:  ; %bb.1:
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT:    s_bcnt1_i32_b32 s3, s3
; GFX10-NEXT:    v_mov_b32_e32 v3, 0
; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v0, s3
; GFX10-NEXT:    v_mul_f32_e32 v2, 4.0, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_load_dword s4, s[0:1], 0x0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_mov_b32_e32 v1, s4
; GFX10-NEXT:  .LBB3_2: ; %atomicrmw.start
; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT:    v_add_f32_e32 v0, v1, v2
; GFX10-NEXT:    global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX10-NEXT:    v_mov_b32_e32 v1, v0
; GFX10-NEXT:    s_or_b32 s2, vcc_lo, s2
; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
; GFX10-NEXT:    s_cbranch_execnz .LBB3_2
; GFX10-NEXT:  .LBB3_3:
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_noret_f32_ieee:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_mov_b32 s2, exec_lo
; GFX11-NEXT:    s_mov_b32 s3, exec_lo
; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11-NEXT:    v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT:    s_cbranch_execz .LBB3_2
; GFX11-NEXT:  ; %bb.1:
; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT:    s_bcnt1_i32_b32 s2, s2
; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v0, s2
; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_mul_f32 v0, 4.0, v0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    global_atomic_add_f32 v1, v0, s[0:1]
; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:  .LBB3_2:
; GFX11-NEXT:    s_endpgm
  %result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @global_atomic_fadd_ret_f32_agent(ptr addrspace(1) %ptr) #0 {
; GFX900-LABEL: global_atomic_fadd_ret_f32_agent:
; GFX900:       ; %bb.0:
; GFX900-NEXT:    s_mov_b64 s[4:5], exec
; GFX900-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX900-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT:    ; implicit-def: $vgpr1
; GFX900-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX900-NEXT:    s_cbranch_execz .LBB4_4
; GFX900-NEXT:  ; %bb.1:
; GFX900-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT:    s_bcnt1_i32_b64 s7, s[4:5]
; GFX900-NEXT:    v_cvt_f32_ubyte0_e32 v1, s7
; GFX900-NEXT:    s_mov_b64 s[4:5], 0
; GFX900-NEXT:    v_mul_f32_e32 v2, 4.0, v1
; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
; GFX900-NEXT:    s_load_dword s6, s[0:1], 0x0
; GFX900-NEXT:    v_mov_b32_e32 v3, 0
; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
; GFX900-NEXT:    v_mov_b32_e32 v1, s6
; GFX900-NEXT:  .LBB4_2: ; %atomicrmw.start
; GFX900-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT:    v_mov_b32_e32 v5, v1
; GFX900-NEXT:    v_add_f32_e32 v4, v5, v2
; GFX900-NEXT:    global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX900-NEXT:    s_waitcnt vmcnt(0)
; GFX900-NEXT:    buffer_wbinvl1_vol
; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v5
; GFX900-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GFX900-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GFX900-NEXT:    s_cbranch_execnz .LBB4_2
; GFX900-NEXT:  ; %bb.3: ; %Flow
; GFX900-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX900-NEXT:  .LBB4_4: ; %Flow1
; GFX900-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX900-NEXT:    v_readfirstlane_b32 s0, v1
; GFX900-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GFX900-NEXT:    v_mad_f32 v0, v0, 4.0, s0
; GFX900-NEXT:    global_store_dword v[0:1], v0, off
; GFX900-NEXT:    s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_ret_f32_agent:
; GFX908:       ; %bb.0:
; GFX908-NEXT:    s_mov_b64 s[4:5], exec
; GFX908-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX908-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT:    ; implicit-def: $vgpr1
; GFX908-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX908-NEXT:    s_cbranch_execz .LBB4_4
; GFX908-NEXT:  ; %bb.1:
; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT:    s_bcnt1_i32_b64 s7, s[4:5]
; GFX908-NEXT:    v_cvt_f32_ubyte0_e32 v1, s7
; GFX908-NEXT:    s_mov_b64 s[4:5], 0
; GFX908-NEXT:    v_mul_f32_e32 v2, 4.0, v1
; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
; GFX908-NEXT:    s_load_dword s6, s[0:1], 0x0
; GFX908-NEXT:    v_mov_b32_e32 v3, 0
; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
; GFX908-NEXT:    v_mov_b32_e32 v1, s6
; GFX908-NEXT:  .LBB4_2: ; %atomicrmw.start
; GFX908-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT:    v_mov_b32_e32 v5, v1
; GFX908-NEXT:    v_add_f32_e32 v4, v5, v2
; GFX908-NEXT:    global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX908-NEXT:    s_waitcnt vmcnt(0)
; GFX908-NEXT:    buffer_wbinvl1_vol
; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v5
; GFX908-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT:    s_cbranch_execnz .LBB4_2
; GFX908-NEXT:  ; %bb.3: ; %Flow
; GFX908-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT:  .LBB4_4: ; %Flow1
; GFX908-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX908-NEXT:    v_readfirstlane_b32 s0, v1
; GFX908-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GFX908-NEXT:    v_mad_f32 v0, v0, 4.0, s0
; GFX908-NEXT:    global_store_dword v[0:1], v0, off
; GFX908-NEXT:    s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_ret_f32_agent:
; GFX90A:       ; %bb.0:
; GFX90A-NEXT:    s_mov_b64 s[4:5], exec
; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT:    ; implicit-def: $vgpr1
; GFX90A-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT:    s_cbranch_execz .LBB4_2
; GFX90A-NEXT:  ; %bb.1:
; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT:    s_bcnt1_i32_b64 s4, s[4:5]
; GFX90A-NEXT:    v_cvt_f32_ubyte0_e32 v2, s4
; GFX90A-NEXT:    v_mov_b32_e32 v1, 0
; GFX90A-NEXT:    v_mul_f32_e32 v2, 4.0, v2
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    global_atomic_add_f32 v1, v1, v2, s[0:1] glc
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    buffer_wbinvl1_vol
; GFX90A-NEXT:  .LBB4_2:
; GFX90A-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX90A-NEXT:    v_readfirstlane_b32 s0, v1
; GFX90A-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GFX90A-NEXT:    v_mad_f32 v0, v0, 4.0, s0
; GFX90A-NEXT:    global_store_dword v[0:1], v0, off
; GFX90A-NEXT:    s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_ret_f32_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_mov_b32 s4, exec_lo
; GFX10-NEXT:    s_mov_b32 s3, 0
; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX10-NEXT:    ; implicit-def: $vgpr1
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT:    s_and_saveexec_b32 s2, vcc_lo
; GFX10-NEXT:    s_cbranch_execz .LBB4_4
; GFX10-NEXT:  ; %bb.1:
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT:    s_bcnt1_i32_b32 s4, s4
; GFX10-NEXT:    v_mov_b32_e32 v3, 0
; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v1, s4
; GFX10-NEXT:    v_mul_f32_e32 v2, 4.0, v1
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_load_dword s5, s[0:1], 0x0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_mov_b32_e32 v1, s5
; GFX10-NEXT:  .LBB4_2: ; %atomicrmw.start
; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT:    v_mov_b32_e32 v5, v1
; GFX10-NEXT:    v_add_f32_e32 v4, v5, v2
; GFX10-NEXT:    global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v5
; GFX10-NEXT:    s_or_b32 s3, vcc_lo, s3
; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-NEXT:    s_cbranch_execnz .LBB4_2
; GFX10-NEXT:  ; %bb.3: ; %Flow
; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s3
; GFX10-NEXT:  .LBB4_4: ; %Flow1
; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s2
; GFX10-NEXT:    v_readfirstlane_b32 s0, v1
; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GFX10-NEXT:    v_mad_f32 v0, v0, 4.0, s0
; GFX10-NEXT:    global_store_dword v[0:1], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_ret_f32_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_mov_b32 s3, exec_lo
; GFX11-NEXT:    s_mov_b32 s2, exec_lo
; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX11-NEXT:    ; implicit-def: $vgpr1
; GFX11-NEXT:    v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT:    s_cbranch_execz .LBB4_2
; GFX11-NEXT:  ; %bb.1:
; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT:    s_bcnt1_i32_b32 s3, s3
; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v1, s3
; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_mul_f32 v1, 4.0, v1
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    global_atomic_add_f32 v1, v2, v1, s[0:1] glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:  .LBB4_2:
; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s2
; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GFX11-NEXT:    v_readfirstlane_b32 s0, v1
; GFX11-NEXT:    v_mul_f32_e32 v0, 4.0, v0
; GFX11-NEXT:    v_add_f32_e32 v0, s0, v0
; GFX11-NEXT:    global_store_b32 v[0:1], v0, off
; GFX11-NEXT:    s_nop 0
; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT:    s_endpgm
  %result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
  store float %result, ptr addrspace(1) undef
  ret void
}

define amdgpu_kernel void @global_atomic_fadd_ret_f32_system(ptr addrspace(1) %ptr) #0 {
; GFX900-LABEL: global_atomic_fadd_ret_f32_system:
; GFX900:       ; %bb.0:
; GFX900-NEXT:    s_mov_b64 s[4:5], exec
; GFX900-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX900-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT:    ; implicit-def: $vgpr1
; GFX900-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX900-NEXT:    s_cbranch_execz .LBB5_4
; GFX900-NEXT:  ; %bb.1:
; GFX900-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT:    s_bcnt1_i32_b64 s7, s[4:5]
; GFX900-NEXT:    v_cvt_f32_ubyte0_e32 v1, s7
; GFX900-NEXT:    s_mov_b64 s[4:5], 0
; GFX900-NEXT:    v_mul_f32_e32 v2, 4.0, v1
; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
; GFX900-NEXT:    s_load_dword s6, s[0:1], 0x0
; GFX900-NEXT:    v_mov_b32_e32 v3, 0
; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
; GFX900-NEXT:    v_mov_b32_e32 v1, s6
; GFX900-NEXT:  .LBB5_2: ; %atomicrmw.start
; GFX900-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT:    v_mov_b32_e32 v5, v1
; GFX900-NEXT:    v_add_f32_e32 v4, v5, v2
; GFX900-NEXT:    global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX900-NEXT:    s_waitcnt vmcnt(0)
; GFX900-NEXT:    buffer_wbinvl1_vol
; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v5
; GFX900-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GFX900-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GFX900-NEXT:    s_cbranch_execnz .LBB5_2
; GFX900-NEXT:  ; %bb.3: ; %Flow
; GFX900-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX900-NEXT:  .LBB5_4: ; %Flow1
; GFX900-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX900-NEXT:    v_readfirstlane_b32 s0, v1
; GFX900-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GFX900-NEXT:    v_mad_f32 v0, v0, 4.0, s0
; GFX900-NEXT:    global_store_dword v[0:1], v0, off
; GFX900-NEXT:    s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_ret_f32_system:
; GFX908:       ; %bb.0:
; GFX908-NEXT:    s_mov_b64 s[4:5], exec
; GFX908-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX908-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT:    ; implicit-def: $vgpr1
; GFX908-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX908-NEXT:    s_cbranch_execz .LBB5_4
; GFX908-NEXT:  ; %bb.1:
; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT:    s_bcnt1_i32_b64 s7, s[4:5]
; GFX908-NEXT:    v_cvt_f32_ubyte0_e32 v1, s7
; GFX908-NEXT:    s_mov_b64 s[4:5], 0
; GFX908-NEXT:    v_mul_f32_e32 v2, 4.0, v1
; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
; GFX908-NEXT:    s_load_dword s6, s[0:1], 0x0
; GFX908-NEXT:    v_mov_b32_e32 v3, 0
; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
; GFX908-NEXT:    v_mov_b32_e32 v1, s6
; GFX908-NEXT:  .LBB5_2: ; %atomicrmw.start
; GFX908-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT:    v_mov_b32_e32 v5, v1
; GFX908-NEXT:    v_add_f32_e32 v4, v5, v2
; GFX908-NEXT:    global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX908-NEXT:    s_waitcnt vmcnt(0)
; GFX908-NEXT:    buffer_wbinvl1_vol
; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v5
; GFX908-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT:    s_cbranch_execnz .LBB5_2
; GFX908-NEXT:  ; %bb.3: ; %Flow
; GFX908-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT:  .LBB5_4: ; %Flow1
; GFX908-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX908-NEXT:    v_readfirstlane_b32 s0, v1
; GFX908-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GFX908-NEXT:    v_mad_f32 v0, v0, 4.0, s0
; GFX908-NEXT:    global_store_dword v[0:1], v0, off
; GFX908-NEXT:    s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_ret_f32_system:
; GFX90A:       ; %bb.0:
; GFX90A-NEXT:    s_mov_b64 s[4:5], exec
; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT:    ; implicit-def: $vgpr1
; GFX90A-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT:    s_cbranch_execz .LBB5_4
; GFX90A-NEXT:  ; %bb.1:
; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT:    s_bcnt1_i32_b64 s7, s[4:5]
; GFX90A-NEXT:    v_cvt_f32_ubyte0_e32 v1, s7
; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
; GFX90A-NEXT:    v_mul_f32_e32 v2, 4.0, v1
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    s_load_dword s6, s[0:1], 0x0
; GFX90A-NEXT:    v_mov_b32_e32 v3, 0
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_mov_b32_e32 v1, s6
; GFX90A-NEXT:  .LBB5_2: ; %atomicrmw.start
; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT:    v_mov_b32_e32 v5, v1
; GFX90A-NEXT:    v_add_f32_e32 v4, v5, v2
; GFX90A-NEXT:    buffer_wbl2
; GFX90A-NEXT:    global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    buffer_invl2
; GFX90A-NEXT:    buffer_wbinvl1_vol
; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v5
; GFX90A-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT:    s_cbranch_execnz .LBB5_2
; GFX90A-NEXT:  ; %bb.3: ; %Flow
; GFX90A-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT:  .LBB5_4: ; %Flow1
; GFX90A-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX90A-NEXT:    v_readfirstlane_b32 s0, v1
; GFX90A-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GFX90A-NEXT:    v_mad_f32 v0, v0, 4.0, s0
; GFX90A-NEXT:    global_store_dword v[0:1], v0, off
; GFX90A-NEXT:    s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_ret_f32_system:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_mov_b32 s4, exec_lo
; GFX10-NEXT:    s_mov_b32 s3, 0
; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX10-NEXT:    ; implicit-def: $vgpr1
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT:    s_and_saveexec_b32 s2, vcc_lo
; GFX10-NEXT:    s_cbranch_execz .LBB5_4
; GFX10-NEXT:  ; %bb.1:
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT:    s_bcnt1_i32_b32 s4, s4
; GFX10-NEXT:    v_mov_b32_e32 v3, 0
; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v1, s4
; GFX10-NEXT:    v_mul_f32_e32 v2, 4.0, v1
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_load_dword s5, s[0:1], 0x0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_mov_b32_e32 v1, s5
; GFX10-NEXT:  .LBB5_2: ; %atomicrmw.start
; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT:    v_mov_b32_e32 v5, v1
; GFX10-NEXT:    v_add_f32_e32 v4, v5, v2
; GFX10-NEXT:    global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v5
; GFX10-NEXT:    s_or_b32 s3, vcc_lo, s3
; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-NEXT:    s_cbranch_execnz .LBB5_2
; GFX10-NEXT:  ; %bb.3: ; %Flow
; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s3
; GFX10-NEXT:  .LBB5_4: ; %Flow1
; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s2
; GFX10-NEXT:    v_readfirstlane_b32 s0, v1
; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GFX10-NEXT:    v_mad_f32 v0, v0, 4.0, s0
; GFX10-NEXT:    global_store_dword v[0:1], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_ret_f32_system:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_mov_b32 s4, exec_lo
; GFX11-NEXT:    s_mov_b32 s3, 0
; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX11-NEXT:    s_mov_b32 s2, exec_lo
; GFX11-NEXT:    ; implicit-def: $vgpr1
; GFX11-NEXT:    v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT:    s_cbranch_execz .LBB5_4
; GFX11-NEXT:  ; %bb.1:
; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT:    s_bcnt1_i32_b32 s4, s4
; GFX11-NEXT:    v_mov_b32_e32 v3, 0
; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v1, s4
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_load_b32 s5, s[0:1], 0x0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_dual_mul_f32 v2, 4.0, v1 :: v_dual_mov_b32 v1, s5
; GFX11-NEXT:  .LBB5_2: ; %atomicrmw.start
; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT:    v_mov_b32_e32 v5, v1
; GFX11-NEXT:    v_add_f32_e32 v4, v5, v2
; GFX11-NEXT:    global_atomic_cmpswap_b32 v1, v3, v[4:5], s[0:1] glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v5
; GFX11-NEXT:    s_or_b32 s3, vcc_lo, s3
; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s3
; GFX11-NEXT:    s_cbranch_execnz .LBB5_2
; GFX11-NEXT:  ; %bb.3: ; %Flow
; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s3
; GFX11-NEXT:  .LBB5_4: ; %Flow1
; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s2
; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GFX11-NEXT:    v_readfirstlane_b32 s0, v1
; GFX11-NEXT:    v_mul_f32_e32 v0, 4.0, v0
; GFX11-NEXT:    v_add_f32_e32 v0, s0, v0
; GFX11-NEXT:    global_store_b32 v[0:1], v0, off
; GFX11-NEXT:    s_nop 0
; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT:    s_endpgm
  %result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("one-as") seq_cst
  store float %result, ptr addrspace(1) undef
  ret void
}

define amdgpu_kernel void @global_atomic_fadd_ret_f32_wrong_subtarget(ptr addrspace(1) %ptr) #1 {
; GCN-LABEL: global_atomic_fadd_ret_f32_wrong_subtarget:
; GCN:       ; %bb.0:
; GCN-NEXT:    s_mov_b64 s[4:5], exec
; GCN-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
; GCN-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT:    ; implicit-def: $vgpr1
; GCN-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GCN-NEXT:    s_cbranch_execz .LBB6_4
; GCN-NEXT:  ; %bb.1:
; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GCN-NEXT:    s_bcnt1_i32_b64 s7, s[4:5]
; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v1, s7
; GCN-NEXT:    s_mov_b64 s[4:5], 0
; GCN-NEXT:    v_mul_f32_e32 v2, 4.0, v1
; GCN-NEXT:    s_waitcnt lgkmcnt(0)
; GCN-NEXT:    s_load_dword s6, s[0:1], 0x0
; GCN-NEXT:    v_mov_b32_e32 v3, 0
; GCN-NEXT:    s_waitcnt lgkmcnt(0)
; GCN-NEXT:    v_mov_b32_e32 v1, s6
; GCN-NEXT:  .LBB6_2: ; %atomicrmw.start
; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
; GCN-NEXT:    v_mov_b32_e32 v5, v1
; GCN-NEXT:    v_add_f32_e32 v4, v5, v2
; GCN-NEXT:    global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    buffer_wbinvl1_vol
; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v5
; GCN-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GCN-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GCN-NEXT:    s_cbranch_execnz .LBB6_2
; GCN-NEXT:  ; %bb.3: ; %Flow
; GCN-NEXT:    s_or_b64 exec, exec, s[4:5]
; GCN-NEXT:  .LBB6_4: ; %Flow1
; GCN-NEXT:    s_or_b64 exec, exec, s[2:3]
; GCN-NEXT:    v_readfirstlane_b32 s0, v1
; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GCN-NEXT:    v_mad_f32 v0, v0, 4.0, s0
; GCN-NEXT:    global_store_dword v[0:1], v0, off
; GCN-NEXT:    s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_ret_f32_wrong_subtarget:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_mov_b64 s[4:5], exec
; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v0, s4, 0
; GFX11-NEXT:    v_mbcnt_hi_u32_b32 v0, s5, v0
; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX11-NEXT:    ; implicit-def: $vgpr1
; GFX11-NEXT:    s_and_saveexec_b64 s[2:3], vcc
; GFX11-NEXT:    s_cbranch_execz .LBB6_4
; GFX11-NEXT:  ; %bb.1:
; GFX11-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX11-NEXT:    s_bcnt1_i32_b64 s7, s[4:5]
; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v1, s7
; GFX11-NEXT:    s_mov_b64 s[4:5], 0
; GFX11-NEXT:    v_mul_f32_e32 v2, 4.0, v1
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_load_dword s6, s[0:1], 0x0
; GFX11-NEXT:    v_mov_b32_e32 v3, 0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_mov_b32_e32 v1, s6
; GFX11-NEXT:  .LBB6_2: ; %atomicrmw.start
; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT:    v_mov_b32_e32 v5, v1
; GFX11-NEXT:    v_add_f32_e32 v4, v5, v2
; GFX11-NEXT:    global_atomic_cmpswap v1, v3, v[4:5], s[0:1] glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    buffer_wbinvl1_vol
; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v5
; GFX11-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
; GFX11-NEXT:    s_andn2_b64 exec, exec, s[4:5]
; GFX11-NEXT:    s_cbranch_execnz .LBB6_2
; GFX11-NEXT:  ; %bb.3: ; %Flow
; GFX11-NEXT:    s_or_b64 exec, exec, s[4:5]
; GFX11-NEXT:  .LBB6_4: ; %Flow1
; GFX11-NEXT:    s_or_b64 exec, exec, s[2:3]
; GFX11-NEXT:    v_readfirstlane_b32 s0, v1
; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v0, v0
; GFX11-NEXT:    v_mad_f32 v0, v0, 4.0, s0
; GFX11-NEXT:    global_store_dword v[0:1], v0, off
; GFX11-NEXT:    s_endpgm
  %result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
  store float %result, ptr addrspace(1) undef
  ret void
}

define amdgpu_kernel void @global_atomic_fadd_noret_f32_wrong_subtarget(ptr addrspace(1) %ptr) #1 {
; GCN-LABEL: global_atomic_fadd_noret_f32_wrong_subtarget:
; GCN:       ; %bb.0:
; GCN-NEXT:    s_mov_b64 s[2:3], exec
; GCN-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GCN-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GCN-NEXT:    s_cbranch_execz .LBB7_2
; GCN-NEXT:  ; %bb.1:
; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GCN-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
; GCN-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
; GCN-NEXT:    v_mov_b32_e32 v0, 0
; GCN-NEXT:    v_mul_f32_e32 v1, 4.0, v1
; GCN-NEXT:    s_waitcnt lgkmcnt(0)
; GCN-NEXT:    global_atomic_add_f32 v0, v1, s[0:1]
; GCN-NEXT:    s_waitcnt vmcnt(0)
; GCN-NEXT:    buffer_wbinvl1_vol
; GCN-NEXT:  .LBB7_2:
; GCN-NEXT:    s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_noret_f32_wrong_subtarget:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_mov_b64 s[2:3], exec
; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX11-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GFX11-NEXT:    s_cbranch_execz .LBB7_2
; GFX11-NEXT:  ; %bb.1:
; GFX11-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX11-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
; GFX11-NEXT:    v_mov_b32_e32 v0, 0
; GFX11-NEXT:    v_mul_f32_e32 v1, 4.0, v1
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    global_atomic_add_f32 v0, v1, s[0:1]
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    buffer_wbinvl1_vol
; GFX11-NEXT:  .LBB7_2:
; GFX11-NEXT:    s_endpgm
  %result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @global_atomic_fadd_noret_f32_safe(ptr addrspace(1) %ptr) {
; GFX900-LABEL: global_atomic_fadd_noret_f32_safe:
; GFX900:       ; %bb.0:
; GFX900-NEXT:    s_mov_b64 s[2:3], exec
; GFX900-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX900-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GFX900-NEXT:    s_cbranch_execz .LBB8_3
; GFX900-NEXT:  ; %bb.1:
; GFX900-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT:    s_bcnt1_i32_b64 s5, s[2:3]
; GFX900-NEXT:    v_cvt_f32_ubyte0_e32 v0, s5
; GFX900-NEXT:    s_mov_b64 s[2:3], 0
; GFX900-NEXT:    v_mul_f32_e32 v2, 4.0, v0
; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
; GFX900-NEXT:    s_load_dword s4, s[0:1], 0x0
; GFX900-NEXT:    v_mov_b32_e32 v3, 0
; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
; GFX900-NEXT:    v_mov_b32_e32 v1, s4
; GFX900-NEXT:  .LBB8_2: ; %atomicrmw.start
; GFX900-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT:    v_add_f32_e32 v0, v1, v2
; GFX900-NEXT:    global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX900-NEXT:    s_waitcnt vmcnt(0)
; GFX900-NEXT:    buffer_wbinvl1_vol
; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GFX900-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
; GFX900-NEXT:    v_mov_b32_e32 v1, v0
; GFX900-NEXT:    s_andn2_b64 exec, exec, s[2:3]
; GFX900-NEXT:    s_cbranch_execnz .LBB8_2
; GFX900-NEXT:  .LBB8_3:
; GFX900-NEXT:    s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_noret_f32_safe:
; GFX908:       ; %bb.0:
; GFX908-NEXT:    s_mov_b64 s[2:3], exec
; GFX908-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX908-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT:    s_cbranch_execz .LBB8_3
; GFX908-NEXT:  ; %bb.1:
; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT:    s_bcnt1_i32_b64 s5, s[2:3]
; GFX908-NEXT:    v_cvt_f32_ubyte0_e32 v0, s5
; GFX908-NEXT:    s_mov_b64 s[2:3], 0
; GFX908-NEXT:    v_mul_f32_e32 v2, 4.0, v0
; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
; GFX908-NEXT:    s_load_dword s4, s[0:1], 0x0
; GFX908-NEXT:    v_mov_b32_e32 v3, 0
; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
; GFX908-NEXT:    v_mov_b32_e32 v1, s4
; GFX908-NEXT:  .LBB8_2: ; %atomicrmw.start
; GFX908-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT:    v_add_f32_e32 v0, v1, v2
; GFX908-NEXT:    global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX908-NEXT:    s_waitcnt vmcnt(0)
; GFX908-NEXT:    buffer_wbinvl1_vol
; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GFX908-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
; GFX908-NEXT:    v_mov_b32_e32 v1, v0
; GFX908-NEXT:    s_andn2_b64 exec, exec, s[2:3]
; GFX908-NEXT:    s_cbranch_execnz .LBB8_2
; GFX908-NEXT:  .LBB8_3:
; GFX908-NEXT:    s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_noret_f32_safe:
; GFX90A:       ; %bb.0:
; GFX90A-NEXT:    s_mov_b64 s[2:3], exec
; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GFX90A-NEXT:    s_cbranch_execz .LBB8_3
; GFX90A-NEXT:  ; %bb.1:
; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT:    s_bcnt1_i32_b64 s5, s[2:3]
; GFX90A-NEXT:    v_cvt_f32_ubyte0_e32 v0, s5
; GFX90A-NEXT:    s_mov_b64 s[2:3], 0
; GFX90A-NEXT:    v_mul_f32_e32 v2, 4.0, v0
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    s_load_dword s4, s[0:1], 0x0
; GFX90A-NEXT:    v_mov_b32_e32 v3, 0
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_mov_b32_e32 v1, s4
; GFX90A-NEXT:  .LBB8_2: ; %atomicrmw.start
; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT:    v_add_f32_e32 v0, v1, v2
; GFX90A-NEXT:    global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    buffer_wbinvl1_vol
; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
; GFX90A-NEXT:    v_mov_b32_e32 v1, v0
; GFX90A-NEXT:    s_andn2_b64 exec, exec, s[2:3]
; GFX90A-NEXT:    s_cbranch_execnz .LBB8_2
; GFX90A-NEXT:  .LBB8_3:
; GFX90A-NEXT:    s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_noret_f32_safe:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_mov_b32 s3, exec_lo
; GFX10-NEXT:    s_mov_b32 s2, 0
; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT:    s_and_saveexec_b32 s4, vcc_lo
; GFX10-NEXT:    s_cbranch_execz .LBB8_3
; GFX10-NEXT:  ; %bb.1:
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT:    s_bcnt1_i32_b32 s3, s3
; GFX10-NEXT:    v_mov_b32_e32 v3, 0
; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v0, s3
; GFX10-NEXT:    v_mul_f32_e32 v2, 4.0, v0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_load_dword s4, s[0:1], 0x0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_mov_b32_e32 v1, s4
; GFX10-NEXT:  .LBB8_2: ; %atomicrmw.start
; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT:    v_add_f32_e32 v0, v1, v2
; GFX10-NEXT:    global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX10-NEXT:    v_mov_b32_e32 v1, v0
; GFX10-NEXT:    s_or_b32 s2, vcc_lo, s2
; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
; GFX10-NEXT:    s_cbranch_execnz .LBB8_2
; GFX10-NEXT:  .LBB8_3:
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_noret_f32_safe:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_mov_b32 s3, exec_lo
; GFX11-NEXT:    s_mov_b32 s2, 0
; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX11-NEXT:    s_mov_b32 s4, exec_lo
; GFX11-NEXT:    v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT:    s_cbranch_execz .LBB8_3
; GFX11-NEXT:  ; %bb.1:
; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT:    s_bcnt1_i32_b32 s3, s3
; GFX11-NEXT:    v_mov_b32_e32 v3, 0
; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v0, s3
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_load_b32 s4, s[0:1], 0x0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_dual_mul_f32 v2, 4.0, v0 :: v_dual_mov_b32 v1, s4
; GFX11-NEXT:  .LBB8_2: ; %atomicrmw.start
; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT:    v_add_f32_e32 v0, v1, v2
; GFX11-NEXT:    global_atomic_cmpswap_b32 v0, v3, v[0:1], s[0:1] glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX11-NEXT:    v_mov_b32_e32 v1, v0
; GFX11-NEXT:    s_or_b32 s2, vcc_lo, s2
; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
; GFX11-NEXT:    s_cbranch_execnz .LBB8_2
; GFX11-NEXT:  .LBB8_3:
; GFX11-NEXT:    s_endpgm
  %result = atomicrmw fadd ptr addrspace(1) %ptr, float 4.0 syncscope("agent") seq_cst
  ret void
}

define amdgpu_kernel void @infer_as_before_atomic(ptr addrspace(4) %arg) #0 {
; GFX900-LABEL: infer_as_before_atomic:
; GFX900:       ; %bb.0:
; GFX900-NEXT:    s_mov_b64 s[2:3], exec
; GFX900-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX900-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX900-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GFX900-NEXT:    s_cbranch_execz .LBB9_3
; GFX900-NEXT:  ; %bb.1:
; GFX900-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX900-NEXT:    s_bcnt1_i32_b64 s5, s[2:3]
; GFX900-NEXT:    s_mov_b64 s[2:3], 0
; GFX900-NEXT:    v_cvt_f32_ubyte0_e32 v2, s5
; GFX900-NEXT:    v_mov_b32_e32 v3, 0
; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
; GFX900-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
; GFX900-NEXT:    s_load_dword s4, s[0:1], 0x0
; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
; GFX900-NEXT:    v_mov_b32_e32 v1, s4
; GFX900-NEXT:  .LBB9_2: ; %atomicrmw.start
; GFX900-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT:    v_add_f32_e32 v0, v1, v2
; GFX900-NEXT:    global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX900-NEXT:    s_waitcnt vmcnt(0)
; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
; GFX900-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
; GFX900-NEXT:    v_mov_b32_e32 v1, v0
; GFX900-NEXT:    s_andn2_b64 exec, exec, s[2:3]
; GFX900-NEXT:    s_cbranch_execnz .LBB9_2
; GFX900-NEXT:  .LBB9_3:
; GFX900-NEXT:    s_endpgm
;
; GFX908-LABEL: infer_as_before_atomic:
; GFX908:       ; %bb.0:
; GFX908-NEXT:    s_mov_b64 s[2:3], exec
; GFX908-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX908-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT:    s_cbranch_execz .LBB9_2
; GFX908-NEXT:  ; %bb.1:
; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX908-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
; GFX908-NEXT:    v_mov_b32_e32 v0, 0
; GFX908-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
; GFX908-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
; GFX908-NEXT:    global_atomic_add_f32 v0, v1, s[0:1]
; GFX908-NEXT:  .LBB9_2:
; GFX908-NEXT:    s_endpgm
;
; GFX90A-LABEL: infer_as_before_atomic:
; GFX90A:       ; %bb.0:
; GFX90A-NEXT:    s_mov_b64 s[2:3], exec
; GFX90A-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX90A-NEXT:    v_mbcnt_hi_u32_b32 v0, s3, v0
; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT:    s_and_saveexec_b64 s[4:5], vcc
; GFX90A-NEXT:    s_cbranch_execz .LBB9_2
; GFX90A-NEXT:  ; %bb.1:
; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT:    s_bcnt1_i32_b64 s2, s[2:3]
; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
; GFX90A-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    global_atomic_add_f32 v0, v1, s[0:1]
; GFX90A-NEXT:  .LBB9_2:
; GFX90A-NEXT:    s_endpgm
;
; GFX10-LABEL: infer_as_before_atomic:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_mov_b32 s3, exec_lo
; GFX10-NEXT:    s_mov_b32 s2, 0
; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v0, s3, 0
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT:    s_and_saveexec_b32 s4, vcc_lo
; GFX10-NEXT:    s_cbranch_execz .LBB9_3
; GFX10-NEXT:  ; %bb.1:
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX10-NEXT:    s_bcnt1_i32_b32 s3, s3
; GFX10-NEXT:    v_mov_b32_e32 v3, 0
; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v2, s3
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_load_dword s4, s[0:1], 0x0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_mov_b32_e32 v1, s4
; GFX10-NEXT:  .LBB9_2: ; %atomicrmw.start
; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT:    v_add_f32_e32 v0, v1, v2
; GFX10-NEXT:    global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX10-NEXT:    v_mov_b32_e32 v1, v0
; GFX10-NEXT:    s_or_b32 s2, vcc_lo, s2
; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
; GFX10-NEXT:    s_cbranch_execnz .LBB9_2
; GFX10-NEXT:  .LBB9_3:
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: infer_as_before_atomic:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_mov_b32 s2, exec_lo
; GFX11-NEXT:    s_mov_b32 s3, exec_lo
; GFX11-NEXT:    v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11-NEXT:    v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT:    s_cbranch_execz .LBB9_2
; GFX11-NEXT:  ; %bb.1:
; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
; GFX11-NEXT:    s_bcnt1_i32_b32 s2, s2
; GFX11-NEXT:    v_mov_b32_e32 v0, 0
; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v1, s2
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    global_atomic_add_f32 v0, v1, s[0:1]
; GFX11-NEXT:  .LBB9_2:
; GFX11-NEXT:    s_nop 0
; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT:    s_endpgm
  %load = load ptr, ptr addrspace(4) %arg
  %v = atomicrmw fadd ptr %load, float 1.0 syncscope("agent-one-as") monotonic, align 4
  ret void
}

define amdgpu_kernel void @global_atomic_fadd_ret_bf16_agent(ptr addrspace(1) %ptr) #0 {
; GFX900-LABEL: global_atomic_fadd_ret_bf16_agent:
; GFX900:       ; %bb.0:
; GFX900-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
; GFX900-NEXT:    s_mov_b64 s[0:1], 0
; GFX900-NEXT:    v_mov_b32_e32 v0, 0
; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
; GFX900-NEXT:    s_and_b32 s2, s4, -4
; GFX900-NEXT:    s_mov_b32 s3, s5
; GFX900-NEXT:    s_load_dword s6, s[2:3], 0x0
; GFX900-NEXT:    s_and_b32 s4, s4, 3
; GFX900-NEXT:    s_lshl_b32 s4, s4, 3
; GFX900-NEXT:    s_lshl_b32 s5, 0xffff, s4
; GFX900-NEXT:    s_not_b32 s5, s5
; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
; GFX900-NEXT:    v_mov_b32_e32 v1, s6
; GFX900-NEXT:  .LBB10_1: ; %atomicrmw.start
; GFX900-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT:    v_mov_b32_e32 v2, v1
; GFX900-NEXT:    v_lshrrev_b32_sdwa v1, s4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX900-NEXT:    v_add_f32_e32 v1, 4.0, v1
; GFX900-NEXT:    v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX900-NEXT:    v_and_or_b32 v1, v2, s5, v1
; GFX900-NEXT:    global_atomic_cmpswap v1, v0, v[1:2], s[2:3] glc
; GFX900-NEXT:    s_waitcnt vmcnt(0)
; GFX900-NEXT:    buffer_wbinvl1_vol
; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
; GFX900-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GFX900-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GFX900-NEXT:    s_cbranch_execnz .LBB10_1
; GFX900-NEXT:  ; %bb.2: ; %atomicrmw.end
; GFX900-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX900-NEXT:    v_lshrrev_b32_e32 v0, s4, v1
; GFX900-NEXT:    global_store_short v[0:1], v0, off
; GFX900-NEXT:    s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_ret_bf16_agent:
; GFX908:       ; %bb.0:
; GFX908-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
; GFX908-NEXT:    s_mov_b64 s[0:1], 0
; GFX908-NEXT:    v_mov_b32_e32 v0, 0
; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
; GFX908-NEXT:    s_and_b32 s2, s4, -4
; GFX908-NEXT:    s_mov_b32 s3, s5
; GFX908-NEXT:    s_load_dword s6, s[2:3], 0x0
; GFX908-NEXT:    s_and_b32 s4, s4, 3
; GFX908-NEXT:    s_lshl_b32 s4, s4, 3
; GFX908-NEXT:    s_lshl_b32 s5, 0xffff, s4
; GFX908-NEXT:    s_not_b32 s5, s5
; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
; GFX908-NEXT:    v_mov_b32_e32 v1, s6
; GFX908-NEXT:  .LBB10_1: ; %atomicrmw.start
; GFX908-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT:    v_mov_b32_e32 v2, v1
; GFX908-NEXT:    v_lshrrev_b32_sdwa v1, s4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX908-NEXT:    v_add_f32_e32 v1, 4.0, v1
; GFX908-NEXT:    v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX908-NEXT:    v_and_or_b32 v1, v2, s5, v1
; GFX908-NEXT:    global_atomic_cmpswap v1, v0, v[1:2], s[2:3] glc
; GFX908-NEXT:    s_waitcnt vmcnt(0)
; GFX908-NEXT:    buffer_wbinvl1_vol
; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
; GFX908-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GFX908-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GFX908-NEXT:    s_cbranch_execnz .LBB10_1
; GFX908-NEXT:  ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX908-NEXT:    v_lshrrev_b32_e32 v0, s4, v1
; GFX908-NEXT:    global_store_short v[0:1], v0, off
; GFX908-NEXT:    s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_ret_bf16_agent:
; GFX90A:       ; %bb.0:
; GFX90A-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
; GFX90A-NEXT:    s_mov_b64 s[0:1], 0
; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    s_and_b32 s2, s4, -4
; GFX90A-NEXT:    s_mov_b32 s3, s5
; GFX90A-NEXT:    s_load_dword s6, s[2:3], 0x0
; GFX90A-NEXT:    s_and_b32 s4, s4, 3
; GFX90A-NEXT:    s_lshl_b32 s4, s4, 3
; GFX90A-NEXT:    s_lshl_b32 s5, 0xffff, s4
; GFX90A-NEXT:    s_not_b32 s5, s5
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_mov_b32_e32 v1, s6
; GFX90A-NEXT:  .LBB10_1: ; %atomicrmw.start
; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT:    v_mov_b32_e32 v3, v1
; GFX90A-NEXT:    v_lshrrev_b32_sdwa v1, s4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX90A-NEXT:    v_add_f32_e32 v1, 4.0, v1
; GFX90A-NEXT:    v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX90A-NEXT:    v_and_or_b32 v2, v3, s5, v1
; GFX90A-NEXT:    global_atomic_cmpswap v1, v0, v[2:3], s[2:3] glc
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    buffer_wbinvl1_vol
; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GFX90A-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT:    s_cbranch_execnz .LBB10_1
; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX90A-NEXT:    v_lshrrev_b32_e32 v0, s4, v1
; GFX90A-NEXT:    global_store_short v[0:1], v0, off
; GFX90A-NEXT:    s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_ret_bf16_agent:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
; GFX10-NEXT:    v_mov_b32_e32 v0, 0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_and_b32 s0, s2, -4
; GFX10-NEXT:    s_mov_b32 s1, s3
; GFX10-NEXT:    s_and_b32 s2, s2, 3
; GFX10-NEXT:    s_load_dword s3, s[0:1], 0x0
; GFX10-NEXT:    s_lshl_b32 s2, s2, 3
; GFX10-NEXT:    s_lshl_b32 s4, 0xffff, s2
; GFX10-NEXT:    s_not_b32 s4, s4
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_mov_b32_e32 v1, s3
; GFX10-NEXT:    s_mov_b32 s3, 0
; GFX10-NEXT:  .LBB10_1: ; %atomicrmw.start
; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT:    v_mov_b32_e32 v2, v1
; GFX10-NEXT:    v_lshrrev_b32_sdwa v1, s2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT:    v_add_f32_e32 v1, 4.0, v1
; GFX10-NEXT:    v_lshlrev_b32_sdwa v1, s2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT:    v_and_or_b32 v1, v2, s4, v1
; GFX10-NEXT:    global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX10-NEXT:    s_or_b32 s3, vcc_lo, s3
; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-NEXT:    s_cbranch_execnz .LBB10_1
; GFX10-NEXT:  ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s3
; GFX10-NEXT:    v_lshrrev_b32_e32 v0, s2, v1
; GFX10-NEXT:    global_store_short v[0:1], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_ret_bf16_agent:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_load_b64 s[2:3], s[0:1], 0x24
; GFX11-NEXT:    v_mov_b32_e32 v0, 0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_and_b32 s0, s2, -4
; GFX11-NEXT:    s_mov_b32 s1, s3
; GFX11-NEXT:    s_and_b32 s2, s2, 3
; GFX11-NEXT:    s_load_b32 s3, s[0:1], 0x0
; GFX11-NEXT:    s_lshl_b32 s2, s2, 3
; GFX11-NEXT:    s_lshl_b32 s4, 0xffff, s2
; GFX11-NEXT:    s_not_b32 s4, s4
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_mov_b32_e32 v1, s3
; GFX11-NEXT:    s_mov_b32 s3, 0
; GFX11-NEXT:  .LBB10_1: ; %atomicrmw.start
; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT:    v_mov_b32_e32 v2, v1
; GFX11-NEXT:    v_lshrrev_b32_e32 v1, s2, v2
; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
; GFX11-NEXT:    v_add_f32_e32 v1, 4.0, v1
; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
; GFX11-NEXT:    v_lshlrev_b32_e32 v1, s2, v1
; GFX11-NEXT:    v_and_or_b32 v1, v2, s4, v1
; GFX11-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX11-NEXT:    s_or_b32 s3, vcc_lo, s3
; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s3
; GFX11-NEXT:    s_cbranch_execnz .LBB10_1
; GFX11-NEXT:  ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s3
; GFX11-NEXT:    v_lshrrev_b32_e32 v0, s2, v1
; GFX11-NEXT:    global_store_b16 v[0:1], v0, off
; GFX11-NEXT:    s_nop 0
; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT:    s_endpgm
  %result = atomicrmw fadd ptr addrspace(1) %ptr, bfloat 4.0 syncscope("agent") seq_cst
  store bfloat %result, ptr addrspace(1) undef
  ret void
}

define amdgpu_kernel void @global_atomic_fadd_ret_bf16_system(ptr addrspace(1) %ptr) #0 {
; GFX900-LABEL: global_atomic_fadd_ret_bf16_system:
; GFX900:       ; %bb.0:
; GFX900-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
; GFX900-NEXT:    s_mov_b64 s[0:1], 0
; GFX900-NEXT:    v_mov_b32_e32 v0, 0
; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
; GFX900-NEXT:    s_and_b32 s2, s4, -4
; GFX900-NEXT:    s_mov_b32 s3, s5
; GFX900-NEXT:    s_load_dword s6, s[2:3], 0x0
; GFX900-NEXT:    s_and_b32 s4, s4, 3
; GFX900-NEXT:    s_lshl_b32 s4, s4, 3
; GFX900-NEXT:    s_lshl_b32 s5, 0xffff, s4
; GFX900-NEXT:    s_not_b32 s5, s5
; GFX900-NEXT:    s_waitcnt lgkmcnt(0)
; GFX900-NEXT:    v_mov_b32_e32 v1, s6
; GFX900-NEXT:  .LBB11_1: ; %atomicrmw.start
; GFX900-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT:    v_mov_b32_e32 v2, v1
; GFX900-NEXT:    v_lshrrev_b32_sdwa v1, s4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX900-NEXT:    v_add_f32_e32 v1, 4.0, v1
; GFX900-NEXT:    v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX900-NEXT:    v_and_or_b32 v1, v2, s5, v1
; GFX900-NEXT:    global_atomic_cmpswap v1, v0, v[1:2], s[2:3] glc
; GFX900-NEXT:    s_waitcnt vmcnt(0)
; GFX900-NEXT:    buffer_wbinvl1_vol
; GFX900-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
; GFX900-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GFX900-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GFX900-NEXT:    s_cbranch_execnz .LBB11_1
; GFX900-NEXT:  ; %bb.2: ; %atomicrmw.end
; GFX900-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX900-NEXT:    v_lshrrev_b32_e32 v0, s4, v1
; GFX900-NEXT:    global_store_short v[0:1], v0, off
; GFX900-NEXT:    s_endpgm
;
; GFX908-LABEL: global_atomic_fadd_ret_bf16_system:
; GFX908:       ; %bb.0:
; GFX908-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
; GFX908-NEXT:    s_mov_b64 s[0:1], 0
; GFX908-NEXT:    v_mov_b32_e32 v0, 0
; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
; GFX908-NEXT:    s_and_b32 s2, s4, -4
; GFX908-NEXT:    s_mov_b32 s3, s5
; GFX908-NEXT:    s_load_dword s6, s[2:3], 0x0
; GFX908-NEXT:    s_and_b32 s4, s4, 3
; GFX908-NEXT:    s_lshl_b32 s4, s4, 3
; GFX908-NEXT:    s_lshl_b32 s5, 0xffff, s4
; GFX908-NEXT:    s_not_b32 s5, s5
; GFX908-NEXT:    s_waitcnt lgkmcnt(0)
; GFX908-NEXT:    v_mov_b32_e32 v1, s6
; GFX908-NEXT:  .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT:    v_mov_b32_e32 v2, v1
; GFX908-NEXT:    v_lshrrev_b32_sdwa v1, s4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX908-NEXT:    v_add_f32_e32 v1, 4.0, v1
; GFX908-NEXT:    v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX908-NEXT:    v_and_or_b32 v1, v2, s5, v1
; GFX908-NEXT:    global_atomic_cmpswap v1, v0, v[1:2], s[2:3] glc
; GFX908-NEXT:    s_waitcnt vmcnt(0)
; GFX908-NEXT:    buffer_wbinvl1_vol
; GFX908-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v2
; GFX908-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GFX908-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GFX908-NEXT:    s_cbranch_execnz .LBB11_1
; GFX908-NEXT:  ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX908-NEXT:    v_lshrrev_b32_e32 v0, s4, v1
; GFX908-NEXT:    global_store_short v[0:1], v0, off
; GFX908-NEXT:    s_endpgm
;
; GFX90A-LABEL: global_atomic_fadd_ret_bf16_system:
; GFX90A:       ; %bb.0:
; GFX90A-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
; GFX90A-NEXT:    s_mov_b64 s[0:1], 0
; GFX90A-NEXT:    v_mov_b32_e32 v0, 0
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    s_and_b32 s2, s4, -4
; GFX90A-NEXT:    s_mov_b32 s3, s5
; GFX90A-NEXT:    s_load_dword s6, s[2:3], 0x0
; GFX90A-NEXT:    s_and_b32 s4, s4, 3
; GFX90A-NEXT:    s_lshl_b32 s4, s4, 3
; GFX90A-NEXT:    s_lshl_b32 s5, 0xffff, s4
; GFX90A-NEXT:    s_not_b32 s5, s5
; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
; GFX90A-NEXT:    v_mov_b32_e32 v1, s6
; GFX90A-NEXT:  .LBB11_1: ; %atomicrmw.start
; GFX90A-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT:    v_mov_b32_e32 v3, v1
; GFX90A-NEXT:    v_lshrrev_b32_sdwa v1, s4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX90A-NEXT:    v_add_f32_e32 v1, 4.0, v1
; GFX90A-NEXT:    v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX90A-NEXT:    v_and_or_b32 v2, v3, s5, v1
; GFX90A-NEXT:    buffer_wbl2
; GFX90A-NEXT:    global_atomic_cmpswap v1, v0, v[2:3], s[2:3] glc
; GFX90A-NEXT:    s_waitcnt vmcnt(0)
; GFX90A-NEXT:    buffer_invl2
; GFX90A-NEXT:    buffer_wbinvl1_vol
; GFX90A-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
; GFX90A-NEXT:    s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT:    s_cbranch_execnz .LBB11_1
; GFX90A-NEXT:  ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT:    s_or_b64 exec, exec, s[0:1]
; GFX90A-NEXT:    v_lshrrev_b32_e32 v0, s4, v1
; GFX90A-NEXT:    global_store_short v[0:1], v0, off
; GFX90A-NEXT:    s_endpgm
;
; GFX10-LABEL: global_atomic_fadd_ret_bf16_system:
; GFX10:       ; %bb.0:
; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
; GFX10-NEXT:    v_mov_b32_e32 v0, 0
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    s_and_b32 s0, s2, -4
; GFX10-NEXT:    s_mov_b32 s1, s3
; GFX10-NEXT:    s_and_b32 s2, s2, 3
; GFX10-NEXT:    s_load_dword s3, s[0:1], 0x0
; GFX10-NEXT:    s_lshl_b32 s2, s2, 3
; GFX10-NEXT:    s_lshl_b32 s4, 0xffff, s2
; GFX10-NEXT:    s_not_b32 s4, s4
; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
; GFX10-NEXT:    v_mov_b32_e32 v1, s3
; GFX10-NEXT:    s_mov_b32 s3, 0
; GFX10-NEXT:  .LBB11_1: ; %atomicrmw.start
; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT:    v_mov_b32_e32 v2, v1
; GFX10-NEXT:    v_lshrrev_b32_sdwa v1, s2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT:    v_add_f32_e32 v1, 4.0, v1
; GFX10-NEXT:    v_lshlrev_b32_sdwa v1, s2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX10-NEXT:    v_and_or_b32 v1, v2, s4, v1
; GFX10-NEXT:    global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
; GFX10-NEXT:    s_waitcnt vmcnt(0)
; GFX10-NEXT:    buffer_gl0_inv
; GFX10-NEXT:    buffer_gl1_inv
; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX10-NEXT:    s_or_b32 s3, vcc_lo, s3
; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-NEXT:    s_cbranch_execnz .LBB11_1
; GFX10-NEXT:  ; %bb.2: ; %atomicrmw.end
; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s3
; GFX10-NEXT:    v_lshrrev_b32_e32 v0, s2, v1
; GFX10-NEXT:    global_store_short v[0:1], v0, off
; GFX10-NEXT:    s_endpgm
;
; GFX11-LABEL: global_atomic_fadd_ret_bf16_system:
; GFX11:       ; %bb.0:
; GFX11-NEXT:    s_load_b64 s[2:3], s[0:1], 0x24
; GFX11-NEXT:    v_mov_b32_e32 v0, 0
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    s_and_b32 s0, s2, -4
; GFX11-NEXT:    s_mov_b32 s1, s3
; GFX11-NEXT:    s_and_b32 s2, s2, 3
; GFX11-NEXT:    s_load_b32 s3, s[0:1], 0x0
; GFX11-NEXT:    s_lshl_b32 s2, s2, 3
; GFX11-NEXT:    s_lshl_b32 s4, 0xffff, s2
; GFX11-NEXT:    s_not_b32 s4, s4
; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
; GFX11-NEXT:    v_mov_b32_e32 v1, s3
; GFX11-NEXT:    s_mov_b32 s3, 0
; GFX11-NEXT:  .LBB11_1: ; %atomicrmw.start
; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT:    v_mov_b32_e32 v2, v1
; GFX11-NEXT:    v_lshrrev_b32_e32 v1, s2, v2
; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
; GFX11-NEXT:    v_add_f32_e32 v1, 4.0, v1
; GFX11-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
; GFX11-NEXT:    v_lshlrev_b32_e32 v1, s2, v1
; GFX11-NEXT:    v_and_or_b32 v1, v2, s4, v1
; GFX11-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] glc
; GFX11-NEXT:    s_waitcnt vmcnt(0)
; GFX11-NEXT:    buffer_gl0_inv
; GFX11-NEXT:    buffer_gl1_inv
; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v1, v2
; GFX11-NEXT:    s_or_b32 s3, vcc_lo, s3
; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s3
; GFX11-NEXT:    s_cbranch_execnz .LBB11_1
; GFX11-NEXT:  ; %bb.2: ; %atomicrmw.end
; GFX11-NEXT:    s_or_b32 exec_lo, exec_lo, s3
; GFX11-NEXT:    v_lshrrev_b32_e32 v0, s2, v1
; GFX11-NEXT:    global_store_b16 v[0:1], v0, off
; GFX11-NEXT:    s_nop 0
; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GFX11-NEXT:    s_endpgm
  %result = atomicrmw fadd ptr addrspace(1) %ptr, bfloat 4.0 syncscope("one-as") seq_cst
  store bfloat %result, ptr addrspace(1) undef
  ret void
}

attributes #0 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "amdgpu-unsafe-fp-atomics"="true" }
attributes #1 = { "denormal-fp-math-f32"="preserve-sign,preserve-sign" "target-cpu"="gfx803" "target-features"="+atomic-fadd-no-rtn-insts" "amdgpu-unsafe-fp-atomics"="true" }
attributes #2 = { "amdgpu-unsafe-fp-atomics"="true" }