; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgcn -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=SI %s
; RUN: llc -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=VI %s

define amdgpu_kernel void @v_ubfe_sub_i32(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) #1 {
; SI-LABEL: v_ubfe_sub_i32:
; SI:       ; %bb.0:
; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
; SI-NEXT:    s_mov_b32 s7, 0xf000
; SI-NEXT:    s_mov_b32 s6, 0
; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; SI-NEXT:    v_mov_b32_e32 v1, 0
; SI-NEXT:    s_waitcnt lgkmcnt(0)
; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
; SI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
; SI-NEXT:    s_waitcnt vmcnt(0)
; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 glc
; SI-NEXT:    s_waitcnt vmcnt(0)
; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
; SI-NEXT:    v_bfe_u32 v2, v2, 0, v3
; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; SI-NEXT:    s_endpgm
;
; VI-LABEL: v_ubfe_sub_i32:
; VI:       ; %bb.0:
; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
; VI-NEXT:    s_waitcnt lgkmcnt(0)
; VI-NEXT:    v_mov_b32_e32 v1, s3
; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT:    flat_load_dword v3, v[0:1] glc
; VI-NEXT:    s_waitcnt vmcnt(0)
; VI-NEXT:    flat_load_dword v4, v[0:1] glc
; VI-NEXT:    s_waitcnt vmcnt(0)
; VI-NEXT:    v_mov_b32_e32 v1, s1
; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT:    v_bfe_u32 v2, v3, 0, v4
; VI-NEXT:    flat_store_dword v[0:1], v2
; VI-NEXT:    s_endpgm
  %id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
  %in0.gep = getelementptr i32, ptr addrspace(1) %in0, i32 %id.x
  %in1.gep = getelementptr i32, ptr addrspace(1) %in1, i32 %id.x
  %out.gep = getelementptr i32, ptr addrspace(1) %out, i32 %id.x
  %src = load volatile i32, ptr addrspace(1) %in0.gep
  %width = load volatile i32, ptr addrspace(1) %in0.gep
  %sub = sub i32 32, %width
  %shl = shl i32 %src, %sub
  %bfe = lshr i32 %shl, %sub
  store i32 %bfe, ptr addrspace(1) %out.gep
  ret void
}

define amdgpu_kernel void @v_ubfe_sub_multi_use_shl_i32(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) #1 {
; SI-LABEL: v_ubfe_sub_multi_use_shl_i32:
; SI:       ; %bb.0:
; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
; SI-NEXT:    s_mov_b32 s6, 0
; SI-NEXT:    s_mov_b32 s7, 0xf000
; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; SI-NEXT:    v_mov_b32_e32 v1, 0
; SI-NEXT:    s_waitcnt lgkmcnt(0)
; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
; SI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
; SI-NEXT:    s_waitcnt vmcnt(0)
; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 glc
; SI-NEXT:    s_waitcnt vmcnt(0)
; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
; SI-NEXT:    s_mov_b32 s6, -1
; SI-NEXT:    v_sub_i32_e32 v3, vcc, 32, v3
; SI-NEXT:    v_lshlrev_b32_e32 v2, v3, v2
; SI-NEXT:    v_lshrrev_b32_e32 v3, v3, v2
; SI-NEXT:    buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 0
; SI-NEXT:    s_waitcnt vmcnt(0)
; SI-NEXT:    s_endpgm
;
; VI-LABEL: v_ubfe_sub_multi_use_shl_i32:
; VI:       ; %bb.0:
; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
; VI-NEXT:    s_waitcnt lgkmcnt(0)
; VI-NEXT:    v_mov_b32_e32 v1, s3
; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT:    flat_load_dword v3, v[0:1] glc
; VI-NEXT:    s_waitcnt vmcnt(0)
; VI-NEXT:    flat_load_dword v4, v[0:1] glc
; VI-NEXT:    s_waitcnt vmcnt(0)
; VI-NEXT:    v_mov_b32_e32 v1, s1
; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT:    v_sub_u32_e32 v2, vcc, 32, v4
; VI-NEXT:    v_lshlrev_b32_e32 v3, v2, v3
; VI-NEXT:    v_lshrrev_b32_e32 v2, v2, v3
; VI-NEXT:    flat_store_dword v[0:1], v2
; VI-NEXT:    flat_store_dword v[0:1], v3
; VI-NEXT:    s_waitcnt vmcnt(0)
; VI-NEXT:    s_endpgm
  %id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
  %in0.gep = getelementptr i32, ptr addrspace(1) %in0, i32 %id.x
  %in1.gep = getelementptr i32, ptr addrspace(1) %in1, i32 %id.x
  %out.gep = getelementptr i32, ptr addrspace(1) %out, i32 %id.x
  %src = load volatile i32, ptr addrspace(1) %in0.gep
  %width = load volatile i32, ptr addrspace(1) %in0.gep
  %sub = sub i32 32, %width
  %shl = shl i32 %src, %sub
  %bfe = lshr i32 %shl, %sub
  store i32 %bfe, ptr addrspace(1) %out.gep
  store volatile i32 %shl, ptr addrspace(1) undef
  ret void
}

define amdgpu_kernel void @s_ubfe_sub_i32(ptr addrspace(1) %out, i32 %src, i32 %width) #1 {
; SI-LABEL: s_ubfe_sub_i32:
; SI:       ; %bb.0:
; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
; SI-NEXT:    s_mov_b32 s7, 0xf000
; SI-NEXT:    s_mov_b32 s6, 0
; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; SI-NEXT:    s_waitcnt lgkmcnt(0)
; SI-NEXT:    s_sub_i32 s3, 32, s3
; SI-NEXT:    s_lshl_b32 s2, s2, s3
; SI-NEXT:    s_lshr_b32 s2, s2, s3
; SI-NEXT:    v_mov_b32_e32 v1, 0
; SI-NEXT:    s_mov_b64 s[4:5], s[0:1]
; SI-NEXT:    v_mov_b32_e32 v2, s2
; SI-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT:    s_endpgm
;
; VI-LABEL: s_ubfe_sub_i32:
; VI:       ; %bb.0:
; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; VI-NEXT:    s_waitcnt lgkmcnt(0)
; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
; VI-NEXT:    s_sub_i32 s0, 32, s3
; VI-NEXT:    v_mov_b32_e32 v1, s1
; VI-NEXT:    s_lshl_b32 s1, s2, s0
; VI-NEXT:    s_lshr_b32 s0, s1, s0
; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT:    v_mov_b32_e32 v2, s0
; VI-NEXT:    flat_store_dword v[0:1], v2
; VI-NEXT:    s_endpgm
  %id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
  %out.gep = getelementptr i32, ptr addrspace(1) %out, i32 %id.x
  %sub = sub i32 32, %width
  %shl = shl i32 %src, %sub
  %bfe = lshr i32 %shl, %sub
  store i32 %bfe, ptr addrspace(1) %out.gep
  ret void
}

define amdgpu_kernel void @s_ubfe_sub_multi_use_shl_i32(ptr addrspace(1) %out, i32 %src, i32 %width) #1 {
; SI-LABEL: s_ubfe_sub_multi_use_shl_i32:
; SI:       ; %bb.0:
; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
; SI-NEXT:    s_mov_b32 s6, 0
; SI-NEXT:    s_mov_b32 s7, 0xf000
; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; SI-NEXT:    s_waitcnt lgkmcnt(0)
; SI-NEXT:    s_sub_i32 s3, 32, s3
; SI-NEXT:    s_lshl_b32 s2, s2, s3
; SI-NEXT:    s_lshr_b32 s3, s2, s3
; SI-NEXT:    v_mov_b32_e32 v1, 0
; SI-NEXT:    s_mov_b64 s[4:5], s[0:1]
; SI-NEXT:    v_mov_b32_e32 v2, s3
; SI-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT:    s_mov_b32 s6, -1
; SI-NEXT:    v_mov_b32_e32 v0, s2
; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT:    s_waitcnt vmcnt(0)
; SI-NEXT:    s_endpgm
;
; VI-LABEL: s_ubfe_sub_multi_use_shl_i32:
; VI:       ; %bb.0:
; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; VI-NEXT:    s_waitcnt lgkmcnt(0)
; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
; VI-NEXT:    s_sub_i32 s0, 32, s3
; VI-NEXT:    v_mov_b32_e32 v1, s1
; VI-NEXT:    s_lshl_b32 s1, s2, s0
; VI-NEXT:    s_lshr_b32 s0, s1, s0
; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT:    v_mov_b32_e32 v2, s0
; VI-NEXT:    flat_store_dword v[0:1], v2
; VI-NEXT:    v_mov_b32_e32 v0, s1
; VI-NEXT:    flat_store_dword v[0:1], v0
; VI-NEXT:    s_waitcnt vmcnt(0)
; VI-NEXT:    s_endpgm
  %id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
  %out.gep = getelementptr i32, ptr addrspace(1) %out, i32 %id.x
  %sub = sub i32 32, %width
  %shl = shl i32 %src, %sub
  %bfe = lshr i32 %shl, %sub
  store i32 %bfe, ptr addrspace(1) %out.gep
  store volatile i32 %shl, ptr addrspace(1) undef
  ret void
}

define amdgpu_kernel void @v_sbfe_sub_i32(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) #1 {
; SI-LABEL: v_sbfe_sub_i32:
; SI:       ; %bb.0:
; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
; SI-NEXT:    s_mov_b32 s7, 0xf000
; SI-NEXT:    s_mov_b32 s6, 0
; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; SI-NEXT:    v_mov_b32_e32 v1, 0
; SI-NEXT:    s_waitcnt lgkmcnt(0)
; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
; SI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
; SI-NEXT:    s_waitcnt vmcnt(0)
; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 glc
; SI-NEXT:    s_waitcnt vmcnt(0)
; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
; SI-NEXT:    v_bfe_i32 v2, v2, 0, v3
; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; SI-NEXT:    s_endpgm
;
; VI-LABEL: v_sbfe_sub_i32:
; VI:       ; %bb.0:
; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
; VI-NEXT:    s_waitcnt lgkmcnt(0)
; VI-NEXT:    v_mov_b32_e32 v1, s3
; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT:    flat_load_dword v3, v[0:1] glc
; VI-NEXT:    s_waitcnt vmcnt(0)
; VI-NEXT:    flat_load_dword v4, v[0:1] glc
; VI-NEXT:    s_waitcnt vmcnt(0)
; VI-NEXT:    v_mov_b32_e32 v1, s1
; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT:    v_bfe_i32 v2, v3, 0, v4
; VI-NEXT:    flat_store_dword v[0:1], v2
; VI-NEXT:    s_endpgm
  %id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
  %in0.gep = getelementptr i32, ptr addrspace(1) %in0, i32 %id.x
  %in1.gep = getelementptr i32, ptr addrspace(1) %in1, i32 %id.x
  %out.gep = getelementptr i32, ptr addrspace(1) %out, i32 %id.x
  %src = load volatile i32, ptr addrspace(1) %in0.gep
  %width = load volatile i32, ptr addrspace(1) %in0.gep
  %sub = sub i32 32, %width
  %shl = shl i32 %src, %sub
  %bfe = ashr i32 %shl, %sub
  store i32 %bfe, ptr addrspace(1) %out.gep
  ret void
}

define amdgpu_kernel void @v_sbfe_sub_multi_use_shl_i32(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) #1 {
; SI-LABEL: v_sbfe_sub_multi_use_shl_i32:
; SI:       ; %bb.0:
; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
; SI-NEXT:    s_mov_b32 s6, 0
; SI-NEXT:    s_mov_b32 s7, 0xf000
; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; SI-NEXT:    v_mov_b32_e32 v1, 0
; SI-NEXT:    s_waitcnt lgkmcnt(0)
; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
; SI-NEXT:    buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 glc
; SI-NEXT:    s_waitcnt vmcnt(0)
; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 glc
; SI-NEXT:    s_waitcnt vmcnt(0)
; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
; SI-NEXT:    s_mov_b32 s6, -1
; SI-NEXT:    v_sub_i32_e32 v3, vcc, 32, v3
; SI-NEXT:    v_lshlrev_b32_e32 v2, v3, v2
; SI-NEXT:    v_ashrrev_i32_e32 v3, v3, v2
; SI-NEXT:    buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 0
; SI-NEXT:    s_waitcnt vmcnt(0)
; SI-NEXT:    s_endpgm
;
; VI-LABEL: v_sbfe_sub_multi_use_shl_i32:
; VI:       ; %bb.0:
; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
; VI-NEXT:    s_waitcnt lgkmcnt(0)
; VI-NEXT:    v_mov_b32_e32 v1, s3
; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT:    flat_load_dword v3, v[0:1] glc
; VI-NEXT:    s_waitcnt vmcnt(0)
; VI-NEXT:    flat_load_dword v4, v[0:1] glc
; VI-NEXT:    s_waitcnt vmcnt(0)
; VI-NEXT:    v_mov_b32_e32 v1, s1
; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT:    v_sub_u32_e32 v2, vcc, 32, v4
; VI-NEXT:    v_lshlrev_b32_e32 v3, v2, v3
; VI-NEXT:    v_ashrrev_i32_e32 v2, v2, v3
; VI-NEXT:    flat_store_dword v[0:1], v2
; VI-NEXT:    flat_store_dword v[0:1], v3
; VI-NEXT:    s_waitcnt vmcnt(0)
; VI-NEXT:    s_endpgm
  %id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
  %in0.gep = getelementptr i32, ptr addrspace(1) %in0, i32 %id.x
  %in1.gep = getelementptr i32, ptr addrspace(1) %in1, i32 %id.x
  %out.gep = getelementptr i32, ptr addrspace(1) %out, i32 %id.x
  %src = load volatile i32, ptr addrspace(1) %in0.gep
  %width = load volatile i32, ptr addrspace(1) %in0.gep
  %sub = sub i32 32, %width
  %shl = shl i32 %src, %sub
  %bfe = ashr i32 %shl, %sub
  store i32 %bfe, ptr addrspace(1) %out.gep
  store volatile i32 %shl, ptr addrspace(1) undef
  ret void
}

define amdgpu_kernel void @s_sbfe_sub_i32(ptr addrspace(1) %out, i32 %src, i32 %width) #1 {
; SI-LABEL: s_sbfe_sub_i32:
; SI:       ; %bb.0:
; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
; SI-NEXT:    s_mov_b32 s7, 0xf000
; SI-NEXT:    s_mov_b32 s6, 0
; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; SI-NEXT:    s_waitcnt lgkmcnt(0)
; SI-NEXT:    s_sub_i32 s3, 32, s3
; SI-NEXT:    s_lshl_b32 s2, s2, s3
; SI-NEXT:    s_ashr_i32 s2, s2, s3
; SI-NEXT:    v_mov_b32_e32 v1, 0
; SI-NEXT:    s_mov_b64 s[4:5], s[0:1]
; SI-NEXT:    v_mov_b32_e32 v2, s2
; SI-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT:    s_endpgm
;
; VI-LABEL: s_sbfe_sub_i32:
; VI:       ; %bb.0:
; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; VI-NEXT:    s_waitcnt lgkmcnt(0)
; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
; VI-NEXT:    s_sub_i32 s0, 32, s3
; VI-NEXT:    v_mov_b32_e32 v1, s1
; VI-NEXT:    s_lshl_b32 s1, s2, s0
; VI-NEXT:    s_ashr_i32 s0, s1, s0
; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT:    v_mov_b32_e32 v2, s0
; VI-NEXT:    flat_store_dword v[0:1], v2
; VI-NEXT:    s_endpgm
  %id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
  %out.gep = getelementptr i32, ptr addrspace(1) %out, i32 %id.x
  %sub = sub i32 32, %width
  %shl = shl i32 %src, %sub
  %bfe = ashr i32 %shl, %sub
  store i32 %bfe, ptr addrspace(1) %out.gep
  ret void
}

define amdgpu_kernel void @s_sbfe_sub_multi_use_shl_i32(ptr addrspace(1) %out, i32 %src, i32 %width) #1 {
; SI-LABEL: s_sbfe_sub_multi_use_shl_i32:
; SI:       ; %bb.0:
; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
; SI-NEXT:    s_mov_b32 s6, 0
; SI-NEXT:    s_mov_b32 s7, 0xf000
; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; SI-NEXT:    s_waitcnt lgkmcnt(0)
; SI-NEXT:    s_sub_i32 s3, 32, s3
; SI-NEXT:    s_lshl_b32 s2, s2, s3
; SI-NEXT:    s_ashr_i32 s3, s2, s3
; SI-NEXT:    v_mov_b32_e32 v1, 0
; SI-NEXT:    s_mov_b64 s[4:5], s[0:1]
; SI-NEXT:    v_mov_b32_e32 v2, s3
; SI-NEXT:    buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT:    s_mov_b32 s6, -1
; SI-NEXT:    v_mov_b32_e32 v0, s2
; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT:    s_waitcnt vmcnt(0)
; SI-NEXT:    s_endpgm
;
; VI-LABEL: s_sbfe_sub_multi_use_shl_i32:
; VI:       ; %bb.0:
; VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
; VI-NEXT:    s_waitcnt lgkmcnt(0)
; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
; VI-NEXT:    s_sub_i32 s0, 32, s3
; VI-NEXT:    v_mov_b32_e32 v1, s1
; VI-NEXT:    s_lshl_b32 s1, s2, s0
; VI-NEXT:    s_ashr_i32 s0, s1, s0
; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT:    v_mov_b32_e32 v2, s0
; VI-NEXT:    flat_store_dword v[0:1], v2
; VI-NEXT:    v_mov_b32_e32 v0, s1
; VI-NEXT:    flat_store_dword v[0:1], v0
; VI-NEXT:    s_waitcnt vmcnt(0)
; VI-NEXT:    s_endpgm
  %id.x = tail call i32 @llvm.amdgcn.workitem.id.x()
  %out.gep = getelementptr i32, ptr addrspace(1) %out, i32 %id.x
  %sub = sub i32 32, %width
  %shl = shl i32 %src, %sub
  %bfe = ashr i32 %shl, %sub
  store i32 %bfe, ptr addrspace(1) %out.gep
  store volatile i32 %shl, ptr addrspace(1) undef
  ret void
}

define amdgpu_kernel void @s_sbfe_or_shl_shl_uniform_i32(ptr addrspace(1) %out, ptr addrspace(1) %in0, ptr addrspace(1) %in1) {
; SI-LABEL: s_sbfe_or_shl_shl_uniform_i32:
; SI:       ; %bb.0:
; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
; SI-NEXT:    s_waitcnt lgkmcnt(0)
; SI-NEXT:    s_load_dword s2, s[6:7], 0x0
; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
; SI-NEXT:    s_mov_b32 s7, 0xf000
; SI-NEXT:    s_waitcnt lgkmcnt(0)
; SI-NEXT:    s_or_b32 s0, s2, s0
; SI-NEXT:    s_bfe_i32 s0, s0, 0xf0000
; SI-NEXT:    s_mov_b32 s6, -1
; SI-NEXT:    v_mov_b32_e32 v0, s0
; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT:    s_endpgm
;
; VI-LABEL: s_sbfe_or_shl_shl_uniform_i32:
; VI:       ; %bb.0:
; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
; VI-NEXT:    s_waitcnt lgkmcnt(0)
; VI-NEXT:    s_load_dword s2, s[6:7], 0x0
; VI-NEXT:    s_load_dword s0, s[0:1], 0x0
; VI-NEXT:    v_mov_b32_e32 v0, s4
; VI-NEXT:    v_mov_b32_e32 v1, s5
; VI-NEXT:    s_waitcnt lgkmcnt(0)
; VI-NEXT:    s_or_b32 s0, s2, s0
; VI-NEXT:    s_bfe_i32 s0, s0, 0xf0000
; VI-NEXT:    v_mov_b32_e32 v2, s0
; VI-NEXT:    flat_store_dword v[0:1], v2
; VI-NEXT:    s_endpgm
  %a0 = load i32, ptr addrspace(1) %in0
  %b0 = load i32, ptr addrspace(1) %in1
  %a1 = shl i32 %a0, 17
  %b1 = shl i32 %b0, 17
  %or = or i32 %a1, %b1
  %result = ashr i32 %or, 17
  store i32 %result, ptr addrspace(1) %out
  ret void
}

; TODO ashr(or(shl(x,c1),shl(y,c2)),c1) -> sign_extend_inreg(or(x,shl(y,c2-c1))) iff c2 >= c1
define amdgpu_kernel void @s_sbfe_or_shl_shl_nonuniform_i32(ptr addrspace(1) %out, ptr addrspace(1) %x, ptr addrspace(1) %y) {
; SI-LABEL: s_sbfe_or_shl_shl_nonuniform_i32:
; SI:       ; %bb.0:
; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
; SI-NEXT:    s_waitcnt lgkmcnt(0)
; SI-NEXT:    s_load_dword s2, s[6:7], 0x0
; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
; SI-NEXT:    s_mov_b32 s7, 0xf000
; SI-NEXT:    s_waitcnt lgkmcnt(0)
; SI-NEXT:    s_lshl_b32 s1, s2, 17
; SI-NEXT:    s_lshl_b32 s0, s0, 19
; SI-NEXT:    s_or_b32 s0, s1, s0
; SI-NEXT:    s_ashr_i32 s0, s0, 17
; SI-NEXT:    s_mov_b32 s6, -1
; SI-NEXT:    v_mov_b32_e32 v0, s0
; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT:    s_endpgm
;
; VI-LABEL: s_sbfe_or_shl_shl_nonuniform_i32:
; VI:       ; %bb.0:
; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
; VI-NEXT:    s_waitcnt lgkmcnt(0)
; VI-NEXT:    s_load_dword s2, s[6:7], 0x0
; VI-NEXT:    s_load_dword s0, s[0:1], 0x0
; VI-NEXT:    v_mov_b32_e32 v0, s4
; VI-NEXT:    v_mov_b32_e32 v1, s5
; VI-NEXT:    s_waitcnt lgkmcnt(0)
; VI-NEXT:    s_lshl_b32 s1, s2, 17
; VI-NEXT:    s_lshl_b32 s0, s0, 19
; VI-NEXT:    s_or_b32 s0, s1, s0
; VI-NEXT:    s_ashr_i32 s0, s0, 17
; VI-NEXT:    v_mov_b32_e32 v2, s0
; VI-NEXT:    flat_store_dword v[0:1], v2
; VI-NEXT:    s_endpgm
  %a0 = load i32, ptr addrspace(1) %x
  %b0 = load i32, ptr addrspace(1) %y
  %a1 = shl i32 %a0, 17
  %b1 = shl i32 %b0, 19
  %or = or i32 %a1, %b1
  %result = ashr i32 %or, 17
  store i32 %result, ptr addrspace(1) %out
  ret void
}

; Don't fold as 'other shl' amount is less than the sign_extend_inreg type.
define amdgpu_kernel void @s_sbfe_or_shl_shl_toosmall_i32(ptr addrspace(1) %out, ptr addrspace(1) %x, ptr addrspace(1) %y) {
; SI-LABEL: s_sbfe_or_shl_shl_toosmall_i32:
; SI:       ; %bb.0:
; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
; SI-NEXT:    s_waitcnt lgkmcnt(0)
; SI-NEXT:    s_load_dword s2, s[6:7], 0x0
; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
; SI-NEXT:    s_mov_b32 s7, 0xf000
; SI-NEXT:    s_waitcnt lgkmcnt(0)
; SI-NEXT:    s_lshl_b32 s1, s2, 17
; SI-NEXT:    s_lshl_b32 s0, s0, 16
; SI-NEXT:    s_or_b32 s0, s1, s0
; SI-NEXT:    s_ashr_i32 s0, s0, 17
; SI-NEXT:    s_mov_b32 s6, -1
; SI-NEXT:    v_mov_b32_e32 v0, s0
; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT:    s_endpgm
;
; VI-LABEL: s_sbfe_or_shl_shl_toosmall_i32:
; VI:       ; %bb.0:
; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
; VI-NEXT:    s_waitcnt lgkmcnt(0)
; VI-NEXT:    s_load_dword s2, s[6:7], 0x0
; VI-NEXT:    s_load_dword s0, s[0:1], 0x0
; VI-NEXT:    v_mov_b32_e32 v0, s4
; VI-NEXT:    v_mov_b32_e32 v1, s5
; VI-NEXT:    s_waitcnt lgkmcnt(0)
; VI-NEXT:    s_lshl_b32 s1, s2, 17
; VI-NEXT:    s_lshl_b32 s0, s0, 16
; VI-NEXT:    s_or_b32 s0, s1, s0
; VI-NEXT:    s_ashr_i32 s0, s0, 17
; VI-NEXT:    v_mov_b32_e32 v2, s0
; VI-NEXT:    flat_store_dword v[0:1], v2
; VI-NEXT:    s_endpgm
  %a0 = load i32, ptr addrspace(1) %x
  %b0 = load i32, ptr addrspace(1) %y
  %a1 = shl i32 %a0, 17
  %b1 = shl i32 %b0, 16
  %or = or i32 %a1, %b1
  %result = ashr i32 %or, 17
  store i32 %result, ptr addrspace(1) %out
  ret void
}

declare i32 @llvm.amdgcn.workitem.id.x() #0

attributes #0 = { nounwind readnone }
attributes #1 = { nounwind }