; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
; Test i128 atomicrmw operations.
;
; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
;
; Test expansion of AtomicRMW instructions, which assume a natural alignment.
; Note that the multiple regmoves inside the CDSG loops hopefully will go away
; when the new i128 support is added.

; Check register exchange.
define i128 @atomicrmw_xchg(ptr %src, i128 %b) {
; CHECK-LABEL: atomicrmw_xchg:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vl %v1, 0(%r4), 3
; CHECK-NEXT:    vl %v0, 0(%r3), 4
; CHECK-NEXT:    vlgvg %r1, %v1, 1
; CHECK-NEXT:    vlgvg %r0, %v1, 0
; CHECK-NEXT:  .LBB0_1: # %atomicrmw.start
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vlgvg %r5, %v0, 1
; CHECK-NEXT:    vlgvg %r4, %v0, 0
; CHECK-NEXT:    cdsg %r4, %r0, 0(%r3)
; CHECK-NEXT:    vlvgp %v0, %r4, %r5
; CHECK-NEXT:    jl .LBB0_1
; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
; CHECK-NEXT:    vst %v0, 0(%r2), 3
; CHECK-NEXT:    br %r14
  %res = atomicrmw xchg ptr %src, i128 %b seq_cst
  ret i128 %res
}

; Check addition of a variable.
define i128 @atomicrmw_add(ptr %src, i128 %b) {
; CHECK-LABEL: atomicrmw_add:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vl %v0, 0(%r4), 3
; CHECK-NEXT:    vl %v1, 0(%r3), 4
; CHECK-NEXT:  .LBB1_1: # %atomicrmw.start
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vaq %v2, %v1, %v0
; CHECK-NEXT:    vlgvg %r1, %v2, 1
; CHECK-NEXT:    vlgvg %r0, %v2, 0
; CHECK-NEXT:    vlgvg %r5, %v1, 1
; CHECK-NEXT:    vlgvg %r4, %v1, 0
; CHECK-NEXT:    cdsg %r4, %r0, 0(%r3)
; CHECK-NEXT:    vlvgp %v1, %r4, %r5
; CHECK-NEXT:    jl .LBB1_1
; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
; CHECK-NEXT:    vst %v1, 0(%r2), 3
; CHECK-NEXT:    br %r14
  %res = atomicrmw add ptr %src, i128 %b seq_cst
  ret i128 %res
}

; Check subtraction of a variable.
define i128 @atomicrmw_sub(ptr %src, i128 %b) {
; CHECK-LABEL: atomicrmw_sub:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vl %v0, 0(%r4), 3
; CHECK-NEXT:    vl %v1, 0(%r3), 4
; CHECK-NEXT:  .LBB2_1: # %atomicrmw.start
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vsq %v2, %v1, %v0
; CHECK-NEXT:    vlgvg %r1, %v2, 1
; CHECK-NEXT:    vlgvg %r0, %v2, 0
; CHECK-NEXT:    vlgvg %r5, %v1, 1
; CHECK-NEXT:    vlgvg %r4, %v1, 0
; CHECK-NEXT:    cdsg %r4, %r0, 0(%r3)
; CHECK-NEXT:    vlvgp %v1, %r4, %r5
; CHECK-NEXT:    jl .LBB2_1
; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
; CHECK-NEXT:    vst %v1, 0(%r2), 3
; CHECK-NEXT:    br %r14
  %res = atomicrmw sub ptr %src, i128 %b seq_cst
  ret i128 %res
}

; Check AND of a variable.
define i128 @atomicrmw_and(ptr %src, i128 %b) {
; CHECK-LABEL: atomicrmw_and:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vl %v0, 0(%r4), 3
; CHECK-NEXT:    vl %v1, 0(%r3), 4
; CHECK-NEXT:  .LBB3_1: # %atomicrmw.start
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vn %v2, %v1, %v0
; CHECK-NEXT:    vlgvg %r1, %v2, 1
; CHECK-NEXT:    vlgvg %r0, %v2, 0
; CHECK-NEXT:    vlgvg %r5, %v1, 1
; CHECK-NEXT:    vlgvg %r4, %v1, 0
; CHECK-NEXT:    cdsg %r4, %r0, 0(%r3)
; CHECK-NEXT:    vlvgp %v1, %r4, %r5
; CHECK-NEXT:    jl .LBB3_1
; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
; CHECK-NEXT:    vst %v1, 0(%r2), 3
; CHECK-NEXT:    br %r14
  %res = atomicrmw and ptr %src, i128 %b seq_cst
  ret i128 %res
}

; Check NAND of a variable.
define i128 @atomicrmw_nand(ptr %src, i128 %b) {
; CHECK-LABEL: atomicrmw_nand:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vl %v0, 0(%r4), 3
; CHECK-NEXT:    vl %v1, 0(%r3), 4
; CHECK-NEXT:  .LBB4_1: # %atomicrmw.start
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vlgvg %r1, %v1, 1
; CHECK-NEXT:    vlgvg %r0, %v1, 0
; CHECK-NEXT:    vnn %v1, %v1, %v0
; CHECK-NEXT:    vlgvg %r5, %v1, 1
; CHECK-NEXT:    vlgvg %r4, %v1, 0
; CHECK-NEXT:    cdsg %r0, %r4, 0(%r3)
; CHECK-NEXT:    vlvgp %v1, %r0, %r1
; CHECK-NEXT:    jl .LBB4_1
; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
; CHECK-NEXT:    vst %v1, 0(%r2), 3
; CHECK-NEXT:    br %r14
  %res = atomicrmw nand ptr %src, i128 %b seq_cst
  ret i128 %res
}

; Check OR of a variable.
define i128 @atomicrmw_or(ptr %src, i128 %b) {
; CHECK-LABEL: atomicrmw_or:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vl %v0, 0(%r4), 3
; CHECK-NEXT:    vl %v1, 0(%r3), 4
; CHECK-NEXT:  .LBB5_1: # %atomicrmw.start
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vo %v2, %v1, %v0
; CHECK-NEXT:    vlgvg %r1, %v2, 1
; CHECK-NEXT:    vlgvg %r0, %v2, 0
; CHECK-NEXT:    vlgvg %r5, %v1, 1
; CHECK-NEXT:    vlgvg %r4, %v1, 0
; CHECK-NEXT:    cdsg %r4, %r0, 0(%r3)
; CHECK-NEXT:    vlvgp %v1, %r4, %r5
; CHECK-NEXT:    jl .LBB5_1
; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
; CHECK-NEXT:    vst %v1, 0(%r2), 3
; CHECK-NEXT:    br %r14
  %res = atomicrmw or ptr %src, i128 %b seq_cst
  ret i128 %res
}

; Check XOR of a variable.
define i128 @atomicrmw_xor(ptr %src, i128 %b) {
; CHECK-LABEL: atomicrmw_xor:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vl %v0, 0(%r4), 3
; CHECK-NEXT:    vl %v1, 0(%r3), 4
; CHECK-NEXT:  .LBB6_1: # %atomicrmw.start
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vx %v2, %v1, %v0
; CHECK-NEXT:    vlgvg %r1, %v2, 1
; CHECK-NEXT:    vlgvg %r0, %v2, 0
; CHECK-NEXT:    vlgvg %r5, %v1, 1
; CHECK-NEXT:    vlgvg %r4, %v1, 0
; CHECK-NEXT:    cdsg %r4, %r0, 0(%r3)
; CHECK-NEXT:    vlvgp %v1, %r4, %r5
; CHECK-NEXT:    jl .LBB6_1
; CHECK-NEXT:  # %bb.2: # %atomicrmw.end
; CHECK-NEXT:    vst %v1, 0(%r2), 3
; CHECK-NEXT:    br %r14
  %res = atomicrmw xor ptr %src, i128 %b seq_cst
  ret i128 %res
}

; Check signed minimum.
define i128 @atomicrmw_min(ptr %src, i128 %b) {
; CHECK-LABEL: atomicrmw_min:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vl %v0, 0(%r4), 3
; CHECK-NEXT:    vl %v1, 0(%r3), 4
; CHECK-NEXT:    j .LBB7_2
; CHECK-NEXT:  .LBB7_1: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB7_2 Depth=1
; CHECK-NEXT:    vlgvg %r1, %v1, 1
; CHECK-NEXT:    vlgvg %r0, %v1, 0
; CHECK-NEXT:    vlgvg %r5, %v2, 1
; CHECK-NEXT:    vlgvg %r4, %v2, 0
; CHECK-NEXT:    cdsg %r0, %r4, 0(%r3)
; CHECK-NEXT:    vlvgp %v1, %r0, %r1
; CHECK-NEXT:    je .LBB7_6
; CHECK-NEXT:  .LBB7_2: # %atomicrmw.start
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vecg %v0, %v1
; CHECK-NEXT:    jlh .LBB7_4
; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB7_2 Depth=1
; CHECK-NEXT:    vchlgs %v2, %v1, %v0
; CHECK-NEXT:  .LBB7_4: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB7_2 Depth=1
; CHECK-NEXT:    vlr %v2, %v1
; CHECK-NEXT:    jnl .LBB7_1
; CHECK-NEXT:  # %bb.5: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB7_2 Depth=1
; CHECK-NEXT:    vlr %v2, %v0
; CHECK-NEXT:    j .LBB7_1
; CHECK-NEXT:  .LBB7_6: # %atomicrmw.end
; CHECK-NEXT:    vst %v1, 0(%r2), 3
; CHECK-NEXT:    br %r14
  %res = atomicrmw min ptr %src, i128 %b seq_cst
  ret i128 %res
}

; Check signed maximum.
define i128 @atomicrmw_max(ptr %src, i128 %b) {
; CHECK-LABEL: atomicrmw_max:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vl %v0, 0(%r4), 3
; CHECK-NEXT:    vl %v1, 0(%r3), 4
; CHECK-NEXT:    j .LBB8_2
; CHECK-NEXT:  .LBB8_1: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB8_2 Depth=1
; CHECK-NEXT:    vlgvg %r1, %v1, 1
; CHECK-NEXT:    vlgvg %r0, %v1, 0
; CHECK-NEXT:    vlgvg %r5, %v2, 1
; CHECK-NEXT:    vlgvg %r4, %v2, 0
; CHECK-NEXT:    cdsg %r0, %r4, 0(%r3)
; CHECK-NEXT:    vlvgp %v1, %r0, %r1
; CHECK-NEXT:    je .LBB8_6
; CHECK-NEXT:  .LBB8_2: # %atomicrmw.start
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vecg %v0, %v1
; CHECK-NEXT:    jlh .LBB8_4
; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB8_2 Depth=1
; CHECK-NEXT:    vchlgs %v2, %v1, %v0
; CHECK-NEXT:  .LBB8_4: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB8_2 Depth=1
; CHECK-NEXT:    vlr %v2, %v1
; CHECK-NEXT:    jl .LBB8_1
; CHECK-NEXT:  # %bb.5: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB8_2 Depth=1
; CHECK-NEXT:    vlr %v2, %v0
; CHECK-NEXT:    j .LBB8_1
; CHECK-NEXT:  .LBB8_6: # %atomicrmw.end
; CHECK-NEXT:    vst %v1, 0(%r2), 3
; CHECK-NEXT:    br %r14
  %res = atomicrmw max ptr %src, i128 %b seq_cst
  ret i128 %res
}

; Check unsigned minimum.
define i128 @atomicrmw_umin(ptr %src, i128 %b) {
; CHECK-LABEL: atomicrmw_umin:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vl %v0, 0(%r4), 3
; CHECK-NEXT:    vl %v1, 0(%r3), 4
; CHECK-NEXT:    j .LBB9_2
; CHECK-NEXT:  .LBB9_1: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB9_2 Depth=1
; CHECK-NEXT:    vlgvg %r1, %v1, 1
; CHECK-NEXT:    vlgvg %r0, %v1, 0
; CHECK-NEXT:    vlgvg %r5, %v2, 1
; CHECK-NEXT:    vlgvg %r4, %v2, 0
; CHECK-NEXT:    cdsg %r0, %r4, 0(%r3)
; CHECK-NEXT:    vlvgp %v1, %r0, %r1
; CHECK-NEXT:    je .LBB9_6
; CHECK-NEXT:  .LBB9_2: # %atomicrmw.start
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    veclg %v0, %v1
; CHECK-NEXT:    jlh .LBB9_4
; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB9_2 Depth=1
; CHECK-NEXT:    vchlgs %v2, %v1, %v0
; CHECK-NEXT:  .LBB9_4: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB9_2 Depth=1
; CHECK-NEXT:    vlr %v2, %v1
; CHECK-NEXT:    jnl .LBB9_1
; CHECK-NEXT:  # %bb.5: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB9_2 Depth=1
; CHECK-NEXT:    vlr %v2, %v0
; CHECK-NEXT:    j .LBB9_1
; CHECK-NEXT:  .LBB9_6: # %atomicrmw.end
; CHECK-NEXT:    vst %v1, 0(%r2), 3
; CHECK-NEXT:    br %r14
  %res = atomicrmw umin ptr %src, i128 %b seq_cst
  ret i128 %res
}

; Check unsigned maximum.
define i128 @atomicrmw_umax(ptr %src, i128 %b) {
; CHECK-LABEL: atomicrmw_umax:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vl %v0, 0(%r4), 3
; CHECK-NEXT:    vl %v1, 0(%r3), 4
; CHECK-NEXT:    j .LBB10_2
; CHECK-NEXT:  .LBB10_1: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB10_2 Depth=1
; CHECK-NEXT:    vlgvg %r1, %v1, 1
; CHECK-NEXT:    vlgvg %r0, %v1, 0
; CHECK-NEXT:    vlgvg %r5, %v2, 1
; CHECK-NEXT:    vlgvg %r4, %v2, 0
; CHECK-NEXT:    cdsg %r0, %r4, 0(%r3)
; CHECK-NEXT:    vlvgp %v1, %r0, %r1
; CHECK-NEXT:    je .LBB10_6
; CHECK-NEXT:  .LBB10_2: # %atomicrmw.start
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    veclg %v0, %v1
; CHECK-NEXT:    jlh .LBB10_4
; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB10_2 Depth=1
; CHECK-NEXT:    vchlgs %v2, %v1, %v0
; CHECK-NEXT:  .LBB10_4: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB10_2 Depth=1
; CHECK-NEXT:    vlr %v2, %v1
; CHECK-NEXT:    jl .LBB10_1
; CHECK-NEXT:  # %bb.5: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB10_2 Depth=1
; CHECK-NEXT:    vlr %v2, %v0
; CHECK-NEXT:    j .LBB10_1
; CHECK-NEXT:  .LBB10_6: # %atomicrmw.end
; CHECK-NEXT:    vst %v1, 0(%r2), 3
; CHECK-NEXT:    br %r14
  %res = atomicrmw umax ptr %src, i128 %b seq_cst
  ret i128 %res
}

; Check increment with wraparound.
define i128 @atomicrmw_uinc_wrap(ptr %src, i128 %b) {
; CHECK-LABEL: atomicrmw_uinc_wrap:
; CHECK:       # %bb.0:
; CHECK-NEXT:    larl %r1, .LCPI11_0
; CHECK-NEXT:    vl %v0, 0(%r4), 3
; CHECK-NEXT:    vl %v2, 0(%r3), 4
; CHECK-NEXT:    vl %v1, 0(%r1), 3
; CHECK-NEXT:    j .LBB11_2
; CHECK-NEXT:  .LBB11_1: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB11_2 Depth=1
; CHECK-NEXT:    vlgvg %r1, %v2, 1
; CHECK-NEXT:    vlgvg %r0, %v2, 0
; CHECK-NEXT:    vlgvg %r5, %v3, 1
; CHECK-NEXT:    vlgvg %r4, %v3, 0
; CHECK-NEXT:    cdsg %r0, %r4, 0(%r3)
; CHECK-NEXT:    vlvgp %v2, %r0, %r1
; CHECK-NEXT:    je .LBB11_6
; CHECK-NEXT:  .LBB11_2: # %atomicrmw.start
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    veclg %v2, %v0
; CHECK-NEXT:    jlh .LBB11_4
; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB11_2 Depth=1
; CHECK-NEXT:    vchlgs %v3, %v0, %v2
; CHECK-NEXT:  .LBB11_4: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB11_2 Depth=1
; CHECK-NEXT:    vgbm %v3, 0
; CHECK-NEXT:    jnl .LBB11_1
; CHECK-NEXT:  # %bb.5: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB11_2 Depth=1
; CHECK-NEXT:    vaq %v3, %v2, %v1
; CHECK-NEXT:    j .LBB11_1
; CHECK-NEXT:  .LBB11_6: # %atomicrmw.end
; CHECK-NEXT:    vst %v2, 0(%r2), 3
; CHECK-NEXT:    br %r14
  %res = atomicrmw uinc_wrap ptr %src, i128 %b seq_cst
  ret i128 %res
}

; Check decrement with wraparound.
define i128 @atomicrmw_udec_wrap(ptr %src, i128 %b) {
; CHECK-LABEL: atomicrmw_udec_wrap:
; CHECK:       # %bb.0:
; CHECK-NEXT:    vl %v0, 0(%r4), 3
; CHECK-NEXT:    vl %v3, 0(%r3), 4
; CHECK-NEXT:    vgbm %v1, 65535
; CHECK-NEXT:    vgbm %v2, 0
; CHECK-NEXT:    j .LBB12_2
; CHECK-NEXT:  .LBB12_1: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
; CHECK-NEXT:    vlgvg %r1, %v3, 1
; CHECK-NEXT:    vlgvg %r0, %v3, 0
; CHECK-NEXT:    vlgvg %r5, %v4, 1
; CHECK-NEXT:    vlgvg %r4, %v4, 0
; CHECK-NEXT:    cdsg %r0, %r4, 0(%r3)
; CHECK-NEXT:    vlvgp %v3, %r0, %r1
; CHECK-NEXT:    je .LBB12_8
; CHECK-NEXT:  .LBB12_2: # %atomicrmw.start
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    veclg %v0, %v3
; CHECK-NEXT:    jlh .LBB12_4
; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
; CHECK-NEXT:    vchlgs %v4, %v3, %v0
; CHECK-NEXT:  .LBB12_4: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
; CHECK-NEXT:    vlr %v5, %v0
; CHECK-NEXT:    jl .LBB12_6
; CHECK-NEXT:  # %bb.5: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
; CHECK-NEXT:    vaq %v5, %v3, %v1
; CHECK-NEXT:  .LBB12_6: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
; CHECK-NEXT:    vceqgs %v4, %v3, %v2
; CHECK-NEXT:    vlr %v4, %v0
; CHECK-NEXT:    je .LBB12_1
; CHECK-NEXT:  # %bb.7: # %atomicrmw.start
; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
; CHECK-NEXT:    vlr %v4, %v5
; CHECK-NEXT:    j .LBB12_1
; CHECK-NEXT:  .LBB12_8: # %atomicrmw.end
; CHECK-NEXT:    vst %v3, 0(%r2), 3
; CHECK-NEXT:    br %r14
  %res = atomicrmw udec_wrap ptr %src, i128 %b seq_cst
  ret i128 %res
}