; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s

define arm_aapcs_vfpcc <4 x i32> @vhadds_v4i32(<4 x i32> %s0, <4 x i32> %s1) {
; CHECK-LABEL: vhadds_v4i32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vhadd.s32 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <4 x i32> %s0 to <4 x i64>
  %s1s = sext <4 x i32> %s1 to <4 x i64>
  %m = add nsw <4 x i64> %s0s, %s1s
  %s = lshr <4 x i64> %m, <i64 1, i64 1, i64 1, i64 1>
  %s2 = trunc <4 x i64> %s to <4 x i32>
  ret <4 x i32> %s2
}

define arm_aapcs_vfpcc <4 x i32> @vhaddu_v4i32(<4 x i32> %s0, <4 x i32> %s1) {
; CHECK-LABEL: vhaddu_v4i32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vhadd.u32 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <4 x i32> %s0 to <4 x i64>
  %s1s = zext <4 x i32> %s1 to <4 x i64>
  %m = add nuw nsw <4 x i64> %s0s, %s1s
  %s = lshr <4 x i64> %m, <i64 1, i64 1, i64 1, i64 1>
  %s2 = trunc <4 x i64> %s to <4 x i32>
  ret <4 x i32> %s2
}

define arm_aapcs_vfpcc <4 x i16> @vhadds_v4i16(<4 x i16> %s0, <4 x i16> %s1) {
; CHECK-LABEL: vhadds_v4i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmovlb.s16 q1, q1
; CHECK-NEXT:    vmovlb.s16 q0, q0
; CHECK-NEXT:    vadd.i32 q0, q0, q1
; CHECK-NEXT:    vshr.u32 q0, q0, #1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <4 x i16> %s0 to <4 x i32>
  %s1s = sext <4 x i16> %s1 to <4 x i32>
  %m = add nsw <4 x i32> %s0s, %s1s
  %s = lshr <4 x i32> %m, <i32 1, i32 1, i32 1, i32 1>
  %s2 = trunc <4 x i32> %s to <4 x i16>
  ret <4 x i16> %s2
}

define arm_aapcs_vfpcc <4 x i16> @vhaddu_v4i16(<4 x i16> %s0, <4 x i16> %s1) {
; CHECK-LABEL: vhaddu_v4i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmovlb.u16 q1, q1
; CHECK-NEXT:    vmovlb.u16 q0, q0
; CHECK-NEXT:    vhadd.u32 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <4 x i16> %s0 to <4 x i32>
  %s1s = zext <4 x i16> %s1 to <4 x i32>
  %m = add nuw nsw <4 x i32> %s0s, %s1s
  %s = lshr <4 x i32> %m, <i32 1, i32 1, i32 1, i32 1>
  %s2 = trunc <4 x i32> %s to <4 x i16>
  ret <4 x i16> %s2
}

define arm_aapcs_vfpcc <8 x i16> @vhadds_v8i16(<8 x i16> %s0, <8 x i16> %s1) {
; CHECK-LABEL: vhadds_v8i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vhadd.s16 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <8 x i16> %s0 to <8 x i32>
  %s1s = sext <8 x i16> %s1 to <8 x i32>
  %m = add nsw <8 x i32> %s0s, %s1s
  %s = lshr <8 x i32> %m, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %s2 = trunc <8 x i32> %s to <8 x i16>
  ret <8 x i16> %s2
}

define arm_aapcs_vfpcc <8 x i16> @vhaddu_v8i16(<8 x i16> %s0, <8 x i16> %s1) {
; CHECK-LABEL: vhaddu_v8i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vhadd.u16 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <8 x i16> %s0 to <8 x i32>
  %s1s = zext <8 x i16> %s1 to <8 x i32>
  %m = add nuw nsw <8 x i32> %s0s, %s1s
  %s = lshr <8 x i32> %m, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %s2 = trunc <8 x i32> %s to <8 x i16>
  ret <8 x i16> %s2
}

define arm_aapcs_vfpcc <4 x i8> @vhadds_v4i8(<4 x i8> %s0, <4 x i8> %s1) {
; CHECK-LABEL: vhadds_v4i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmovlb.s8 q1, q1
; CHECK-NEXT:    vmovlb.s8 q0, q0
; CHECK-NEXT:    vmovlb.s16 q1, q1
; CHECK-NEXT:    vmovlb.s16 q0, q0
; CHECK-NEXT:    vadd.i32 q0, q0, q1
; CHECK-NEXT:    vmovlb.u16 q0, q0
; CHECK-NEXT:    vshr.u32 q0, q0, #1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <4 x i8> %s0 to <4 x i16>
  %s1s = sext <4 x i8> %s1 to <4 x i16>
  %m = add nsw <4 x i16> %s0s, %s1s
  %s = lshr <4 x i16> %m, <i16 1, i16 1, i16 1, i16 1>
  %s2 = trunc <4 x i16> %s to <4 x i8>
  ret <4 x i8> %s2
}

define arm_aapcs_vfpcc <4 x i8> @vhaddu_v4i8(<4 x i8> %s0, <4 x i8> %s1) {
; CHECK-LABEL: vhaddu_v4i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmov.i32 q2, #0xff
; CHECK-NEXT:    vand q1, q1, q2
; CHECK-NEXT:    vand q0, q0, q2
; CHECK-NEXT:    vhadd.u32 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <4 x i8> %s0 to <4 x i16>
  %s1s = zext <4 x i8> %s1 to <4 x i16>
  %m = add nuw nsw <4 x i16> %s0s, %s1s
  %s = lshr <4 x i16> %m, <i16 1, i16 1, i16 1, i16 1>
  %s2 = trunc <4 x i16> %s to <4 x i8>
  ret <4 x i8> %s2
}

define arm_aapcs_vfpcc <8 x i8> @vhadds_v8i8(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-LABEL: vhadds_v8i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmovlb.s8 q1, q1
; CHECK-NEXT:    vmovlb.s8 q0, q0
; CHECK-NEXT:    vadd.i16 q0, q0, q1
; CHECK-NEXT:    vshr.u16 q0, q0, #1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <8 x i8> %s0 to <8 x i16>
  %s1s = sext <8 x i8> %s1 to <8 x i16>
  %m = add nsw <8 x i16> %s0s, %s1s
  %s = lshr <8 x i16> %m, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %s2 = trunc <8 x i16> %s to <8 x i8>
  ret <8 x i8> %s2
}

define arm_aapcs_vfpcc <8 x i8> @vhaddu_v8i8(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-LABEL: vhaddu_v8i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmovlb.u8 q1, q1
; CHECK-NEXT:    vmovlb.u8 q0, q0
; CHECK-NEXT:    vhadd.u16 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <8 x i8> %s0 to <8 x i16>
  %s1s = zext <8 x i8> %s1 to <8 x i16>
  %m = add nuw nsw <8 x i16> %s0s, %s1s
  %s = lshr <8 x i16> %m, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %s2 = trunc <8 x i16> %s to <8 x i8>
  ret <8 x i8> %s2
}

define arm_aapcs_vfpcc <16 x i8> @vhadds_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
; CHECK-LABEL: vhadds_v16i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vhadd.s8 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <16 x i8> %s0 to <16 x i16>
  %s1s = sext <16 x i8> %s1 to <16 x i16>
  %m = add nsw <16 x i16> %s0s, %s1s
  %s = lshr <16 x i16> %m, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %s2 = trunc <16 x i16> %s to <16 x i8>
  ret <16 x i8> %s2
}

define arm_aapcs_vfpcc <16 x i8> @vhaddu_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
; CHECK-LABEL: vhaddu_v16i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vhadd.u8 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <16 x i8> %s0 to <16 x i16>
  %s1s = zext <16 x i8> %s1 to <16 x i16>
  %m = add nuw nsw <16 x i16> %s0s, %s1s
  %s = lshr <16 x i16> %m, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %s2 = trunc <16 x i16> %s to <16 x i8>
  ret <16 x i8> %s2
}

define arm_aapcs_vfpcc <4 x i32> @vrhadds_v4i32(<4 x i32> %s0, <4 x i32> %s1) {
; CHECK-LABEL: vrhadds_v4i32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vrhadd.s32 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <4 x i32> %s0 to <4 x i64>
  %s1s = sext <4 x i32> %s1 to <4 x i64>
  %add = add nsw <4 x i64> %s0s, <i64 1, i64 1, i64 1, i64 1>
  %add2 = add nsw <4 x i64> %add, %s1s
  %s = lshr <4 x i64> %add2, <i64 1, i64 1, i64 1, i64 1>
  %result = trunc <4 x i64> %s to <4 x i32>
  ret <4 x i32> %result
}

define arm_aapcs_vfpcc <4 x i32> @vrhaddu_v4i32(<4 x i32> %s0, <4 x i32> %s1) {
; CHECK-LABEL: vrhaddu_v4i32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vrhadd.u32 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <4 x i32> %s0 to <4 x i64>
  %s1s = zext <4 x i32> %s1 to <4 x i64>
  %add = add nuw nsw <4 x i64> %s0s, <i64 1, i64 1, i64 1, i64 1>
  %add2 = add nuw nsw <4 x i64> %add, %s1s
  %s = lshr <4 x i64> %add2, <i64 1, i64 1, i64 1, i64 1>
  %result = trunc <4 x i64> %s to <4 x i32>
  ret <4 x i32> %result
}

define arm_aapcs_vfpcc <4 x i16> @vrhadds_v4i16(<4 x i16> %s0, <4 x i16> %s1) {
; CHECK-LABEL: vrhadds_v4i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmovlb.s16 q1, q1
; CHECK-NEXT:    vmovlb.s16 q0, q0
; CHECK-NEXT:    vadd.i32 q0, q0, q1
; CHECK-NEXT:    movs r0, #1
; CHECK-NEXT:    vadd.i32 q0, q0, r0
; CHECK-NEXT:    vshr.u32 q0, q0, #1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <4 x i16> %s0 to <4 x i32>
  %s1s = sext <4 x i16> %s1 to <4 x i32>
  %add = add nsw <4 x i32> %s0s, <i32 1, i32 1, i32 1, i32 1>
  %add2 = add nsw <4 x i32> %add, %s1s
  %s = lshr <4 x i32> %add2, <i32 1, i32 1, i32 1, i32 1>
  %result = trunc <4 x i32> %s to <4 x i16>
  ret <4 x i16> %result
}

define arm_aapcs_vfpcc <4 x i16> @vrhaddu_v4i16(<4 x i16> %s0, <4 x i16> %s1) {
; CHECK-LABEL: vrhaddu_v4i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmovlb.u16 q1, q1
; CHECK-NEXT:    vmovlb.u16 q0, q0
; CHECK-NEXT:    vrhadd.u32 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <4 x i16> %s0 to <4 x i32>
  %s1s = zext <4 x i16> %s1 to <4 x i32>
  %add = add nuw nsw <4 x i32> %s0s, <i32 1, i32 1, i32 1, i32 1>
  %add2 = add nuw nsw <4 x i32> %add, %s1s
  %s = lshr <4 x i32> %add2, <i32 1, i32 1, i32 1, i32 1>
  %result = trunc <4 x i32> %s to <4 x i16>
  ret <4 x i16> %result
}

define arm_aapcs_vfpcc <8 x i16> @vrhadds_v8i16(<8 x i16> %s0, <8 x i16> %s1) {
; CHECK-LABEL: vrhadds_v8i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vrhadd.s16 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <8 x i16> %s0 to <8 x i32>
  %s1s = sext <8 x i16> %s1 to <8 x i32>
  %add = add nsw <8 x i32> %s0s, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %add2 = add nsw <8 x i32> %add, %s1s
  %s = lshr <8 x i32> %add2, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %result = trunc <8 x i32> %s to <8 x i16>
  ret <8 x i16> %result
}

define arm_aapcs_vfpcc <8 x i16> @vrhaddu_v8i16(<8 x i16> %s0, <8 x i16> %s1) {
; CHECK-LABEL: vrhaddu_v8i16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vrhadd.u16 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <8 x i16> %s0 to <8 x i32>
  %s1s = zext <8 x i16> %s1 to <8 x i32>
  %add = add nuw nsw <8 x i32> %s0s, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %add2 = add nuw nsw <8 x i32> %add, %s1s
  %s = lshr <8 x i32> %add2, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %result = trunc <8 x i32> %s to <8 x i16>
  ret <8 x i16> %result
}

define arm_aapcs_vfpcc <4 x i8> @vrhadds_v4i8(<4 x i8> %s0, <4 x i8> %s1) {
; CHECK-LABEL: vrhadds_v4i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmovlb.s8 q1, q1
; CHECK-NEXT:    vmovlb.s8 q0, q0
; CHECK-NEXT:    vmovlb.s16 q1, q1
; CHECK-NEXT:    vmovlb.s16 q0, q0
; CHECK-NEXT:    vadd.i32 q0, q0, q1
; CHECK-NEXT:    movs r0, #1
; CHECK-NEXT:    vadd.i32 q0, q0, r0
; CHECK-NEXT:    vmovlb.u16 q0, q0
; CHECK-NEXT:    vshr.u32 q0, q0, #1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <4 x i8> %s0 to <4 x i16>
  %s1s = sext <4 x i8> %s1 to <4 x i16>
  %add = add nsw <4 x i16> %s0s, <i16 1, i16 1, i16 1, i16 1>
  %add2 = add nsw <4 x i16> %add, %s1s
  %s = lshr <4 x i16> %add2, <i16 1, i16 1, i16 1, i16 1>
  %result = trunc <4 x i16> %s to <4 x i8>
  ret <4 x i8> %result
}

define arm_aapcs_vfpcc <4 x i8> @vrhaddu_v4i8(<4 x i8> %s0, <4 x i8> %s1) {
; CHECK-LABEL: vrhaddu_v4i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmov.i32 q2, #0xff
; CHECK-NEXT:    vand q1, q1, q2
; CHECK-NEXT:    vand q0, q0, q2
; CHECK-NEXT:    vrhadd.u32 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <4 x i8> %s0 to <4 x i16>
  %s1s = zext <4 x i8> %s1 to <4 x i16>
  %add = add nuw nsw <4 x i16> %s0s, <i16 1, i16 1, i16 1, i16 1>
  %add2 = add nuw nsw <4 x i16> %add, %s1s
  %s = lshr <4 x i16> %add2, <i16 1, i16 1, i16 1, i16 1>
  %result = trunc <4 x i16> %s to <4 x i8>
  ret <4 x i8> %result
}

define arm_aapcs_vfpcc <8 x i8> @vrhadds_v8i8(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-LABEL: vrhadds_v8i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmovlb.s8 q1, q1
; CHECK-NEXT:    vmovlb.s8 q0, q0
; CHECK-NEXT:    vadd.i16 q0, q0, q1
; CHECK-NEXT:    movs r0, #1
; CHECK-NEXT:    vadd.i16 q0, q0, r0
; CHECK-NEXT:    vshr.u16 q0, q0, #1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <8 x i8> %s0 to <8 x i16>
  %s1s = sext <8 x i8> %s1 to <8 x i16>
  %add = add nsw <8 x i16> %s0s, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %add2 = add nsw <8 x i16> %add, %s1s
  %s = lshr <8 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %result = trunc <8 x i16> %s to <8 x i8>
  ret <8 x i8> %result
}

define arm_aapcs_vfpcc <8 x i8> @vrhaddu_v8i8(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-LABEL: vrhaddu_v8i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmovlb.u8 q1, q1
; CHECK-NEXT:    vmovlb.u8 q0, q0
; CHECK-NEXT:    vrhadd.u16 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <8 x i8> %s0 to <8 x i16>
  %s1s = zext <8 x i8> %s1 to <8 x i16>
  %add = add nuw nsw <8 x i16> %s0s, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %add2 = add nuw nsw <8 x i16> %add, %s1s
  %s = lshr <8 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %result = trunc <8 x i16> %s to <8 x i8>
  ret <8 x i8> %result
}

define arm_aapcs_vfpcc <16 x i8> @vrhadds_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
; CHECK-LABEL: vrhadds_v16i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vrhadd.s8 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <16 x i8> %s0 to <16 x i16>
  %s1s = sext <16 x i8> %s1 to <16 x i16>
  %add = add nsw <16 x i16> %s0s, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %add2 = add nsw <16 x i16> %add, %s1s
  %s = lshr <16 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %result = trunc <16 x i16> %s to <16 x i8>
  ret <16 x i8> %result
}

define arm_aapcs_vfpcc <16 x i8> @vrhaddu_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
; CHECK-LABEL: vrhaddu_v16i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vrhadd.u8 q0, q0, q1
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <16 x i8> %s0 to <16 x i16>
  %s1s = zext <16 x i8> %s1 to <16 x i16>
  %add = add nuw nsw <16 x i16> %s0s, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %add2 = add nuw nsw <16 x i16> %add, %s1s
  %s = lshr <16 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %result = trunc <16 x i16> %s to <16 x i8>
  ret <16 x i8> %result
}

define void @vhadd_loop_s8(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {
; CHECK-LABEL: vhadd_loop_s8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #64
; CHECK-NEXT:  .LBB24_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrb.u8 q0, [r0], #16
; CHECK-NEXT:    vldrb.u8 q1, [r1], #16
; CHECK-NEXT:    vhadd.s8 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB24_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i8, ptr %x, i32 %index
  %wide.load = load <16 x i8>, ptr %0, align 1
  %1 = sext <16 x i8> %wide.load to <16 x i16>
  %2 = getelementptr inbounds i8, ptr %y, i32 %index
  %wide.load16 = load <16 x i8>, ptr %2, align 1
  %3 = sext <16 x i8> %wide.load16 to <16 x i16>
  %4 = add nsw <16 x i16> %3, %1
  %5 = lshr <16 x i16> %4, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %6 = trunc <16 x i16> %5 to <16 x i8>
  %7 = getelementptr inbounds i8, ptr %z, i32 %index
  store <16 x i8> %6, ptr %7, align 1
  %index.next = add i32 %index, 16
  %8 = icmp eq i32 %index.next, 1024
  br i1 %8, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vhadd_loop_s16(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {
; CHECK-LABEL: vhadd_loop_s16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #128
; CHECK-NEXT:  .LBB25_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
; CHECK-NEXT:    vldrh.u16 q1, [r1], #16
; CHECK-NEXT:    vhadd.s16 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB25_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i16, ptr %x, i32 %index
  %wide.load = load <8 x i16>, ptr %0, align 2
  %1 = sext <8 x i16> %wide.load to <8 x i32>
  %2 = getelementptr inbounds i16, ptr %y, i32 %index
  %wide.load16 = load <8 x i16>, ptr %2, align 2
  %3 = sext <8 x i16> %wide.load16 to <8 x i32>
  %4 = add nsw <8 x i32> %3, %1
  %5 = lshr <8 x i32> %4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %6 = trunc <8 x i32> %5 to <8 x i16>
  %7 = getelementptr inbounds i16, ptr %z, i32 %index
  store <8 x i16> %6, ptr %7, align 2
  %index.next = add i32 %index, 8
  %8 = icmp eq i32 %index.next, 1024
  br i1 %8, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vhadd_loop_s32(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {
; CHECK-LABEL: vhadd_loop_s32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #256
; CHECK-NEXT:  .LBB26_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
; CHECK-NEXT:    vhadd.s32 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB26_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i32, ptr %x, i32 %index
  %wide.load = load <4 x i32>, ptr %0, align 4
  %1 = sext <4 x i32> %wide.load to <4 x i64>
  %2 = getelementptr inbounds i32, ptr %y, i32 %index
  %wide.load16 = load <4 x i32>, ptr %2, align 4
  %3 = sext <4 x i32> %wide.load16 to <4 x i64>
  %4 = add nsw <4 x i64> %3, %1
  %5 = lshr <4 x i64> %4, <i64 1, i64 1, i64 1, i64 1>
  %6 = trunc <4 x i64> %5 to <4 x i32>
  %7 = getelementptr inbounds i32, ptr %z, i32 %index
  store <4 x i32> %6, ptr %7, align 4
  %index.next = add i32 %index, 4
  %8 = icmp eq i32 %index.next, 1024
  br i1 %8, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vhadd_loop_u8(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {
; CHECK-LABEL: vhadd_loop_u8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #64
; CHECK-NEXT:  .LBB27_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrb.u8 q0, [r0], #16
; CHECK-NEXT:    vldrb.u8 q1, [r1], #16
; CHECK-NEXT:    vhadd.u8 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB27_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i8, ptr %x, i32 %index
  %wide.load = load <16 x i8>, ptr %0, align 1
  %1 = zext <16 x i8> %wide.load to <16 x i16>
  %2 = getelementptr inbounds i8, ptr %y, i32 %index
  %wide.load16 = load <16 x i8>, ptr %2, align 1
  %3 = zext <16 x i8> %wide.load16 to <16 x i16>
  %4 = add nuw nsw <16 x i16> %3, %1
  %5 = lshr <16 x i16> %4, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %6 = trunc <16 x i16> %5 to <16 x i8>
  %7 = getelementptr inbounds i8, ptr %z, i32 %index
  store <16 x i8> %6, ptr %7, align 1
  %index.next = add i32 %index, 16
  %8 = icmp eq i32 %index.next, 1024
  br i1 %8, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vhadd_loop_u16(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {
; CHECK-LABEL: vhadd_loop_u16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #128
; CHECK-NEXT:  .LBB28_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrh.u16 q0, [r0], #16
; CHECK-NEXT:    vldrh.u16 q1, [r1], #16
; CHECK-NEXT:    vhadd.u16 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB28_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i16, ptr %x, i32 %index
  %wide.load = load <8 x i16>, ptr %0, align 2
  %1 = zext <8 x i16> %wide.load to <8 x i32>
  %2 = getelementptr inbounds i16, ptr %y, i32 %index
  %wide.load16 = load <8 x i16>, ptr %2, align 2
  %3 = zext <8 x i16> %wide.load16 to <8 x i32>
  %4 = add nuw nsw <8 x i32> %3, %1
  %5 = lshr <8 x i32> %4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %6 = trunc <8 x i32> %5 to <8 x i16>
  %7 = getelementptr inbounds i16, ptr %z, i32 %index
  store <8 x i16> %6, ptr %7, align 2
  %index.next = add i32 %index, 8
  %8 = icmp eq i32 %index.next, 1024
  br i1 %8, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vhadd_loop_u32(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {
; CHECK-LABEL: vhadd_loop_u32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #256
; CHECK-NEXT:  .LBB29_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrw.u32 q0, [r0], #16
; CHECK-NEXT:    vldrw.u32 q1, [r1], #16
; CHECK-NEXT:    vhadd.u32 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB29_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i32, ptr %x, i32 %index
  %wide.load = load <4 x i32>, ptr %0, align 4
  %1 = zext <4 x i32> %wide.load to <4 x i64>
  %2 = getelementptr inbounds i32, ptr %y, i32 %index
  %wide.load16 = load <4 x i32>, ptr %2, align 4
  %3 = zext <4 x i32> %wide.load16 to <4 x i64>
  %4 = add nuw nsw <4 x i64> %3, %1
  %5 = lshr <4 x i64> %4, <i64 1, i64 1, i64 1, i64 1>
  %6 = trunc <4 x i64> %5 to <4 x i32>
  %7 = getelementptr inbounds i32, ptr %z, i32 %index
  store <4 x i32> %6, ptr %7, align 4
  %index.next = add i32 %index, 4
  %8 = icmp eq i32 %index.next, 1024
  br i1 %8, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vrhadd_loop_s8(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {
; CHECK-LABEL: vrhadd_loop_s8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #64
; CHECK-NEXT:  .LBB30_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrb.u8 q0, [r1], #16
; CHECK-NEXT:    vldrb.u8 q1, [r0], #16
; CHECK-NEXT:    vrhadd.u8 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB30_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i8, ptr %x, i32 %index
  %wide.load = load <16 x i8>, ptr %0, align 1
  %1 = zext <16 x i8> %wide.load to <16 x i16>
  %2 = getelementptr inbounds i8, ptr %y, i32 %index
  %wide.load16 = load <16 x i8>, ptr %2, align 1
  %3 = zext <16 x i8> %wide.load16 to <16 x i16>
  %4 = add nuw nsw <16 x i16> %1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %5 = add nuw nsw <16 x i16> %4, %3
  %6 = lshr <16 x i16> %5, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %7 = trunc <16 x i16> %6 to <16 x i8>
  %8 = getelementptr inbounds i8, ptr %z, i32 %index
  store <16 x i8> %7, ptr %8, align 1
  %index.next = add i32 %index, 16
  %9 = icmp eq i32 %index.next, 1024
  br i1 %9, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vrhadd_loop_s16(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {
; CHECK-LABEL: vrhadd_loop_s16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #128
; CHECK-NEXT:  .LBB31_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrh.u16 q0, [r1], #16
; CHECK-NEXT:    vldrh.u16 q1, [r0], #16
; CHECK-NEXT:    vrhadd.u16 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB31_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i16, ptr %x, i32 %index
  %wide.load = load <8 x i16>, ptr %0, align 2
  %1 = zext <8 x i16> %wide.load to <8 x i32>
  %2 = getelementptr inbounds i16, ptr %y, i32 %index
  %wide.load16 = load <8 x i16>, ptr %2, align 2
  %3 = zext <8 x i16> %wide.load16 to <8 x i32>
  %4 = add nuw nsw <8 x i32> %1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %5 = add nuw nsw <8 x i32> %4, %3
  %6 = lshr <8 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %7 = trunc <8 x i32> %6 to <8 x i16>
  %8 = getelementptr inbounds i16, ptr %z, i32 %index
  store <8 x i16> %7, ptr %8, align 2
  %index.next = add i32 %index, 8
  %9 = icmp eq i32 %index.next, 1024
  br i1 %9, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vrhadd_loop_s32(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {
; CHECK-LABEL: vrhadd_loop_s32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #256
; CHECK-NEXT:  .LBB32_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
; CHECK-NEXT:    vrhadd.u32 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB32_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i32, ptr %x, i32 %index
  %wide.load = load <4 x i32>, ptr %0, align 4
  %1 = zext <4 x i32> %wide.load to <4 x i64>
  %2 = getelementptr inbounds i32, ptr %y, i32 %index
  %wide.load16 = load <4 x i32>, ptr %2, align 4
  %3 = zext <4 x i32> %wide.load16 to <4 x i64>
  %4 = add nuw nsw <4 x i64> %1, <i64 1, i64 1, i64 1, i64 1>
  %5 = add nuw nsw <4 x i64> %4, %3
  %6 = lshr <4 x i64> %5, <i64 1, i64 1, i64 1, i64 1>
  %7 = trunc <4 x i64> %6 to <4 x i32>
  %8 = getelementptr inbounds i32, ptr %z, i32 %index
  store <4 x i32> %7, ptr %8, align 4
  %index.next = add i32 %index, 4
  %9 = icmp eq i32 %index.next, 1024
  br i1 %9, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vrhadd_loop_u8(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {
; CHECK-LABEL: vrhadd_loop_u8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #64
; CHECK-NEXT:  .LBB33_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrb.u8 q0, [r1], #16
; CHECK-NEXT:    vldrb.u8 q1, [r0], #16
; CHECK-NEXT:    vrhadd.u8 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB33_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i8, ptr %x, i32 %index
  %wide.load = load <16 x i8>, ptr %0, align 1
  %1 = zext <16 x i8> %wide.load to <16 x i16>
  %2 = getelementptr inbounds i8, ptr %y, i32 %index
  %wide.load16 = load <16 x i8>, ptr %2, align 1
  %3 = zext <16 x i8> %wide.load16 to <16 x i16>
  %4 = add nuw nsw <16 x i16> %1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %5 = add nuw nsw <16 x i16> %4, %3
  %6 = lshr <16 x i16> %5, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %7 = trunc <16 x i16> %6 to <16 x i8>
  %8 = getelementptr inbounds i8, ptr %z, i32 %index
  store <16 x i8> %7, ptr %8, align 1
  %index.next = add i32 %index, 16
  %9 = icmp eq i32 %index.next, 1024
  br i1 %9, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vrhadd_loop_u16(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {
; CHECK-LABEL: vrhadd_loop_u16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #128
; CHECK-NEXT:  .LBB34_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrh.u16 q0, [r1], #16
; CHECK-NEXT:    vldrh.u16 q1, [r0], #16
; CHECK-NEXT:    vrhadd.u16 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB34_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i16, ptr %x, i32 %index
  %wide.load = load <8 x i16>, ptr %0, align 2
  %1 = zext <8 x i16> %wide.load to <8 x i32>
  %2 = getelementptr inbounds i16, ptr %y, i32 %index
  %wide.load16 = load <8 x i16>, ptr %2, align 2
  %3 = zext <8 x i16> %wide.load16 to <8 x i32>
  %4 = add nuw nsw <8 x i32> %1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %5 = add nuw nsw <8 x i32> %4, %3
  %6 = lshr <8 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
  %7 = trunc <8 x i32> %6 to <8 x i16>
  %8 = getelementptr inbounds i16, ptr %z, i32 %index
  store <8 x i16> %7, ptr %8, align 2
  %index.next = add i32 %index, 8
  %9 = icmp eq i32 %index.next, 1024
  br i1 %9, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}

define void @vrhadd_loop_u32(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture writeonly %z, i32 %n) {
; CHECK-LABEL: vrhadd_loop_u32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    mov.w lr, #256
; CHECK-NEXT:  .LBB35_1: @ %vector.body
; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
; CHECK-NEXT:    vldrw.u32 q1, [r0], #16
; CHECK-NEXT:    vrhadd.u32 q0, q1, q0
; CHECK-NEXT:    vstrb.8 q0, [r2], #16
; CHECK-NEXT:    le lr, .LBB35_1
; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
; CHECK-NEXT:    pop {r7, pc}
entry:
  br label %vector.body

vector.body:                                      ; preds = %vector.body, %entry
  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
  %0 = getelementptr inbounds i32, ptr %x, i32 %index
  %wide.load = load <4 x i32>, ptr %0, align 4
  %1 = zext <4 x i32> %wide.load to <4 x i64>
  %2 = getelementptr inbounds i32, ptr %y, i32 %index
  %wide.load16 = load <4 x i32>, ptr %2, align 4
  %3 = zext <4 x i32> %wide.load16 to <4 x i64>
  %4 = add nuw nsw <4 x i64> %1, <i64 1, i64 1, i64 1, i64 1>
  %5 = add nuw nsw <4 x i64> %4, %3
  %6 = lshr <4 x i64> %5, <i64 1, i64 1, i64 1, i64 1>
  %7 = trunc <4 x i64> %6 to <4 x i32>
  %8 = getelementptr inbounds i32, ptr %z, i32 %index
  store <4 x i32> %7, ptr %8, align 4
  %index.next = add i32 %index, 4
  %9 = icmp eq i32 %index.next, 1024
  br i1 %9, label %for.cond.cleanup, label %vector.body

for.cond.cleanup:                                 ; preds = %vector.body
  ret void
}


define arm_aapcs_vfpcc i16 @vhadds_reduce_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
; CHECK-LABEL: vhadds_reduce_v16i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vhadd.s8 q0, q0, q1
; CHECK-NEXT:    vaddv.s8 r0, q0
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <16 x i8> %s0 to <16 x i16>
  %s1s = sext <16 x i8> %s1 to <16 x i16>
  %add = add <16 x i16> %s0s, %s1s
  %s = ashr <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %result = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %s)
  ret i16 %result
}

define arm_aapcs_vfpcc i16 @vhaddu_reduce_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
; CHECK-LABEL: vhaddu_reduce_v16i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vhadd.u8 q0, q0, q1
; CHECK-NEXT:    vaddv.u8 r0, q0
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <16 x i8> %s0 to <16 x i16>
  %s1s = zext <16 x i8> %s1 to <16 x i16>
  %add = add <16 x i16> %s0s, %s1s
  %s = lshr <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %result = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %s)
  ret i16 %result
}

define arm_aapcs_vfpcc i16 @vrhadds_reduce_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
; CHECK-LABEL: vrhadds_reduce_v16i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vrhadd.s8 q0, q0, q1
; CHECK-NEXT:    vaddv.s8 r0, q0
; CHECK-NEXT:    bx lr
entry:
  %s0s = sext <16 x i8> %s0 to <16 x i16>
  %s1s = sext <16 x i8> %s1 to <16 x i16>
  %add = add <16 x i16> %s0s, %s1s
  %add2 = add <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %s = ashr <16 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %result = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %s)
  ret i16 %result
}

define arm_aapcs_vfpcc i16 @vrhaddu_reduce_v16i8(<16 x i8> %s0, <16 x i8> %s1) {
; CHECK-LABEL: vrhaddu_reduce_v16i8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vrhadd.u8 q0, q0, q1
; CHECK-NEXT:    vaddv.u8 r0, q0
; CHECK-NEXT:    bx lr
entry:
  %s0s = zext <16 x i8> %s0 to <16 x i16>
  %s1s = zext <16 x i8> %s1 to <16 x i16>
  %add = add <16 x i16> %s0s, %s1s
  %add2 = add <16 x i16> %add, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %s = lshr <16 x i16> %add2, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
  %result = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %s)
  ret i16 %result
}

declare i16 @llvm.vector.reduce.add.v16i16(<16 x i16>)