; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s

define arm_aapcs_vfpcc <8 x half> @test_vld1q_f16(ptr %base) {
; CHECK-LABEL: test_vld1q_f16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrh.u16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <8 x half>, ptr %base, align 2
  ret <8 x half> %0
}

define arm_aapcs_vfpcc <4 x float> @test_vld1q_f32(ptr %base) {
; CHECK-LABEL: test_vld1q_f32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrw.u32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <4 x float>, ptr %base, align 4
  ret <4 x float> %0
}

define arm_aapcs_vfpcc <16 x i8> @test_vld1q_s8(ptr %base) {
; CHECK-LABEL: test_vld1q_s8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrb.u8 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <16 x i8>, ptr %base, align 1
  ret <16 x i8> %0
}

define arm_aapcs_vfpcc <8 x i16> @test_vld1q_s16(ptr %base) {
; CHECK-LABEL: test_vld1q_s16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrh.u16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <8 x i16>, ptr %base, align 2
  ret <8 x i16> %0
}

define arm_aapcs_vfpcc <4 x i32> @test_vld1q_s32(ptr %base) {
; CHECK-LABEL: test_vld1q_s32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrw.u32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <4 x i32>, ptr %base, align 4
  ret <4 x i32> %0
}

define arm_aapcs_vfpcc <16 x i8> @test_vld1q_u8(ptr %base) {
; CHECK-LABEL: test_vld1q_u8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrb.u8 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <16 x i8>, ptr %base, align 1
  ret <16 x i8> %0
}

define arm_aapcs_vfpcc <8 x i16> @test_vld1q_u16(ptr %base) {
; CHECK-LABEL: test_vld1q_u16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrh.u16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <8 x i16>, ptr %base, align 2
  ret <8 x i16> %0
}

define arm_aapcs_vfpcc <4 x i32> @test_vld1q_u32(ptr %base) {
; CHECK-LABEL: test_vld1q_u32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrw.u32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <4 x i32>, ptr %base, align 4
  ret <4 x i32> %0
}

define arm_aapcs_vfpcc <8 x half> @test_vld1q_z_f16(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vld1q_z_f16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrht.u16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
  %2 = call <8 x half> @llvm.masked.load.v8f16.p0(ptr %base, i32 2, <8 x i1> %1, <8 x half> zeroinitializer)
  ret <8 x half> %2
}

declare <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32)

declare <8 x half> @llvm.masked.load.v8f16.p0(ptr, i32 immarg, <8 x i1>, <8 x half>)

define arm_aapcs_vfpcc <4 x float> @test_vld1q_z_f32(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vld1q_z_f32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrwt.u32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
  %2 = call <4 x float> @llvm.masked.load.v4f32.p0(ptr %base, i32 4, <4 x i1> %1, <4 x float> zeroinitializer)
  ret <4 x float> %2
}

declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)

declare <4 x float> @llvm.masked.load.v4f32.p0(ptr, i32 immarg, <4 x i1>, <4 x float>)

define arm_aapcs_vfpcc <16 x i8> @test_vld1q_z_s8(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vld1q_z_s8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrbt.u8 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
  %2 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %base, i32 1, <16 x i1> %1, <16 x i8> zeroinitializer)
  ret <16 x i8> %2
}

declare <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32)

declare <16 x i8> @llvm.masked.load.v16i8.p0(ptr, i32 immarg, <16 x i1>, <16 x i8>)

define arm_aapcs_vfpcc <8 x i16> @test_vld1q_z_s16(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vld1q_z_s16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrht.u16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
  %2 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %base, i32 2, <8 x i1> %1, <8 x i16> zeroinitializer)
  ret <8 x i16> %2
}

declare <8 x i16> @llvm.masked.load.v8i16.p0(ptr, i32 immarg, <8 x i1>, <8 x i16>)

define arm_aapcs_vfpcc <4 x i32> @test_vld1q_z_s32(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vld1q_z_s32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrwt.u32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
  %2 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %base, i32 4, <4 x i1> %1, <4 x i32> zeroinitializer)
  ret <4 x i32> %2
}

declare <4 x i32> @llvm.masked.load.v4i32.p0(ptr, i32 immarg, <4 x i1>, <4 x i32>)

define arm_aapcs_vfpcc <16 x i8> @test_vld1q_z_u8(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vld1q_z_u8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrbt.u8 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
  %2 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %base, i32 1, <16 x i1> %1, <16 x i8> zeroinitializer)
  ret <16 x i8> %2
}

define arm_aapcs_vfpcc <8 x i16> @test_vld1q_z_u16(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vld1q_z_u16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrht.u16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
  %2 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %base, i32 2, <8 x i1> %1, <8 x i16> zeroinitializer)
  ret <8 x i16> %2
}

define arm_aapcs_vfpcc <4 x i32> @test_vld1q_z_u32(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vld1q_z_u32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrwt.u32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
  %2 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %base, i32 4, <4 x i1> %1, <4 x i32> zeroinitializer)
  ret <4 x i32> %2
}

define arm_aapcs_vfpcc <16 x i8> @test_vldrbq_s8(ptr %base) {
; CHECK-LABEL: test_vldrbq_s8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrb.u8 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <16 x i8>, ptr %base, align 1
  ret <16 x i8> %0
}

define arm_aapcs_vfpcc <8 x i16> @test_vldrbq_s16(ptr %base) {
; CHECK-LABEL: test_vldrbq_s16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrb.s16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <8 x i8>, ptr %base, align 1
  %1 = sext <8 x i8> %0 to <8 x i16>
  ret <8 x i16> %1
}

define arm_aapcs_vfpcc <4 x i32> @test_vldrbq_s32(ptr %base) {
; CHECK-LABEL: test_vldrbq_s32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrb.s32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <4 x i8>, ptr %base, align 1
  %1 = sext <4 x i8> %0 to <4 x i32>
  ret <4 x i32> %1
}

define arm_aapcs_vfpcc <16 x i8> @test_vldrbq_u8(ptr %base) {
; CHECK-LABEL: test_vldrbq_u8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrb.u8 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <16 x i8>, ptr %base, align 1
  ret <16 x i8> %0
}

define arm_aapcs_vfpcc <8 x i16> @test_vldrbq_u16(ptr %base) {
; CHECK-LABEL: test_vldrbq_u16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrb.u16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <8 x i8>, ptr %base, align 1
  %1 = zext <8 x i8> %0 to <8 x i16>
  ret <8 x i16> %1
}

define arm_aapcs_vfpcc <4 x i32> @test_vldrbq_u32(ptr %base) {
; CHECK-LABEL: test_vldrbq_u32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrb.u32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <4 x i8>, ptr %base, align 1
  %1 = zext <4 x i8> %0 to <4 x i32>
  ret <4 x i32> %1
}

define arm_aapcs_vfpcc <16 x i8> @test_vldrbq_z_s8(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vldrbq_z_s8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrbt.u8 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
  %2 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %base, i32 1, <16 x i1> %1, <16 x i8> zeroinitializer)
  ret <16 x i8> %2
}

define arm_aapcs_vfpcc <8 x i16> @test_vldrbq_z_s16(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vldrbq_z_s16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrbt.s16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
  %2 = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr %base, i32 1, <8 x i1> %1, <8 x i8> zeroinitializer)
  %3 = sext <8 x i8> %2 to <8 x i16>
  ret <8 x i16> %3
}

declare <8 x i8> @llvm.masked.load.v8i8.p0(ptr, i32 immarg, <8 x i1>, <8 x i8>)

define arm_aapcs_vfpcc <4 x i32> @test_vldrbq_z_s32(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vldrbq_z_s32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrbt.s32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
  %2 = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr %base, i32 1, <4 x i1> %1, <4 x i8> zeroinitializer)
  %3 = sext <4 x i8> %2 to <4 x i32>
  ret <4 x i32> %3
}

declare <4 x i8> @llvm.masked.load.v4i8.p0(ptr, i32 immarg, <4 x i1>, <4 x i8>)

define arm_aapcs_vfpcc <16 x i8> @test_vldrbq_z_u8(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vldrbq_z_u8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrbt.u8 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
  %2 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %base, i32 1, <16 x i1> %1, <16 x i8> zeroinitializer)
  ret <16 x i8> %2
}

define arm_aapcs_vfpcc <8 x i16> @test_vldrbq_z_u16(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vldrbq_z_u16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrbt.u16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
  %2 = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr %base, i32 1, <8 x i1> %1, <8 x i8> zeroinitializer)
  %3 = zext <8 x i8> %2 to <8 x i16>
  ret <8 x i16> %3
}

define arm_aapcs_vfpcc <4 x i32> @test_vldrbq_z_u32(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vldrbq_z_u32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrbt.u32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
  %2 = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr %base, i32 1, <4 x i1> %1, <4 x i8> zeroinitializer)
  %3 = zext <4 x i8> %2 to <4 x i32>
  ret <4 x i32> %3
}

define arm_aapcs_vfpcc <8 x half> @test_vldrhq_f16(ptr %base) {
; CHECK-LABEL: test_vldrhq_f16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrh.u16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <8 x half>, ptr %base, align 2
  ret <8 x half> %0
}

define arm_aapcs_vfpcc <8 x i16> @test_vldrhq_s16(ptr %base) {
; CHECK-LABEL: test_vldrhq_s16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrh.u16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <8 x i16>, ptr %base, align 2
  ret <8 x i16> %0
}

define arm_aapcs_vfpcc <4 x i32> @test_vldrhq_s32(ptr %base) {
; CHECK-LABEL: test_vldrhq_s32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrh.s32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <4 x i16>, ptr %base, align 2
  %1 = sext <4 x i16> %0 to <4 x i32>
  ret <4 x i32> %1
}

define arm_aapcs_vfpcc <8 x i16> @test_vldrhq_u16(ptr %base) {
; CHECK-LABEL: test_vldrhq_u16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrh.u16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <8 x i16>, ptr %base, align 2
  ret <8 x i16> %0
}

define arm_aapcs_vfpcc <4 x i32> @test_vldrhq_u32(ptr %base) {
; CHECK-LABEL: test_vldrhq_u32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrh.u32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <4 x i16>, ptr %base, align 2
  %1 = zext <4 x i16> %0 to <4 x i32>
  ret <4 x i32> %1
}

define arm_aapcs_vfpcc <8 x half> @test_vldrhq_z_f16(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vldrhq_z_f16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrht.u16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
  %2 = call <8 x half> @llvm.masked.load.v8f16.p0(ptr %base, i32 2, <8 x i1> %1, <8 x half> zeroinitializer)
  ret <8 x half> %2
}

define arm_aapcs_vfpcc <8 x i16> @test_vldrhq_z_s16(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vldrhq_z_s16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrht.u16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
  %2 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %base, i32 2, <8 x i1> %1, <8 x i16> zeroinitializer)
  ret <8 x i16> %2
}

define arm_aapcs_vfpcc <4 x i32> @test_vldrhq_z_s32(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vldrhq_z_s32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrht.s32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
  %2 = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr %base, i32 2, <4 x i1> %1, <4 x i16> zeroinitializer)
  %3 = sext <4 x i16> %2 to <4 x i32>
  ret <4 x i32> %3
}

declare <4 x i16> @llvm.masked.load.v4i16.p0(ptr, i32 immarg, <4 x i1>, <4 x i16>)

define arm_aapcs_vfpcc <8 x i16> @test_vldrhq_z_u16(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vldrhq_z_u16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrht.u16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
  %2 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %base, i32 2, <8 x i1> %1, <8 x i16> zeroinitializer)
  ret <8 x i16> %2
}

define arm_aapcs_vfpcc <4 x i32> @test_vldrhq_z_u32(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vldrhq_z_u32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrht.u32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
  %2 = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr %base, i32 2, <4 x i1> %1, <4 x i16> zeroinitializer)
  %3 = zext <4 x i16> %2 to <4 x i32>
  ret <4 x i32> %3
}

define arm_aapcs_vfpcc <4 x float> @test_vldrwq_f32(ptr %base) {
; CHECK-LABEL: test_vldrwq_f32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrw.u32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <4 x float>, ptr %base, align 4
  ret <4 x float> %0
}

define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_s32(ptr %base) {
; CHECK-LABEL: test_vldrwq_s32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrw.u32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <4 x i32>, ptr %base, align 4
  ret <4 x i32> %0
}

define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_u32(ptr %base) {
; CHECK-LABEL: test_vldrwq_u32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vldrw.u32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = load <4 x i32>, ptr %base, align 4
  ret <4 x i32> %0
}

define arm_aapcs_vfpcc <4 x float> @test_vldrwq_z_f32(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vldrwq_z_f32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrwt.u32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
  %2 = call <4 x float> @llvm.masked.load.v4f32.p0(ptr %base, i32 4, <4 x i1> %1, <4 x float> zeroinitializer)
  ret <4 x float> %2
}

define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_z_s32(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vldrwq_z_s32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrwt.u32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
  %2 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %base, i32 4, <4 x i1> %1, <4 x i32> zeroinitializer)
  ret <4 x i32> %2
}

define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_z_u32(ptr %base, i16 zeroext %p) {
; CHECK-LABEL: test_vldrwq_z_u32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vldrwt.u32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
  %2 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %base, i32 4, <4 x i1> %1, <4 x i32> zeroinitializer)
  ret <4 x i32> %2
}

define arm_aapcs_vfpcc void @test_vst1q_f16(ptr %base, <8 x half> %value) {
; CHECK-LABEL: test_vst1q_f16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrh.16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  store <8 x half> %value, ptr %base, align 2
  ret void
}

define arm_aapcs_vfpcc void @test_vst1q_f32(ptr %base, <4 x float> %value) {
; CHECK-LABEL: test_vst1q_f32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrw.32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  store <4 x float> %value, ptr %base, align 4
  ret void
}

define arm_aapcs_vfpcc void @test_vst1q_s8(ptr %base, <16 x i8> %value) {
; CHECK-LABEL: test_vst1q_s8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrb.8 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  store <16 x i8> %value, ptr %base, align 1
  ret void
}

define arm_aapcs_vfpcc void @test_vst1q_s16(ptr %base, <8 x i16> %value) {
; CHECK-LABEL: test_vst1q_s16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrh.16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  store <8 x i16> %value, ptr %base, align 2
  ret void
}

define arm_aapcs_vfpcc void @test_vst1q_s32(ptr %base, <4 x i32> %value) {
; CHECK-LABEL: test_vst1q_s32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrw.32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  store <4 x i32> %value, ptr %base, align 4
  ret void
}

define arm_aapcs_vfpcc void @test_vst1q_u8(ptr %base, <16 x i8> %value) {
; CHECK-LABEL: test_vst1q_u8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrb.8 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  store <16 x i8> %value, ptr %base, align 1
  ret void
}

define arm_aapcs_vfpcc void @test_vst1q_u16(ptr %base, <8 x i16> %value) {
; CHECK-LABEL: test_vst1q_u16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrh.16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  store <8 x i16> %value, ptr %base, align 2
  ret void
}

define arm_aapcs_vfpcc void @test_vst1q_u32(ptr %base, <4 x i32> %value) {
; CHECK-LABEL: test_vst1q_u32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrw.32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  store <4 x i32> %value, ptr %base, align 4
  ret void
}

define arm_aapcs_vfpcc void @test_vst1q_p_f16(ptr %base, <8 x half> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vst1q_p_f16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrht.16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
  call void @llvm.masked.store.v8f16.p0(<8 x half> %value, ptr %base, i32 2, <8 x i1> %1)
  ret void
}

declare void @llvm.masked.store.v8f16.p0(<8 x half>, ptr, i32 immarg, <8 x i1>)

define arm_aapcs_vfpcc void @test_vst1q_p_f32(ptr %base, <4 x float> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vst1q_p_f32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrwt.32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
  call void @llvm.masked.store.v4f32.p0(<4 x float> %value, ptr %base, i32 4, <4 x i1> %1)
  ret void
}

declare void @llvm.masked.store.v4f32.p0(<4 x float>, ptr, i32 immarg, <4 x i1>)

define arm_aapcs_vfpcc void @test_vst1q_p_s8(ptr %base, <16 x i8> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vst1q_p_s8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrbt.8 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
  call void @llvm.masked.store.v16i8.p0(<16 x i8> %value, ptr %base, i32 1, <16 x i1> %1)
  ret void
}

declare void @llvm.masked.store.v16i8.p0(<16 x i8>, ptr, i32 immarg, <16 x i1>)

define arm_aapcs_vfpcc void @test_vst1q_p_s16(ptr %base, <8 x i16> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vst1q_p_s16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrht.16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
  call void @llvm.masked.store.v8i16.p0(<8 x i16> %value, ptr %base, i32 2, <8 x i1> %1)
  ret void
}

declare void @llvm.masked.store.v8i16.p0(<8 x i16>, ptr, i32 immarg, <8 x i1>)

define arm_aapcs_vfpcc void @test_vst1q_p_s32(ptr %base, <4 x i32> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vst1q_p_s32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrwt.32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
  call void @llvm.masked.store.v4i32.p0(<4 x i32> %value, ptr %base, i32 4, <4 x i1> %1)
  ret void
}

declare void @llvm.masked.store.v4i32.p0(<4 x i32>, ptr, i32 immarg, <4 x i1>)

define arm_aapcs_vfpcc void @test_vst1q_p_u8(ptr %base, <16 x i8> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vst1q_p_u8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrbt.8 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
  call void @llvm.masked.store.v16i8.p0(<16 x i8> %value, ptr %base, i32 1, <16 x i1> %1)
  ret void
}

define arm_aapcs_vfpcc void @test_vst1q_p_u16(ptr %base, <8 x i16> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vst1q_p_u16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrht.16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
  call void @llvm.masked.store.v8i16.p0(<8 x i16> %value, ptr %base, i32 2, <8 x i1> %1)
  ret void
}

define arm_aapcs_vfpcc void @test_vst1q_p_u32(ptr %base, <4 x i32> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vst1q_p_u32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrwt.32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
  call void @llvm.masked.store.v4i32.p0(<4 x i32> %value, ptr %base, i32 4, <4 x i1> %1)
  ret void
}

define arm_aapcs_vfpcc void @test_vstrbq_s8(ptr %base, <16 x i8> %value) {
; CHECK-LABEL: test_vstrbq_s8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrb.8 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  store <16 x i8> %value, ptr %base, align 1
  ret void
}

define arm_aapcs_vfpcc void @test_vstrbq_s16(ptr %base, <8 x i16> %value) {
; CHECK-LABEL: test_vstrbq_s16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrb.16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = trunc <8 x i16> %value to <8 x i8>
  store <8 x i8> %0, ptr %base, align 1
  ret void
}

define arm_aapcs_vfpcc void @test_vstrbq_s32(ptr %base, <4 x i32> %value) {
; CHECK-LABEL: test_vstrbq_s32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrb.32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = trunc <4 x i32> %value to <4 x i8>
  store <4 x i8> %0, ptr %base, align 1
  ret void
}

define arm_aapcs_vfpcc void @test_vstrbq_u8(ptr %base, <16 x i8> %value) {
; CHECK-LABEL: test_vstrbq_u8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrb.8 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  store <16 x i8> %value, ptr %base, align 1
  ret void
}

define arm_aapcs_vfpcc void @test_vstrbq_u16(ptr %base, <8 x i16> %value) {
; CHECK-LABEL: test_vstrbq_u16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrb.16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = trunc <8 x i16> %value to <8 x i8>
  store <8 x i8> %0, ptr %base, align 1
  ret void
}

define arm_aapcs_vfpcc void @test_vstrbq_u32(ptr %base, <4 x i32> %value) {
; CHECK-LABEL: test_vstrbq_u32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrb.32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = trunc <4 x i32> %value to <4 x i8>
  store <4 x i8> %0, ptr %base, align 1
  ret void
}

define arm_aapcs_vfpcc void @test_vstrbq_p_s8(ptr %base, <16 x i8> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vstrbq_p_s8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrbt.8 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
  call void @llvm.masked.store.v16i8.p0(<16 x i8> %value, ptr %base, i32 1, <16 x i1> %1)
  ret void
}

define arm_aapcs_vfpcc void @test_vstrbq_p_s16(ptr %base, <8 x i16> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vstrbq_p_s16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrbt.16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = trunc <8 x i16> %value to <8 x i8>
  %1 = zext i16 %p to i32
  %2 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %1)
  call void @llvm.masked.store.v8i8.p0(<8 x i8> %0, ptr %base, i32 1, <8 x i1> %2)
  ret void
}

declare void @llvm.masked.store.v8i8.p0(<8 x i8>, ptr, i32 immarg, <8 x i1>)

define arm_aapcs_vfpcc void @test_vstrbq_p_s32(ptr %base, <4 x i32> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vstrbq_p_s32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrbt.32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = trunc <4 x i32> %value to <4 x i8>
  %1 = zext i16 %p to i32
  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
  call void @llvm.masked.store.v4i8.p0(<4 x i8> %0, ptr %base, i32 1, <4 x i1> %2)
  ret void
}

declare void @llvm.masked.store.v4i8.p0(<4 x i8>, ptr, i32 immarg, <4 x i1>)

define arm_aapcs_vfpcc void @test_vstrbq_p_u8(ptr %base, <16 x i8> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vstrbq_p_u8:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrbt.8 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 %0)
  call void @llvm.masked.store.v16i8.p0(<16 x i8> %value, ptr %base, i32 1, <16 x i1> %1)
  ret void
}

define arm_aapcs_vfpcc void @test_vstrbq_p_u16(ptr %base, <8 x i16> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vstrbq_p_u16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrbt.16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = trunc <8 x i16> %value to <8 x i8>
  %1 = zext i16 %p to i32
  %2 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %1)
  call void @llvm.masked.store.v8i8.p0(<8 x i8> %0, ptr %base, i32 1, <8 x i1> %2)
  ret void
}

define arm_aapcs_vfpcc void @test_vstrbq_p_u32(ptr %base, <4 x i32> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vstrbq_p_u32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrbt.32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = trunc <4 x i32> %value to <4 x i8>
  %1 = zext i16 %p to i32
  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
  call void @llvm.masked.store.v4i8.p0(<4 x i8> %0, ptr %base, i32 1, <4 x i1> %2)
  ret void
}

define arm_aapcs_vfpcc void @test_vstrhq_f16(ptr %base, <8 x half> %value) {
; CHECK-LABEL: test_vstrhq_f16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrh.16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  store <8 x half> %value, ptr %base, align 2
  ret void
}

define arm_aapcs_vfpcc void @test_vstrhq_s16(ptr %base, <8 x i16> %value) {
; CHECK-LABEL: test_vstrhq_s16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrh.16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  store <8 x i16> %value, ptr %base, align 2
  ret void
}

define arm_aapcs_vfpcc void @test_vstrhq_s32(ptr %base, <4 x i32> %value) {
; CHECK-LABEL: test_vstrhq_s32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrh.32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = trunc <4 x i32> %value to <4 x i16>
  store <4 x i16> %0, ptr %base, align 2
  ret void
}

define arm_aapcs_vfpcc void @test_vstrhq_u16(ptr %base, <8 x i16> %value) {
; CHECK-LABEL: test_vstrhq_u16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrh.16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  store <8 x i16> %value, ptr %base, align 2
  ret void
}

define arm_aapcs_vfpcc void @test_vstrhq_u32(ptr %base, <4 x i32> %value) {
; CHECK-LABEL: test_vstrhq_u32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrh.32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = trunc <4 x i32> %value to <4 x i16>
  store <4 x i16> %0, ptr %base, align 2
  ret void
}

define arm_aapcs_vfpcc void @test_vstrhq_p_f16(ptr %base, <8 x half> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vstrhq_p_f16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrht.16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
  call void @llvm.masked.store.v8f16.p0(<8 x half> %value, ptr %base, i32 2, <8 x i1> %1)
  ret void
}

define arm_aapcs_vfpcc void @test_vstrhq_p_s16(ptr %base, <8 x i16> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vstrhq_p_s16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrht.16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
  call void @llvm.masked.store.v8i16.p0(<8 x i16> %value, ptr %base, i32 2, <8 x i1> %1)
  ret void
}

define arm_aapcs_vfpcc void @test_vstrhq_p_s32(ptr %base, <4 x i32> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vstrhq_p_s32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrht.32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = trunc <4 x i32> %value to <4 x i16>
  %1 = zext i16 %p to i32
  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
  call void @llvm.masked.store.v4i16.p0(<4 x i16> %0, ptr %base, i32 2, <4 x i1> %2)
  ret void
}

declare void @llvm.masked.store.v4i16.p0(<4 x i16>, ptr, i32 immarg, <4 x i1>)

define arm_aapcs_vfpcc void @test_vstrhq_p_u16(ptr %base, <8 x i16> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vstrhq_p_u16:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrht.16 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 %0)
  call void @llvm.masked.store.v8i16.p0(<8 x i16> %value, ptr %base, i32 2, <8 x i1> %1)
  ret void
}

define arm_aapcs_vfpcc void @test_vstrhq_p_u32(ptr %base, <4 x i32> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vstrhq_p_u32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrht.32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = trunc <4 x i32> %value to <4 x i16>
  %1 = zext i16 %p to i32
  %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
  call void @llvm.masked.store.v4i16.p0(<4 x i16> %0, ptr %base, i32 2, <4 x i1> %2)
  ret void
}

define arm_aapcs_vfpcc void @test_vstrwq_f32(ptr %base, <4 x float> %value) {
; CHECK-LABEL: test_vstrwq_f32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrw.32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  store <4 x float> %value, ptr %base, align 4
  ret void
}

define arm_aapcs_vfpcc void @test_vstrwq_s32(ptr %base, <4 x i32> %value) {
; CHECK-LABEL: test_vstrwq_s32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrw.32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  store <4 x i32> %value, ptr %base, align 4
  ret void
}

define arm_aapcs_vfpcc void @test_vstrwq_u32(ptr %base, <4 x i32> %value) {
; CHECK-LABEL: test_vstrwq_u32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vstrw.32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  store <4 x i32> %value, ptr %base, align 4
  ret void
}

define arm_aapcs_vfpcc void @test_vstrwq_p_f32(ptr %base, <4 x float> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vstrwq_p_f32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrwt.32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
  call void @llvm.masked.store.v4f32.p0(<4 x float> %value, ptr %base, i32 4, <4 x i1> %1)
  ret void
}

define arm_aapcs_vfpcc void @test_vstrwq_p_s32(ptr %base, <4 x i32> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vstrwq_p_s32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrwt.32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
  call void @llvm.masked.store.v4i32.p0(<4 x i32> %value, ptr %base, i32 4, <4 x i1> %1)
  ret void
}

define arm_aapcs_vfpcc void @test_vstrwq_p_u32(ptr %base, <4 x i32> %value, i16 zeroext %p) {
; CHECK-LABEL: test_vstrwq_p_u32:
; CHECK:       @ %bb.0: @ %entry
; CHECK-NEXT:    vmsr p0, r1
; CHECK-NEXT:    vpst
; CHECK-NEXT:    vstrwt.32 q0, [r0]
; CHECK-NEXT:    bx lr
entry:
  %0 = zext i16 %p to i32
  %1 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %0)
  call void @llvm.masked.store.v4i32.p0(<4 x i32> %value, ptr %base, i32 4, <4 x i1> %1)
  ret void
}