; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK define arm_aapcs_vfpcc <2 x i32> @vmulhs_v2i32(<2 x i32> %s0, <2 x i32> %s1) { ; CHECK-LABEL: vmulhs_v2i32: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: vmullb.s32 q2, q0, q1 ; CHECK-NEXT: vmov r0, s11 ; CHECK-NEXT: vmov r1, s9 ; CHECK-NEXT: vmov q0[2], q0[0], r1, r0 ; CHECK-NEXT: asrs r0, r0, #31 ; CHECK-NEXT: asrs r1, r1, #31 ; CHECK-NEXT: vmov q0[3], q0[1], r1, r0 ; CHECK-NEXT: bx lr entry: %s0s = sext <2 x i32> %s0 to <2 x i64> %s1s = sext <2 x i32> %s1 to <2 x i64> %m = mul <2 x i64> %s0s, %s1s %s = ashr <2 x i64> %m, %s2 = trunc <2 x i64> %s to <2 x i32> ret <2 x i32> %s2 } define arm_aapcs_vfpcc <2 x i32> @vmulhu_v2i32(<2 x i32> %s0, <2 x i32> %s1) { ; CHECK-LABEL: vmulhu_v2i32: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: vmullb.u32 q2, q0, q1 ; CHECK-NEXT: vldr s1, .LCPI1_0 ; CHECK-NEXT: vmov.f32 s0, s9 ; CHECK-NEXT: vmov.f32 s2, s11 ; CHECK-NEXT: vmov.f32 s3, s1 ; CHECK-NEXT: bx lr ; CHECK-NEXT: .p2align 2 ; CHECK-NEXT: @ %bb.1: ; CHECK-NEXT: .LCPI1_0: ; CHECK-NEXT: .long 0x00000000 @ float 0 entry: %s0s = zext <2 x i32> %s0 to <2 x i64> %s1s = zext <2 x i32> %s1 to <2 x i64> %m = mul <2 x i64> %s0s, %s1s %s = lshr <2 x i64> %m, %s2 = trunc <2 x i64> %s to <2 x i32> ret <2 x i32> %s2 } define arm_aapcs_vfpcc <4 x i32> @vmulhs_v4i32(<4 x i32> %s0, <4 x i32> %s1) { ; CHECK-LABEL: vmulhs_v4i32: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: vmulh.s32 q0, q0, q1 ; CHECK-NEXT: bx lr entry: %s0s = sext <4 x i32> %s0 to <4 x i64> %s1s = sext <4 x i32> %s1 to <4 x i64> %m = mul <4 x i64> %s0s, %s1s %s = ashr <4 x i64> %m, %s2 = trunc <4 x i64> %s to <4 x i32> ret <4 x i32> %s2 } define arm_aapcs_vfpcc <4 x i32> @vmulhu_v4i32(<4 x i32> %s0, <4 x i32> %s1) { ; CHECK-LABEL: vmulhu_v4i32: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: vmulh.u32 q0, q0, q1 ; CHECK-NEXT: bx lr entry: %s0s = zext <4 x i32> %s0 to <4 x i64> %s1s = zext <4 x i32> %s1 to <4 x i64> %m = mul <4 x i64> %s0s, %s1s %s = lshr <4 x i64> %m, %s2 = trunc <4 x i64> %s to <4 x i32> ret <4 x i32> %s2 } define arm_aapcs_vfpcc <4 x i16> @vmulhs_v4i16(<4 x i16> %s0, <4 x i16> %s1) { ; CHECK-LABEL: vmulhs_v4i16: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: vmullb.s16 q0, q0, q1 ; CHECK-NEXT: vshr.s32 q0, q0, #16 ; CHECK-NEXT: bx lr entry: %s0s = sext <4 x i16> %s0 to <4 x i32> %s1s = sext <4 x i16> %s1 to <4 x i32> %m = mul <4 x i32> %s0s, %s1s %s = ashr <4 x i32> %m, %s2 = trunc <4 x i32> %s to <4 x i16> ret <4 x i16> %s2 } define arm_aapcs_vfpcc <4 x i16> @vmulhu_v4i16(<4 x i16> %s0, <4 x i16> %s1) { ; CHECK-LABEL: vmulhu_v4i16: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: vmullb.u16 q0, q0, q1 ; CHECK-NEXT: vshr.u32 q0, q0, #16 ; CHECK-NEXT: bx lr entry: %s0s = zext <4 x i16> %s0 to <4 x i32> %s1s = zext <4 x i16> %s1 to <4 x i32> %m = mul <4 x i32> %s0s, %s1s %s = lshr <4 x i32> %m, %s2 = trunc <4 x i32> %s to <4 x i16> ret <4 x i16> %s2 } define arm_aapcs_vfpcc <8 x i16> @vmulhs_v8i16(<8 x i16> %s0, <8 x i16> %s1) { ; CHECK-LABEL: vmulhs_v8i16: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: vmulh.s16 q0, q0, q1 ; CHECK-NEXT: bx lr entry: %s0s = sext <8 x i16> %s0 to <8 x i32> %s1s = sext <8 x i16> %s1 to <8 x i32> %m = mul <8 x i32> %s0s, %s1s %s = ashr <8 x i32> %m, %s2 = trunc <8 x i32> %s to <8 x i16> ret <8 x i16> %s2 } define arm_aapcs_vfpcc <8 x i16> @vmulhu_v8i16(<8 x i16> %s0, <8 x i16> %s1) { ; CHECK-LABEL: vmulhu_v8i16: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: vmulh.u16 q0, q0, q1 ; CHECK-NEXT: bx lr entry: %s0s = zext <8 x i16> %s0 to <8 x i32> %s1s = zext <8 x i16> %s1 to <8 x i32> %m = mul <8 x i32> %s0s, %s1s %s = lshr <8 x i32> %m, %s2 = trunc <8 x i32> %s to <8 x i16> ret <8 x i16> %s2 } define arm_aapcs_vfpcc <4 x i8> @vmulhs_v4i8(<4 x i8> %s0, <4 x i8> %s1) { ; CHECK-LABEL: vmulhs_v4i8: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: vmovlb.s8 q1, q1 ; CHECK-NEXT: vmovlb.s8 q0, q0 ; CHECK-NEXT: vmovlb.s16 q1, q1 ; CHECK-NEXT: vmovlb.s16 q0, q0 ; CHECK-NEXT: vmul.i32 q0, q0, q1 ; CHECK-NEXT: vshr.s32 q0, q0, #8 ; CHECK-NEXT: bx lr entry: %s0s = sext <4 x i8> %s0 to <4 x i16> %s1s = sext <4 x i8> %s1 to <4 x i16> %m = mul <4 x i16> %s0s, %s1s %s = ashr <4 x i16> %m, %s2 = trunc <4 x i16> %s to <4 x i8> ret <4 x i8> %s2 } define arm_aapcs_vfpcc <4 x i8> @vmulhu_v4i8(<4 x i8> %s0, <4 x i8> %s1) { ; CHECK-LABEL: vmulhu_v4i8: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: vmov.i32 q2, #0xff ; CHECK-NEXT: vand q1, q1, q2 ; CHECK-NEXT: vand q0, q0, q2 ; CHECK-NEXT: vmul.i32 q0, q0, q1 ; CHECK-NEXT: vshr.u32 q0, q0, #8 ; CHECK-NEXT: bx lr entry: %s0s = zext <4 x i8> %s0 to <4 x i16> %s1s = zext <4 x i8> %s1 to <4 x i16> %m = mul <4 x i16> %s0s, %s1s %s = lshr <4 x i16> %m, %s2 = trunc <4 x i16> %s to <4 x i8> ret <4 x i8> %s2 } define arm_aapcs_vfpcc <8 x i8> @vmulhs_v8i8(<8 x i8> %s0, <8 x i8> %s1) { ; CHECK-LABEL: vmulhs_v8i8: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: vmullb.s8 q0, q0, q1 ; CHECK-NEXT: vshr.s16 q0, q0, #8 ; CHECK-NEXT: bx lr entry: %s0s = sext <8 x i8> %s0 to <8 x i16> %s1s = sext <8 x i8> %s1 to <8 x i16> %m = mul <8 x i16> %s0s, %s1s %s = ashr <8 x i16> %m, %s2 = trunc <8 x i16> %s to <8 x i8> ret <8 x i8> %s2 } define arm_aapcs_vfpcc <8 x i8> @vmulhu_v8i8(<8 x i8> %s0, <8 x i8> %s1) { ; CHECK-LABEL: vmulhu_v8i8: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: vmullb.u8 q0, q0, q1 ; CHECK-NEXT: vshr.u16 q0, q0, #8 ; CHECK-NEXT: bx lr entry: %s0s = zext <8 x i8> %s0 to <8 x i16> %s1s = zext <8 x i8> %s1 to <8 x i16> %m = mul <8 x i16> %s0s, %s1s %s = lshr <8 x i16> %m, %s2 = trunc <8 x i16> %s to <8 x i8> ret <8 x i8> %s2 } define arm_aapcs_vfpcc <16 x i8> @vmulhs_v16i8(<16 x i8> %s0, <16 x i8> %s1) { ; CHECK-LABEL: vmulhs_v16i8: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: vmulh.s8 q0, q0, q1 ; CHECK-NEXT: bx lr entry: %s0s = sext <16 x i8> %s0 to <16 x i16> %s1s = sext <16 x i8> %s1 to <16 x i16> %m = mul <16 x i16> %s0s, %s1s %s = ashr <16 x i16> %m, %s2 = trunc <16 x i16> %s to <16 x i8> ret <16 x i8> %s2 } define arm_aapcs_vfpcc <16 x i8> @vmulhu_v16i8(<16 x i8> %s0, <16 x i8> %s1) { ; CHECK-LABEL: vmulhu_v16i8: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: vmulh.u8 q0, q0, q1 ; CHECK-NEXT: bx lr entry: %s0s = zext <16 x i8> %s0 to <16 x i16> %s1s = zext <16 x i8> %s1 to <16 x i16> %m = mul <16 x i16> %s0s, %s1s %s = lshr <16 x i16> %m, %s2 = trunc <16 x i16> %s to <16 x i8> ret <16 x i8> %s2 } define void @vmulh_s8(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture %z, i32 %n) { ; CHECK-LABEL: vmulh_s8: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r7, lr} ; CHECK-NEXT: push {r7, lr} ; CHECK-NEXT: mov.w lr, #64 ; CHECK-NEXT: .LBB14_1: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrb.u8 q0, [r0], #16 ; CHECK-NEXT: vldrb.u8 q1, [r1], #16 ; CHECK-NEXT: vmulh.s8 q0, q1, q0 ; CHECK-NEXT: vstrb.8 q0, [r2], #16 ; CHECK-NEXT: le lr, .LBB14_1 ; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup ; CHECK-NEXT: pop {r7, pc} entry: br label %vector.body vector.body: ; preds = %vector.body, %entry %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ] %0 = getelementptr inbounds i8, ptr %x, i32 %index %wide.load = load <16 x i8>, ptr %0, align 1 %1 = sext <16 x i8> %wide.load to <16 x i16> %2 = getelementptr inbounds i8, ptr %y, i32 %index %wide.load17 = load <16 x i8>, ptr %2, align 1 %3 = sext <16 x i8> %wide.load17 to <16 x i16> %4 = mul nsw <16 x i16> %3, %1 %5 = lshr <16 x i16> %4, %6 = trunc <16 x i16> %5 to <16 x i8> %7 = getelementptr inbounds i8, ptr %z, i32 %index store <16 x i8> %6, ptr %7, align 1 %index.next = add i32 %index, 16 %8 = icmp eq i32 %index.next, 1024 br i1 %8, label %for.cond.cleanup, label %vector.body for.cond.cleanup: ; preds = %vector.body ret void } define void @vmulh_s16(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture %z, i32 %n) { ; CHECK-LABEL: vmulh_s16: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r7, lr} ; CHECK-NEXT: push {r7, lr} ; CHECK-NEXT: mov.w lr, #128 ; CHECK-NEXT: .LBB15_1: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrh.u16 q0, [r0], #16 ; CHECK-NEXT: vldrh.u16 q1, [r1], #16 ; CHECK-NEXT: vmulh.s16 q0, q1, q0 ; CHECK-NEXT: vstrb.8 q0, [r2], #16 ; CHECK-NEXT: le lr, .LBB15_1 ; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup ; CHECK-NEXT: pop {r7, pc} entry: br label %vector.body vector.body: ; preds = %vector.body, %entry %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ] %0 = getelementptr inbounds i16, ptr %x, i32 %index %wide.load = load <8 x i16>, ptr %0, align 2 %1 = sext <8 x i16> %wide.load to <8 x i32> %2 = getelementptr inbounds i16, ptr %y, i32 %index %wide.load17 = load <8 x i16>, ptr %2, align 2 %3 = sext <8 x i16> %wide.load17 to <8 x i32> %4 = mul nsw <8 x i32> %3, %1 %5 = lshr <8 x i32> %4, %6 = trunc <8 x i32> %5 to <8 x i16> %7 = getelementptr inbounds i16, ptr %z, i32 %index store <8 x i16> %6, ptr %7, align 2 %index.next = add i32 %index, 8 %8 = icmp eq i32 %index.next, 1024 br i1 %8, label %for.cond.cleanup, label %vector.body for.cond.cleanup: ; preds = %vector.body ret void } define void @vmulh_s32(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture %z, i32 %n) { ; CHECK-LABEL: vmulh_s32: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r7, lr} ; CHECK-NEXT: push {r7, lr} ; CHECK-NEXT: mov.w lr, #256 ; CHECK-NEXT: .LBB16_1: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrw.u32 q0, [r0], #16 ; CHECK-NEXT: vldrw.u32 q1, [r1], #16 ; CHECK-NEXT: vmulh.s32 q0, q1, q0 ; CHECK-NEXT: vstrb.8 q0, [r2], #16 ; CHECK-NEXT: le lr, .LBB16_1 ; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup ; CHECK-NEXT: pop {r7, pc} entry: br label %vector.body vector.body: ; preds = %vector.body, %entry %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ] %0 = getelementptr inbounds i32, ptr %x, i32 %index %wide.load = load <4 x i32>, ptr %0, align 4 %1 = sext <4 x i32> %wide.load to <4 x i64> %2 = getelementptr inbounds i32, ptr %y, i32 %index %wide.load17 = load <4 x i32>, ptr %2, align 4 %3 = sext <4 x i32> %wide.load17 to <4 x i64> %4 = mul nsw <4 x i64> %3, %1 %5 = lshr <4 x i64> %4, %6 = trunc <4 x i64> %5 to <4 x i32> %7 = getelementptr inbounds i32, ptr %z, i32 %index store <4 x i32> %6, ptr %7, align 4 %index.next = add i32 %index, 4 %8 = icmp eq i32 %index.next, 1024 br i1 %8, label %for.cond.cleanup, label %vector.body for.cond.cleanup: ; preds = %vector.body ret void } define void @vmulh_u8(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture %z, i32 %n) { ; CHECK-LABEL: vmulh_u8: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r7, lr} ; CHECK-NEXT: push {r7, lr} ; CHECK-NEXT: mov.w lr, #64 ; CHECK-NEXT: .LBB17_1: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrb.u8 q0, [r0], #16 ; CHECK-NEXT: vldrb.u8 q1, [r1], #16 ; CHECK-NEXT: vmulh.u8 q0, q1, q0 ; CHECK-NEXT: vstrb.8 q0, [r2], #16 ; CHECK-NEXT: le lr, .LBB17_1 ; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup ; CHECK-NEXT: pop {r7, pc} entry: br label %vector.body vector.body: ; preds = %vector.body, %entry %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ] %0 = getelementptr inbounds i8, ptr %x, i32 %index %wide.load = load <16 x i8>, ptr %0, align 1 %1 = zext <16 x i8> %wide.load to <16 x i16> %2 = getelementptr inbounds i8, ptr %y, i32 %index %wide.load17 = load <16 x i8>, ptr %2, align 1 %3 = zext <16 x i8> %wide.load17 to <16 x i16> %4 = mul nuw <16 x i16> %3, %1 %5 = lshr <16 x i16> %4, %6 = trunc <16 x i16> %5 to <16 x i8> %7 = getelementptr inbounds i8, ptr %z, i32 %index store <16 x i8> %6, ptr %7, align 1 %index.next = add i32 %index, 16 %8 = icmp eq i32 %index.next, 1024 br i1 %8, label %for.cond.cleanup, label %vector.body for.cond.cleanup: ; preds = %vector.body ret void } define void @vmulh_u16(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture %z, i32 %n) { ; CHECK-LABEL: vmulh_u16: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r7, lr} ; CHECK-NEXT: push {r7, lr} ; CHECK-NEXT: mov.w lr, #128 ; CHECK-NEXT: .LBB18_1: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrh.u16 q0, [r0], #16 ; CHECK-NEXT: vldrh.u16 q1, [r1], #16 ; CHECK-NEXT: vmulh.u16 q0, q1, q0 ; CHECK-NEXT: vstrb.8 q0, [r2], #16 ; CHECK-NEXT: le lr, .LBB18_1 ; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup ; CHECK-NEXT: pop {r7, pc} entry: br label %vector.body vector.body: ; preds = %vector.body, %entry %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ] %0 = getelementptr inbounds i16, ptr %x, i32 %index %wide.load = load <8 x i16>, ptr %0, align 2 %1 = zext <8 x i16> %wide.load to <8 x i32> %2 = getelementptr inbounds i16, ptr %y, i32 %index %wide.load17 = load <8 x i16>, ptr %2, align 2 %3 = zext <8 x i16> %wide.load17 to <8 x i32> %4 = mul nuw <8 x i32> %3, %1 %5 = lshr <8 x i32> %4, %6 = trunc <8 x i32> %5 to <8 x i16> %7 = getelementptr inbounds i16, ptr %z, i32 %index store <8 x i16> %6, ptr %7, align 2 %index.next = add i32 %index, 8 %8 = icmp eq i32 %index.next, 1024 br i1 %8, label %for.cond.cleanup, label %vector.body for.cond.cleanup: ; preds = %vector.body ret void } define void @vmulh_u32(ptr nocapture readonly %x, ptr nocapture readonly %y, ptr noalias nocapture %z, i32 %n) { ; CHECK-LABEL: vmulh_u32: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r7, lr} ; CHECK-NEXT: push {r7, lr} ; CHECK-NEXT: mov.w lr, #256 ; CHECK-NEXT: .LBB19_1: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrw.u32 q0, [r0], #16 ; CHECK-NEXT: vldrw.u32 q1, [r1], #16 ; CHECK-NEXT: vmulh.u32 q0, q1, q0 ; CHECK-NEXT: vstrb.8 q0, [r2], #16 ; CHECK-NEXT: le lr, .LBB19_1 ; CHECK-NEXT: @ %bb.2: @ %for.cond.cleanup ; CHECK-NEXT: pop {r7, pc} entry: br label %vector.body vector.body: ; preds = %vector.body, %entry %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ] %0 = getelementptr inbounds i32, ptr %x, i32 %index %wide.load = load <4 x i32>, ptr %0, align 4 %1 = zext <4 x i32> %wide.load to <4 x i64> %2 = getelementptr inbounds i32, ptr %y, i32 %index %wide.load17 = load <4 x i32>, ptr %2, align 4 %3 = zext <4 x i32> %wide.load17 to <4 x i64> %4 = mul nuw <4 x i64> %3, %1 %5 = lshr <4 x i64> %4, %6 = trunc <4 x i64> %5 to <4 x i32> %7 = getelementptr inbounds i32, ptr %z, i32 %index store <4 x i32> %6, ptr %7, align 4 %index.next = add i32 %index, 4 %8 = icmp eq i32 %index.next, 1024 br i1 %8, label %for.cond.cleanup, label %vector.body for.cond.cleanup: ; preds = %vector.body ret void } define void @vmulh_s32_pred(ptr noalias nocapture %d, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y, i32 %n) { ; CHECK-LABEL: vmulh_s32_pred: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r7, lr} ; CHECK-NEXT: push {r7, lr} ; CHECK-NEXT: cmp r3, #1 ; CHECK-NEXT: it lt ; CHECK-NEXT: poplt {r7, pc} ; CHECK-NEXT: .LBB20_1: @ %vector.ph ; CHECK-NEXT: dlstp.32 lr, r3 ; CHECK-NEXT: .LBB20_2: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrw.u32 q0, [r1], #16 ; CHECK-NEXT: vldrw.u32 q1, [r2], #16 ; CHECK-NEXT: vmulh.s32 q0, q1, q0 ; CHECK-NEXT: vstrw.32 q0, [r0], #16 ; CHECK-NEXT: letp lr, .LBB20_2 ; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup ; CHECK-NEXT: pop {r7, pc} entry: %cmp10 = icmp sgt i32 %n, 0 br i1 %cmp10, label %vector.ph, label %for.cond.cleanup vector.ph: ; preds = %entry %n.rnd.up = add i32 %n, 3 %n.vec = and i32 %n.rnd.up, -4 br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %n) %0 = getelementptr inbounds i32, ptr %x, i32 %index %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %0, i32 4, <4 x i1> %active.lane.mask, <4 x i32> poison) %1 = sext <4 x i32> %wide.masked.load to <4 x i64> %2 = getelementptr inbounds i32, ptr %y, i32 %index %wide.masked.load12 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %2, i32 4, <4 x i1> %active.lane.mask, <4 x i32> poison) %3 = sext <4 x i32> %wide.masked.load12 to <4 x i64> %4 = mul nsw <4 x i64> %3, %1 %5 = lshr <4 x i64> %4, %6 = trunc <4 x i64> %5 to <4 x i32> %7 = getelementptr inbounds i32, ptr %d, i32 %index call void @llvm.masked.store.v4i32.p0(<4 x i32> %6, ptr %7, i32 4, <4 x i1> %active.lane.mask) %index.next = add i32 %index, 4 %8 = icmp eq i32 %index.next, %n.vec br i1 %8, label %for.cond.cleanup, label %vector.body for.cond.cleanup: ; preds = %vector.body, %entry ret void } define void @vmulh_u32_pred(ptr noalias nocapture %d, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y, i32 %n) { ; CHECK-LABEL: vmulh_u32_pred: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r7, lr} ; CHECK-NEXT: push {r7, lr} ; CHECK-NEXT: cmp r3, #1 ; CHECK-NEXT: it lt ; CHECK-NEXT: poplt {r7, pc} ; CHECK-NEXT: .LBB21_1: @ %vector.ph ; CHECK-NEXT: dlstp.32 lr, r3 ; CHECK-NEXT: .LBB21_2: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrw.u32 q0, [r1], #16 ; CHECK-NEXT: vldrw.u32 q1, [r2], #16 ; CHECK-NEXT: vmulh.u32 q0, q1, q0 ; CHECK-NEXT: vstrw.32 q0, [r0], #16 ; CHECK-NEXT: letp lr, .LBB21_2 ; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup ; CHECK-NEXT: pop {r7, pc} entry: %cmp10 = icmp sgt i32 %n, 0 br i1 %cmp10, label %vector.ph, label %for.cond.cleanup vector.ph: ; preds = %entry %n.rnd.up = add i32 %n, 3 %n.vec = and i32 %n.rnd.up, -4 br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %n) %0 = getelementptr inbounds i32, ptr %x, i32 %index %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %0, i32 4, <4 x i1> %active.lane.mask, <4 x i32> poison) %1 = zext <4 x i32> %wide.masked.load to <4 x i64> %2 = getelementptr inbounds i32, ptr %y, i32 %index %wide.masked.load12 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %2, i32 4, <4 x i1> %active.lane.mask, <4 x i32> poison) %3 = zext <4 x i32> %wide.masked.load12 to <4 x i64> %4 = mul nuw <4 x i64> %3, %1 %5 = lshr <4 x i64> %4, %6 = trunc <4 x i64> %5 to <4 x i32> %7 = getelementptr inbounds i32, ptr %d, i32 %index call void @llvm.masked.store.v4i32.p0(<4 x i32> %6, ptr %7, i32 4, <4 x i1> %active.lane.mask) %index.next = add i32 %index, 4 %8 = icmp eq i32 %index.next, %n.vec br i1 %8, label %for.cond.cleanup, label %vector.body for.cond.cleanup: ; preds = %vector.body, %entry ret void } define void @vmulh_s16_pred(ptr noalias nocapture %d, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y, i32 %n) { ; CHECK-LABEL: vmulh_s16_pred: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r7, lr} ; CHECK-NEXT: push {r7, lr} ; CHECK-NEXT: cmp r3, #1 ; CHECK-NEXT: it lt ; CHECK-NEXT: poplt {r7, pc} ; CHECK-NEXT: .LBB22_1: @ %vector.ph ; CHECK-NEXT: dlstp.16 lr, r3 ; CHECK-NEXT: .LBB22_2: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrh.u16 q0, [r1], #16 ; CHECK-NEXT: vldrh.u16 q1, [r2], #16 ; CHECK-NEXT: vmulh.s16 q0, q1, q0 ; CHECK-NEXT: vstrh.16 q0, [r0], #16 ; CHECK-NEXT: letp lr, .LBB22_2 ; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup ; CHECK-NEXT: pop {r7, pc} entry: %cmp10 = icmp sgt i32 %n, 0 br i1 %cmp10, label %vector.ph, label %for.cond.cleanup vector.ph: ; preds = %entry %n.rnd.up = add i32 %n, 7 %n.vec = and i32 %n.rnd.up, -8 br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %n) %0 = getelementptr inbounds i16, ptr %x, i32 %index %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %0, i32 2, <8 x i1> %active.lane.mask, <8 x i16> poison) %1 = sext <8 x i16> %wide.masked.load to <8 x i32> %2 = getelementptr inbounds i16, ptr %y, i32 %index %wide.masked.load12 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %2, i32 2, <8 x i1> %active.lane.mask, <8 x i16> poison) %3 = sext <8 x i16> %wide.masked.load12 to <8 x i32> %4 = mul nsw <8 x i32> %3, %1 %5 = lshr <8 x i32> %4, %6 = trunc <8 x i32> %5 to <8 x i16> %7 = getelementptr inbounds i16, ptr %d, i32 %index call void @llvm.masked.store.v8i16.p0(<8 x i16> %6, ptr %7, i32 2, <8 x i1> %active.lane.mask) %index.next = add i32 %index, 8 %8 = icmp eq i32 %index.next, %n.vec br i1 %8, label %for.cond.cleanup, label %vector.body for.cond.cleanup: ; preds = %vector.body, %entry ret void } define void @vmulh_u16_pred(ptr noalias nocapture %d, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y, i32 %n) { ; CHECK-LABEL: vmulh_u16_pred: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r7, lr} ; CHECK-NEXT: push {r7, lr} ; CHECK-NEXT: cmp r3, #1 ; CHECK-NEXT: it lt ; CHECK-NEXT: poplt {r7, pc} ; CHECK-NEXT: .LBB23_1: @ %vector.ph ; CHECK-NEXT: dlstp.16 lr, r3 ; CHECK-NEXT: .LBB23_2: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrh.u16 q0, [r1], #16 ; CHECK-NEXT: vldrh.u16 q1, [r2], #16 ; CHECK-NEXT: vmulh.u16 q0, q1, q0 ; CHECK-NEXT: vstrh.16 q0, [r0], #16 ; CHECK-NEXT: letp lr, .LBB23_2 ; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup ; CHECK-NEXT: pop {r7, pc} entry: %cmp10 = icmp sgt i32 %n, 0 br i1 %cmp10, label %vector.ph, label %for.cond.cleanup vector.ph: ; preds = %entry %n.rnd.up = add i32 %n, 7 %n.vec = and i32 %n.rnd.up, -8 br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 %index, i32 %n) %0 = getelementptr inbounds i16, ptr %x, i32 %index %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %0, i32 2, <8 x i1> %active.lane.mask, <8 x i16> poison) %1 = zext <8 x i16> %wide.masked.load to <8 x i32> %2 = getelementptr inbounds i16, ptr %y, i32 %index %wide.masked.load12 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %2, i32 2, <8 x i1> %active.lane.mask, <8 x i16> poison) %3 = zext <8 x i16> %wide.masked.load12 to <8 x i32> %4 = mul nuw <8 x i32> %3, %1 %5 = lshr <8 x i32> %4, %6 = trunc <8 x i32> %5 to <8 x i16> %7 = getelementptr inbounds i16, ptr %d, i32 %index call void @llvm.masked.store.v8i16.p0(<8 x i16> %6, ptr %7, i32 2, <8 x i1> %active.lane.mask) %index.next = add i32 %index, 8 %8 = icmp eq i32 %index.next, %n.vec br i1 %8, label %for.cond.cleanup, label %vector.body for.cond.cleanup: ; preds = %vector.body, %entry ret void } define void @vmulh_s8_pred(ptr noalias nocapture %d, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y, i32 %n) { ; CHECK-LABEL: vmulh_s8_pred: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r7, lr} ; CHECK-NEXT: push {r7, lr} ; CHECK-NEXT: cmp r3, #1 ; CHECK-NEXT: it lt ; CHECK-NEXT: poplt {r7, pc} ; CHECK-NEXT: .LBB24_1: @ %vector.ph ; CHECK-NEXT: dlstp.8 lr, r3 ; CHECK-NEXT: .LBB24_2: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrb.u8 q0, [r1], #16 ; CHECK-NEXT: vldrb.u8 q1, [r2], #16 ; CHECK-NEXT: vmulh.s8 q0, q1, q0 ; CHECK-NEXT: vstrb.8 q0, [r0], #16 ; CHECK-NEXT: letp lr, .LBB24_2 ; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup ; CHECK-NEXT: pop {r7, pc} entry: %cmp10 = icmp sgt i32 %n, 0 br i1 %cmp10, label %vector.ph, label %for.cond.cleanup vector.ph: ; preds = %entry %n.rnd.up = add i32 %n, 15 %n.vec = and i32 %n.rnd.up, -16 br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %n) %0 = getelementptr inbounds i8, ptr %x, i32 %index %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %0, i32 1, <16 x i1> %active.lane.mask, <16 x i8> poison) %1 = sext <16 x i8> %wide.masked.load to <16 x i16> %2 = getelementptr inbounds i8, ptr %y, i32 %index %wide.masked.load12 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %2, i32 1, <16 x i1> %active.lane.mask, <16 x i8> poison) %3 = sext <16 x i8> %wide.masked.load12 to <16 x i16> %4 = mul nsw <16 x i16> %3, %1 %5 = lshr <16 x i16> %4, %6 = trunc <16 x i16> %5 to <16 x i8> %7 = getelementptr inbounds i8, ptr %d, i32 %index call void @llvm.masked.store.v16i8.p0(<16 x i8> %6, ptr %7, i32 1, <16 x i1> %active.lane.mask) %index.next = add i32 %index, 16 %8 = icmp eq i32 %index.next, %n.vec br i1 %8, label %for.cond.cleanup, label %vector.body for.cond.cleanup: ; preds = %vector.body, %entry ret void } define void @vmulh_u8_pred(ptr noalias nocapture %d, ptr noalias nocapture readonly %x, ptr noalias nocapture readonly %y, i32 %n) { ; CHECK-LABEL: vmulh_u8_pred: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r7, lr} ; CHECK-NEXT: push {r7, lr} ; CHECK-NEXT: cmp r3, #1 ; CHECK-NEXT: it lt ; CHECK-NEXT: poplt {r7, pc} ; CHECK-NEXT: .LBB25_1: @ %vector.ph ; CHECK-NEXT: dlstp.8 lr, r3 ; CHECK-NEXT: .LBB25_2: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrb.u8 q0, [r1], #16 ; CHECK-NEXT: vldrb.u8 q1, [r2], #16 ; CHECK-NEXT: vmulh.u8 q0, q1, q0 ; CHECK-NEXT: vstrb.8 q0, [r0], #16 ; CHECK-NEXT: letp lr, .LBB25_2 ; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup ; CHECK-NEXT: pop {r7, pc} entry: %cmp10 = icmp sgt i32 %n, 0 br i1 %cmp10, label %vector.ph, label %for.cond.cleanup vector.ph: ; preds = %entry %n.rnd.up = add i32 %n, 15 %n.vec = and i32 %n.rnd.up, -16 br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 %index, i32 %n) %0 = getelementptr inbounds i8, ptr %x, i32 %index %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %0, i32 1, <16 x i1> %active.lane.mask, <16 x i8> poison) %1 = zext <16 x i8> %wide.masked.load to <16 x i16> %2 = getelementptr inbounds i8, ptr %y, i32 %index %wide.masked.load12 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %2, i32 1, <16 x i1> %active.lane.mask, <16 x i8> poison) %3 = zext <16 x i8> %wide.masked.load12 to <16 x i16> %4 = mul nuw <16 x i16> %3, %1 %5 = lshr <16 x i16> %4, %6 = trunc <16 x i16> %5 to <16 x i8> %7 = getelementptr inbounds i8, ptr %d, i32 %index call void @llvm.masked.store.v16i8.p0(<16 x i8> %6, ptr %7, i32 1, <16 x i1> %active.lane.mask) %index.next = add i32 %index, 16 %8 = icmp eq i32 %index.next, %n.vec br i1 %8, label %for.cond.cleanup, label %vector.body for.cond.cleanup: ; preds = %vector.body, %entry ret void } define arm_aapcs_vfpcc i16 @vmulhs_reduce_v16i8(<16 x i8> %s0, <16 x i8> %s1) { ; CHECK-LABEL: vmulhs_reduce_v16i8: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: vmulh.s8 q0, q0, q1 ; CHECK-NEXT: vaddv.s8 r0, q0 ; CHECK-NEXT: bx lr entry: %s0s = sext <16 x i8> %s0 to <16 x i16> %s1s = sext <16 x i8> %s1 to <16 x i16> %m = mul <16 x i16> %s0s, %s1s %s = ashr <16 x i16> %m, %result = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %s) ret i16 %result } define arm_aapcs_vfpcc i16 @vmulhu_reduce_v16i8(<16 x i8> %s0, <16 x i8> %s1) { ; CHECK-LABEL: vmulhu_reduce_v16i8: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: vmulh.u8 q0, q0, q1 ; CHECK-NEXT: vaddv.s8 r0, q0 ; CHECK-NEXT: bx lr entry: %s0s = zext <16 x i8> %s0 to <16 x i16> %s1s = zext <16 x i8> %s1 to <16 x i16> %m = mul <16 x i16> %s0s, %s1s %s = ashr <16 x i16> %m, %result = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %s) ret i16 %result } declare i16 @llvm.vector.reduce.add.v16i16(<16 x i16>) declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32) declare <4 x i32> @llvm.masked.load.v4i32.p0(ptr, i32 immarg, <4 x i1>, <4 x i32>) declare void @llvm.masked.store.v4i32.p0(<4 x i32>, ptr, i32 immarg, <4 x i1>) declare <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32, i32) declare <8 x i16> @llvm.masked.load.v8i16.p0(ptr, i32 immarg, <8 x i1>, <8 x i16>) declare void @llvm.masked.store.v8i16.p0(<8 x i16>, ptr, i32 immarg, <8 x i1>) declare <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32, i32) declare <16 x i8> @llvm.masked.load.v16i8.p0(ptr, i32 immarg, <16 x i1>, <16 x i8>) declare void @llvm.masked.store.v16i8.p0(<16 x i8>, ptr, i32 immarg, <16 x i1>)