; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s define arm_aapcs_vfpcc void @ssatmul_s_q31(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: ssatmul_s_q31: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr} ; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr} ; CHECK-NEXT: .pad #12 ; CHECK-NEXT: sub sp, #12 ; CHECK-NEXT: cmp r3, #0 ; CHECK-NEXT: beq.w .LBB0_8 ; CHECK-NEXT: @ %bb.1: @ %entry ; CHECK-NEXT: cmp r3, #1 ; CHECK-NEXT: bne .LBB0_3 ; CHECK-NEXT: @ %bb.2: ; CHECK-NEXT: movs r7, #0 ; CHECK-NEXT: mov r12, r0 ; CHECK-NEXT: mov r8, r1 ; CHECK-NEXT: mov r10, r2 ; CHECK-NEXT: b .LBB0_6 ; CHECK-NEXT: .LBB0_3: @ %vector.ph ; CHECK-NEXT: str r3, [sp, #4] @ 4-byte Spill ; CHECK-NEXT: bic r3, r3, #1 ; CHECK-NEXT: subs r7, r3, #2 ; CHECK-NEXT: movs r6, #1 ; CHECK-NEXT: adr r4, .LCPI0_0 ; CHECK-NEXT: str r3, [sp] @ 4-byte Spill ; CHECK-NEXT: add.w lr, r6, r7, lsr #1 ; CHECK-NEXT: add.w r10, r2, r3, lsl #2 ; CHECK-NEXT: add.w r8, r1, r3, lsl #2 ; CHECK-NEXT: add.w r12, r0, r3, lsl #2 ; CHECK-NEXT: vldrw.u32 q0, [r4] ; CHECK-NEXT: vmvn.i32 q1, #0x80000000 ; CHECK-NEXT: .LBB0_4: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldrd r4, r3, [r0], #8 ; CHECK-NEXT: movs r5, #0 ; CHECK-NEXT: ldrd r7, r6, [r1], #8 ; CHECK-NEXT: str r3, [sp, #8] @ 4-byte Spill ; CHECK-NEXT: smull r4, r11, r7, r4 ; CHECK-NEXT: asrl r4, r11, #31 ; CHECK-NEXT: rsbs.w r9, r4, #-2147483648 ; CHECK-NEXT: mov.w r9, #-1 ; CHECK-NEXT: sbcs.w r3, r9, r11 ; CHECK-NEXT: csetm r3, lt ; CHECK-NEXT: bfi r5, r3, #0, #8 ; CHECK-NEXT: ldr r3, [sp, #8] @ 4-byte Reload ; CHECK-NEXT: smull r6, r3, r6, r3 ; CHECK-NEXT: asrl r6, r3, #31 ; CHECK-NEXT: rsbs.w r7, r6, #-2147483648 ; CHECK-NEXT: vmov q2[2], q2[0], r4, r6 ; CHECK-NEXT: sbcs.w r7, r9, r3 ; CHECK-NEXT: vmov q2[3], q2[1], r11, r3 ; CHECK-NEXT: csetm r7, lt ; CHECK-NEXT: mvn r6, #-2147483648 ; CHECK-NEXT: bfi r5, r7, #8, #8 ; CHECK-NEXT: vmsr p0, r5 ; CHECK-NEXT: vpsel q2, q2, q0 ; CHECK-NEXT: vmov r3, r4, d4 ; CHECK-NEXT: subs r3, r3, r6 ; CHECK-NEXT: sbcs r3, r4, #0 ; CHECK-NEXT: mov.w r4, #0 ; CHECK-NEXT: csetm r3, lt ; CHECK-NEXT: bfi r4, r3, #0, #8 ; CHECK-NEXT: vmov r3, r5, d5 ; CHECK-NEXT: subs r3, r3, r6 ; CHECK-NEXT: sbcs r3, r5, #0 ; CHECK-NEXT: csetm r3, lt ; CHECK-NEXT: bfi r4, r3, #8, #8 ; CHECK-NEXT: vmsr p0, r4 ; CHECK-NEXT: vpsel q2, q2, q1 ; CHECK-NEXT: vmov r3, s10 ; CHECK-NEXT: vmov r4, s8 ; CHECK-NEXT: strd r4, r3, [r2], #8 ; CHECK-NEXT: le lr, .LBB0_4 ; CHECK-NEXT: @ %bb.5: @ %middle.block ; CHECK-NEXT: ldrd r7, r3, [sp] @ 8-byte Folded Reload ; CHECK-NEXT: cmp r7, r3 ; CHECK-NEXT: beq .LBB0_8 ; CHECK-NEXT: .LBB0_6: @ %for.body.preheader ; CHECK-NEXT: sub.w lr, r3, r7 ; CHECK-NEXT: mov.w r0, #-1 ; CHECK-NEXT: mov.w r1, #-2147483648 ; CHECK-NEXT: mvn r2, #-2147483648 ; CHECK-NEXT: .LBB0_7: @ %for.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldr r3, [r12], #4 ; CHECK-NEXT: ldr r4, [r8], #4 ; CHECK-NEXT: smull r4, r3, r4, r3 ; CHECK-NEXT: asrl r4, r3, #31 ; CHECK-NEXT: subs r5, r1, r4 ; CHECK-NEXT: sbcs.w r5, r0, r3 ; CHECK-NEXT: cset r5, lt ; CHECK-NEXT: cmp r5, #0 ; CHECK-NEXT: csel r4, r4, r1, ne ; CHECK-NEXT: csel r3, r3, r0, ne ; CHECK-NEXT: subs r5, r4, r2 ; CHECK-NEXT: sbcs r3, r3, #0 ; CHECK-NEXT: csel r3, r4, r2, lt ; CHECK-NEXT: str r3, [r10], #4 ; CHECK-NEXT: le lr, .LBB0_7 ; CHECK-NEXT: .LBB0_8: @ %for.cond.cleanup ; CHECK-NEXT: add sp, #12 ; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc} ; CHECK-NEXT: .p2align 4 ; CHECK-NEXT: @ %bb.9: ; CHECK-NEXT: .LCPI0_0: ; CHECK-NEXT: .long 2147483648 @ 0x80000000 ; CHECK-NEXT: .long 4294967295 @ 0xffffffff ; CHECK-NEXT: .long 2147483648 @ 0x80000000 ; CHECK-NEXT: .long 4294967295 @ 0xffffffff entry: switch i32 %N, label %vector.ph [ i32 0, label %for.cond.cleanup i32 1, label %for.body.preheader ] vector.ph: ; preds = %entry %n.vec = and i32 %N, -2 %ind.end = getelementptr i32, ptr %pSrcA, i32 %n.vec %ind.end15 = getelementptr i32, ptr %pSrcB, i32 %n.vec %ind.end17 = getelementptr i32, ptr %pDst, i32 %n.vec br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %next.gep = getelementptr i32, ptr %pSrcA, i32 %index %next.gep18 = getelementptr i32, ptr %pSrcB, i32 %index %next.gep19 = getelementptr i32, ptr %pDst, i32 %index %wide.load = load <2 x i32>, ptr %next.gep, align 4 %0 = sext <2 x i32> %wide.load to <2 x i64> %wide.load20 = load <2 x i32>, ptr %next.gep18, align 4 %1 = sext <2 x i32> %wide.load20 to <2 x i64> %2 = mul nsw <2 x i64> %1, %0 %3 = ashr <2 x i64> %2, %4 = icmp sgt <2 x i64> %3, %5 = select <2 x i1> %4, <2 x i64> %3, <2 x i64> %6 = icmp slt <2 x i64> %5, %7 = select <2 x i1> %6, <2 x i64> %5, <2 x i64> %8 = trunc <2 x i64> %7 to <2 x i32> store <2 x i32> %8, ptr %next.gep19, align 4 %index.next = add i32 %index, 2 %9 = icmp eq i32 %index.next, %n.vec br i1 %9, label %middle.block, label %vector.body middle.block: ; preds = %vector.body %cmp.n = icmp eq i32 %n.vec, %N br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader for.body.preheader: ; preds = %entry, %middle.block %i.012.ph = phi i32 [ 0, %entry ], [ %n.vec, %middle.block ] %pSrcA.addr.011.ph = phi ptr [ %pSrcA, %entry ], [ %ind.end, %middle.block ] %pSrcB.addr.010.ph = phi ptr [ %pSrcB, %entry ], [ %ind.end15, %middle.block ] %pDst.addr.09.ph = phi ptr [ %pDst, %entry ], [ %ind.end17, %middle.block ] br label %for.body for.cond.cleanup: ; preds = %for.body, %middle.block, %entry ret void for.body: ; preds = %for.body.preheader, %for.body %i.012 = phi i32 [ %inc, %for.body ], [ %i.012.ph, %for.body.preheader ] %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.011.ph, %for.body.preheader ] %pSrcB.addr.010 = phi ptr [ %incdec.ptr1, %for.body ], [ %pSrcB.addr.010.ph, %for.body.preheader ] %pDst.addr.09 = phi ptr [ %incdec.ptr4, %for.body ], [ %pDst.addr.09.ph, %for.body.preheader ] %incdec.ptr = getelementptr inbounds i32, ptr %pSrcA.addr.011, i32 1 %10 = load i32, ptr %pSrcA.addr.011, align 4 %conv = sext i32 %10 to i64 %incdec.ptr1 = getelementptr inbounds i32, ptr %pSrcB.addr.010, i32 1 %11 = load i32, ptr %pSrcB.addr.010, align 4 %conv2 = sext i32 %11 to i64 %mul = mul nsw i64 %conv2, %conv %shr = ashr i64 %mul, 31 %12 = icmp sgt i64 %shr, -2147483648 %.val.i = select i1 %12, i64 %shr, i64 -2147483648 %13 = icmp slt i64 %.val.i, 2147483647 %retval.0.i = select i1 %13, i64 %.val.i, i64 2147483647 %conv3 = trunc i64 %retval.0.i to i32 %incdec.ptr4 = getelementptr inbounds i32, ptr %pDst.addr.09, i32 1 store i32 %conv3, ptr %pDst.addr.09, align 4 %inc = add nuw i32 %i.012, 1 %exitcond = icmp eq i32 %inc, %N br i1 %exitcond, label %for.cond.cleanup, label %for.body } define arm_aapcs_vfpcc void @ssatmul_4_q31(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: ssatmul_4_q31: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr} ; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr} ; CHECK-NEXT: .pad #4 ; CHECK-NEXT: sub sp, #4 ; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13} ; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13} ; CHECK-NEXT: .pad #16 ; CHECK-NEXT: sub sp, #16 ; CHECK-NEXT: cmp r3, #0 ; CHECK-NEXT: beq.w .LBB1_8 ; CHECK-NEXT: @ %bb.1: @ %for.body.preheader ; CHECK-NEXT: mov r5, r1 ; CHECK-NEXT: movs r1, #0 ; CHECK-NEXT: cmp r3, #3 ; CHECK-NEXT: bhi .LBB1_3 ; CHECK-NEXT: @ %bb.2: ; CHECK-NEXT: mov r12, r0 ; CHECK-NEXT: mov r9, r5 ; CHECK-NEXT: mov r11, r2 ; CHECK-NEXT: b .LBB1_6 ; CHECK-NEXT: .LBB1_3: @ %vector.ph ; CHECK-NEXT: bic r1, r3, #3 ; CHECK-NEXT: adr r4, .LCPI1_0 ; CHECK-NEXT: subs r7, r1, #4 ; CHECK-NEXT: movs r6, #1 ; CHECK-NEXT: vldrw.u32 q0, [r4] ; CHECK-NEXT: adr r4, .LCPI1_1 ; CHECK-NEXT: str r3, [sp, #4] @ 4-byte Spill ; CHECK-NEXT: add.w lr, r6, r7, lsr #2 ; CHECK-NEXT: str r1, [sp] @ 4-byte Spill ; CHECK-NEXT: add.w r11, r2, r1, lsl #2 ; CHECK-NEXT: add.w r9, r5, r1, lsl #2 ; CHECK-NEXT: add.w r12, r0, r1, lsl #2 ; CHECK-NEXT: vldrw.u32 q1, [r4] ; CHECK-NEXT: .LBB1_4: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrw.u32 q4, [r5], #16 ; CHECK-NEXT: vldrw.u32 q3, [r0], #16 ; CHECK-NEXT: str r2, [sp, #12] @ 4-byte Spill ; CHECK-NEXT: mov.w r2, #-1 ; CHECK-NEXT: vmov.f32 s8, s14 ; CHECK-NEXT: str r5, [sp, #8] @ 4-byte Spill ; CHECK-NEXT: vmov.f32 s20, s18 ; CHECK-NEXT: mov.w r8, #0 ; CHECK-NEXT: vmov.f32 s10, s15 ; CHECK-NEXT: vmov.f32 s22, s19 ; CHECK-NEXT: vmullb.s32 q6, q5, q2 ; CHECK-NEXT: vmov.f32 s18, s17 ; CHECK-NEXT: vmov r4, r7, d12 ; CHECK-NEXT: asrl r4, r7, #31 ; CHECK-NEXT: vmov.f32 s14, s13 ; CHECK-NEXT: rsbs.w r5, r4, #-2147483648 ; CHECK-NEXT: sbcs.w r5, r2, r7 ; CHECK-NEXT: csetm r5, lt ; CHECK-NEXT: bfi r8, r5, #0, #8 ; CHECK-NEXT: vmov r10, r5, d13 ; CHECK-NEXT: asrl r10, r5, #31 ; CHECK-NEXT: vmov r6, s18 ; CHECK-NEXT: rsbs.w r3, r10, #-2147483648 ; CHECK-NEXT: vmov q2[2], q2[0], r4, r10 ; CHECK-NEXT: sbcs.w r3, r2, r5 ; CHECK-NEXT: vmov q2[3], q2[1], r7, r5 ; CHECK-NEXT: csetm r3, lt ; CHECK-NEXT: bfi r8, r3, #8, #8 ; CHECK-NEXT: vmsr p0, r8 ; CHECK-NEXT: mvn r8, #-2147483648 ; CHECK-NEXT: vpsel q2, q2, q0 ; CHECK-NEXT: vmov r3, r4, d4 ; CHECK-NEXT: subs.w r3, r3, r8 ; CHECK-NEXT: sbcs r3, r4, #0 ; CHECK-NEXT: mov.w r4, #0 ; CHECK-NEXT: csetm r3, lt ; CHECK-NEXT: bfi r4, r3, #0, #8 ; CHECK-NEXT: vmov r3, r5, d5 ; CHECK-NEXT: subs.w r3, r3, r8 ; CHECK-NEXT: sbcs r3, r5, #0 ; CHECK-NEXT: mov.w r5, #0 ; CHECK-NEXT: csetm r3, lt ; CHECK-NEXT: bfi r4, r3, #8, #8 ; CHECK-NEXT: vmov r3, s12 ; CHECK-NEXT: vmsr p0, r4 ; CHECK-NEXT: vmov r4, s16 ; CHECK-NEXT: vpsel q2, q2, q1 ; CHECK-NEXT: smull r4, r7, r4, r3 ; CHECK-NEXT: asrl r4, r7, #31 ; CHECK-NEXT: rsbs.w r3, r4, #-2147483648 ; CHECK-NEXT: sbcs.w r3, r2, r7 ; CHECK-NEXT: csetm r3, lt ; CHECK-NEXT: bfi r5, r3, #0, #8 ; CHECK-NEXT: vmov r3, s14 ; CHECK-NEXT: smull r6, r3, r6, r3 ; CHECK-NEXT: asrl r6, r3, #31 ; CHECK-NEXT: rsbs.w r1, r6, #-2147483648 ; CHECK-NEXT: vmov q3[2], q3[0], r4, r6 ; CHECK-NEXT: sbcs.w r1, r2, r3 ; CHECK-NEXT: vmov q3[3], q3[1], r7, r3 ; CHECK-NEXT: csetm r1, lt ; CHECK-NEXT: bfi r5, r1, #8, #8 ; CHECK-NEXT: vmsr p0, r5 ; CHECK-NEXT: ldrd r5, r2, [sp, #8] @ 8-byte Folded Reload ; CHECK-NEXT: vpsel q3, q3, q0 ; CHECK-NEXT: vmov r1, r3, d6 ; CHECK-NEXT: subs.w r1, r1, r8 ; CHECK-NEXT: sbcs r1, r3, #0 ; CHECK-NEXT: mov.w r3, #0 ; CHECK-NEXT: csetm r1, lt ; CHECK-NEXT: bfi r3, r1, #0, #8 ; CHECK-NEXT: vmov r1, r4, d7 ; CHECK-NEXT: subs.w r1, r1, r8 ; CHECK-NEXT: sbcs r1, r4, #0 ; CHECK-NEXT: csetm r1, lt ; CHECK-NEXT: bfi r3, r1, #8, #8 ; CHECK-NEXT: vmsr p0, r3 ; CHECK-NEXT: vpsel q3, q3, q1 ; CHECK-NEXT: vmov.f32 s13, s14 ; CHECK-NEXT: vmov.f32 s14, s8 ; CHECK-NEXT: vmov.f32 s15, s10 ; CHECK-NEXT: vstrb.8 q3, [r2], #16 ; CHECK-NEXT: le lr, .LBB1_4 ; CHECK-NEXT: @ %bb.5: @ %middle.block ; CHECK-NEXT: ldrd r1, r3, [sp] @ 8-byte Folded Reload ; CHECK-NEXT: cmp r1, r3 ; CHECK-NEXT: beq .LBB1_8 ; CHECK-NEXT: .LBB1_6: @ %for.body.preheader21 ; CHECK-NEXT: sub.w lr, r3, r1 ; CHECK-NEXT: mov.w r0, #-1 ; CHECK-NEXT: mov.w r3, #-2147483648 ; CHECK-NEXT: mvn r2, #-2147483648 ; CHECK-NEXT: .LBB1_7: @ %for.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldr r1, [r12], #4 ; CHECK-NEXT: ldr r4, [r9], #4 ; CHECK-NEXT: smull r4, r1, r4, r1 ; CHECK-NEXT: asrl r4, r1, #31 ; CHECK-NEXT: subs r5, r3, r4 ; CHECK-NEXT: sbcs.w r5, r0, r1 ; CHECK-NEXT: cset r5, lt ; CHECK-NEXT: cmp r5, #0 ; CHECK-NEXT: csel r4, r4, r3, ne ; CHECK-NEXT: csel r1, r1, r0, ne ; CHECK-NEXT: subs r5, r4, r2 ; CHECK-NEXT: sbcs r1, r1, #0 ; CHECK-NEXT: csel r1, r4, r2, lt ; CHECK-NEXT: str r1, [r11], #4 ; CHECK-NEXT: le lr, .LBB1_7 ; CHECK-NEXT: .LBB1_8: @ %for.cond.cleanup ; CHECK-NEXT: add sp, #16 ; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13} ; CHECK-NEXT: add sp, #4 ; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc} ; CHECK-NEXT: .p2align 4 ; CHECK-NEXT: @ %bb.9: ; CHECK-NEXT: .LCPI1_0: ; CHECK-NEXT: .long 2147483648 @ 0x80000000 ; CHECK-NEXT: .long 4294967295 @ 0xffffffff ; CHECK-NEXT: .long 2147483648 @ 0x80000000 ; CHECK-NEXT: .long 4294967295 @ 0xffffffff ; CHECK-NEXT: .LCPI1_1: ; CHECK-NEXT: .long 2147483647 @ 0x7fffffff ; CHECK-NEXT: .long 0 @ 0x0 ; CHECK-NEXT: .long 2147483647 @ 0x7fffffff ; CHECK-NEXT: .long 0 @ 0x0 entry: %cmp8 = icmp eq i32 %N, 0 br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader for.body.preheader: ; preds = %entry %min.iters.check = icmp ult i32 %N, 4 br i1 %min.iters.check, label %for.body.preheader21, label %vector.ph for.body.preheader21: ; preds = %middle.block, %for.body.preheader %i.012.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ] %pSrcA.addr.011.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ] %pSrcB.addr.010.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end15, %middle.block ] %pDst.addr.09.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end17, %middle.block ] br label %for.body vector.ph: ; preds = %for.body.preheader %n.vec = and i32 %N, -4 %ind.end = getelementptr i32, ptr %pSrcA, i32 %n.vec %ind.end15 = getelementptr i32, ptr %pSrcB, i32 %n.vec %ind.end17 = getelementptr i32, ptr %pDst, i32 %n.vec br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %next.gep = getelementptr i32, ptr %pSrcA, i32 %index %next.gep18 = getelementptr i32, ptr %pSrcB, i32 %index %next.gep19 = getelementptr i32, ptr %pDst, i32 %index %wide.load = load <4 x i32>, ptr %next.gep, align 4 %0 = sext <4 x i32> %wide.load to <4 x i64> %wide.load20 = load <4 x i32>, ptr %next.gep18, align 4 %1 = sext <4 x i32> %wide.load20 to <4 x i64> %2 = mul nsw <4 x i64> %1, %0 %3 = ashr <4 x i64> %2, %4 = icmp sgt <4 x i64> %3, %5 = select <4 x i1> %4, <4 x i64> %3, <4 x i64> %6 = icmp slt <4 x i64> %5, %7 = select <4 x i1> %6, <4 x i64> %5, <4 x i64> %8 = trunc <4 x i64> %7 to <4 x i32> store <4 x i32> %8, ptr %next.gep19, align 4 %index.next = add i32 %index, 4 %9 = icmp eq i32 %index.next, %n.vec br i1 %9, label %middle.block, label %vector.body middle.block: ; preds = %vector.body %cmp.n = icmp eq i32 %n.vec, %N br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader21 for.cond.cleanup: ; preds = %for.body, %middle.block, %entry ret void for.body: ; preds = %for.body.preheader21, %for.body %i.012 = phi i32 [ %inc, %for.body ], [ %i.012.ph, %for.body.preheader21 ] %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.011.ph, %for.body.preheader21 ] %pSrcB.addr.010 = phi ptr [ %incdec.ptr1, %for.body ], [ %pSrcB.addr.010.ph, %for.body.preheader21 ] %pDst.addr.09 = phi ptr [ %incdec.ptr4, %for.body ], [ %pDst.addr.09.ph, %for.body.preheader21 ] %incdec.ptr = getelementptr inbounds i32, ptr %pSrcA.addr.011, i32 1 %10 = load i32, ptr %pSrcA.addr.011, align 4 %conv = sext i32 %10 to i64 %incdec.ptr1 = getelementptr inbounds i32, ptr %pSrcB.addr.010, i32 1 %11 = load i32, ptr %pSrcB.addr.010, align 4 %conv2 = sext i32 %11 to i64 %mul = mul nsw i64 %conv2, %conv %shr = ashr i64 %mul, 31 %12 = icmp sgt i64 %shr, -2147483648 %.val.i = select i1 %12, i64 %shr, i64 -2147483648 %13 = icmp slt i64 %.val.i, 2147483647 %retval.0.i = select i1 %13, i64 %.val.i, i64 2147483647 %conv3 = trunc i64 %retval.0.i to i32 %incdec.ptr4 = getelementptr inbounds i32, ptr %pDst.addr.09, i32 1 store i32 %conv3, ptr %pDst.addr.09, align 4 %inc = add nuw i32 %i.012, 1 %exitcond = icmp eq i32 %inc, %N br i1 %exitcond, label %for.cond.cleanup, label %for.body } define arm_aapcs_vfpcc void @ssatmul_4t_q31(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: ssatmul_4t_q31: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, lr} ; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, lr} ; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15} ; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15} ; CHECK-NEXT: .pad #24 ; CHECK-NEXT: sub sp, #24 ; CHECK-NEXT: cmp r3, #0 ; CHECK-NEXT: beq.w .LBB2_3 ; CHECK-NEXT: @ %bb.1: @ %vector.ph ; CHECK-NEXT: adds r6, r3, #3 ; CHECK-NEXT: movs r5, #1 ; CHECK-NEXT: bic r6, r6, #3 ; CHECK-NEXT: adr r4, .LCPI2_1 ; CHECK-NEXT: subs r6, #4 ; CHECK-NEXT: vldrw.u32 q2, [r4] ; CHECK-NEXT: mov.w r9, #0 ; CHECK-NEXT: mov.w r12, #-1 ; CHECK-NEXT: add.w lr, r5, r6, lsr #2 ; CHECK-NEXT: adr r5, .LCPI2_0 ; CHECK-NEXT: vldrw.u32 q0, [r5] ; CHECK-NEXT: adr r5, .LCPI2_2 ; CHECK-NEXT: subs r6, r3, #1 ; CHECK-NEXT: vldrw.u32 q3, [r5] ; CHECK-NEXT: vdup.32 q1, r6 ; CHECK-NEXT: mvn r8, #-2147483648 ; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill ; CHECK-NEXT: .LBB2_2: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrw.u32 q0, [sp] @ 16-byte Reload ; CHECK-NEXT: vdup.32 q4, r9 ; CHECK-NEXT: movs r4, #0 ; CHECK-NEXT: add.w r9, r9, #4 ; CHECK-NEXT: vorr q4, q4, q0 ; CHECK-NEXT: vcmp.u32 cs, q1, q4 ; CHECK-NEXT: vstr p0, [sp, #20] @ 4-byte Spill ; CHECK-NEXT: vpstt ; CHECK-NEXT: vldrwt.u32 q5, [r0], #16 ; CHECK-NEXT: vldrwt.u32 q6, [r1], #16 ; CHECK-NEXT: vmov.f32 s16, s22 ; CHECK-NEXT: vmov.f32 s18, s23 ; CHECK-NEXT: vmov.f32 s28, s26 ; CHECK-NEXT: vmov.f32 s30, s27 ; CHECK-NEXT: vmullb.s32 q0, q7, q4 ; CHECK-NEXT: vmov.f32 s22, s25 ; CHECK-NEXT: vmov r10, r5, d0 ; CHECK-NEXT: asrl r10, r5, #31 ; CHECK-NEXT: rsbs.w r7, r10, #-2147483648 ; CHECK-NEXT: sbcs.w r7, r12, r5 ; CHECK-NEXT: csetm r7, lt ; CHECK-NEXT: bfi r4, r7, #0, #8 ; CHECK-NEXT: vmov r6, r7, d1 ; CHECK-NEXT: asrl r6, r7, #31 ; CHECK-NEXT: rsbs.w r3, r6, #-2147483648 ; CHECK-NEXT: vmov q0[2], q0[0], r10, r6 ; CHECK-NEXT: sbcs.w r3, r12, r7 ; CHECK-NEXT: vmov q0[3], q0[1], r5, r7 ; CHECK-NEXT: csetm r3, lt ; CHECK-NEXT: vmov r7, s22 ; CHECK-NEXT: bfi r4, r3, #8, #8 ; CHECK-NEXT: vmsr p0, r4 ; CHECK-NEXT: vpsel q0, q0, q2 ; CHECK-NEXT: vmov r3, r4, d0 ; CHECK-NEXT: subs.w r3, r3, r8 ; CHECK-NEXT: sbcs r3, r4, #0 ; CHECK-NEXT: mov.w r4, #0 ; CHECK-NEXT: csetm r3, lt ; CHECK-NEXT: bfi r4, r3, #0, #8 ; CHECK-NEXT: vmov r3, r5, d1 ; CHECK-NEXT: subs.w r3, r3, r8 ; CHECK-NEXT: sbcs r3, r5, #0 ; CHECK-NEXT: csetm r3, lt ; CHECK-NEXT: bfi r4, r3, #8, #8 ; CHECK-NEXT: vmov r3, s20 ; CHECK-NEXT: vmsr p0, r4 ; CHECK-NEXT: vmov r4, s24 ; CHECK-NEXT: vpsel q4, q0, q3 ; CHECK-NEXT: vmov.f32 s2, s21 ; CHECK-NEXT: smull r10, r5, r4, r3 ; CHECK-NEXT: movs r4, #0 ; CHECK-NEXT: asrl r10, r5, #31 ; CHECK-NEXT: rsbs.w r3, r10, #-2147483648 ; CHECK-NEXT: sbcs.w r3, r12, r5 ; CHECK-NEXT: csetm r3, lt ; CHECK-NEXT: bfi r4, r3, #0, #8 ; CHECK-NEXT: vmov r3, s2 ; CHECK-NEXT: smull r6, r3, r7, r3 ; CHECK-NEXT: asrl r6, r3, #31 ; CHECK-NEXT: rsbs.w r7, r6, #-2147483648 ; CHECK-NEXT: vmov q0[2], q0[0], r10, r6 ; CHECK-NEXT: sbcs.w r7, r12, r3 ; CHECK-NEXT: vmov q0[3], q0[1], r5, r3 ; CHECK-NEXT: csetm r7, lt ; CHECK-NEXT: bfi r4, r7, #8, #8 ; CHECK-NEXT: vmsr p0, r4 ; CHECK-NEXT: vpsel q0, q0, q2 ; CHECK-NEXT: vmov r3, r4, d0 ; CHECK-NEXT: subs.w r3, r3, r8 ; CHECK-NEXT: sbcs r3, r4, #0 ; CHECK-NEXT: mov.w r4, #0 ; CHECK-NEXT: csetm r3, lt ; CHECK-NEXT: bfi r4, r3, #0, #8 ; CHECK-NEXT: vmov r3, r5, d1 ; CHECK-NEXT: subs.w r3, r3, r8 ; CHECK-NEXT: sbcs r3, r5, #0 ; CHECK-NEXT: csetm r3, lt ; CHECK-NEXT: bfi r4, r3, #8, #8 ; CHECK-NEXT: vmsr p0, r4 ; CHECK-NEXT: vpsel q0, q0, q3 ; CHECK-NEXT: vldr p0, [sp, #20] @ 4-byte Reload ; CHECK-NEXT: vmov.f32 s1, s2 ; CHECK-NEXT: vmov.f32 s2, s16 ; CHECK-NEXT: vmov.f32 s3, s18 ; CHECK-NEXT: vpst ; CHECK-NEXT: vstrwt.32 q0, [r2], #16 ; CHECK-NEXT: le lr, .LBB2_2 ; CHECK-NEXT: .LBB2_3: @ %for.cond.cleanup ; CHECK-NEXT: add sp, #24 ; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15} ; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, pc} ; CHECK-NEXT: .p2align 4 ; CHECK-NEXT: @ %bb.4: ; CHECK-NEXT: .LCPI2_0: ; CHECK-NEXT: .long 0 @ 0x0 ; CHECK-NEXT: .long 1 @ 0x1 ; CHECK-NEXT: .long 2 @ 0x2 ; CHECK-NEXT: .long 3 @ 0x3 ; CHECK-NEXT: .LCPI2_1: ; CHECK-NEXT: .long 2147483648 @ 0x80000000 ; CHECK-NEXT: .long 4294967295 @ 0xffffffff ; CHECK-NEXT: .long 2147483648 @ 0x80000000 ; CHECK-NEXT: .long 4294967295 @ 0xffffffff ; CHECK-NEXT: .LCPI2_2: ; CHECK-NEXT: .long 2147483647 @ 0x7fffffff ; CHECK-NEXT: .long 0 @ 0x0 ; CHECK-NEXT: .long 2147483647 @ 0x7fffffff ; CHECK-NEXT: .long 0 @ 0x0 entry: %cmp8 = icmp eq i32 %N, 0 br i1 %cmp8, label %for.cond.cleanup, label %vector.ph vector.ph: ; preds = %entry %n.rnd.up = add i32 %N, 3 %n.vec = and i32 %n.rnd.up, -4 %trip.count.minus.1 = add i32 %N, -1 %broadcast.splatinsert20 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0 %broadcast.splat21 = shufflevector <4 x i32> %broadcast.splatinsert20, <4 x i32> undef, <4 x i32> zeroinitializer br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0 %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer %induction = or <4 x i32> %broadcast.splat, %next.gep = getelementptr i32, ptr %pSrcA, i32 %index %next.gep18 = getelementptr i32, ptr %pSrcB, i32 %index %next.gep19 = getelementptr i32, ptr %pDst, i32 %index %0 = icmp ule <4 x i32> %induction, %broadcast.splat21 %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %next.gep, i32 4, <4 x i1> %0, <4 x i32> undef) %1 = sext <4 x i32> %wide.masked.load to <4 x i64> %wide.masked.load22 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %next.gep18, i32 4, <4 x i1> %0, <4 x i32> undef) %2 = sext <4 x i32> %wide.masked.load22 to <4 x i64> %3 = mul nsw <4 x i64> %2, %1 %4 = ashr <4 x i64> %3, %5 = icmp sgt <4 x i64> %4, %6 = select <4 x i1> %5, <4 x i64> %4, <4 x i64> %7 = icmp slt <4 x i64> %6, %8 = select <4 x i1> %7, <4 x i64> %6, <4 x i64> %9 = trunc <4 x i64> %8 to <4 x i32> call void @llvm.masked.store.v4i32.p0(<4 x i32> %9, ptr %next.gep19, i32 4, <4 x i1> %0) %index.next = add i32 %index, 4 %10 = icmp eq i32 %index.next, %n.vec br i1 %10, label %for.cond.cleanup, label %vector.body for.cond.cleanup: ; preds = %vector.body, %entry ret void } define arm_aapcs_vfpcc void @usatmul_2_q31(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: usatmul_2_q31: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr} ; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr} ; CHECK-NEXT: .pad #4 ; CHECK-NEXT: sub sp, #4 ; CHECK-NEXT: cmp r3, #0 ; CHECK-NEXT: beq .LBB3_8 ; CHECK-NEXT: @ %bb.1: @ %entry ; CHECK-NEXT: mov r8, r2 ; CHECK-NEXT: cmp r3, #1 ; CHECK-NEXT: bne .LBB3_3 ; CHECK-NEXT: @ %bb.2: ; CHECK-NEXT: movs r7, #0 ; CHECK-NEXT: mov r12, r0 ; CHECK-NEXT: mov r11, r1 ; CHECK-NEXT: mov r2, r8 ; CHECK-NEXT: b .LBB3_6 ; CHECK-NEXT: .LBB3_3: @ %vector.ph ; CHECK-NEXT: bic r5, r3, #1 ; CHECK-NEXT: movs r6, #1 ; CHECK-NEXT: subs r7, r5, #2 ; CHECK-NEXT: str r5, [sp] @ 4-byte Spill ; CHECK-NEXT: add.w r2, r8, r5, lsl #2 ; CHECK-NEXT: add.w r11, r1, r5, lsl #2 ; CHECK-NEXT: add.w lr, r6, r7, lsr #1 ; CHECK-NEXT: add.w r12, r0, r5, lsl #2 ; CHECK-NEXT: vmov.i8 q0, #0xff ; CHECK-NEXT: .LBB3_4: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldrd r4, r9, [r0], #8 ; CHECK-NEXT: ldrd r5, r10, [r1], #8 ; CHECK-NEXT: umull r4, r5, r5, r4 ; CHECK-NEXT: lsrl r4, r5, #31 ; CHECK-NEXT: subs.w r6, r4, #-1 ; CHECK-NEXT: sbcs r5, r5, #0 ; CHECK-NEXT: mov.w r6, #0 ; CHECK-NEXT: csetm r5, lo ; CHECK-NEXT: bfi r6, r5, #0, #8 ; CHECK-NEXT: umull r10, r5, r10, r9 ; CHECK-NEXT: lsrl r10, r5, #31 ; CHECK-NEXT: subs.w r7, r10, #-1 ; CHECK-NEXT: vmov q1[2], q1[0], r4, r10 ; CHECK-NEXT: sbcs r5, r5, #0 ; CHECK-NEXT: csetm r5, lo ; CHECK-NEXT: bfi r6, r5, #8, #8 ; CHECK-NEXT: vmsr p0, r6 ; CHECK-NEXT: vpsel q1, q1, q0 ; CHECK-NEXT: vmov r4, s6 ; CHECK-NEXT: vmov r5, s4 ; CHECK-NEXT: strd r5, r4, [r8], #8 ; CHECK-NEXT: le lr, .LBB3_4 ; CHECK-NEXT: @ %bb.5: @ %middle.block ; CHECK-NEXT: ldr r7, [sp] @ 4-byte Reload ; CHECK-NEXT: cmp r7, r3 ; CHECK-NEXT: beq .LBB3_8 ; CHECK-NEXT: .LBB3_6: @ %for.body.preheader ; CHECK-NEXT: sub.w lr, r3, r7 ; CHECK-NEXT: .LBB3_7: @ %for.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldr r0, [r12], #4 ; CHECK-NEXT: ldr r1, [r11], #4 ; CHECK-NEXT: umull r0, r1, r1, r0 ; CHECK-NEXT: lsrl r0, r1, #31 ; CHECK-NEXT: subs.w r3, r0, #-1 ; CHECK-NEXT: sbcs r1, r1, #0 ; CHECK-NEXT: it hs ; CHECK-NEXT: movhs.w r0, #-1 ; CHECK-NEXT: str r0, [r2], #4 ; CHECK-NEXT: le lr, .LBB3_7 ; CHECK-NEXT: .LBB3_8: @ %for.cond.cleanup ; CHECK-NEXT: add sp, #4 ; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc} entry: switch i32 %N, label %vector.ph [ i32 0, label %for.cond.cleanup i32 1, label %for.body.preheader ] vector.ph: ; preds = %entry %n.vec = and i32 %N, -2 %ind.end = getelementptr i32, ptr %pSrcA, i32 %n.vec %ind.end15 = getelementptr i32, ptr %pSrcB, i32 %n.vec %ind.end17 = getelementptr i32, ptr %pDst, i32 %n.vec br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %next.gep = getelementptr i32, ptr %pSrcA, i32 %index %next.gep18 = getelementptr i32, ptr %pSrcB, i32 %index %next.gep19 = getelementptr i32, ptr %pDst, i32 %index %wide.load = load <2 x i32>, ptr %next.gep, align 4 %0 = zext <2 x i32> %wide.load to <2 x i64> %wide.load20 = load <2 x i32>, ptr %next.gep18, align 4 %1 = zext <2 x i32> %wide.load20 to <2 x i64> %2 = mul nuw <2 x i64> %1, %0 %3 = lshr <2 x i64> %2, %4 = icmp ult <2 x i64> %3, %5 = select <2 x i1> %4, <2 x i64> %3, <2 x i64> %6 = trunc <2 x i64> %5 to <2 x i32> store <2 x i32> %6, ptr %next.gep19, align 4 %index.next = add i32 %index, 2 %7 = icmp eq i32 %index.next, %n.vec br i1 %7, label %middle.block, label %vector.body middle.block: ; preds = %vector.body %cmp.n = icmp eq i32 %n.vec, %N br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader for.body.preheader: ; preds = %entry, %middle.block %i.012.ph = phi i32 [ 0, %entry ], [ %n.vec, %middle.block ] %pSrcA.addr.011.ph = phi ptr [ %pSrcA, %entry ], [ %ind.end, %middle.block ] %pSrcB.addr.010.ph = phi ptr [ %pSrcB, %entry ], [ %ind.end15, %middle.block ] %pDst.addr.09.ph = phi ptr [ %pDst, %entry ], [ %ind.end17, %middle.block ] br label %for.body for.cond.cleanup: ; preds = %for.body, %middle.block, %entry ret void for.body: ; preds = %for.body.preheader, %for.body %i.012 = phi i32 [ %inc, %for.body ], [ %i.012.ph, %for.body.preheader ] %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.011.ph, %for.body.preheader ] %pSrcB.addr.010 = phi ptr [ %incdec.ptr1, %for.body ], [ %pSrcB.addr.010.ph, %for.body.preheader ] %pDst.addr.09 = phi ptr [ %incdec.ptr4, %for.body ], [ %pDst.addr.09.ph, %for.body.preheader ] %incdec.ptr = getelementptr inbounds i32, ptr %pSrcA.addr.011, i32 1 %8 = load i32, ptr %pSrcA.addr.011, align 4 %conv = zext i32 %8 to i64 %incdec.ptr1 = getelementptr inbounds i32, ptr %pSrcB.addr.010, i32 1 %9 = load i32, ptr %pSrcB.addr.010, align 4 %conv2 = zext i32 %9 to i64 %mul = mul nuw i64 %conv2, %conv %shr = lshr i64 %mul, 31 %10 = icmp ult i64 %shr, 4294967295 %retval.0.i = select i1 %10, i64 %shr, i64 4294967295 %conv3 = trunc i64 %retval.0.i to i32 %incdec.ptr4 = getelementptr inbounds i32, ptr %pDst.addr.09, i32 1 store i32 %conv3, ptr %pDst.addr.09, align 4 %inc = add nuw i32 %i.012, 1 %exitcond = icmp eq i32 %inc, %N br i1 %exitcond, label %for.cond.cleanup, label %for.body } define arm_aapcs_vfpcc void @usatmul_4_q31(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: usatmul_4_q31: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr} ; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr} ; CHECK-NEXT: .pad #4 ; CHECK-NEXT: sub sp, #4 ; CHECK-NEXT: .vsave {d8, d9, d10, d11} ; CHECK-NEXT: vpush {d8, d9, d10, d11} ; CHECK-NEXT: cmp r3, #0 ; CHECK-NEXT: beq.w .LBB4_8 ; CHECK-NEXT: @ %bb.1: @ %for.body.preheader ; CHECK-NEXT: mov.w r8, #0 ; CHECK-NEXT: cmp r3, #3 ; CHECK-NEXT: bhi .LBB4_3 ; CHECK-NEXT: @ %bb.2: ; CHECK-NEXT: mov r12, r0 ; CHECK-NEXT: mov r9, r1 ; CHECK-NEXT: mov r11, r2 ; CHECK-NEXT: b .LBB4_6 ; CHECK-NEXT: .LBB4_3: @ %vector.ph ; CHECK-NEXT: bic r8, r3, #3 ; CHECK-NEXT: movs r6, #1 ; CHECK-NEXT: sub.w r7, r8, #4 ; CHECK-NEXT: vmov.i64 q0, #0xffffffff ; CHECK-NEXT: add.w r11, r2, r8, lsl #2 ; CHECK-NEXT: add.w r9, r1, r8, lsl #2 ; CHECK-NEXT: add.w lr, r6, r7, lsr #2 ; CHECK-NEXT: add.w r12, r0, r8, lsl #2 ; CHECK-NEXT: .LBB4_4: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrw.u32 q1, [r0], #16 ; CHECK-NEXT: vldrw.u32 q3, [r1], #16 ; CHECK-NEXT: vmov.f32 s8, s6 ; CHECK-NEXT: vmov.f32 s16, s14 ; CHECK-NEXT: vmov.f32 s10, s7 ; CHECK-NEXT: vmov.f32 s18, s15 ; CHECK-NEXT: vmullb.u32 q5, q4, q2 ; CHECK-NEXT: vmov.f32 s6, s5 ; CHECK-NEXT: vmov r10, r5, d10 ; CHECK-NEXT: lsrl r10, r5, #31 ; CHECK-NEXT: vmov.f32 s14, s13 ; CHECK-NEXT: subs.w r6, r10, #-1 ; CHECK-NEXT: vmullb.u32 q4, q3, q1 ; CHECK-NEXT: sbcs r5, r5, #0 ; CHECK-NEXT: mov.w r6, #0 ; CHECK-NEXT: csetm r5, lo ; CHECK-NEXT: bfi r6, r5, #0, #8 ; CHECK-NEXT: vmov r4, r5, d11 ; CHECK-NEXT: lsrl r4, r5, #31 ; CHECK-NEXT: subs.w r7, r4, #-1 ; CHECK-NEXT: vmov q2[2], q2[0], r10, r4 ; CHECK-NEXT: sbcs r5, r5, #0 ; CHECK-NEXT: csetm r5, lo ; CHECK-NEXT: bfi r6, r5, #8, #8 ; CHECK-NEXT: vmov r10, r5, d8 ; CHECK-NEXT: lsrl r10, r5, #31 ; CHECK-NEXT: vmsr p0, r6 ; CHECK-NEXT: subs.w r6, r10, #-1 ; CHECK-NEXT: vpsel q2, q2, q0 ; CHECK-NEXT: sbcs r5, r5, #0 ; CHECK-NEXT: mov.w r6, #0 ; CHECK-NEXT: csetm r5, lo ; CHECK-NEXT: bfi r6, r5, #0, #8 ; CHECK-NEXT: vmov r4, r5, d9 ; CHECK-NEXT: lsrl r4, r5, #31 ; CHECK-NEXT: subs.w r7, r4, #-1 ; CHECK-NEXT: vmov q1[2], q1[0], r10, r4 ; CHECK-NEXT: sbcs r5, r5, #0 ; CHECK-NEXT: csetm r5, lo ; CHECK-NEXT: bfi r6, r5, #8, #8 ; CHECK-NEXT: vmsr p0, r6 ; CHECK-NEXT: vpsel q1, q1, q0 ; CHECK-NEXT: vmov.f32 s5, s6 ; CHECK-NEXT: vmov.f32 s6, s8 ; CHECK-NEXT: vmov.f32 s7, s10 ; CHECK-NEXT: vstrb.8 q1, [r2], #16 ; CHECK-NEXT: le lr, .LBB4_4 ; CHECK-NEXT: @ %bb.5: @ %middle.block ; CHECK-NEXT: cmp r8, r3 ; CHECK-NEXT: beq .LBB4_8 ; CHECK-NEXT: .LBB4_6: @ %for.body.preheader21 ; CHECK-NEXT: sub.w lr, r3, r8 ; CHECK-NEXT: .LBB4_7: @ %for.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldr r0, [r12], #4 ; CHECK-NEXT: ldr r1, [r9], #4 ; CHECK-NEXT: umull r0, r1, r1, r0 ; CHECK-NEXT: lsrl r0, r1, #31 ; CHECK-NEXT: subs.w r2, r0, #-1 ; CHECK-NEXT: sbcs r1, r1, #0 ; CHECK-NEXT: it hs ; CHECK-NEXT: movhs.w r0, #-1 ; CHECK-NEXT: str r0, [r11], #4 ; CHECK-NEXT: le lr, .LBB4_7 ; CHECK-NEXT: .LBB4_8: @ %for.cond.cleanup ; CHECK-NEXT: vpop {d8, d9, d10, d11} ; CHECK-NEXT: add sp, #4 ; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc} entry: %cmp8 = icmp eq i32 %N, 0 br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader for.body.preheader: ; preds = %entry %min.iters.check = icmp ult i32 %N, 4 br i1 %min.iters.check, label %for.body.preheader21, label %vector.ph for.body.preheader21: ; preds = %middle.block, %for.body.preheader %i.012.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ] %pSrcA.addr.011.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ] %pSrcB.addr.010.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end15, %middle.block ] %pDst.addr.09.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end17, %middle.block ] br label %for.body vector.ph: ; preds = %for.body.preheader %n.vec = and i32 %N, -4 %ind.end = getelementptr i32, ptr %pSrcA, i32 %n.vec %ind.end15 = getelementptr i32, ptr %pSrcB, i32 %n.vec %ind.end17 = getelementptr i32, ptr %pDst, i32 %n.vec br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %next.gep = getelementptr i32, ptr %pSrcA, i32 %index %next.gep18 = getelementptr i32, ptr %pSrcB, i32 %index %next.gep19 = getelementptr i32, ptr %pDst, i32 %index %wide.load = load <4 x i32>, ptr %next.gep, align 4 %0 = zext <4 x i32> %wide.load to <4 x i64> %wide.load20 = load <4 x i32>, ptr %next.gep18, align 4 %1 = zext <4 x i32> %wide.load20 to <4 x i64> %2 = mul nuw <4 x i64> %1, %0 %3 = lshr <4 x i64> %2, %4 = icmp ult <4 x i64> %3, %5 = select <4 x i1> %4, <4 x i64> %3, <4 x i64> %6 = trunc <4 x i64> %5 to <4 x i32> store <4 x i32> %6, ptr %next.gep19, align 4 %index.next = add i32 %index, 4 %7 = icmp eq i32 %index.next, %n.vec br i1 %7, label %middle.block, label %vector.body middle.block: ; preds = %vector.body %cmp.n = icmp eq i32 %n.vec, %N br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader21 for.cond.cleanup: ; preds = %for.body, %middle.block, %entry ret void for.body: ; preds = %for.body.preheader21, %for.body %i.012 = phi i32 [ %inc, %for.body ], [ %i.012.ph, %for.body.preheader21 ] %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.011.ph, %for.body.preheader21 ] %pSrcB.addr.010 = phi ptr [ %incdec.ptr1, %for.body ], [ %pSrcB.addr.010.ph, %for.body.preheader21 ] %pDst.addr.09 = phi ptr [ %incdec.ptr4, %for.body ], [ %pDst.addr.09.ph, %for.body.preheader21 ] %incdec.ptr = getelementptr inbounds i32, ptr %pSrcA.addr.011, i32 1 %8 = load i32, ptr %pSrcA.addr.011, align 4 %conv = zext i32 %8 to i64 %incdec.ptr1 = getelementptr inbounds i32, ptr %pSrcB.addr.010, i32 1 %9 = load i32, ptr %pSrcB.addr.010, align 4 %conv2 = zext i32 %9 to i64 %mul = mul nuw i64 %conv2, %conv %shr = lshr i64 %mul, 31 %10 = icmp ult i64 %shr, 4294967295 %retval.0.i = select i1 %10, i64 %shr, i64 4294967295 %conv3 = trunc i64 %retval.0.i to i32 %incdec.ptr4 = getelementptr inbounds i32, ptr %pDst.addr.09, i32 1 store i32 %conv3, ptr %pDst.addr.09, align 4 %inc = add nuw i32 %i.012, 1 %exitcond = icmp eq i32 %inc, %N br i1 %exitcond, label %for.cond.cleanup, label %for.body } ; i16 define arm_aapcs_vfpcc void @ssatmul_4_q15(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: ssatmul_4_q15: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r6, lr} ; CHECK-NEXT: push {r4, r5, r6, lr} ; CHECK-NEXT: cbz r3, .LBB5_8 ; CHECK-NEXT: @ %bb.1: @ %for.body.preheader ; CHECK-NEXT: cmp r3, #3 ; CHECK-NEXT: bhi .LBB5_3 ; CHECK-NEXT: @ %bb.2: ; CHECK-NEXT: movs r5, #0 ; CHECK-NEXT: mov r12, r0 ; CHECK-NEXT: mov r6, r1 ; CHECK-NEXT: mov r4, r2 ; CHECK-NEXT: b .LBB5_6 ; CHECK-NEXT: .LBB5_3: @ %vector.ph ; CHECK-NEXT: bic r5, r3, #3 ; CHECK-NEXT: movs r4, #1 ; CHECK-NEXT: subs r6, r5, #4 ; CHECK-NEXT: add.w r12, r0, r5, lsl #1 ; CHECK-NEXT: add.w lr, r4, r6, lsr #2 ; CHECK-NEXT: add.w r4, r2, r5, lsl #1 ; CHECK-NEXT: add.w r6, r1, r5, lsl #1 ; CHECK-NEXT: .LBB5_4: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrh.s32 q0, [r0], #8 ; CHECK-NEXT: vldrh.s32 q1, [r1], #8 ; CHECK-NEXT: vmul.i32 q0, q1, q0 ; CHECK-NEXT: vqshrnb.s32 q0, q0, #15 ; CHECK-NEXT: vstrh.32 q0, [r2], #8 ; CHECK-NEXT: le lr, .LBB5_4 ; CHECK-NEXT: @ %bb.5: @ %middle.block ; CHECK-NEXT: cmp r5, r3 ; CHECK-NEXT: it eq ; CHECK-NEXT: popeq {r4, r5, r6, pc} ; CHECK-NEXT: .LBB5_6: @ %for.body.preheader21 ; CHECK-NEXT: sub.w lr, r3, r5 ; CHECK-NEXT: .LBB5_7: @ %for.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldrsh r0, [r12], #2 ; CHECK-NEXT: ldrsh r1, [r6], #2 ; CHECK-NEXT: muls r0, r1, r0 ; CHECK-NEXT: ssat r0, #16, r0, asr #15 ; CHECK-NEXT: strh r0, [r4], #2 ; CHECK-NEXT: le lr, .LBB5_7 ; CHECK-NEXT: .LBB5_8: @ %for.cond.cleanup ; CHECK-NEXT: pop {r4, r5, r6, pc} entry: %cmp8 = icmp eq i32 %N, 0 br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader for.body.preheader: ; preds = %entry %min.iters.check = icmp ult i32 %N, 4 br i1 %min.iters.check, label %for.body.preheader21, label %vector.ph for.body.preheader21: ; preds = %middle.block, %for.body.preheader %i.012.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ] %pSrcA.addr.011.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ] %pSrcB.addr.010.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end15, %middle.block ] %pDst.addr.09.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end17, %middle.block ] br label %for.body vector.ph: ; preds = %for.body.preheader %n.vec = and i32 %N, -4 %ind.end = getelementptr i16, ptr %pSrcA, i32 %n.vec %ind.end15 = getelementptr i16, ptr %pSrcB, i32 %n.vec %ind.end17 = getelementptr i16, ptr %pDst, i32 %n.vec br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %next.gep = getelementptr i16, ptr %pSrcA, i32 %index %next.gep18 = getelementptr i16, ptr %pSrcB, i32 %index %next.gep19 = getelementptr i16, ptr %pDst, i32 %index %wide.load = load <4 x i16>, ptr %next.gep, align 2 %0 = sext <4 x i16> %wide.load to <4 x i32> %wide.load20 = load <4 x i16>, ptr %next.gep18, align 2 %1 = sext <4 x i16> %wide.load20 to <4 x i32> %2 = mul nsw <4 x i32> %1, %0 %3 = ashr <4 x i32> %2, %4 = icmp sgt <4 x i32> %3, %5 = select <4 x i1> %4, <4 x i32> %3, <4 x i32> %6 = icmp slt <4 x i32> %5, %7 = select <4 x i1> %6, <4 x i32> %5, <4 x i32> %8 = trunc <4 x i32> %7 to <4 x i16> store <4 x i16> %8, ptr %next.gep19, align 2 %index.next = add i32 %index, 4 %9 = icmp eq i32 %index.next, %n.vec br i1 %9, label %middle.block, label %vector.body middle.block: ; preds = %vector.body %cmp.n = icmp eq i32 %n.vec, %N br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader21 for.cond.cleanup: ; preds = %for.body, %middle.block, %entry ret void for.body: ; preds = %for.body.preheader21, %for.body %i.012 = phi i32 [ %inc, %for.body ], [ %i.012.ph, %for.body.preheader21 ] %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.011.ph, %for.body.preheader21 ] %pSrcB.addr.010 = phi ptr [ %incdec.ptr1, %for.body ], [ %pSrcB.addr.010.ph, %for.body.preheader21 ] %pDst.addr.09 = phi ptr [ %incdec.ptr4, %for.body ], [ %pDst.addr.09.ph, %for.body.preheader21 ] %incdec.ptr = getelementptr inbounds i16, ptr %pSrcA.addr.011, i32 1 %10 = load i16, ptr %pSrcA.addr.011, align 2 %conv = sext i16 %10 to i32 %incdec.ptr1 = getelementptr inbounds i16, ptr %pSrcB.addr.010, i32 1 %11 = load i16, ptr %pSrcB.addr.010, align 2 %conv2 = sext i16 %11 to i32 %mul = mul nsw i32 %conv2, %conv %shr = ashr i32 %mul, 15 %12 = icmp sgt i32 %shr, -32768 %.val.i = select i1 %12, i32 %shr, i32 -32768 %13 = icmp slt i32 %.val.i, 32767 %retval.0.i = select i1 %13, i32 %.val.i, i32 32767 %conv3 = trunc i32 %retval.0.i to i16 %incdec.ptr4 = getelementptr inbounds i16, ptr %pDst.addr.09, i32 1 store i16 %conv3, ptr %pDst.addr.09, align 2 %inc = add nuw i32 %i.012, 1 %exitcond = icmp eq i32 %inc, %N br i1 %exitcond, label %for.cond.cleanup, label %for.body } define arm_aapcs_vfpcc void @ssatmul_8_q15(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: ssatmul_8_q15: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r6, lr} ; CHECK-NEXT: push {r4, r5, r6, lr} ; CHECK-NEXT: cbz r3, .LBB6_8 ; CHECK-NEXT: @ %bb.1: @ %for.body.preheader ; CHECK-NEXT: cmp r3, #7 ; CHECK-NEXT: bhi .LBB6_3 ; CHECK-NEXT: @ %bb.2: ; CHECK-NEXT: movs r5, #0 ; CHECK-NEXT: mov r12, r0 ; CHECK-NEXT: mov r6, r1 ; CHECK-NEXT: mov r4, r2 ; CHECK-NEXT: b .LBB6_6 ; CHECK-NEXT: .LBB6_3: @ %vector.ph ; CHECK-NEXT: bic r5, r3, #7 ; CHECK-NEXT: movs r4, #1 ; CHECK-NEXT: sub.w r6, r5, #8 ; CHECK-NEXT: add.w r12, r0, r5, lsl #1 ; CHECK-NEXT: add.w lr, r4, r6, lsr #3 ; CHECK-NEXT: add.w r4, r2, r5, lsl #1 ; CHECK-NEXT: add.w r6, r1, r5, lsl #1 ; CHECK-NEXT: .LBB6_4: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrh.u16 q0, [r0], #16 ; CHECK-NEXT: vldrh.u16 q1, [r1], #16 ; CHECK-NEXT: vmullt.s16 q2, q1, q0 ; CHECK-NEXT: vmullb.s16 q0, q1, q0 ; CHECK-NEXT: vqshrnb.s32 q0, q0, #15 ; CHECK-NEXT: vqshrnt.s32 q0, q2, #15 ; CHECK-NEXT: vstrb.8 q0, [r2], #16 ; CHECK-NEXT: le lr, .LBB6_4 ; CHECK-NEXT: @ %bb.5: @ %middle.block ; CHECK-NEXT: cmp r5, r3 ; CHECK-NEXT: it eq ; CHECK-NEXT: popeq {r4, r5, r6, pc} ; CHECK-NEXT: .LBB6_6: @ %for.body.preheader21 ; CHECK-NEXT: sub.w lr, r3, r5 ; CHECK-NEXT: .LBB6_7: @ %for.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldrsh r0, [r12], #2 ; CHECK-NEXT: ldrsh r1, [r6], #2 ; CHECK-NEXT: muls r0, r1, r0 ; CHECK-NEXT: ssat r0, #16, r0, asr #15 ; CHECK-NEXT: strh r0, [r4], #2 ; CHECK-NEXT: le lr, .LBB6_7 ; CHECK-NEXT: .LBB6_8: @ %for.cond.cleanup ; CHECK-NEXT: pop {r4, r5, r6, pc} entry: %cmp8 = icmp eq i32 %N, 0 br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader for.body.preheader: ; preds = %entry %min.iters.check = icmp ult i32 %N, 8 br i1 %min.iters.check, label %for.body.preheader21, label %vector.ph for.body.preheader21: ; preds = %middle.block, %for.body.preheader %i.012.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ] %pSrcA.addr.011.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ] %pSrcB.addr.010.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end15, %middle.block ] %pDst.addr.09.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end17, %middle.block ] br label %for.body vector.ph: ; preds = %for.body.preheader %n.vec = and i32 %N, -8 %ind.end = getelementptr i16, ptr %pSrcA, i32 %n.vec %ind.end15 = getelementptr i16, ptr %pSrcB, i32 %n.vec %ind.end17 = getelementptr i16, ptr %pDst, i32 %n.vec br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %next.gep = getelementptr i16, ptr %pSrcA, i32 %index %next.gep18 = getelementptr i16, ptr %pSrcB, i32 %index %next.gep19 = getelementptr i16, ptr %pDst, i32 %index %wide.load = load <8 x i16>, ptr %next.gep, align 2 %0 = sext <8 x i16> %wide.load to <8 x i32> %wide.load20 = load <8 x i16>, ptr %next.gep18, align 2 %1 = sext <8 x i16> %wide.load20 to <8 x i32> %2 = mul nsw <8 x i32> %1, %0 %3 = ashr <8 x i32> %2, %4 = icmp sgt <8 x i32> %3, %5 = select <8 x i1> %4, <8 x i32> %3, <8 x i32> %6 = icmp slt <8 x i32> %5, %7 = select <8 x i1> %6, <8 x i32> %5, <8 x i32> %8 = trunc <8 x i32> %7 to <8 x i16> store <8 x i16> %8, ptr %next.gep19, align 2 %index.next = add i32 %index, 8 %9 = icmp eq i32 %index.next, %n.vec br i1 %9, label %middle.block, label %vector.body middle.block: ; preds = %vector.body %cmp.n = icmp eq i32 %n.vec, %N br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader21 for.cond.cleanup: ; preds = %for.body, %middle.block, %entry ret void for.body: ; preds = %for.body.preheader21, %for.body %i.012 = phi i32 [ %inc, %for.body ], [ %i.012.ph, %for.body.preheader21 ] %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.011.ph, %for.body.preheader21 ] %pSrcB.addr.010 = phi ptr [ %incdec.ptr1, %for.body ], [ %pSrcB.addr.010.ph, %for.body.preheader21 ] %pDst.addr.09 = phi ptr [ %incdec.ptr4, %for.body ], [ %pDst.addr.09.ph, %for.body.preheader21 ] %incdec.ptr = getelementptr inbounds i16, ptr %pSrcA.addr.011, i32 1 %10 = load i16, ptr %pSrcA.addr.011, align 2 %conv = sext i16 %10 to i32 %incdec.ptr1 = getelementptr inbounds i16, ptr %pSrcB.addr.010, i32 1 %11 = load i16, ptr %pSrcB.addr.010, align 2 %conv2 = sext i16 %11 to i32 %mul = mul nsw i32 %conv2, %conv %shr = ashr i32 %mul, 15 %12 = icmp sgt i32 %shr, -32768 %.val.i = select i1 %12, i32 %shr, i32 -32768 %13 = icmp slt i32 %.val.i, 32767 %retval.0.i = select i1 %13, i32 %.val.i, i32 32767 %conv3 = trunc i32 %retval.0.i to i16 %incdec.ptr4 = getelementptr inbounds i16, ptr %pDst.addr.09, i32 1 store i16 %conv3, ptr %pDst.addr.09, align 2 %inc = add nuw i32 %i.012, 1 %exitcond = icmp eq i32 %inc, %N br i1 %exitcond, label %for.cond.cleanup, label %for.body } define arm_aapcs_vfpcc void @ssatmul_8i_q15(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: ssatmul_8i_q15: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r6, lr} ; CHECK-NEXT: push {r4, r5, r6, lr} ; CHECK-NEXT: cbz r3, .LBB7_8 ; CHECK-NEXT: @ %bb.1: @ %for.body.preheader ; CHECK-NEXT: cmp r3, #7 ; CHECK-NEXT: bhi .LBB7_3 ; CHECK-NEXT: @ %bb.2: ; CHECK-NEXT: movs r5, #0 ; CHECK-NEXT: mov r12, r0 ; CHECK-NEXT: mov r6, r1 ; CHECK-NEXT: mov r4, r2 ; CHECK-NEXT: b .LBB7_6 ; CHECK-NEXT: .LBB7_3: @ %vector.ph ; CHECK-NEXT: bic r5, r3, #7 ; CHECK-NEXT: movs r4, #1 ; CHECK-NEXT: sub.w r6, r5, #8 ; CHECK-NEXT: add.w r12, r0, r5, lsl #1 ; CHECK-NEXT: add.w lr, r4, r6, lsr #3 ; CHECK-NEXT: add.w r4, r2, r5, lsl #1 ; CHECK-NEXT: add.w r6, r1, r5, lsl #1 ; CHECK-NEXT: .LBB7_4: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrh.u16 q0, [r0], #16 ; CHECK-NEXT: vldrh.u16 q1, [r1], #16 ; CHECK-NEXT: vmullt.s16 q2, q1, q0 ; CHECK-NEXT: vmullb.s16 q0, q1, q0 ; CHECK-NEXT: vqshrnb.s32 q0, q0, #15 ; CHECK-NEXT: vqshrnt.s32 q0, q2, #15 ; CHECK-NEXT: vstrb.8 q0, [r2], #16 ; CHECK-NEXT: le lr, .LBB7_4 ; CHECK-NEXT: @ %bb.5: @ %middle.block ; CHECK-NEXT: cmp r5, r3 ; CHECK-NEXT: it eq ; CHECK-NEXT: popeq {r4, r5, r6, pc} ; CHECK-NEXT: .LBB7_6: @ %for.body.preheader21 ; CHECK-NEXT: sub.w lr, r3, r5 ; CHECK-NEXT: .LBB7_7: @ %for.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldrsh r0, [r12], #2 ; CHECK-NEXT: ldrsh r1, [r6], #2 ; CHECK-NEXT: muls r0, r1, r0 ; CHECK-NEXT: ssat r0, #16, r0, asr #15 ; CHECK-NEXT: strh r0, [r4], #2 ; CHECK-NEXT: le lr, .LBB7_7 ; CHECK-NEXT: .LBB7_8: @ %for.cond.cleanup ; CHECK-NEXT: pop {r4, r5, r6, pc} entry: %cmp8 = icmp eq i32 %N, 0 br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader for.body.preheader: ; preds = %entry %min.iters.check = icmp ult i32 %N, 8 br i1 %min.iters.check, label %for.body.preheader21, label %vector.ph for.body.preheader21: ; preds = %middle.block, %for.body.preheader %i.012.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ] %pSrcA.addr.011.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ] %pSrcB.addr.010.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end15, %middle.block ] %pDst.addr.09.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end17, %middle.block ] br label %for.body vector.ph: ; preds = %for.body.preheader %n.vec = and i32 %N, -8 %ind.end = getelementptr i16, ptr %pSrcA, i32 %n.vec %ind.end15 = getelementptr i16, ptr %pSrcB, i32 %n.vec %ind.end17 = getelementptr i16, ptr %pDst, i32 %n.vec br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %next.gep = getelementptr i16, ptr %pSrcA, i32 %index %next.gep18 = getelementptr i16, ptr %pSrcB, i32 %index %next.gep19 = getelementptr i16, ptr %pDst, i32 %index %wide.load = load <8 x i16>, ptr %next.gep, align 2 %0 = shufflevector <8 x i16> %wide.load, <8 x i16> %wide.load, <4 x i32> %1 = shufflevector <8 x i16> %wide.load, <8 x i16> %wide.load, <4 x i32> %2 = sext <4 x i16> %0 to <4 x i32> %3 = sext <4 x i16> %1 to <4 x i32> %wide.load20 = load <8 x i16>, ptr %next.gep18, align 2 %4 = shufflevector <8 x i16> %wide.load20, <8 x i16> %wide.load20, <4 x i32> %5 = shufflevector <8 x i16> %wide.load20, <8 x i16> %wide.load20, <4 x i32> %6 = sext <4 x i16> %4 to <4 x i32> %7 = sext <4 x i16> %5 to <4 x i32> %8 = mul <4 x i32> %6, %2 %9 = mul <4 x i32> %7, %3 %10 = ashr <4 x i32> %8, %11 = ashr <4 x i32> %9, %12 = icmp sgt <4 x i32> %10, %13 = icmp sgt <4 x i32> %11, %14 = select <4 x i1> %12, <4 x i32> %10, <4 x i32> %15 = select <4 x i1> %13, <4 x i32> %11, <4 x i32> %16 = icmp slt <4 x i32> %14, %17 = icmp slt <4 x i32> %15, %18 = select <4 x i1> %16, <4 x i32> %14, <4 x i32> %19 = select <4 x i1> %17, <4 x i32> %15, <4 x i32> %20 = shufflevector <4 x i32> %18, <4 x i32> %19, <8 x i32> %21 = trunc <8 x i32> %20 to <8 x i16> store <8 x i16> %21, ptr %next.gep19, align 2 %index.next = add i32 %index, 8 %22 = icmp eq i32 %index.next, %n.vec br i1 %22, label %middle.block, label %vector.body middle.block: ; preds = %vector.body %cmp.n = icmp eq i32 %n.vec, %N br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader21 for.cond.cleanup: ; preds = %for.body, %middle.block, %entry ret void for.body: ; preds = %for.body, %for.body.preheader21 %i.012 = phi i32 [ %inc, %for.body ], [ %i.012.ph, %for.body.preheader21 ] %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.011.ph, %for.body.preheader21 ] %pSrcB.addr.010 = phi ptr [ %incdec.ptr1, %for.body ], [ %pSrcB.addr.010.ph, %for.body.preheader21 ] %pDst.addr.09 = phi ptr [ %incdec.ptr4, %for.body ], [ %pDst.addr.09.ph, %for.body.preheader21 ] %incdec.ptr = getelementptr inbounds i16, ptr %pSrcA.addr.011, i32 1 %23 = load i16, ptr %pSrcA.addr.011, align 2 %conv = sext i16 %23 to i32 %incdec.ptr1 = getelementptr inbounds i16, ptr %pSrcB.addr.010, i32 1 %24 = load i16, ptr %pSrcB.addr.010, align 2 %conv2 = sext i16 %24 to i32 %mul = mul nsw i32 %conv2, %conv %shr = ashr i32 %mul, 15 %25 = icmp sgt i32 %shr, -32768 %.val.i = select i1 %25, i32 %shr, i32 -32768 %26 = icmp slt i32 %.val.i, 32767 %retval.0.i = select i1 %26, i32 %.val.i, i32 32767 %conv3 = trunc i32 %retval.0.i to i16 %incdec.ptr4 = getelementptr inbounds i16, ptr %pDst.addr.09, i32 1 store i16 %conv3, ptr %pDst.addr.09, align 2 %inc = add nuw i32 %i.012, 1 %exitcond = icmp eq i32 %inc, %N br i1 %exitcond, label %for.cond.cleanup, label %for.body } define arm_aapcs_vfpcc void @ssatmul_s4t_q15(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: ssatmul_s4t_q15: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, lr} ; CHECK-NEXT: push {r4, lr} ; CHECK-NEXT: cmp r3, #0 ; CHECK-NEXT: it eq ; CHECK-NEXT: popeq {r4, pc} ; CHECK-NEXT: .LBB8_1: @ %vector.ph ; CHECK-NEXT: add.w r12, r3, #3 ; CHECK-NEXT: mov.w lr, #1 ; CHECK-NEXT: bic r12, r12, #3 ; CHECK-NEXT: adr r4, .LCPI8_0 ; CHECK-NEXT: sub.w r12, r12, #4 ; CHECK-NEXT: vldrw.u32 q0, [r4] ; CHECK-NEXT: add.w lr, lr, r12, lsr #2 ; CHECK-NEXT: sub.w r12, r3, #1 ; CHECK-NEXT: movs r3, #0 ; CHECK-NEXT: vdup.32 q1, r12 ; CHECK-NEXT: .LBB8_2: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vdup.32 q2, r3 ; CHECK-NEXT: adds r3, #4 ; CHECK-NEXT: vorr q2, q2, q0 ; CHECK-NEXT: vptt.u32 cs, q1, q2 ; CHECK-NEXT: vldrht.s32 q2, [r0], #8 ; CHECK-NEXT: vldrht.s32 q3, [r1], #8 ; CHECK-NEXT: vmul.i32 q2, q3, q2 ; CHECK-NEXT: vqshrnb.s32 q2, q2, #15 ; CHECK-NEXT: vpst ; CHECK-NEXT: vstrht.32 q2, [r2], #8 ; CHECK-NEXT: le lr, .LBB8_2 ; CHECK-NEXT: @ %bb.3: @ %for.cond.cleanup ; CHECK-NEXT: pop {r4, pc} ; CHECK-NEXT: .p2align 4 ; CHECK-NEXT: @ %bb.4: ; CHECK-NEXT: .LCPI8_0: ; CHECK-NEXT: .long 0 @ 0x0 ; CHECK-NEXT: .long 1 @ 0x1 ; CHECK-NEXT: .long 2 @ 0x2 ; CHECK-NEXT: .long 3 @ 0x3 entry: %cmp8 = icmp eq i32 %N, 0 br i1 %cmp8, label %for.cond.cleanup, label %vector.ph vector.ph: ; preds = %entry %n.rnd.up = add i32 %N, 3 %n.vec = and i32 %n.rnd.up, -4 %trip.count.minus.1 = add i32 %N, -1 %broadcast.splatinsert20 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0 %broadcast.splat21 = shufflevector <4 x i32> %broadcast.splatinsert20, <4 x i32> undef, <4 x i32> zeroinitializer br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0 %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer %induction = or <4 x i32> %broadcast.splat, %next.gep = getelementptr i16, ptr %pSrcA, i32 %index %next.gep18 = getelementptr i16, ptr %pSrcB, i32 %index %next.gep19 = getelementptr i16, ptr %pDst, i32 %index %0 = icmp ule <4 x i32> %induction, %broadcast.splat21 %wide.masked.load = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr %next.gep, i32 2, <4 x i1> %0, <4 x i16> undef) %1 = sext <4 x i16> %wide.masked.load to <4 x i32> %wide.masked.load22 = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr %next.gep18, i32 2, <4 x i1> %0, <4 x i16> undef) %2 = sext <4 x i16> %wide.masked.load22 to <4 x i32> %3 = mul nsw <4 x i32> %2, %1 %4 = ashr <4 x i32> %3, %5 = icmp sgt <4 x i32> %4, %6 = select <4 x i1> %5, <4 x i32> %4, <4 x i32> %7 = icmp slt <4 x i32> %6, %8 = select <4 x i1> %7, <4 x i32> %6, <4 x i32> %9 = trunc <4 x i32> %8 to <4 x i16> call void @llvm.masked.store.v4i16.p0(<4 x i16> %9, ptr %next.gep19, i32 2, <4 x i1> %0) %index.next = add i32 %index, 4 %10 = icmp eq i32 %index.next, %n.vec br i1 %10, label %for.cond.cleanup, label %vector.body for.cond.cleanup: ; preds = %vector.body, %entry ret void } define arm_aapcs_vfpcc void @ssatmul_8t_q15(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: ssatmul_8t_q15: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r7, lr} ; CHECK-NEXT: push {r4, r5, r7, lr} ; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15} ; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15} ; CHECK-NEXT: .pad #16 ; CHECK-NEXT: sub sp, #16 ; CHECK-NEXT: cmp r3, #0 ; CHECK-NEXT: beq .LBB9_3 ; CHECK-NEXT: @ %bb.1: @ %vector.ph ; CHECK-NEXT: adds r4, r3, #7 ; CHECK-NEXT: vmov.i8 q2, #0x0 ; CHECK-NEXT: bic r4, r4, #7 ; CHECK-NEXT: vmov.i8 q3, #0xff ; CHECK-NEXT: sub.w r12, r4, #8 ; CHECK-NEXT: movs r4, #1 ; CHECK-NEXT: mov r5, sp ; CHECK-NEXT: add.w lr, r4, r12, lsr #3 ; CHECK-NEXT: adr r4, .LCPI9_0 ; CHECK-NEXT: vldrw.u32 q0, [r4] ; CHECK-NEXT: adr r4, .LCPI9_1 ; CHECK-NEXT: sub.w r12, r3, #1 ; CHECK-NEXT: vldrw.u32 q4, [r4] ; CHECK-NEXT: movs r3, #0 ; CHECK-NEXT: vdup.32 q1, r12 ; CHECK-NEXT: .LBB9_2: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vdup.32 q5, r3 ; CHECK-NEXT: adds r3, #8 ; CHECK-NEXT: vorr q6, q5, q0 ; CHECK-NEXT: vorr q5, q5, q4 ; CHECK-NEXT: vcmp.u32 cs, q1, q6 ; CHECK-NEXT: vpsel q6, q3, q2 ; CHECK-NEXT: vcmp.u32 cs, q1, q5 ; CHECK-NEXT: vpsel q5, q3, q2 ; CHECK-NEXT: vstrh.32 q6, [r5, #8] ; CHECK-NEXT: vstrh.32 q5, [r5] ; CHECK-NEXT: vldrw.u32 q5, [r5] ; CHECK-NEXT: vptt.i16 ne, q5, zr ; CHECK-NEXT: vldrht.u16 q5, [r0], #16 ; CHECK-NEXT: vldrht.u16 q6, [r1], #16 ; CHECK-NEXT: vmullt.s16 q7, q6, q5 ; CHECK-NEXT: vmullb.s16 q5, q6, q5 ; CHECK-NEXT: vqshrnb.s32 q5, q5, #15 ; CHECK-NEXT: vqshrnt.s32 q5, q7, #15 ; CHECK-NEXT: vpst ; CHECK-NEXT: vstrht.16 q5, [r2], #16 ; CHECK-NEXT: le lr, .LBB9_2 ; CHECK-NEXT: .LBB9_3: @ %for.cond.cleanup ; CHECK-NEXT: add sp, #16 ; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15} ; CHECK-NEXT: pop {r4, r5, r7, pc} ; CHECK-NEXT: .p2align 4 ; CHECK-NEXT: @ %bb.4: ; CHECK-NEXT: .LCPI9_0: ; CHECK-NEXT: .long 4 @ 0x4 ; CHECK-NEXT: .long 5 @ 0x5 ; CHECK-NEXT: .long 6 @ 0x6 ; CHECK-NEXT: .long 7 @ 0x7 ; CHECK-NEXT: .LCPI9_1: ; CHECK-NEXT: .long 0 @ 0x0 ; CHECK-NEXT: .long 1 @ 0x1 ; CHECK-NEXT: .long 2 @ 0x2 ; CHECK-NEXT: .long 3 @ 0x3 entry: %cmp8 = icmp eq i32 %N, 0 br i1 %cmp8, label %for.cond.cleanup, label %vector.ph vector.ph: ; preds = %entry %n.rnd.up = add i32 %N, 7 %n.vec = and i32 %n.rnd.up, -8 %trip.count.minus.1 = add i32 %N, -1 %broadcast.splatinsert20 = insertelement <8 x i32> undef, i32 %trip.count.minus.1, i32 0 %broadcast.splat21 = shufflevector <8 x i32> %broadcast.splatinsert20, <8 x i32> undef, <8 x i32> zeroinitializer br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0 %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer %induction = or <8 x i32> %broadcast.splat, %next.gep = getelementptr i16, ptr %pSrcA, i32 %index %next.gep18 = getelementptr i16, ptr %pSrcB, i32 %index %next.gep19 = getelementptr i16, ptr %pDst, i32 %index %0 = icmp ule <8 x i32> %induction, %broadcast.splat21 %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %next.gep, i32 2, <8 x i1> %0, <8 x i16> undef) %1 = sext <8 x i16> %wide.masked.load to <8 x i32> %wide.masked.load22 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %next.gep18, i32 2, <8 x i1> %0, <8 x i16> undef) %2 = sext <8 x i16> %wide.masked.load22 to <8 x i32> %3 = mul nsw <8 x i32> %2, %1 %4 = ashr <8 x i32> %3, %5 = icmp sgt <8 x i32> %4, %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> %7 = icmp slt <8 x i32> %6, %8 = select <8 x i1> %7, <8 x i32> %6, <8 x i32> %9 = trunc <8 x i32> %8 to <8 x i16> call void @llvm.masked.store.v8i16.p0(<8 x i16> %9, ptr %next.gep19, i32 2, <8 x i1> %0) %index.next = add i32 %index, 8 %10 = icmp eq i32 %index.next, %n.vec br i1 %10, label %for.cond.cleanup, label %vector.body for.cond.cleanup: ; preds = %vector.body, %entry ret void } define arm_aapcs_vfpcc void @ssatmul_8ti_q15(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: ssatmul_8ti_q15: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r7, lr} ; CHECK-NEXT: push {r4, r5, r7, lr} ; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15} ; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15} ; CHECK-NEXT: .pad #16 ; CHECK-NEXT: sub sp, #16 ; CHECK-NEXT: cmp r3, #0 ; CHECK-NEXT: beq .LBB10_3 ; CHECK-NEXT: @ %bb.1: @ %vector.ph ; CHECK-NEXT: adds r4, r3, #7 ; CHECK-NEXT: vmov.i8 q2, #0x0 ; CHECK-NEXT: bic r4, r4, #7 ; CHECK-NEXT: vmov.i8 q3, #0xff ; CHECK-NEXT: sub.w r12, r4, #8 ; CHECK-NEXT: movs r4, #1 ; CHECK-NEXT: mov r5, sp ; CHECK-NEXT: add.w lr, r4, r12, lsr #3 ; CHECK-NEXT: adr r4, .LCPI10_0 ; CHECK-NEXT: vldrw.u32 q0, [r4] ; CHECK-NEXT: adr r4, .LCPI10_1 ; CHECK-NEXT: sub.w r12, r3, #1 ; CHECK-NEXT: vldrw.u32 q4, [r4] ; CHECK-NEXT: movs r3, #0 ; CHECK-NEXT: vdup.32 q1, r12 ; CHECK-NEXT: .LBB10_2: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vdup.32 q5, r3 ; CHECK-NEXT: adds r3, #8 ; CHECK-NEXT: vorr q6, q5, q0 ; CHECK-NEXT: vorr q5, q5, q4 ; CHECK-NEXT: vcmp.u32 cs, q1, q6 ; CHECK-NEXT: vpsel q6, q3, q2 ; CHECK-NEXT: vcmp.u32 cs, q1, q5 ; CHECK-NEXT: vpsel q5, q3, q2 ; CHECK-NEXT: vstrh.32 q6, [r5, #8] ; CHECK-NEXT: vstrh.32 q5, [r5] ; CHECK-NEXT: vldrw.u32 q5, [r5] ; CHECK-NEXT: vptt.i16 ne, q5, zr ; CHECK-NEXT: vldrht.u16 q5, [r0], #16 ; CHECK-NEXT: vldrht.u16 q6, [r1], #16 ; CHECK-NEXT: vmullt.s16 q7, q6, q5 ; CHECK-NEXT: vmullb.s16 q5, q6, q5 ; CHECK-NEXT: vqshrnb.s32 q5, q5, #15 ; CHECK-NEXT: vqshrnt.s32 q5, q7, #15 ; CHECK-NEXT: vpst ; CHECK-NEXT: vstrht.16 q5, [r2], #16 ; CHECK-NEXT: le lr, .LBB10_2 ; CHECK-NEXT: .LBB10_3: @ %for.cond.cleanup ; CHECK-NEXT: add sp, #16 ; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15} ; CHECK-NEXT: pop {r4, r5, r7, pc} ; CHECK-NEXT: .p2align 4 ; CHECK-NEXT: @ %bb.4: ; CHECK-NEXT: .LCPI10_0: ; CHECK-NEXT: .long 4 @ 0x4 ; CHECK-NEXT: .long 5 @ 0x5 ; CHECK-NEXT: .long 6 @ 0x6 ; CHECK-NEXT: .long 7 @ 0x7 ; CHECK-NEXT: .LCPI10_1: ; CHECK-NEXT: .long 0 @ 0x0 ; CHECK-NEXT: .long 1 @ 0x1 ; CHECK-NEXT: .long 2 @ 0x2 ; CHECK-NEXT: .long 3 @ 0x3 entry: %cmp8 = icmp eq i32 %N, 0 br i1 %cmp8, label %for.cond.cleanup, label %vector.ph vector.ph: ; preds = %entry %n.rnd.up = add i32 %N, 7 %n.vec = and i32 %n.rnd.up, -8 %trip.count.minus.1 = add i32 %N, -1 %broadcast.splatinsert20 = insertelement <8 x i32> undef, i32 %trip.count.minus.1, i32 0 %broadcast.splat21 = shufflevector <8 x i32> %broadcast.splatinsert20, <8 x i32> undef, <8 x i32> zeroinitializer br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0 %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer %induction = or <8 x i32> %broadcast.splat, %next.gep = getelementptr i16, ptr %pSrcA, i32 %index %next.gep18 = getelementptr i16, ptr %pSrcB, i32 %index %next.gep19 = getelementptr i16, ptr %pDst, i32 %index %0 = icmp ule <8 x i32> %induction, %broadcast.splat21 %wide.masked.load = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %next.gep, i32 2, <8 x i1> %0, <8 x i16> undef) %1 = shufflevector <8 x i16> %wide.masked.load, <8 x i16> %wide.masked.load, <4 x i32> %2 = shufflevector <8 x i16> %wide.masked.load, <8 x i16> %wide.masked.load, <4 x i32> %3 = sext <4 x i16> %1 to <4 x i32> %4 = sext <4 x i16> %2 to <4 x i32> %wide.masked.load22 = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr %next.gep18, i32 2, <8 x i1> %0, <8 x i16> undef) %5 = shufflevector <8 x i16> %wide.masked.load22, <8 x i16> %wide.masked.load22, <4 x i32> %6 = shufflevector <8 x i16> %wide.masked.load22, <8 x i16> %wide.masked.load22, <4 x i32> %7 = sext <4 x i16> %5 to <4 x i32> %8 = sext <4 x i16> %6 to <4 x i32> %9 = mul <4 x i32> %7, %3 %10 = mul <4 x i32> %8, %4 %11 = ashr <4 x i32> %9, %12 = ashr <4 x i32> %10, %13 = icmp sgt <4 x i32> %11, %14 = icmp sgt <4 x i32> %12, %15 = select <4 x i1> %13, <4 x i32> %11, <4 x i32> %16 = select <4 x i1> %14, <4 x i32> %12, <4 x i32> %17 = icmp slt <4 x i32> %15, %18 = icmp slt <4 x i32> %16, %19 = select <4 x i1> %17, <4 x i32> %15, <4 x i32> %20 = select <4 x i1> %18, <4 x i32> %16, <4 x i32> %21 = shufflevector <4 x i32> %19, <4 x i32> %20, <8 x i32> %22 = trunc <8 x i32> %21 to <8 x i16> call void @llvm.masked.store.v8i16.p0(<8 x i16> %22, ptr %next.gep19, i32 2, <8 x i1> %0) %index.next = add i32 %index, 8 %23 = icmp eq i32 %index.next, %n.vec br i1 %23, label %for.cond.cleanup, label %vector.body for.cond.cleanup: ; preds = %vector.body, %entry ret void } define arm_aapcs_vfpcc void @usatmul_4_q15(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: usatmul_4_q15: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r6, lr} ; CHECK-NEXT: push {r4, r5, r6, lr} ; CHECK-NEXT: cmp r3, #0 ; CHECK-NEXT: beq .LBB11_8 ; CHECK-NEXT: @ %bb.1: @ %for.body.preheader ; CHECK-NEXT: cmp r3, #3 ; CHECK-NEXT: bhi .LBB11_3 ; CHECK-NEXT: @ %bb.2: ; CHECK-NEXT: movs r5, #0 ; CHECK-NEXT: mov r12, r0 ; CHECK-NEXT: mov r6, r1 ; CHECK-NEXT: mov r4, r2 ; CHECK-NEXT: b .LBB11_6 ; CHECK-NEXT: .LBB11_3: @ %vector.ph ; CHECK-NEXT: bic r5, r3, #3 ; CHECK-NEXT: movs r4, #1 ; CHECK-NEXT: subs r6, r5, #4 ; CHECK-NEXT: add.w r12, r0, r5, lsl #1 ; CHECK-NEXT: add.w lr, r4, r6, lsr #2 ; CHECK-NEXT: add.w r4, r2, r5, lsl #1 ; CHECK-NEXT: add.w r6, r1, r5, lsl #1 ; CHECK-NEXT: .LBB11_4: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrh.u32 q0, [r0], #8 ; CHECK-NEXT: vldrh.u32 q1, [r1], #8 ; CHECK-NEXT: vmul.i32 q0, q1, q0 ; CHECK-NEXT: vqshrnb.u32 q0, q0, #15 ; CHECK-NEXT: vstrh.32 q0, [r2], #8 ; CHECK-NEXT: le lr, .LBB11_4 ; CHECK-NEXT: @ %bb.5: @ %middle.block ; CHECK-NEXT: cmp r5, r3 ; CHECK-NEXT: it eq ; CHECK-NEXT: popeq {r4, r5, r6, pc} ; CHECK-NEXT: .LBB11_6: @ %for.body.preheader21 ; CHECK-NEXT: sub.w lr, r3, r5 ; CHECK-NEXT: movw r0, #65535 ; CHECK-NEXT: .LBB11_7: @ %for.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldrh r1, [r12], #2 ; CHECK-NEXT: ldrh r2, [r6], #2 ; CHECK-NEXT: muls r1, r2, r1 ; CHECK-NEXT: lsrs r2, r1, #15 ; CHECK-NEXT: cmp r2, r0 ; CHECK-NEXT: movw r2, #65535 ; CHECK-NEXT: it lo ; CHECK-NEXT: lsrlo r2, r1, #15 ; CHECK-NEXT: strh r2, [r4], #2 ; CHECK-NEXT: le lr, .LBB11_7 ; CHECK-NEXT: .LBB11_8: @ %for.cond.cleanup ; CHECK-NEXT: pop {r4, r5, r6, pc} entry: %cmp8 = icmp eq i32 %N, 0 br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader for.body.preheader: ; preds = %entry %min.iters.check = icmp ult i32 %N, 4 br i1 %min.iters.check, label %for.body.preheader21, label %vector.ph for.body.preheader21: ; preds = %middle.block, %for.body.preheader %i.012.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ] %pSrcA.addr.011.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ] %pSrcB.addr.010.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end15, %middle.block ] %pDst.addr.09.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end17, %middle.block ] br label %for.body vector.ph: ; preds = %for.body.preheader %n.vec = and i32 %N, -4 %ind.end = getelementptr i16, ptr %pSrcA, i32 %n.vec %ind.end15 = getelementptr i16, ptr %pSrcB, i32 %n.vec %ind.end17 = getelementptr i16, ptr %pDst, i32 %n.vec br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %next.gep = getelementptr i16, ptr %pSrcA, i32 %index %next.gep18 = getelementptr i16, ptr %pSrcB, i32 %index %next.gep19 = getelementptr i16, ptr %pDst, i32 %index %wide.load = load <4 x i16>, ptr %next.gep, align 2 %0 = zext <4 x i16> %wide.load to <4 x i32> %wide.load20 = load <4 x i16>, ptr %next.gep18, align 2 %1 = zext <4 x i16> %wide.load20 to <4 x i32> %2 = mul nuw <4 x i32> %1, %0 %3 = lshr <4 x i32> %2, %4 = icmp ult <4 x i32> %3, %5 = select <4 x i1> %4, <4 x i32> %3, <4 x i32> %6 = trunc <4 x i32> %5 to <4 x i16> store <4 x i16> %6, ptr %next.gep19, align 2 %index.next = add i32 %index, 4 %7 = icmp eq i32 %index.next, %n.vec br i1 %7, label %middle.block, label %vector.body middle.block: ; preds = %vector.body %cmp.n = icmp eq i32 %n.vec, %N br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader21 for.cond.cleanup: ; preds = %for.body, %middle.block, %entry ret void for.body: ; preds = %for.body.preheader21, %for.body %i.012 = phi i32 [ %inc, %for.body ], [ %i.012.ph, %for.body.preheader21 ] %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.011.ph, %for.body.preheader21 ] %pSrcB.addr.010 = phi ptr [ %incdec.ptr1, %for.body ], [ %pSrcB.addr.010.ph, %for.body.preheader21 ] %pDst.addr.09 = phi ptr [ %incdec.ptr4, %for.body ], [ %pDst.addr.09.ph, %for.body.preheader21 ] %incdec.ptr = getelementptr inbounds i16, ptr %pSrcA.addr.011, i32 1 %8 = load i16, ptr %pSrcA.addr.011, align 2 %conv = zext i16 %8 to i32 %incdec.ptr1 = getelementptr inbounds i16, ptr %pSrcB.addr.010, i32 1 %9 = load i16, ptr %pSrcB.addr.010, align 2 %conv2 = zext i16 %9 to i32 %mul = mul nuw i32 %conv2, %conv %shr = lshr i32 %mul, 15 %10 = icmp ult i32 %shr, 65535 %retval.0.i = select i1 %10, i32 %shr, i32 65535 %conv3 = trunc i32 %retval.0.i to i16 %incdec.ptr4 = getelementptr inbounds i16, ptr %pDst.addr.09, i32 1 store i16 %conv3, ptr %pDst.addr.09, align 2 %inc = add nuw i32 %i.012, 1 %exitcond = icmp eq i32 %inc, %N br i1 %exitcond, label %for.cond.cleanup, label %for.body } define arm_aapcs_vfpcc void @usatmul_8_q15(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: usatmul_8_q15: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r6, lr} ; CHECK-NEXT: push {r4, r5, r6, lr} ; CHECK-NEXT: cmp r3, #0 ; CHECK-NEXT: beq .LBB12_8 ; CHECK-NEXT: @ %bb.1: @ %for.body.preheader ; CHECK-NEXT: cmp r3, #7 ; CHECK-NEXT: bhi .LBB12_3 ; CHECK-NEXT: @ %bb.2: ; CHECK-NEXT: movs r5, #0 ; CHECK-NEXT: mov r12, r0 ; CHECK-NEXT: mov r6, r1 ; CHECK-NEXT: mov r4, r2 ; CHECK-NEXT: b .LBB12_6 ; CHECK-NEXT: .LBB12_3: @ %vector.ph ; CHECK-NEXT: bic r5, r3, #7 ; CHECK-NEXT: movs r4, #1 ; CHECK-NEXT: sub.w r6, r5, #8 ; CHECK-NEXT: add.w r12, r0, r5, lsl #1 ; CHECK-NEXT: add.w lr, r4, r6, lsr #3 ; CHECK-NEXT: add.w r4, r2, r5, lsl #1 ; CHECK-NEXT: add.w r6, r1, r5, lsl #1 ; CHECK-NEXT: .LBB12_4: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrh.u16 q0, [r0], #16 ; CHECK-NEXT: vldrh.u16 q1, [r1], #16 ; CHECK-NEXT: vmullt.u16 q2, q1, q0 ; CHECK-NEXT: vmullb.u16 q0, q1, q0 ; CHECK-NEXT: vqshrnb.u32 q0, q0, #15 ; CHECK-NEXT: vqshrnt.u32 q0, q2, #15 ; CHECK-NEXT: vstrb.8 q0, [r2], #16 ; CHECK-NEXT: le lr, .LBB12_4 ; CHECK-NEXT: @ %bb.5: @ %middle.block ; CHECK-NEXT: cmp r5, r3 ; CHECK-NEXT: it eq ; CHECK-NEXT: popeq {r4, r5, r6, pc} ; CHECK-NEXT: .LBB12_6: @ %for.body.preheader21 ; CHECK-NEXT: sub.w lr, r3, r5 ; CHECK-NEXT: movw r0, #65535 ; CHECK-NEXT: .LBB12_7: @ %for.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldrh r1, [r12], #2 ; CHECK-NEXT: ldrh r2, [r6], #2 ; CHECK-NEXT: muls r1, r2, r1 ; CHECK-NEXT: lsrs r2, r1, #15 ; CHECK-NEXT: cmp r2, r0 ; CHECK-NEXT: movw r2, #65535 ; CHECK-NEXT: it lo ; CHECK-NEXT: lsrlo r2, r1, #15 ; CHECK-NEXT: strh r2, [r4], #2 ; CHECK-NEXT: le lr, .LBB12_7 ; CHECK-NEXT: .LBB12_8: @ %for.cond.cleanup ; CHECK-NEXT: pop {r4, r5, r6, pc} entry: %cmp8 = icmp eq i32 %N, 0 br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader for.body.preheader: ; preds = %entry %min.iters.check = icmp ult i32 %N, 8 br i1 %min.iters.check, label %for.body.preheader21, label %vector.ph for.body.preheader21: ; preds = %middle.block, %for.body.preheader %i.012.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ] %pSrcA.addr.011.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ] %pSrcB.addr.010.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end15, %middle.block ] %pDst.addr.09.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end17, %middle.block ] br label %for.body vector.ph: ; preds = %for.body.preheader %n.vec = and i32 %N, -8 %ind.end = getelementptr i16, ptr %pSrcA, i32 %n.vec %ind.end15 = getelementptr i16, ptr %pSrcB, i32 %n.vec %ind.end17 = getelementptr i16, ptr %pDst, i32 %n.vec br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %next.gep = getelementptr i16, ptr %pSrcA, i32 %index %next.gep18 = getelementptr i16, ptr %pSrcB, i32 %index %next.gep19 = getelementptr i16, ptr %pDst, i32 %index %wide.load = load <8 x i16>, ptr %next.gep, align 2 %0 = zext <8 x i16> %wide.load to <8 x i32> %wide.load20 = load <8 x i16>, ptr %next.gep18, align 2 %1 = zext <8 x i16> %wide.load20 to <8 x i32> %2 = mul nuw <8 x i32> %1, %0 %3 = lshr <8 x i32> %2, %4 = icmp ult <8 x i32> %3, %5 = select <8 x i1> %4, <8 x i32> %3, <8 x i32> %6 = trunc <8 x i32> %5 to <8 x i16> store <8 x i16> %6, ptr %next.gep19, align 2 %index.next = add i32 %index, 8 %7 = icmp eq i32 %index.next, %n.vec br i1 %7, label %middle.block, label %vector.body middle.block: ; preds = %vector.body %cmp.n = icmp eq i32 %n.vec, %N br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader21 for.cond.cleanup: ; preds = %for.body, %middle.block, %entry ret void for.body: ; preds = %for.body.preheader21, %for.body %i.012 = phi i32 [ %inc, %for.body ], [ %i.012.ph, %for.body.preheader21 ] %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.011.ph, %for.body.preheader21 ] %pSrcB.addr.010 = phi ptr [ %incdec.ptr1, %for.body ], [ %pSrcB.addr.010.ph, %for.body.preheader21 ] %pDst.addr.09 = phi ptr [ %incdec.ptr4, %for.body ], [ %pDst.addr.09.ph, %for.body.preheader21 ] %incdec.ptr = getelementptr inbounds i16, ptr %pSrcA.addr.011, i32 1 %8 = load i16, ptr %pSrcA.addr.011, align 2 %conv = zext i16 %8 to i32 %incdec.ptr1 = getelementptr inbounds i16, ptr %pSrcB.addr.010, i32 1 %9 = load i16, ptr %pSrcB.addr.010, align 2 %conv2 = zext i16 %9 to i32 %mul = mul nuw i32 %conv2, %conv %shr = lshr i32 %mul, 15 %10 = icmp ult i32 %shr, 65535 %retval.0.i = select i1 %10, i32 %shr, i32 65535 %conv3 = trunc i32 %retval.0.i to i16 %incdec.ptr4 = getelementptr inbounds i16, ptr %pDst.addr.09, i32 1 store i16 %conv3, ptr %pDst.addr.09, align 2 %inc = add nuw i32 %i.012, 1 %exitcond = icmp eq i32 %inc, %N br i1 %exitcond, label %for.cond.cleanup, label %for.body } ; i8 define arm_aapcs_vfpcc void @ssatmul_4_q7(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: ssatmul_4_q7: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r6, lr} ; CHECK-NEXT: push {r4, r5, r6, lr} ; CHECK-NEXT: cmp r3, #0 ; CHECK-NEXT: beq .LBB13_8 ; CHECK-NEXT: @ %bb.1: @ %for.body.preheader ; CHECK-NEXT: cmp r3, #3 ; CHECK-NEXT: bhi .LBB13_3 ; CHECK-NEXT: @ %bb.2: ; CHECK-NEXT: movs r5, #0 ; CHECK-NEXT: mov r12, r0 ; CHECK-NEXT: mov r6, r1 ; CHECK-NEXT: mov r4, r2 ; CHECK-NEXT: b .LBB13_6 ; CHECK-NEXT: .LBB13_3: @ %vector.ph ; CHECK-NEXT: bic r5, r3, #3 ; CHECK-NEXT: movs r4, #1 ; CHECK-NEXT: subs r6, r5, #4 ; CHECK-NEXT: add.w r12, r0, r5 ; CHECK-NEXT: vmvn.i32 q0, #0x7f ; CHECK-NEXT: vmov.i32 q1, #0x7f ; CHECK-NEXT: add.w lr, r4, r6, lsr #2 ; CHECK-NEXT: adds r4, r2, r5 ; CHECK-NEXT: adds r6, r1, r5 ; CHECK-NEXT: .LBB13_4: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrb.s32 q2, [r0], #4 ; CHECK-NEXT: vldrb.s32 q3, [r1], #4 ; CHECK-NEXT: vmul.i32 q2, q3, q2 ; CHECK-NEXT: vshr.s32 q2, q2, #7 ; CHECK-NEXT: vmax.s32 q2, q2, q0 ; CHECK-NEXT: vmin.s32 q2, q2, q1 ; CHECK-NEXT: vstrb.32 q2, [r2], #4 ; CHECK-NEXT: le lr, .LBB13_4 ; CHECK-NEXT: @ %bb.5: @ %middle.block ; CHECK-NEXT: cmp r5, r3 ; CHECK-NEXT: it eq ; CHECK-NEXT: popeq {r4, r5, r6, pc} ; CHECK-NEXT: .LBB13_6: @ %for.body.preheader21 ; CHECK-NEXT: sub.w lr, r3, r5 ; CHECK-NEXT: .LBB13_7: @ %for.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldrsb r0, [r12], #1 ; CHECK-NEXT: ldrsb r1, [r6], #1 ; CHECK-NEXT: muls r0, r1, r0 ; CHECK-NEXT: ssat r0, #8, r0, asr #7 ; CHECK-NEXT: strb r0, [r4], #1 ; CHECK-NEXT: le lr, .LBB13_7 ; CHECK-NEXT: .LBB13_8: @ %for.cond.cleanup ; CHECK-NEXT: pop {r4, r5, r6, pc} entry: %cmp8 = icmp eq i32 %N, 0 br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader for.body.preheader: ; preds = %entry %min.iters.check = icmp ult i32 %N, 4 br i1 %min.iters.check, label %for.body.preheader21, label %vector.ph for.body.preheader21: ; preds = %middle.block, %for.body.preheader %i.012.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ] %pSrcA.addr.011.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ] %pSrcB.addr.010.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end15, %middle.block ] %pDst.addr.09.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end17, %middle.block ] br label %for.body vector.ph: ; preds = %for.body.preheader %n.vec = and i32 %N, -4 %ind.end = getelementptr i8, ptr %pSrcA, i32 %n.vec %ind.end15 = getelementptr i8, ptr %pSrcB, i32 %n.vec %ind.end17 = getelementptr i8, ptr %pDst, i32 %n.vec br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %next.gep = getelementptr i8, ptr %pSrcA, i32 %index %next.gep18 = getelementptr i8, ptr %pSrcB, i32 %index %next.gep19 = getelementptr i8, ptr %pDst, i32 %index %wide.load = load <4 x i8>, ptr %next.gep, align 1 %0 = sext <4 x i8> %wide.load to <4 x i32> %wide.load20 = load <4 x i8>, ptr %next.gep18, align 1 %1 = sext <4 x i8> %wide.load20 to <4 x i32> %2 = mul nsw <4 x i32> %1, %0 %3 = ashr <4 x i32> %2, %4 = icmp sgt <4 x i32> %3, %5 = select <4 x i1> %4, <4 x i32> %3, <4 x i32> %6 = icmp slt <4 x i32> %5, %7 = select <4 x i1> %6, <4 x i32> %5, <4 x i32> %8 = trunc <4 x i32> %7 to <4 x i8> store <4 x i8> %8, ptr %next.gep19, align 1 %index.next = add i32 %index, 4 %9 = icmp eq i32 %index.next, %n.vec br i1 %9, label %middle.block, label %vector.body middle.block: ; preds = %vector.body %cmp.n = icmp eq i32 %n.vec, %N br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader21 for.cond.cleanup: ; preds = %for.body, %middle.block, %entry ret void for.body: ; preds = %for.body.preheader21, %for.body %i.012 = phi i32 [ %inc, %for.body ], [ %i.012.ph, %for.body.preheader21 ] %pSrcA.addr.011 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.011.ph, %for.body.preheader21 ] %pSrcB.addr.010 = phi ptr [ %incdec.ptr1, %for.body ], [ %pSrcB.addr.010.ph, %for.body.preheader21 ] %pDst.addr.09 = phi ptr [ %incdec.ptr4, %for.body ], [ %pDst.addr.09.ph, %for.body.preheader21 ] %incdec.ptr = getelementptr inbounds i8, ptr %pSrcA.addr.011, i32 1 %10 = load i8, ptr %pSrcA.addr.011, align 1 %conv = sext i8 %10 to i32 %incdec.ptr1 = getelementptr inbounds i8, ptr %pSrcB.addr.010, i32 1 %11 = load i8, ptr %pSrcB.addr.010, align 1 %conv2 = sext i8 %11 to i32 %mul = mul nsw i32 %conv2, %conv %shr = ashr i32 %mul, 7 %12 = icmp sgt i32 %shr, -128 %.val.i = select i1 %12, i32 %shr, i32 -128 %13 = icmp slt i32 %.val.i, 127 %retval.0.i = select i1 %13, i32 %.val.i, i32 127 %conv3 = trunc i32 %retval.0.i to i8 %incdec.ptr4 = getelementptr inbounds i8, ptr %pDst.addr.09, i32 1 store i8 %conv3, ptr %pDst.addr.09, align 1 %inc = add nuw i32 %i.012, 1 %exitcond = icmp eq i32 %inc, %N br i1 %exitcond, label %for.cond.cleanup, label %for.body } define arm_aapcs_vfpcc void @ssatmul_8_q7(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: ssatmul_8_q7: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r6, lr} ; CHECK-NEXT: push {r4, r5, r6, lr} ; CHECK-NEXT: cbz r3, .LBB14_8 ; CHECK-NEXT: @ %bb.1: @ %for.body.preheader ; CHECK-NEXT: cmp r3, #7 ; CHECK-NEXT: bhi .LBB14_3 ; CHECK-NEXT: @ %bb.2: ; CHECK-NEXT: movs r5, #0 ; CHECK-NEXT: mov r12, r0 ; CHECK-NEXT: mov r6, r1 ; CHECK-NEXT: mov r4, r2 ; CHECK-NEXT: b .LBB14_6 ; CHECK-NEXT: .LBB14_3: @ %vector.ph ; CHECK-NEXT: bic r5, r3, #7 ; CHECK-NEXT: movs r4, #1 ; CHECK-NEXT: sub.w r6, r5, #8 ; CHECK-NEXT: add.w r12, r0, r5 ; CHECK-NEXT: add.w lr, r4, r6, lsr #3 ; CHECK-NEXT: adds r4, r2, r5 ; CHECK-NEXT: adds r6, r1, r5 ; CHECK-NEXT: .LBB14_4: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrb.s16 q0, [r0], #8 ; CHECK-NEXT: vldrb.s16 q1, [r1], #8 ; CHECK-NEXT: vmul.i16 q0, q1, q0 ; CHECK-NEXT: vqshrnb.s16 q0, q0, #7 ; CHECK-NEXT: vstrb.16 q0, [r2], #8 ; CHECK-NEXT: le lr, .LBB14_4 ; CHECK-NEXT: @ %bb.5: @ %middle.block ; CHECK-NEXT: cmp r5, r3 ; CHECK-NEXT: it eq ; CHECK-NEXT: popeq {r4, r5, r6, pc} ; CHECK-NEXT: .LBB14_6: @ %for.body.preheader23 ; CHECK-NEXT: sub.w lr, r3, r5 ; CHECK-NEXT: .LBB14_7: @ %for.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldrsb r0, [r12], #1 ; CHECK-NEXT: ldrsb r1, [r6], #1 ; CHECK-NEXT: muls r0, r1, r0 ; CHECK-NEXT: ssat r0, #8, r0, asr #7 ; CHECK-NEXT: strb r0, [r4], #1 ; CHECK-NEXT: le lr, .LBB14_7 ; CHECK-NEXT: .LBB14_8: @ %for.cond.cleanup ; CHECK-NEXT: pop {r4, r5, r6, pc} entry: %cmp10 = icmp eq i32 %N, 0 br i1 %cmp10, label %for.cond.cleanup, label %for.body.preheader for.body.preheader: ; preds = %entry %min.iters.check = icmp ult i32 %N, 8 br i1 %min.iters.check, label %for.body.preheader23, label %vector.ph for.body.preheader23: ; preds = %middle.block, %for.body.preheader %i.014.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ] %pSrcA.addr.013.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ] %pSrcB.addr.012.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end17, %middle.block ] %pDst.addr.011.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end19, %middle.block ] br label %for.body vector.ph: ; preds = %for.body.preheader %n.vec = and i32 %N, -8 %ind.end = getelementptr i8, ptr %pSrcA, i32 %n.vec %ind.end17 = getelementptr i8, ptr %pSrcB, i32 %n.vec %ind.end19 = getelementptr i8, ptr %pDst, i32 %n.vec br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %next.gep = getelementptr i8, ptr %pSrcA, i32 %index %next.gep20 = getelementptr i8, ptr %pSrcB, i32 %index %next.gep21 = getelementptr i8, ptr %pDst, i32 %index %wide.load = load <8 x i8>, ptr %next.gep, align 1 %0 = sext <8 x i8> %wide.load to <8 x i16> %wide.load22 = load <8 x i8>, ptr %next.gep20, align 1 %1 = sext <8 x i8> %wide.load22 to <8 x i16> %2 = mul nsw <8 x i16> %1, %0 %3 = ashr <8 x i16> %2, %4 = icmp sgt <8 x i16> %3, %5 = select <8 x i1> %4, <8 x i16> %3, <8 x i16> %6 = icmp slt <8 x i16> %5, %7 = select <8 x i1> %6, <8 x i16> %5, <8 x i16> %8 = trunc <8 x i16> %7 to <8 x i8> store <8 x i8> %8, ptr %next.gep21, align 1 %index.next = add i32 %index, 8 %9 = icmp eq i32 %index.next, %n.vec br i1 %9, label %middle.block, label %vector.body middle.block: ; preds = %vector.body %cmp.n = icmp eq i32 %n.vec, %N br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader23 for.cond.cleanup: ; preds = %for.body, %middle.block, %entry ret void for.body: ; preds = %for.body.preheader23, %for.body %i.014 = phi i32 [ %inc, %for.body ], [ %i.014.ph, %for.body.preheader23 ] %pSrcA.addr.013 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.013.ph, %for.body.preheader23 ] %pSrcB.addr.012 = phi ptr [ %incdec.ptr2, %for.body ], [ %pSrcB.addr.012.ph, %for.body.preheader23 ] %pDst.addr.011 = phi ptr [ %incdec.ptr6, %for.body ], [ %pDst.addr.011.ph, %for.body.preheader23 ] %incdec.ptr = getelementptr inbounds i8, ptr %pSrcA.addr.013, i32 1 %10 = load i8, ptr %pSrcA.addr.013, align 1 %conv1 = sext i8 %10 to i16 %incdec.ptr2 = getelementptr inbounds i8, ptr %pSrcB.addr.012, i32 1 %11 = load i8, ptr %pSrcB.addr.012, align 1 %conv3 = sext i8 %11 to i16 %mul = mul nsw i16 %conv3, %conv1 %shr = ashr i16 %mul, 7 %12 = icmp sgt i16 %shr, -128 %.val.i = select i1 %12, i16 %shr, i16 -128 %13 = icmp slt i16 %.val.i, 127 %retval.0.i = select i1 %13, i16 %.val.i, i16 127 %conv5 = trunc i16 %retval.0.i to i8 %incdec.ptr6 = getelementptr inbounds i8, ptr %pDst.addr.011, i32 1 store i8 %conv5, ptr %pDst.addr.011, align 1 %inc = add nuw i32 %i.014, 1 %exitcond = icmp eq i32 %inc, %N br i1 %exitcond, label %for.cond.cleanup, label %for.body } define arm_aapcs_vfpcc void @ssatmul_16_q7(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: ssatmul_16_q7: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r6, lr} ; CHECK-NEXT: push {r4, r5, r6, lr} ; CHECK-NEXT: cbz r3, .LBB15_8 ; CHECK-NEXT: @ %bb.1: @ %for.body.preheader ; CHECK-NEXT: cmp r3, #15 ; CHECK-NEXT: bhi .LBB15_3 ; CHECK-NEXT: @ %bb.2: ; CHECK-NEXT: movs r5, #0 ; CHECK-NEXT: mov r12, r0 ; CHECK-NEXT: mov r6, r1 ; CHECK-NEXT: mov r4, r2 ; CHECK-NEXT: b .LBB15_6 ; CHECK-NEXT: .LBB15_3: @ %vector.ph ; CHECK-NEXT: bic r5, r3, #15 ; CHECK-NEXT: movs r4, #1 ; CHECK-NEXT: sub.w r6, r5, #16 ; CHECK-NEXT: add.w r12, r0, r5 ; CHECK-NEXT: add.w lr, r4, r6, lsr #4 ; CHECK-NEXT: adds r4, r2, r5 ; CHECK-NEXT: adds r6, r1, r5 ; CHECK-NEXT: .LBB15_4: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrb.u8 q0, [r0], #16 ; CHECK-NEXT: vldrb.u8 q1, [r1], #16 ; CHECK-NEXT: vmullt.s8 q2, q1, q0 ; CHECK-NEXT: vmullb.s8 q0, q1, q0 ; CHECK-NEXT: vqshrnb.s16 q0, q0, #7 ; CHECK-NEXT: vqshrnt.s16 q0, q2, #7 ; CHECK-NEXT: vstrb.8 q0, [r2], #16 ; CHECK-NEXT: le lr, .LBB15_4 ; CHECK-NEXT: @ %bb.5: @ %middle.block ; CHECK-NEXT: cmp r5, r3 ; CHECK-NEXT: it eq ; CHECK-NEXT: popeq {r4, r5, r6, pc} ; CHECK-NEXT: .LBB15_6: @ %for.body.preheader23 ; CHECK-NEXT: sub.w lr, r3, r5 ; CHECK-NEXT: .LBB15_7: @ %for.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldrsb r0, [r12], #1 ; CHECK-NEXT: ldrsb r1, [r6], #1 ; CHECK-NEXT: muls r0, r1, r0 ; CHECK-NEXT: ssat r0, #8, r0, asr #7 ; CHECK-NEXT: strb r0, [r4], #1 ; CHECK-NEXT: le lr, .LBB15_7 ; CHECK-NEXT: .LBB15_8: @ %for.cond.cleanup ; CHECK-NEXT: pop {r4, r5, r6, pc} entry: %cmp10 = icmp eq i32 %N, 0 br i1 %cmp10, label %for.cond.cleanup, label %for.body.preheader for.body.preheader: ; preds = %entry %min.iters.check = icmp ult i32 %N, 16 br i1 %min.iters.check, label %for.body.preheader23, label %vector.ph for.body.preheader23: ; preds = %middle.block, %for.body.preheader %i.014.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ] %pSrcA.addr.013.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ] %pSrcB.addr.012.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end17, %middle.block ] %pDst.addr.011.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end19, %middle.block ] br label %for.body vector.ph: ; preds = %for.body.preheader %n.vec = and i32 %N, -16 %ind.end = getelementptr i8, ptr %pSrcA, i32 %n.vec %ind.end17 = getelementptr i8, ptr %pSrcB, i32 %n.vec %ind.end19 = getelementptr i8, ptr %pDst, i32 %n.vec br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %next.gep = getelementptr i8, ptr %pSrcA, i32 %index %next.gep20 = getelementptr i8, ptr %pSrcB, i32 %index %next.gep21 = getelementptr i8, ptr %pDst, i32 %index %wide.load = load <16 x i8>, ptr %next.gep, align 1 %0 = sext <16 x i8> %wide.load to <16 x i16> %wide.load22 = load <16 x i8>, ptr %next.gep20, align 1 %1 = sext <16 x i8> %wide.load22 to <16 x i16> %2 = mul nsw <16 x i16> %1, %0 %3 = ashr <16 x i16> %2, %4 = icmp sgt <16 x i16> %3, %5 = select <16 x i1> %4, <16 x i16> %3, <16 x i16> %6 = icmp slt <16 x i16> %5, %7 = select <16 x i1> %6, <16 x i16> %5, <16 x i16> %8 = trunc <16 x i16> %7 to <16 x i8> store <16 x i8> %8, ptr %next.gep21, align 1 %index.next = add i32 %index, 16 %9 = icmp eq i32 %index.next, %n.vec br i1 %9, label %middle.block, label %vector.body middle.block: ; preds = %vector.body %cmp.n = icmp eq i32 %n.vec, %N br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader23 for.cond.cleanup: ; preds = %for.body, %middle.block, %entry ret void for.body: ; preds = %for.body.preheader23, %for.body %i.014 = phi i32 [ %inc, %for.body ], [ %i.014.ph, %for.body.preheader23 ] %pSrcA.addr.013 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.013.ph, %for.body.preheader23 ] %pSrcB.addr.012 = phi ptr [ %incdec.ptr2, %for.body ], [ %pSrcB.addr.012.ph, %for.body.preheader23 ] %pDst.addr.011 = phi ptr [ %incdec.ptr6, %for.body ], [ %pDst.addr.011.ph, %for.body.preheader23 ] %incdec.ptr = getelementptr inbounds i8, ptr %pSrcA.addr.013, i32 1 %10 = load i8, ptr %pSrcA.addr.013, align 1 %conv1 = sext i8 %10 to i16 %incdec.ptr2 = getelementptr inbounds i8, ptr %pSrcB.addr.012, i32 1 %11 = load i8, ptr %pSrcB.addr.012, align 1 %conv3 = sext i8 %11 to i16 %mul = mul nsw i16 %conv3, %conv1 %shr = ashr i16 %mul, 7 %12 = icmp sgt i16 %shr, -128 %.val.i = select i1 %12, i16 %shr, i16 -128 %13 = icmp slt i16 %.val.i, 127 %retval.0.i = select i1 %13, i16 %.val.i, i16 127 %conv5 = trunc i16 %retval.0.i to i8 %incdec.ptr6 = getelementptr inbounds i8, ptr %pDst.addr.011, i32 1 store i8 %conv5, ptr %pDst.addr.011, align 1 %inc = add nuw i32 %i.014, 1 %exitcond = icmp eq i32 %inc, %N br i1 %exitcond, label %for.cond.cleanup, label %for.body } define arm_aapcs_vfpcc void @ssatmul_16i_q7(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: ssatmul_16i_q7: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r6, lr} ; CHECK-NEXT: push {r4, r5, r6, lr} ; CHECK-NEXT: cbz r3, .LBB16_8 ; CHECK-NEXT: @ %bb.1: @ %for.body.preheader ; CHECK-NEXT: cmp r3, #15 ; CHECK-NEXT: bhi .LBB16_3 ; CHECK-NEXT: @ %bb.2: ; CHECK-NEXT: movs r5, #0 ; CHECK-NEXT: mov r12, r0 ; CHECK-NEXT: mov r6, r1 ; CHECK-NEXT: mov r4, r2 ; CHECK-NEXT: b .LBB16_6 ; CHECK-NEXT: .LBB16_3: @ %vector.ph ; CHECK-NEXT: bic r5, r3, #15 ; CHECK-NEXT: movs r4, #1 ; CHECK-NEXT: sub.w r6, r5, #16 ; CHECK-NEXT: add.w r12, r0, r5 ; CHECK-NEXT: add.w lr, r4, r6, lsr #4 ; CHECK-NEXT: adds r4, r2, r5 ; CHECK-NEXT: adds r6, r1, r5 ; CHECK-NEXT: .LBB16_4: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrb.u8 q0, [r0], #16 ; CHECK-NEXT: vldrb.u8 q1, [r1], #16 ; CHECK-NEXT: vmullt.s8 q2, q1, q0 ; CHECK-NEXT: vmullb.s8 q0, q1, q0 ; CHECK-NEXT: vqshrnb.s16 q0, q0, #7 ; CHECK-NEXT: vqshrnt.s16 q0, q2, #7 ; CHECK-NEXT: vstrb.8 q0, [r2], #16 ; CHECK-NEXT: le lr, .LBB16_4 ; CHECK-NEXT: @ %bb.5: @ %middle.block ; CHECK-NEXT: cmp r5, r3 ; CHECK-NEXT: it eq ; CHECK-NEXT: popeq {r4, r5, r6, pc} ; CHECK-NEXT: .LBB16_6: @ %for.body.preheader23 ; CHECK-NEXT: sub.w lr, r3, r5 ; CHECK-NEXT: .LBB16_7: @ %for.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldrsb r0, [r12], #1 ; CHECK-NEXT: ldrsb r1, [r6], #1 ; CHECK-NEXT: muls r0, r1, r0 ; CHECK-NEXT: ssat r0, #8, r0, asr #7 ; CHECK-NEXT: strb r0, [r4], #1 ; CHECK-NEXT: le lr, .LBB16_7 ; CHECK-NEXT: .LBB16_8: @ %for.cond.cleanup ; CHECK-NEXT: pop {r4, r5, r6, pc} entry: %cmp10 = icmp eq i32 %N, 0 br i1 %cmp10, label %for.cond.cleanup, label %for.body.preheader for.body.preheader: ; preds = %entry %min.iters.check = icmp ult i32 %N, 16 br i1 %min.iters.check, label %for.body.preheader23, label %vector.ph for.body.preheader23: ; preds = %middle.block, %for.body.preheader %i.014.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ] %pSrcA.addr.013.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ] %pSrcB.addr.012.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end17, %middle.block ] %pDst.addr.011.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end19, %middle.block ] br label %for.body vector.ph: ; preds = %for.body.preheader %n.vec = and i32 %N, -16 %ind.end = getelementptr i8, ptr %pSrcA, i32 %n.vec %ind.end17 = getelementptr i8, ptr %pSrcB, i32 %n.vec %ind.end19 = getelementptr i8, ptr %pDst, i32 %n.vec br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %next.gep = getelementptr i8, ptr %pSrcA, i32 %index %next.gep20 = getelementptr i8, ptr %pSrcB, i32 %index %next.gep21 = getelementptr i8, ptr %pDst, i32 %index %wide.load = load <16 x i8>, ptr %next.gep, align 1 %0 = shufflevector <16 x i8> %wide.load, <16 x i8> %wide.load, <8 x i32> %1 = shufflevector <16 x i8> %wide.load, <16 x i8> %wide.load, <8 x i32> %2 = sext <8 x i8> %0 to <8 x i16> %3 = sext <8 x i8> %1 to <8 x i16> %wide.load22 = load <16 x i8>, ptr %next.gep20, align 1 %4 = shufflevector <16 x i8> %wide.load22, <16 x i8> %wide.load22, <8 x i32> %5 = shufflevector <16 x i8> %wide.load22, <16 x i8> %wide.load22, <8 x i32> %6 = sext <8 x i8> %4 to <8 x i16> %7 = sext <8 x i8> %5 to <8 x i16> %8 = mul <8 x i16> %6, %2 %9 = mul <8 x i16> %7, %3 %10 = ashr <8 x i16> %8, %11 = ashr <8 x i16> %9, %12 = icmp sgt <8 x i16> %10, %13 = icmp sgt <8 x i16> %11, %14 = select <8 x i1> %12, <8 x i16> %10, <8 x i16> %15 = select <8 x i1> %13, <8 x i16> %11, <8 x i16> %16 = icmp slt <8 x i16> %14, %17 = icmp slt <8 x i16> %15, %18 = select <8 x i1> %16, <8 x i16> %14, <8 x i16> %19 = select <8 x i1> %17, <8 x i16> %15, <8 x i16> %20 = shufflevector <8 x i16> %18, <8 x i16> %19, <16 x i32> %21 = trunc <16 x i16> %20 to <16 x i8> store <16 x i8> %21, ptr %next.gep21, align 1 %index.next = add i32 %index, 16 %22 = icmp eq i32 %index.next, %n.vec br i1 %22, label %middle.block, label %vector.body middle.block: ; preds = %vector.body %cmp.n = icmp eq i32 %n.vec, %N br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader23 for.cond.cleanup: ; preds = %for.body, %middle.block, %entry ret void for.body: ; preds = %for.body, %for.body.preheader23 %i.014 = phi i32 [ %inc, %for.body ], [ %i.014.ph, %for.body.preheader23 ] %pSrcA.addr.013 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.013.ph, %for.body.preheader23 ] %pSrcB.addr.012 = phi ptr [ %incdec.ptr2, %for.body ], [ %pSrcB.addr.012.ph, %for.body.preheader23 ] %pDst.addr.011 = phi ptr [ %incdec.ptr6, %for.body ], [ %pDst.addr.011.ph, %for.body.preheader23 ] %incdec.ptr = getelementptr inbounds i8, ptr %pSrcA.addr.013, i32 1 %23 = load i8, ptr %pSrcA.addr.013, align 1 %conv1 = sext i8 %23 to i16 %incdec.ptr2 = getelementptr inbounds i8, ptr %pSrcB.addr.012, i32 1 %24 = load i8, ptr %pSrcB.addr.012, align 1 %conv3 = sext i8 %24 to i16 %mul = mul nsw i16 %conv3, %conv1 %shr = ashr i16 %mul, 7 %25 = icmp sgt i16 %shr, -128 %.val.i = select i1 %25, i16 %shr, i16 -128 %26 = icmp slt i16 %.val.i, 127 %retval.0.i = select i1 %26, i16 %.val.i, i16 127 %conv5 = trunc i16 %retval.0.i to i8 %incdec.ptr6 = getelementptr inbounds i8, ptr %pDst.addr.011, i32 1 store i8 %conv5, ptr %pDst.addr.011, align 1 %inc = add nuw i32 %i.014, 1 %exitcond = icmp eq i32 %inc, %N br i1 %exitcond, label %for.cond.cleanup, label %for.body } define arm_aapcs_vfpcc void @ssatmul_8t_q7(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: ssatmul_8t_q7: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r7, lr} ; CHECK-NEXT: push {r4, r5, r7, lr} ; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13} ; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13} ; CHECK-NEXT: .pad #16 ; CHECK-NEXT: sub sp, #16 ; CHECK-NEXT: cmp r3, #0 ; CHECK-NEXT: beq .LBB17_3 ; CHECK-NEXT: @ %bb.1: @ %vector.ph ; CHECK-NEXT: adds r4, r3, #7 ; CHECK-NEXT: vmov.i8 q2, #0x0 ; CHECK-NEXT: bic r4, r4, #7 ; CHECK-NEXT: vmov.i8 q3, #0xff ; CHECK-NEXT: sub.w r12, r4, #8 ; CHECK-NEXT: movs r4, #1 ; CHECK-NEXT: mov r5, sp ; CHECK-NEXT: add.w lr, r4, r12, lsr #3 ; CHECK-NEXT: adr r4, .LCPI17_0 ; CHECK-NEXT: vldrw.u32 q0, [r4] ; CHECK-NEXT: adr r4, .LCPI17_1 ; CHECK-NEXT: sub.w r12, r3, #1 ; CHECK-NEXT: vldrw.u32 q4, [r4] ; CHECK-NEXT: movs r3, #0 ; CHECK-NEXT: vdup.32 q1, r12 ; CHECK-NEXT: .LBB17_2: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vdup.32 q5, r3 ; CHECK-NEXT: adds r3, #8 ; CHECK-NEXT: vorr q6, q5, q0 ; CHECK-NEXT: vorr q5, q5, q4 ; CHECK-NEXT: vcmp.u32 cs, q1, q6 ; CHECK-NEXT: vpsel q6, q3, q2 ; CHECK-NEXT: vcmp.u32 cs, q1, q5 ; CHECK-NEXT: vpsel q5, q3, q2 ; CHECK-NEXT: vstrh.32 q6, [r5, #8] ; CHECK-NEXT: vstrh.32 q5, [r5] ; CHECK-NEXT: vldrw.u32 q5, [r5] ; CHECK-NEXT: vptt.i16 ne, q5, zr ; CHECK-NEXT: vldrbt.s16 q5, [r0], #8 ; CHECK-NEXT: vldrbt.s16 q6, [r1], #8 ; CHECK-NEXT: vmul.i16 q5, q6, q5 ; CHECK-NEXT: vqshrnb.s16 q5, q5, #7 ; CHECK-NEXT: vpst ; CHECK-NEXT: vstrbt.16 q5, [r2], #8 ; CHECK-NEXT: le lr, .LBB17_2 ; CHECK-NEXT: .LBB17_3: @ %for.cond.cleanup ; CHECK-NEXT: add sp, #16 ; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13} ; CHECK-NEXT: pop {r4, r5, r7, pc} ; CHECK-NEXT: .p2align 4 ; CHECK-NEXT: @ %bb.4: ; CHECK-NEXT: .LCPI17_0: ; CHECK-NEXT: .long 4 @ 0x4 ; CHECK-NEXT: .long 5 @ 0x5 ; CHECK-NEXT: .long 6 @ 0x6 ; CHECK-NEXT: .long 7 @ 0x7 ; CHECK-NEXT: .LCPI17_1: ; CHECK-NEXT: .long 0 @ 0x0 ; CHECK-NEXT: .long 1 @ 0x1 ; CHECK-NEXT: .long 2 @ 0x2 ; CHECK-NEXT: .long 3 @ 0x3 entry: %cmp10 = icmp eq i32 %N, 0 br i1 %cmp10, label %for.cond.cleanup, label %vector.ph vector.ph: ; preds = %entry %n.rnd.up = add i32 %N, 7 %n.vec = and i32 %n.rnd.up, -8 %trip.count.minus.1 = add i32 %N, -1 %broadcast.splatinsert22 = insertelement <8 x i32> undef, i32 %trip.count.minus.1, i32 0 %broadcast.splat23 = shufflevector <8 x i32> %broadcast.splatinsert22, <8 x i32> undef, <8 x i32> zeroinitializer br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %index, i32 0 %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer %induction = or <8 x i32> %broadcast.splat, %next.gep = getelementptr i8, ptr %pSrcA, i32 %index %next.gep20 = getelementptr i8, ptr %pSrcB, i32 %index %next.gep21 = getelementptr i8, ptr %pDst, i32 %index %0 = icmp ule <8 x i32> %induction, %broadcast.splat23 %wide.masked.load = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr %next.gep, i32 1, <8 x i1> %0, <8 x i8> undef) %1 = sext <8 x i8> %wide.masked.load to <8 x i16> %wide.masked.load24 = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr %next.gep20, i32 1, <8 x i1> %0, <8 x i8> undef) %2 = sext <8 x i8> %wide.masked.load24 to <8 x i16> %3 = mul nsw <8 x i16> %2, %1 %4 = ashr <8 x i16> %3, %5 = icmp sgt <8 x i16> %4, %6 = select <8 x i1> %5, <8 x i16> %4, <8 x i16> %7 = icmp slt <8 x i16> %6, %8 = select <8 x i1> %7, <8 x i16> %6, <8 x i16> %9 = trunc <8 x i16> %8 to <8 x i8> call void @llvm.masked.store.v8i8.p0(<8 x i8> %9, ptr %next.gep21, i32 1, <8 x i1> %0) %index.next = add i32 %index, 8 %10 = icmp eq i32 %index.next, %n.vec br i1 %10, label %for.cond.cleanup, label %vector.body for.cond.cleanup: ; preds = %vector.body, %entry ret void } define arm_aapcs_vfpcc void @ssatmul_16t_q7(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: ssatmul_16t_q7: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r6, lr} ; CHECK-NEXT: push {r4, r5, r6, lr} ; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15} ; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15} ; CHECK-NEXT: .pad #80 ; CHECK-NEXT: sub sp, #80 ; CHECK-NEXT: cmp r3, #0 ; CHECK-NEXT: beq .LBB18_3 ; CHECK-NEXT: @ %bb.1: @ %vector.ph ; CHECK-NEXT: add.w r6, r3, #15 ; CHECK-NEXT: movs r5, #1 ; CHECK-NEXT: bic r6, r6, #15 ; CHECK-NEXT: add r4, sp, #48 ; CHECK-NEXT: subs r6, #16 ; CHECK-NEXT: vmov.i8 q2, #0x0 ; CHECK-NEXT: vmov.i8 q3, #0xff ; CHECK-NEXT: add.w lr, r5, r6, lsr #4 ; CHECK-NEXT: adr r5, .LCPI18_0 ; CHECK-NEXT: subs r6, r3, #1 ; CHECK-NEXT: vldrw.u32 q0, [r5] ; CHECK-NEXT: vdup.32 q1, r6 ; CHECK-NEXT: adr r6, .LCPI18_1 ; CHECK-NEXT: vstrw.32 q0, [sp, #16] @ 16-byte Spill ; CHECK-NEXT: vldrw.u32 q0, [r6] ; CHECK-NEXT: adr r6, .LCPI18_2 ; CHECK-NEXT: vldrw.u32 q5, [r6] ; CHECK-NEXT: adr r6, .LCPI18_3 ; CHECK-NEXT: vldrw.u32 q6, [r6] ; CHECK-NEXT: add r5, sp, #32 ; CHECK-NEXT: add r6, sp, #64 ; CHECK-NEXT: movs r3, #0 ; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill ; CHECK-NEXT: .LBB18_2: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrw.u32 q0, [sp, #16] @ 16-byte Reload ; CHECK-NEXT: vdup.32 q7, r3 ; CHECK-NEXT: adds r3, #16 ; CHECK-NEXT: vorr q0, q7, q0 ; CHECK-NEXT: vcmp.u32 cs, q1, q0 ; CHECK-NEXT: vpsel q0, q3, q2 ; CHECK-NEXT: vstrh.32 q0, [r4, #8] ; CHECK-NEXT: vldrw.u32 q0, [sp] @ 16-byte Reload ; CHECK-NEXT: vorr q0, q7, q0 ; CHECK-NEXT: vcmp.u32 cs, q1, q0 ; CHECK-NEXT: vpsel q0, q3, q2 ; CHECK-NEXT: vstrh.32 q0, [r4] ; CHECK-NEXT: vorr q0, q7, q5 ; CHECK-NEXT: vcmp.u32 cs, q1, q0 ; CHECK-NEXT: vpsel q0, q3, q2 ; CHECK-NEXT: vstrh.32 q0, [r5, #8] ; CHECK-NEXT: vorr q0, q7, q6 ; CHECK-NEXT: vcmp.u32 cs, q1, q0 ; CHECK-NEXT: vpsel q0, q3, q2 ; CHECK-NEXT: vstrh.32 q0, [r5] ; CHECK-NEXT: vldrw.u32 q0, [r4] ; CHECK-NEXT: vcmp.i16 ne, q0, zr ; CHECK-NEXT: vpsel q0, q3, q2 ; CHECK-NEXT: vstrb.16 q0, [r6, #8] ; CHECK-NEXT: vldrw.u32 q0, [r5] ; CHECK-NEXT: vcmp.i16 ne, q0, zr ; CHECK-NEXT: vpsel q0, q3, q2 ; CHECK-NEXT: vstrb.16 q0, [r6] ; CHECK-NEXT: vldrw.u32 q0, [r6] ; CHECK-NEXT: vptt.i8 ne, q0, zr ; CHECK-NEXT: vldrbt.u8 q0, [r0], #16 ; CHECK-NEXT: vldrbt.u8 q7, [r1], #16 ; CHECK-NEXT: vmullt.s8 q4, q7, q0 ; CHECK-NEXT: vmullb.s8 q0, q7, q0 ; CHECK-NEXT: vqshrnb.s16 q0, q0, #7 ; CHECK-NEXT: vqshrnt.s16 q0, q4, #7 ; CHECK-NEXT: vpst ; CHECK-NEXT: vstrbt.8 q0, [r2], #16 ; CHECK-NEXT: le lr, .LBB18_2 ; CHECK-NEXT: .LBB18_3: @ %for.cond.cleanup ; CHECK-NEXT: add sp, #80 ; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15} ; CHECK-NEXT: pop {r4, r5, r6, pc} ; CHECK-NEXT: .p2align 4 ; CHECK-NEXT: @ %bb.4: ; CHECK-NEXT: .LCPI18_0: ; CHECK-NEXT: .long 12 @ 0xc ; CHECK-NEXT: .long 13 @ 0xd ; CHECK-NEXT: .long 14 @ 0xe ; CHECK-NEXT: .long 15 @ 0xf ; CHECK-NEXT: .LCPI18_1: ; CHECK-NEXT: .long 8 @ 0x8 ; CHECK-NEXT: .long 9 @ 0x9 ; CHECK-NEXT: .long 10 @ 0xa ; CHECK-NEXT: .long 11 @ 0xb ; CHECK-NEXT: .LCPI18_2: ; CHECK-NEXT: .long 4 @ 0x4 ; CHECK-NEXT: .long 5 @ 0x5 ; CHECK-NEXT: .long 6 @ 0x6 ; CHECK-NEXT: .long 7 @ 0x7 ; CHECK-NEXT: .LCPI18_3: ; CHECK-NEXT: .long 0 @ 0x0 ; CHECK-NEXT: .long 1 @ 0x1 ; CHECK-NEXT: .long 2 @ 0x2 ; CHECK-NEXT: .long 3 @ 0x3 entry: %cmp10 = icmp eq i32 %N, 0 br i1 %cmp10, label %for.cond.cleanup, label %vector.ph vector.ph: ; preds = %entry %n.rnd.up = add i32 %N, 15 %n.vec = and i32 %n.rnd.up, -16 %trip.count.minus.1 = add i32 %N, -1 %broadcast.splatinsert22 = insertelement <16 x i32> undef, i32 %trip.count.minus.1, i32 0 %broadcast.splat23 = shufflevector <16 x i32> %broadcast.splatinsert22, <16 x i32> undef, <16 x i32> zeroinitializer br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %broadcast.splatinsert = insertelement <16 x i32> undef, i32 %index, i32 0 %broadcast.splat = shufflevector <16 x i32> %broadcast.splatinsert, <16 x i32> undef, <16 x i32> zeroinitializer %induction = or <16 x i32> %broadcast.splat, %next.gep = getelementptr i8, ptr %pSrcA, i32 %index %next.gep20 = getelementptr i8, ptr %pSrcB, i32 %index %next.gep21 = getelementptr i8, ptr %pDst, i32 %index %0 = icmp ule <16 x i32> %induction, %broadcast.splat23 %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %next.gep, i32 1, <16 x i1> %0, <16 x i8> undef) %1 = sext <16 x i8> %wide.masked.load to <16 x i16> %wide.masked.load24 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %next.gep20, i32 1, <16 x i1> %0, <16 x i8> undef) %2 = sext <16 x i8> %wide.masked.load24 to <16 x i16> %3 = mul nsw <16 x i16> %2, %1 %4 = ashr <16 x i16> %3, %5 = icmp sgt <16 x i16> %4, %6 = select <16 x i1> %5, <16 x i16> %4, <16 x i16> %7 = icmp slt <16 x i16> %6, %8 = select <16 x i1> %7, <16 x i16> %6, <16 x i16> %9 = trunc <16 x i16> %8 to <16 x i8> call void @llvm.masked.store.v16i8.p0(<16 x i8> %9, ptr %next.gep21, i32 1, <16 x i1> %0) %index.next = add i32 %index, 16 %10 = icmp eq i32 %index.next, %n.vec br i1 %10, label %for.cond.cleanup, label %vector.body for.cond.cleanup: ; preds = %vector.body, %entry ret void } define arm_aapcs_vfpcc void @ssatmul_16ti_q7(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: ssatmul_16ti_q7: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r6, lr} ; CHECK-NEXT: push {r4, r5, r6, lr} ; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15} ; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15} ; CHECK-NEXT: .pad #80 ; CHECK-NEXT: sub sp, #80 ; CHECK-NEXT: cmp r3, #0 ; CHECK-NEXT: beq .LBB19_3 ; CHECK-NEXT: @ %bb.1: @ %vector.ph ; CHECK-NEXT: add.w r6, r3, #15 ; CHECK-NEXT: movs r5, #1 ; CHECK-NEXT: bic r6, r6, #15 ; CHECK-NEXT: add r4, sp, #48 ; CHECK-NEXT: subs r6, #16 ; CHECK-NEXT: vmov.i8 q2, #0x0 ; CHECK-NEXT: vmov.i8 q3, #0xff ; CHECK-NEXT: add.w lr, r5, r6, lsr #4 ; CHECK-NEXT: adr r5, .LCPI19_0 ; CHECK-NEXT: subs r6, r3, #1 ; CHECK-NEXT: vldrw.u32 q0, [r5] ; CHECK-NEXT: vdup.32 q1, r6 ; CHECK-NEXT: adr r6, .LCPI19_1 ; CHECK-NEXT: vstrw.32 q0, [sp, #16] @ 16-byte Spill ; CHECK-NEXT: vldrw.u32 q0, [r6] ; CHECK-NEXT: adr r6, .LCPI19_2 ; CHECK-NEXT: vldrw.u32 q5, [r6] ; CHECK-NEXT: adr r6, .LCPI19_3 ; CHECK-NEXT: vldrw.u32 q6, [r6] ; CHECK-NEXT: add r5, sp, #32 ; CHECK-NEXT: add r6, sp, #64 ; CHECK-NEXT: movs r3, #0 ; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill ; CHECK-NEXT: .LBB19_2: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrw.u32 q0, [sp, #16] @ 16-byte Reload ; CHECK-NEXT: vdup.32 q7, r3 ; CHECK-NEXT: adds r3, #16 ; CHECK-NEXT: vorr q0, q7, q0 ; CHECK-NEXT: vcmp.u32 cs, q1, q0 ; CHECK-NEXT: vpsel q0, q3, q2 ; CHECK-NEXT: vstrh.32 q0, [r4, #8] ; CHECK-NEXT: vldrw.u32 q0, [sp] @ 16-byte Reload ; CHECK-NEXT: vorr q0, q7, q0 ; CHECK-NEXT: vcmp.u32 cs, q1, q0 ; CHECK-NEXT: vpsel q0, q3, q2 ; CHECK-NEXT: vstrh.32 q0, [r4] ; CHECK-NEXT: vorr q0, q7, q5 ; CHECK-NEXT: vcmp.u32 cs, q1, q0 ; CHECK-NEXT: vpsel q0, q3, q2 ; CHECK-NEXT: vstrh.32 q0, [r5, #8] ; CHECK-NEXT: vorr q0, q7, q6 ; CHECK-NEXT: vcmp.u32 cs, q1, q0 ; CHECK-NEXT: vpsel q0, q3, q2 ; CHECK-NEXT: vstrh.32 q0, [r5] ; CHECK-NEXT: vldrw.u32 q0, [r4] ; CHECK-NEXT: vcmp.i16 ne, q0, zr ; CHECK-NEXT: vpsel q0, q3, q2 ; CHECK-NEXT: vstrb.16 q0, [r6, #8] ; CHECK-NEXT: vldrw.u32 q0, [r5] ; CHECK-NEXT: vcmp.i16 ne, q0, zr ; CHECK-NEXT: vpsel q0, q3, q2 ; CHECK-NEXT: vstrb.16 q0, [r6] ; CHECK-NEXT: vldrw.u32 q0, [r6] ; CHECK-NEXT: vptt.i8 ne, q0, zr ; CHECK-NEXT: vldrbt.u8 q0, [r0], #16 ; CHECK-NEXT: vldrbt.u8 q7, [r1], #16 ; CHECK-NEXT: vmullt.s8 q4, q7, q0 ; CHECK-NEXT: vmullb.s8 q0, q7, q0 ; CHECK-NEXT: vqshrnb.s16 q0, q0, #7 ; CHECK-NEXT: vqshrnt.s16 q0, q4, #7 ; CHECK-NEXT: vpst ; CHECK-NEXT: vstrbt.8 q0, [r2], #16 ; CHECK-NEXT: le lr, .LBB19_2 ; CHECK-NEXT: .LBB19_3: @ %for.cond.cleanup ; CHECK-NEXT: add sp, #80 ; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15} ; CHECK-NEXT: pop {r4, r5, r6, pc} ; CHECK-NEXT: .p2align 4 ; CHECK-NEXT: @ %bb.4: ; CHECK-NEXT: .LCPI19_0: ; CHECK-NEXT: .long 12 @ 0xc ; CHECK-NEXT: .long 13 @ 0xd ; CHECK-NEXT: .long 14 @ 0xe ; CHECK-NEXT: .long 15 @ 0xf ; CHECK-NEXT: .LCPI19_1: ; CHECK-NEXT: .long 8 @ 0x8 ; CHECK-NEXT: .long 9 @ 0x9 ; CHECK-NEXT: .long 10 @ 0xa ; CHECK-NEXT: .long 11 @ 0xb ; CHECK-NEXT: .LCPI19_2: ; CHECK-NEXT: .long 4 @ 0x4 ; CHECK-NEXT: .long 5 @ 0x5 ; CHECK-NEXT: .long 6 @ 0x6 ; CHECK-NEXT: .long 7 @ 0x7 ; CHECK-NEXT: .LCPI19_3: ; CHECK-NEXT: .long 0 @ 0x0 ; CHECK-NEXT: .long 1 @ 0x1 ; CHECK-NEXT: .long 2 @ 0x2 ; CHECK-NEXT: .long 3 @ 0x3 entry: %cmp10 = icmp eq i32 %N, 0 br i1 %cmp10, label %for.cond.cleanup, label %vector.ph vector.ph: ; preds = %entry %n.rnd.up = add i32 %N, 15 %n.vec = and i32 %n.rnd.up, -16 %trip.count.minus.1 = add i32 %N, -1 %broadcast.splatinsert22 = insertelement <16 x i32> undef, i32 %trip.count.minus.1, i32 0 %broadcast.splat23 = shufflevector <16 x i32> %broadcast.splatinsert22, <16 x i32> undef, <16 x i32> zeroinitializer br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %broadcast.splatinsert = insertelement <16 x i32> undef, i32 %index, i32 0 %broadcast.splat = shufflevector <16 x i32> %broadcast.splatinsert, <16 x i32> undef, <16 x i32> zeroinitializer %induction = or <16 x i32> %broadcast.splat, %next.gep = getelementptr i8, ptr %pSrcA, i32 %index %next.gep20 = getelementptr i8, ptr %pSrcB, i32 %index %next.gep21 = getelementptr i8, ptr %pDst, i32 %index %0 = icmp ule <16 x i32> %induction, %broadcast.splat23 %wide.masked.load = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %next.gep, i32 1, <16 x i1> %0, <16 x i8> undef) %1 = shufflevector <16 x i8> %wide.masked.load, <16 x i8> %wide.masked.load, <8 x i32> %2 = shufflevector <16 x i8> %wide.masked.load, <16 x i8> %wide.masked.load, <8 x i32> %3 = sext <8 x i8> %1 to <8 x i16> %4 = sext <8 x i8> %2 to <8 x i16> %wide.masked.load24 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr %next.gep20, i32 1, <16 x i1> %0, <16 x i8> undef) %5 = shufflevector <16 x i8> %wide.masked.load24, <16 x i8> %wide.masked.load24, <8 x i32> %6 = shufflevector <16 x i8> %wide.masked.load24, <16 x i8> %wide.masked.load24, <8 x i32> %7 = sext <8 x i8> %5 to <8 x i16> %8 = sext <8 x i8> %6 to <8 x i16> %9 = mul <8 x i16> %7, %3 %10 = mul <8 x i16> %8, %4 %11 = ashr <8 x i16> %9, %12 = ashr <8 x i16> %10, %13 = icmp sgt <8 x i16> %11, %14 = icmp sgt <8 x i16> %12, %15 = select <8 x i1> %13, <8 x i16> %11, <8 x i16> %16 = select <8 x i1> %14, <8 x i16> %12, <8 x i16> %17 = icmp slt <8 x i16> %15, %18 = icmp slt <8 x i16> %16, %19 = select <8 x i1> %17, <8 x i16> %15, <8 x i16> %20 = select <8 x i1> %18, <8 x i16> %16, <8 x i16> %21 = shufflevector <8 x i16> %19, <8 x i16> %20, <16 x i32> %22 = trunc <16 x i16> %21 to <16 x i8> call void @llvm.masked.store.v16i8.p0(<16 x i8> %22, ptr %next.gep21, i32 1, <16 x i1> %0) %index.next = add i32 %index, 16 %23 = icmp eq i32 %index.next, %n.vec br i1 %23, label %for.cond.cleanup, label %vector.body for.cond.cleanup: ; preds = %vector.body, %entry ret void } define arm_aapcs_vfpcc void @usatmul_8_q7(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: usatmul_8_q7: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r6, lr} ; CHECK-NEXT: push {r4, r5, r6, lr} ; CHECK-NEXT: cbz r3, .LBB20_8 ; CHECK-NEXT: @ %bb.1: @ %for.body.preheader ; CHECK-NEXT: cmp r3, #7 ; CHECK-NEXT: bhi .LBB20_3 ; CHECK-NEXT: @ %bb.2: ; CHECK-NEXT: movs r5, #0 ; CHECK-NEXT: mov r12, r0 ; CHECK-NEXT: mov r6, r1 ; CHECK-NEXT: mov r4, r2 ; CHECK-NEXT: b .LBB20_6 ; CHECK-NEXT: .LBB20_3: @ %vector.ph ; CHECK-NEXT: bic r5, r3, #7 ; CHECK-NEXT: movs r4, #1 ; CHECK-NEXT: sub.w r6, r5, #8 ; CHECK-NEXT: add.w r12, r0, r5 ; CHECK-NEXT: add.w lr, r4, r6, lsr #3 ; CHECK-NEXT: adds r4, r2, r5 ; CHECK-NEXT: adds r6, r1, r5 ; CHECK-NEXT: .LBB20_4: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrb.u16 q0, [r0], #8 ; CHECK-NEXT: vldrb.u16 q1, [r1], #8 ; CHECK-NEXT: vmul.i16 q0, q1, q0 ; CHECK-NEXT: vqshrnb.u16 q0, q0, #7 ; CHECK-NEXT: vstrb.16 q0, [r2], #8 ; CHECK-NEXT: le lr, .LBB20_4 ; CHECK-NEXT: @ %bb.5: @ %middle.block ; CHECK-NEXT: cmp r5, r3 ; CHECK-NEXT: it eq ; CHECK-NEXT: popeq {r4, r5, r6, pc} ; CHECK-NEXT: .LBB20_6: @ %for.body.preheader23 ; CHECK-NEXT: sub.w lr, r3, r5 ; CHECK-NEXT: .LBB20_7: @ %for.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldrb r0, [r12], #1 ; CHECK-NEXT: ldrb r1, [r6], #1 ; CHECK-NEXT: muls r0, r1, r0 ; CHECK-NEXT: lsrs r1, r0, #7 ; CHECK-NEXT: cmp r1, #255 ; CHECK-NEXT: mov.w r1, #255 ; CHECK-NEXT: it lo ; CHECK-NEXT: lsrlo r1, r0, #7 ; CHECK-NEXT: strb r1, [r4], #1 ; CHECK-NEXT: le lr, .LBB20_7 ; CHECK-NEXT: .LBB20_8: @ %for.cond.cleanup ; CHECK-NEXT: pop {r4, r5, r6, pc} entry: %cmp10 = icmp eq i32 %N, 0 br i1 %cmp10, label %for.cond.cleanup, label %for.body.preheader for.body.preheader: ; preds = %entry %min.iters.check = icmp ult i32 %N, 8 br i1 %min.iters.check, label %for.body.preheader23, label %vector.ph for.body.preheader23: ; preds = %middle.block, %for.body.preheader %i.014.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ] %pSrcA.addr.013.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ] %pSrcB.addr.012.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end17, %middle.block ] %pDst.addr.011.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end19, %middle.block ] br label %for.body vector.ph: ; preds = %for.body.preheader %n.vec = and i32 %N, -8 %ind.end = getelementptr i8, ptr %pSrcA, i32 %n.vec %ind.end17 = getelementptr i8, ptr %pSrcB, i32 %n.vec %ind.end19 = getelementptr i8, ptr %pDst, i32 %n.vec br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %next.gep = getelementptr i8, ptr %pSrcA, i32 %index %next.gep20 = getelementptr i8, ptr %pSrcB, i32 %index %next.gep21 = getelementptr i8, ptr %pDst, i32 %index %wide.load = load <8 x i8>, ptr %next.gep, align 1 %0 = zext <8 x i8> %wide.load to <8 x i16> %wide.load22 = load <8 x i8>, ptr %next.gep20, align 1 %1 = zext <8 x i8> %wide.load22 to <8 x i16> %2 = mul nuw <8 x i16> %1, %0 %3 = lshr <8 x i16> %2, %4 = icmp ult <8 x i16> %3, %5 = select <8 x i1> %4, <8 x i16> %3, <8 x i16> %6 = trunc <8 x i16> %5 to <8 x i8> store <8 x i8> %6, ptr %next.gep21, align 1 %index.next = add i32 %index, 8 %7 = icmp eq i32 %index.next, %n.vec br i1 %7, label %middle.block, label %vector.body middle.block: ; preds = %vector.body %cmp.n = icmp eq i32 %n.vec, %N br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader23 for.cond.cleanup: ; preds = %for.body, %middle.block, %entry ret void for.body: ; preds = %for.body.preheader23, %for.body %i.014 = phi i32 [ %inc, %for.body ], [ %i.014.ph, %for.body.preheader23 ] %pSrcA.addr.013 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.013.ph, %for.body.preheader23 ] %pSrcB.addr.012 = phi ptr [ %incdec.ptr2, %for.body ], [ %pSrcB.addr.012.ph, %for.body.preheader23 ] %pDst.addr.011 = phi ptr [ %incdec.ptr6, %for.body ], [ %pDst.addr.011.ph, %for.body.preheader23 ] %incdec.ptr = getelementptr inbounds i8, ptr %pSrcA.addr.013, i32 1 %8 = load i8, ptr %pSrcA.addr.013, align 1 %conv1 = zext i8 %8 to i16 %incdec.ptr2 = getelementptr inbounds i8, ptr %pSrcB.addr.012, i32 1 %9 = load i8, ptr %pSrcB.addr.012, align 1 %conv3 = zext i8 %9 to i16 %mul = mul nuw i16 %conv3, %conv1 %10 = lshr i16 %mul, 7 %11 = icmp ult i16 %10, 255 %retval.0.i = select i1 %11, i16 %10, i16 255 %conv5 = trunc i16 %retval.0.i to i8 %incdec.ptr6 = getelementptr inbounds i8, ptr %pDst.addr.011, i32 1 store i8 %conv5, ptr %pDst.addr.011, align 1 %inc = add nuw i32 %i.014, 1 %exitcond = icmp eq i32 %inc, %N br i1 %exitcond, label %for.cond.cleanup, label %for.body } define arm_aapcs_vfpcc void @usatmul_16_q7(ptr nocapture readonly %pSrcA, ptr nocapture readonly %pSrcB, ptr noalias nocapture %pDst, i32 %N) { ; CHECK-LABEL: usatmul_16_q7: ; CHECK: @ %bb.0: @ %entry ; CHECK-NEXT: .save {r4, r5, r6, lr} ; CHECK-NEXT: push {r4, r5, r6, lr} ; CHECK-NEXT: cmp r3, #0 ; CHECK-NEXT: beq .LBB21_8 ; CHECK-NEXT: @ %bb.1: @ %for.body.preheader ; CHECK-NEXT: cmp r3, #15 ; CHECK-NEXT: bhi .LBB21_3 ; CHECK-NEXT: @ %bb.2: ; CHECK-NEXT: movs r5, #0 ; CHECK-NEXT: mov r12, r0 ; CHECK-NEXT: mov r6, r1 ; CHECK-NEXT: mov r4, r2 ; CHECK-NEXT: b .LBB21_6 ; CHECK-NEXT: .LBB21_3: @ %vector.ph ; CHECK-NEXT: bic r5, r3, #15 ; CHECK-NEXT: movs r4, #1 ; CHECK-NEXT: sub.w r6, r5, #16 ; CHECK-NEXT: add.w r12, r0, r5 ; CHECK-NEXT: add.w lr, r4, r6, lsr #4 ; CHECK-NEXT: adds r4, r2, r5 ; CHECK-NEXT: adds r6, r1, r5 ; CHECK-NEXT: .LBB21_4: @ %vector.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: vldrb.u8 q0, [r0], #16 ; CHECK-NEXT: vldrb.u8 q1, [r1], #16 ; CHECK-NEXT: vmullt.u8 q2, q1, q0 ; CHECK-NEXT: vmullb.u8 q0, q1, q0 ; CHECK-NEXT: vqshrnb.u16 q0, q0, #7 ; CHECK-NEXT: vqshrnt.u16 q0, q2, #7 ; CHECK-NEXT: vstrb.8 q0, [r2], #16 ; CHECK-NEXT: le lr, .LBB21_4 ; CHECK-NEXT: @ %bb.5: @ %middle.block ; CHECK-NEXT: cmp r5, r3 ; CHECK-NEXT: it eq ; CHECK-NEXT: popeq {r4, r5, r6, pc} ; CHECK-NEXT: .LBB21_6: @ %for.body.preheader23 ; CHECK-NEXT: sub.w lr, r3, r5 ; CHECK-NEXT: .LBB21_7: @ %for.body ; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldrb r0, [r12], #1 ; CHECK-NEXT: ldrb r1, [r6], #1 ; CHECK-NEXT: muls r0, r1, r0 ; CHECK-NEXT: lsrs r1, r0, #7 ; CHECK-NEXT: cmp r1, #255 ; CHECK-NEXT: mov.w r1, #255 ; CHECK-NEXT: it lo ; CHECK-NEXT: lsrlo r1, r0, #7 ; CHECK-NEXT: strb r1, [r4], #1 ; CHECK-NEXT: le lr, .LBB21_7 ; CHECK-NEXT: .LBB21_8: @ %for.cond.cleanup ; CHECK-NEXT: pop {r4, r5, r6, pc} entry: %cmp10 = icmp eq i32 %N, 0 br i1 %cmp10, label %for.cond.cleanup, label %for.body.preheader for.body.preheader: ; preds = %entry %min.iters.check = icmp ult i32 %N, 16 br i1 %min.iters.check, label %for.body.preheader23, label %vector.ph for.body.preheader23: ; preds = %middle.block, %for.body.preheader %i.014.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ] %pSrcA.addr.013.ph = phi ptr [ %pSrcA, %for.body.preheader ], [ %ind.end, %middle.block ] %pSrcB.addr.012.ph = phi ptr [ %pSrcB, %for.body.preheader ], [ %ind.end17, %middle.block ] %pDst.addr.011.ph = phi ptr [ %pDst, %for.body.preheader ], [ %ind.end19, %middle.block ] br label %for.body vector.ph: ; preds = %for.body.preheader %n.vec = and i32 %N, -16 %ind.end = getelementptr i8, ptr %pSrcA, i32 %n.vec %ind.end17 = getelementptr i8, ptr %pSrcB, i32 %n.vec %ind.end19 = getelementptr i8, ptr %pDst, i32 %n.vec br label %vector.body vector.body: ; preds = %vector.body, %vector.ph %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] %next.gep = getelementptr i8, ptr %pSrcA, i32 %index %next.gep20 = getelementptr i8, ptr %pSrcB, i32 %index %next.gep21 = getelementptr i8, ptr %pDst, i32 %index %wide.load = load <16 x i8>, ptr %next.gep, align 1 %0 = zext <16 x i8> %wide.load to <16 x i16> %wide.load22 = load <16 x i8>, ptr %next.gep20, align 1 %1 = zext <16 x i8> %wide.load22 to <16 x i16> %2 = mul nuw <16 x i16> %1, %0 %3 = lshr <16 x i16> %2, %4 = icmp ult <16 x i16> %3, %5 = select <16 x i1> %4, <16 x i16> %3, <16 x i16> %6 = trunc <16 x i16> %5 to <16 x i8> store <16 x i8> %6, ptr %next.gep21, align 1 %index.next = add i32 %index, 16 %7 = icmp eq i32 %index.next, %n.vec br i1 %7, label %middle.block, label %vector.body middle.block: ; preds = %vector.body %cmp.n = icmp eq i32 %n.vec, %N br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader23 for.cond.cleanup: ; preds = %for.body, %middle.block, %entry ret void for.body: ; preds = %for.body.preheader23, %for.body %i.014 = phi i32 [ %inc, %for.body ], [ %i.014.ph, %for.body.preheader23 ] %pSrcA.addr.013 = phi ptr [ %incdec.ptr, %for.body ], [ %pSrcA.addr.013.ph, %for.body.preheader23 ] %pSrcB.addr.012 = phi ptr [ %incdec.ptr2, %for.body ], [ %pSrcB.addr.012.ph, %for.body.preheader23 ] %pDst.addr.011 = phi ptr [ %incdec.ptr6, %for.body ], [ %pDst.addr.011.ph, %for.body.preheader23 ] %incdec.ptr = getelementptr inbounds i8, ptr %pSrcA.addr.013, i32 1 %8 = load i8, ptr %pSrcA.addr.013, align 1 %conv1 = zext i8 %8 to i16 %incdec.ptr2 = getelementptr inbounds i8, ptr %pSrcB.addr.012, i32 1 %9 = load i8, ptr %pSrcB.addr.012, align 1 %conv3 = zext i8 %9 to i16 %mul = mul nuw i16 %conv3, %conv1 %10 = lshr i16 %mul, 7 %11 = icmp ult i16 %10, 255 %retval.0.i = select i1 %11, i16 %10, i16 255 %conv5 = trunc i16 %retval.0.i to i8 %incdec.ptr6 = getelementptr inbounds i8, ptr %pDst.addr.011, i32 1 store i8 %conv5, ptr %pDst.addr.011, align 1 %inc = add nuw i32 %i.014, 1 %exitcond = icmp eq i32 %inc, %N br i1 %exitcond, label %for.cond.cleanup, label %for.body } declare <4 x i32> @llvm.masked.load.v4i32.p0(ptr, i32, <4 x i1>, <4 x i32>) declare <8 x i16> @llvm.masked.load.v8i16.p0(ptr, i32, <8 x i1>, <8 x i16>) declare <4 x i16> @llvm.masked.load.v4i16.p0(ptr, i32, <4 x i1>, <4 x i16>) declare <8 x i8> @llvm.masked.load.v8i8.p0(ptr, i32, <8 x i1>, <8 x i8>) declare <16 x i8> @llvm.masked.load.v16i8.p0(ptr, i32, <16 x i1>, <16 x i8>) declare void @llvm.masked.store.v4i32.p0(<4 x i32>, ptr, i32, <4 x i1>) declare void @llvm.masked.store.v8i16.p0(<8 x i16>, ptr, i32, <8 x i1>) declare void @llvm.masked.store.v4i16.p0(<4 x i16>, ptr, i32, <4 x i1>) declare void @llvm.masked.store.v8i8.p0(<8 x i8>, ptr, i32, <8 x i1>) declare void @llvm.masked.store.v16i8.p0(<16 x i8>, ptr, i32, <16 x i1>)