; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=aarch64 -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP16
; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -o - %s | FileCheck %s --check-prefixes=CHECK,CHECK-FP16

define ptr @mul_v16i8(ptr %p, ptr %ps, <16 x i8> %t) {
; CHECK-LABEL: mul_v16i8:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ld1r { v1.16b }, [x0], #1
; CHECK-NEXT:    mul v0.16b, v1.16b, v0.16b
; CHECK-NEXT:    str q0, [x1]
; CHECK-NEXT:    ret
  %l = load i8, ptr %p
  %i = insertelement <16 x i8> undef, i8 %l, i32 0
  %s = shufflevector <16 x i8> %i, <16 x i8> undef, <16 x i32> zeroinitializer
  %m = mul <16 x i8> %s, %t
  store <16 x i8> %m, ptr %ps
  %g = getelementptr i8, ptr %p, i64 1
  ret ptr %g
}

define ptr @mul_v8i16(ptr %p, ptr %ps, <8 x i16> %t) {
; CHECK-LABEL: mul_v8i16:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ld1r { v1.8h }, [x0], #2
; CHECK-NEXT:    mul v0.8h, v1.8h, v0.8h
; CHECK-NEXT:    str q0, [x1]
; CHECK-NEXT:    ret
  %l = load i16, ptr %p
  %i = insertelement <8 x i16> undef, i16 %l, i32 0
  %s = shufflevector <8 x i16> %i, <8 x i16> undef, <8 x i32> zeroinitializer
  %m = mul <8 x i16> %s, %t
  store <8 x i16> %m, ptr %ps
  %g = getelementptr i16, ptr %p, i64 1
  ret ptr %g
}

define ptr @mul_v4i32(ptr %p, ptr %ps, <4 x i32> %t) {
; CHECK-LABEL: mul_v4i32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ld1r { v1.4s }, [x0], #4
; CHECK-NEXT:    mul v0.4s, v1.4s, v0.4s
; CHECK-NEXT:    str q0, [x1]
; CHECK-NEXT:    ret
  %l = load i32, ptr %p
  %i = insertelement <4 x i32> undef, i32 %l, i32 0
  %s = shufflevector <4 x i32> %i, <4 x i32> undef, <4 x i32> zeroinitializer
  %m = mul <4 x i32> %s, %t
  store <4 x i32> %m, ptr %ps
  %g = getelementptr i32, ptr %p, i64 1
  ret ptr %g
}

define ptr @fmul_v4f16(ptr %p, ptr %ps, <4 x half> %t) {
; CHECK-NOFP16-LABEL: fmul_v4f16:
; CHECK-NOFP16:       // %bb.0:
; CHECK-NOFP16-NEXT:    ld1r { v1.4h }, [x0], #2
; CHECK-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
; CHECK-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
; CHECK-NOFP16-NEXT:    fmul v0.4s, v1.4s, v0.4s
; CHECK-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
; CHECK-NOFP16-NEXT:    str d0, [x1]
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: fmul_v4f16:
; CHECK-FP16:       // %bb.0:
; CHECK-FP16-NEXT:    ldr h1, [x0], #2
; CHECK-FP16-NEXT:    fmul v0.4h, v0.4h, v1.h[0]
; CHECK-FP16-NEXT:    str d0, [x1]
; CHECK-FP16-NEXT:    ret
  %l = load half, ptr %p
  %i = insertelement <4 x half> undef, half %l, i32 0
  %s = shufflevector <4 x half> %i, <4 x half> undef, <4 x i32> zeroinitializer
  %m = fmul <4 x half> %s, %t
  store <4 x half> %m, ptr %ps
  %g = getelementptr half, ptr %p, i64 1
  ret ptr %g
}

define ptr @fmla_v4f16(ptr %p, ptr %ps, <4 x half> %t, <4 x half> %u) {
; CHECK-NOFP16-LABEL: fmla_v4f16:
; CHECK-NOFP16:       // %bb.0:
; CHECK-NOFP16-NEXT:    ld1r { v2.4h }, [x0], #2
; CHECK-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
; CHECK-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
; CHECK-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
; CHECK-NOFP16-NEXT:    fmul v0.4s, v2.4s, v0.4s
; CHECK-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
; CHECK-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
; CHECK-NOFP16-NEXT:    fadd v0.4s, v0.4s, v1.4s
; CHECK-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
; CHECK-NOFP16-NEXT:    str d0, [x1]
; CHECK-NOFP16-NEXT:    ret
;
; CHECK-FP16-LABEL: fmla_v4f16:
; CHECK-FP16:       // %bb.0:
; CHECK-FP16-NEXT:    ldr h2, [x0], #2
; CHECK-FP16-NEXT:    fmla v1.4h, v0.4h, v2.h[0]
; CHECK-FP16-NEXT:    str d1, [x1]
; CHECK-FP16-NEXT:    ret
  %l = load half, ptr %p
  %i = insertelement <4 x half> undef, half %l, i32 0
  %s = shufflevector <4 x half> %i, <4 x half> undef, <4 x i32> zeroinitializer
  %m = fmul fast <4 x half> %s, %t
  %a = fadd fast <4 x half> %m, %u
  store <4 x half> %a, ptr %ps
  %g = getelementptr half, ptr %p, i64 1
  ret ptr %g
}

define ptr @fmul_v4f32(ptr %p, ptr %ps, <4 x float> %t) {
; CHECK-LABEL: fmul_v4f32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr s1, [x0], #4
; CHECK-NEXT:    fmul v0.4s, v0.4s, v1.s[0]
; CHECK-NEXT:    str q0, [x1]
; CHECK-NEXT:    ret
  %l = load float, ptr %p
  %i = insertelement <4 x float> undef, float %l, i32 0
  %s = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
  %m = fmul <4 x float> %s, %t
  store <4 x float> %m, ptr %ps
  %g = getelementptr float, ptr %p, i64 1
  ret ptr %g
}

define ptr @fmla_v4f32(ptr %p, ptr %ps, <4 x float> %t, <4 x float> %u) {
; CHECK-LABEL: fmla_v4f32:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr s2, [x0], #4
; CHECK-NEXT:    fmla v1.4s, v0.4s, v2.s[0]
; CHECK-NEXT:    str q1, [x1]
; CHECK-NEXT:    ret
  %l = load float, ptr %p
  %i = insertelement <4 x float> undef, float %l, i32 0
  %s = shufflevector <4 x float> %i, <4 x float> undef, <4 x i32> zeroinitializer
  %m = fmul fast <4 x float> %s, %t
  %a = fadd fast <4 x float> %m, %u
  store <4 x float> %a, ptr %ps
  %g = getelementptr float, ptr %p, i64 1
  ret ptr %g
}

define ptr @fmul_v2f64(ptr %p, ptr %ps, <2 x double> %t) {
; CHECK-LABEL: fmul_v2f64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d1, [x0], #8
; CHECK-NEXT:    fmul v0.2d, v0.2d, v1.d[0]
; CHECK-NEXT:    str q0, [x1]
; CHECK-NEXT:    ret
  %l = load double, ptr %p
  %i = insertelement <2 x double> undef, double %l, i32 0
  %s = shufflevector <2 x double> %i, <2 x double> undef, <2 x i32> zeroinitializer
  %m = fmul <2 x double> %s, %t
  store <2 x double> %m, ptr %ps
  %g = getelementptr double, ptr %p, i64 1
  ret ptr %g
}

define ptr @fmla_v2f64(ptr %p, ptr %ps, <2 x double> %t, <2 x double> %u) {
; CHECK-LABEL: fmla_v2f64:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ldr d2, [x0], #8
; CHECK-NEXT:    fmla v1.2d, v0.2d, v2.d[0]
; CHECK-NEXT:    str q1, [x1]
; CHECK-NEXT:    ret
  %l = load double, ptr %p
  %i = insertelement <2 x double> undef, double %l, i32 0
  %s = shufflevector <2 x double> %i, <2 x double> undef, <2 x i32> zeroinitializer
  %m = fmul fast <2 x double> %s, %t
  %a = fadd fast <2 x double> %m, %u
  store <2 x double> %a, ptr %ps
  %g = getelementptr double, ptr %p, i64 1
  ret ptr %g
}