; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s

;
; CMPEQ
;

define <vscale x 16 x i1> @cmpeq_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: cmpeq_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpeq p0.b, p0/z, z0.b, z1.b
; CHECK-NEXT:    ret
  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpeq.nxv16i8(<vscale x 16 x i1> %pg,
                                                                 <vscale x 16 x i8> %a,
                                                                 <vscale x 16 x i8> %b)
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmpeq_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
; CHECK-LABEL: cmpeq_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpeq p0.h, p0/z, z0.h, z1.h
; CHECK-NEXT:    ret
  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpeq.nxv8i16(<vscale x 8 x i1> %pg,
                                                                <vscale x 8 x i16> %a,
                                                                <vscale x 8 x i16> %b)
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmpeq_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: cmpeq_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpeq p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT:    ret
  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpeq.nxv4i32(<vscale x 4 x i1> %pg,
                                                                <vscale x 4 x i32> %a,
                                                                <vscale x 4 x i32> %b)
  ret <vscale x 4 x i1> %out
}

define <vscale x 2 x i1> @cmpeq_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpeq_d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpeq p0.d, p0/z, z0.d, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpeq.nxv2i64(<vscale x 2 x i1> %pg,
                                                                <vscale x 2 x i64> %a,
                                                                <vscale x 2 x i64> %b)
  ret <vscale x 2 x i1> %out
}

define <vscale x 16 x i1> @cmpeq_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpeq_wide_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpeq p0.b, p0/z, z0.b, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv16i8(<vscale x 16 x i1> %pg,
                                                                      <vscale x 16 x i8> %a,
                                                                      <vscale x 2 x i64> %b)
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmpeq_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpeq_wide_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpeq p0.h, p0/z, z0.h, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv8i16(<vscale x 8 x i1> %pg,
                                                                     <vscale x 8 x i16> %a,
                                                                     <vscale x 2 x i64> %b)
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmpeq_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpeq_wide_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpeq p0.s, p0/z, z0.s, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv4i32(<vscale x 4 x i1> %pg,
                                                                     <vscale x 4 x i32> %a,
                                                                     <vscale x 2 x i64> %b)
  ret <vscale x 4 x i1> %out
}

define <vscale x 16 x i1> @cmpeq_ir_b(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: cmpeq_ir_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.b
; CHECK-NEXT:    cmpeq p0.b, p0/z, z0.b, z1.b
; CHECK-NEXT:    ret
  %out = icmp eq <vscale x 16 x i8> %a, %b
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmpeq_ir_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
; CHECK-LABEL: cmpeq_ir_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    cmpeq p0.h, p0/z, z0.h, z1.h
; CHECK-NEXT:    ret
  %out = icmp eq <vscale x 8 x i16> %a, %b
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmpeq_ir_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: cmpeq_ir_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    cmpeq p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT:    ret
  %out = icmp eq <vscale x 4 x i32> %a, %b
  ret <vscale x 4 x i1> %out
}

define <vscale x 2 x i1> @cmpeq_ir_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpeq_ir_d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    cmpeq p0.d, p0/z, z0.d, z1.d
; CHECK-NEXT:    ret
  %out = icmp eq <vscale x 2 x i64> %a, %b
  ret <vscale x 2 x i1> %out
}

;
; CMPGE
;

define <vscale x 16 x i1> @cmpge_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: cmpge_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpge p0.b, p0/z, z0.b, z1.b
; CHECK-NEXT:    ret
  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.nxv16i8(<vscale x 16 x i1> %pg,
                                                                 <vscale x 16 x i8> %a,
                                                                 <vscale x 16 x i8> %b)
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmpge_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
; CHECK-LABEL: cmpge_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpge p0.h, p0/z, z0.h, z1.h
; CHECK-NEXT:    ret
  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.nxv8i16(<vscale x 8 x i1> %pg,
                                                                <vscale x 8 x i16> %a,
                                                                <vscale x 8 x i16> %b)
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmpge_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: cmpge_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpge p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT:    ret
  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.nxv4i32(<vscale x 4 x i1> %pg,
                                                                <vscale x 4 x i32> %a,
                                                                <vscale x 4 x i32> %b)
  ret <vscale x 4 x i1> %out
}

define <vscale x 2 x i1> @cmpge_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpge_d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpge p0.d, p0/z, z0.d, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpge.nxv2i64(<vscale x 2 x i1> %pg,
                                                                <vscale x 2 x i64> %a,
                                                                <vscale x 2 x i64> %b)
  ret <vscale x 2 x i1> %out
}

define <vscale x 16 x i1> @cmpge_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpge_wide_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpge p0.b, p0/z, z0.b, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.wide.nxv16i8(<vscale x 16 x i1> %pg,
                                                                      <vscale x 16 x i8> %a,
                                                                      <vscale x 2 x i64> %b)
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmpge_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpge_wide_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpge p0.h, p0/z, z0.h, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.wide.nxv8i16(<vscale x 8 x i1> %pg,
                                                                     <vscale x 8 x i16> %a,
                                                                     <vscale x 2 x i64> %b)
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmpge_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpge_wide_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpge p0.s, p0/z, z0.s, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.wide.nxv4i32(<vscale x 4 x i1> %pg,
                                                                     <vscale x 4 x i32> %a,
                                                                     <vscale x 2 x i64> %b)
  ret <vscale x 4 x i1> %out
}

define <vscale x 16 x i1> @cmpge_ir_b(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: cmpge_ir_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.b
; CHECK-NEXT:    cmpge p0.b, p0/z, z0.b, z1.b
; CHECK-NEXT:    ret
  %out = icmp sge <vscale x 16 x i8> %a, %b
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmpge_ir_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
; CHECK-LABEL: cmpge_ir_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    cmpge p0.h, p0/z, z0.h, z1.h
; CHECK-NEXT:    ret
  %out = icmp sge <vscale x 8 x i16> %a, %b
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmpge_ir_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: cmpge_ir_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    cmpge p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT:    ret
  %out = icmp sge <vscale x 4 x i32> %a, %b
  ret <vscale x 4 x i1> %out
}

define <vscale x 2 x i1> @cmpge_ir_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpge_ir_d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    cmpge p0.d, p0/z, z0.d, z1.d
; CHECK-NEXT:    ret
  %out = icmp sge <vscale x 2 x i64> %a, %b
  ret <vscale x 2 x i1> %out
}

define <vscale x 16 x i1> @cmpge_ir_comm_b(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: cmpge_ir_comm_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.b
; CHECK-NEXT:    cmpge p0.b, p0/z, z1.b, z0.b
; CHECK-NEXT:    ret
  %out = icmp sle <vscale x 16 x i8> %a, %b
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmpge_ir_comm_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
; CHECK-LABEL: cmpge_ir_comm_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    cmpge p0.h, p0/z, z1.h, z0.h
; CHECK-NEXT:    ret
  %out = icmp sle <vscale x 8 x i16> %a, %b
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmpge_ir_comm_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: cmpge_ir_comm_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    cmpge p0.s, p0/z, z1.s, z0.s
; CHECK-NEXT:    ret
  %out = icmp sle <vscale x 4 x i32> %a, %b
  ret <vscale x 4 x i1> %out
}

define <vscale x 2 x i1> @cmpge_ir_comm_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpge_ir_comm_d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    cmpge p0.d, p0/z, z1.d, z0.d
; CHECK-NEXT:    ret
  %out = icmp sle <vscale x 2 x i64> %a, %b
  ret <vscale x 2 x i1> %out
}

;
; CMPGT
;

define <vscale x 16 x i1> @cmpgt_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: cmpgt_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpgt p0.b, p0/z, z0.b, z1.b
; CHECK-NEXT:    ret
  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.nxv16i8(<vscale x 16 x i1> %pg,
                                                                 <vscale x 16 x i8> %a,
                                                                 <vscale x 16 x i8> %b)
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmpgt_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
; CHECK-LABEL: cmpgt_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpgt p0.h, p0/z, z0.h, z1.h
; CHECK-NEXT:    ret
  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.nxv8i16(<vscale x 8 x i1> %pg,
                                                                <vscale x 8 x i16> %a,
                                                                <vscale x 8 x i16> %b)
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmpgt_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: cmpgt_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpgt p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT:    ret
  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.nxv4i32(<vscale x 4 x i1> %pg,
                                                                <vscale x 4 x i32> %a,
                                                                <vscale x 4 x i32> %b)
  ret <vscale x 4 x i1> %out
}

define <vscale x 2 x i1> @cmpgt_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpgt_d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpgt p0.d, p0/z, z0.d, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpgt.nxv2i64(<vscale x 2 x i1> %pg,
                                                                <vscale x 2 x i64> %a,
                                                                <vscale x 2 x i64> %b)
  ret <vscale x 2 x i1> %out
}

define <vscale x 16 x i1> @cmpgt_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpgt_wide_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpgt p0.b, p0/z, z0.b, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv16i8(<vscale x 16 x i1> %pg,
                                                                      <vscale x 16 x i8> %a,
                                                                      <vscale x 2 x i64> %b)
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmpgt_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpgt_wide_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpgt p0.h, p0/z, z0.h, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv8i16(<vscale x 8 x i1> %pg,
                                                                     <vscale x 8 x i16> %a,
                                                                     <vscale x 2 x i64> %b)
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmpgt_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpgt_wide_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpgt p0.s, p0/z, z0.s, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv4i32(<vscale x 4 x i1> %pg,
                                                                     <vscale x 4 x i32> %a,
                                                                     <vscale x 2 x i64> %b)
  ret <vscale x 4 x i1> %out
}

define <vscale x 16 x i1> @cmpgt_ir_b(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: cmpgt_ir_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.b
; CHECK-NEXT:    cmpgt p0.b, p0/z, z0.b, z1.b
; CHECK-NEXT:    ret
  %out = icmp sgt <vscale x 16 x i8> %a, %b
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmpgt_ir_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
; CHECK-LABEL: cmpgt_ir_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    cmpgt p0.h, p0/z, z0.h, z1.h
; CHECK-NEXT:    ret
  %out = icmp sgt <vscale x 8 x i16> %a, %b
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmpgt_ir_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: cmpgt_ir_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    cmpgt p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT:    ret
  %out = icmp sgt <vscale x 4 x i32> %a, %b
  ret <vscale x 4 x i1> %out
}

define <vscale x 2 x i1> @cmpgt_ir_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpgt_ir_d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    cmpgt p0.d, p0/z, z0.d, z1.d
; CHECK-NEXT:    ret
  %out = icmp sgt <vscale x 2 x i64> %a, %b
  ret <vscale x 2 x i1> %out
}

define <vscale x 16 x i1> @cmpgt_ir_comm_b(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: cmpgt_ir_comm_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.b
; CHECK-NEXT:    cmpgt p0.b, p0/z, z1.b, z0.b
; CHECK-NEXT:    ret
  %out = icmp slt <vscale x 16 x i8> %a, %b
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmpgt_ir_comm_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
; CHECK-LABEL: cmpgt_ir_comm_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    cmpgt p0.h, p0/z, z1.h, z0.h
; CHECK-NEXT:    ret
  %out = icmp slt <vscale x 8 x i16> %a, %b
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmpgt_ir_comm_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: cmpgt_ir_comm_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    cmpgt p0.s, p0/z, z1.s, z0.s
; CHECK-NEXT:    ret
  %out = icmp slt <vscale x 4 x i32> %a, %b
  ret <vscale x 4 x i1> %out
}

define <vscale x 2 x i1> @cmpgt_ir_comm_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpgt_ir_comm_d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    cmpgt p0.d, p0/z, z1.d, z0.d
; CHECK-NEXT:    ret
  %out = icmp slt <vscale x 2 x i64> %a, %b
  ret <vscale x 2 x i1> %out
}

;
; CMPHI
;

define <vscale x 16 x i1> @cmphi_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: cmphi_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmphi p0.b, p0/z, z0.b, z1.b
; CHECK-NEXT:    ret
  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.nxv16i8(<vscale x 16 x i1> %pg,
                                                                 <vscale x 16 x i8> %a,
                                                                 <vscale x 16 x i8> %b)
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmphi_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
; CHECK-LABEL: cmphi_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmphi p0.h, p0/z, z0.h, z1.h
; CHECK-NEXT:    ret
  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.nxv8i16(<vscale x 8 x i1> %pg,
                                                                <vscale x 8 x i16> %a,
                                                                <vscale x 8 x i16> %b)
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmphi_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: cmphi_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmphi p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT:    ret
  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.nxv4i32(<vscale x 4 x i1> %pg,
                                                                <vscale x 4 x i32> %a,
                                                                <vscale x 4 x i32> %b)
  ret <vscale x 4 x i1> %out
}

define <vscale x 2 x i1> @cmphi_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmphi_d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmphi p0.d, p0/z, z0.d, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmphi.nxv2i64(<vscale x 2 x i1> %pg,
                                                                <vscale x 2 x i64> %a,
                                                                <vscale x 2 x i64> %b)
  ret <vscale x 2 x i1> %out
}

define <vscale x 16 x i1> @cmphi_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmphi_wide_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmphi p0.b, p0/z, z0.b, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.wide.nxv16i8(<vscale x 16 x i1> %pg,
                                                                      <vscale x 16 x i8> %a,
                                                                      <vscale x 2 x i64> %b)
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmphi_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmphi_wide_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmphi p0.h, p0/z, z0.h, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.wide.nxv8i16(<vscale x 8 x i1> %pg,
                                                                     <vscale x 8 x i16> %a,
                                                                     <vscale x 2 x i64> %b)
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmphi_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmphi_wide_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmphi p0.s, p0/z, z0.s, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.wide.nxv4i32(<vscale x 4 x i1> %pg,
                                                                     <vscale x 4 x i32> %a,
                                                                     <vscale x 2 x i64> %b)
  ret <vscale x 4 x i1> %out
}

define <vscale x 16 x i1> @cmphi_ir_b(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: cmphi_ir_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.b
; CHECK-NEXT:    cmphi p0.b, p0/z, z0.b, z1.b
; CHECK-NEXT:    ret
  %out = icmp ugt <vscale x 16 x i8> %a, %b
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmphi_ir_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
; CHECK-LABEL: cmphi_ir_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    cmphi p0.h, p0/z, z0.h, z1.h
; CHECK-NEXT:    ret
  %out = icmp ugt <vscale x 8 x i16> %a, %b
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmphi_ir_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: cmphi_ir_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    cmphi p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT:    ret
  %out = icmp ugt <vscale x 4 x i32> %a, %b
  ret <vscale x 4 x i1> %out
}

define <vscale x 2 x i1> @cmphi_ir_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmphi_ir_d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    cmphi p0.d, p0/z, z0.d, z1.d
; CHECK-NEXT:    ret
  %out = icmp ugt <vscale x 2 x i64> %a, %b
  ret <vscale x 2 x i1> %out
}

define <vscale x 16 x i1> @cmphi_ir_comm_b(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: cmphi_ir_comm_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.b
; CHECK-NEXT:    cmphi p0.b, p0/z, z1.b, z0.b
; CHECK-NEXT:    ret
  %out = icmp ult <vscale x 16 x i8> %a, %b
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmphi_ir_comm_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
; CHECK-LABEL: cmphi_ir_comm_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    cmphi p0.h, p0/z, z1.h, z0.h
; CHECK-NEXT:    ret
  %out = icmp ult <vscale x 8 x i16> %a, %b
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmphi_ir_comm_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: cmphi_ir_comm_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    cmphi p0.s, p0/z, z1.s, z0.s
; CHECK-NEXT:    ret
  %out = icmp ult <vscale x 4 x i32> %a, %b
  ret <vscale x 4 x i1> %out
}

define <vscale x 2 x i1> @cmphi_ir_comm_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmphi_ir_comm_d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    cmphi p0.d, p0/z, z1.d, z0.d
; CHECK-NEXT:    ret
  %out = icmp ult <vscale x 2 x i64> %a, %b
  ret <vscale x 2 x i1> %out
}

;
; CMPHS
;

define <vscale x 16 x i1> @cmphs_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: cmphs_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmphs p0.b, p0/z, z0.b, z1.b
; CHECK-NEXT:    ret
  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.nxv16i8(<vscale x 16 x i1> %pg,
                                                                 <vscale x 16 x i8> %a,
                                                                 <vscale x 16 x i8> %b)
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmphs_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
; CHECK-LABEL: cmphs_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmphs p0.h, p0/z, z0.h, z1.h
; CHECK-NEXT:    ret
  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.nxv8i16(<vscale x 8 x i1> %pg,
                                                                <vscale x 8 x i16> %a,
                                                                <vscale x 8 x i16> %b)
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmphs_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: cmphs_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmphs p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT:    ret
  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.nxv4i32(<vscale x 4 x i1> %pg,
                                                                <vscale x 4 x i32> %a,
                                                                <vscale x 4 x i32> %b)
  ret <vscale x 4 x i1> %out
}

define <vscale x 2 x i1> @cmphs_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmphs_d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmphs p0.d, p0/z, z0.d, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmphs.nxv2i64(<vscale x 2 x i1> %pg,
                                                                <vscale x 2 x i64> %a,
                                                                <vscale x 2 x i64> %b)
  ret <vscale x 2 x i1> %out
}

define <vscale x 16 x i1> @cmphs_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmphs_wide_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmphs p0.b, p0/z, z0.b, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.wide.nxv16i8(<vscale x 16 x i1> %pg,
                                                                      <vscale x 16 x i8> %a,
                                                                      <vscale x 2 x i64> %b)
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmphs_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmphs_wide_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmphs p0.h, p0/z, z0.h, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.wide.nxv8i16(<vscale x 8 x i1> %pg,
                                                                     <vscale x 8 x i16> %a,
                                                                     <vscale x 2 x i64> %b)
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmphs_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmphs_wide_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmphs p0.s, p0/z, z0.s, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.wide.nxv4i32(<vscale x 4 x i1> %pg,
                                                                     <vscale x 4 x i32> %a,
                                                                     <vscale x 2 x i64> %b)
  ret <vscale x 4 x i1> %out
}

define <vscale x 16 x i1> @cmphs_ir_b(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: cmphs_ir_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.b
; CHECK-NEXT:    cmphs p0.b, p0/z, z0.b, z1.b
; CHECK-NEXT:    ret
  %out = icmp uge <vscale x 16 x i8> %a, %b
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmphs_ir_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
; CHECK-LABEL: cmphs_ir_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    cmphs p0.h, p0/z, z0.h, z1.h
; CHECK-NEXT:    ret
  %out = icmp uge <vscale x 8 x i16> %a, %b
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmphs_ir_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: cmphs_ir_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    cmphs p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT:    ret
  %out = icmp uge <vscale x 4 x i32> %a, %b
  ret <vscale x 4 x i1> %out
}

define <vscale x 2 x i1> @cmphs_ir_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmphs_ir_d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    cmphs p0.d, p0/z, z0.d, z1.d
; CHECK-NEXT:    ret
  %out = icmp uge <vscale x 2 x i64> %a, %b
  ret <vscale x 2 x i1> %out
}

define <vscale x 16 x i1> @cmphs_ir_comm_b(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: cmphs_ir_comm_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.b
; CHECK-NEXT:    cmphs p0.b, p0/z, z1.b, z0.b
; CHECK-NEXT:    ret
  %out = icmp ule <vscale x 16 x i8> %a, %b
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmphs_ir_comm_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
; CHECK-LABEL: cmphs_ir_comm_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    cmphs p0.h, p0/z, z1.h, z0.h
; CHECK-NEXT:    ret
  %out = icmp ule <vscale x 8 x i16> %a, %b
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmphs_ir_comm_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: cmphs_ir_comm_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    cmphs p0.s, p0/z, z1.s, z0.s
; CHECK-NEXT:    ret
  %out = icmp ule <vscale x 4 x i32> %a, %b
  ret <vscale x 4 x i1> %out
}

define <vscale x 2 x i1> @cmphs_ir_comm_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmphs_ir_comm_d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    cmphs p0.d, p0/z, z1.d, z0.d
; CHECK-NEXT:    ret
  %out = icmp ule <vscale x 2 x i64> %a, %b
  ret <vscale x 2 x i1> %out
}

;
; CMPLE
;

define <vscale x 16 x i1> @cmple_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmple_wide_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmple p0.b, p0/z, z0.b, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmple.wide.nxv16i8(<vscale x 16 x i1> %pg,
                                                                      <vscale x 16 x i8> %a,
                                                                      <vscale x 2 x i64> %b)
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmple_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmple_wide_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmple p0.h, p0/z, z0.h, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmple.wide.nxv8i16(<vscale x 8 x i1> %pg,
                                                                     <vscale x 8 x i16> %a,
                                                                     <vscale x 2 x i64> %b)
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmple_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmple_wide_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmple p0.s, p0/z, z0.s, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmple.wide.nxv4i32(<vscale x 4 x i1> %pg,
                                                                     <vscale x 4 x i32> %a,
                                                                     <vscale x 2 x i64> %b)
  ret <vscale x 4 x i1> %out
}

;
; CMPLO
;

define <vscale x 16 x i1> @cmplo_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmplo_wide_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmplo p0.b, p0/z, z0.b, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmplo.wide.nxv16i8(<vscale x 16 x i1> %pg,
                                                                      <vscale x 16 x i8> %a,
                                                                      <vscale x 2 x i64> %b)
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmplo_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmplo_wide_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmplo p0.h, p0/z, z0.h, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmplo.wide.nxv8i16(<vscale x 8 x i1> %pg,
                                                                     <vscale x 8 x i16> %a,
                                                                     <vscale x 2 x i64> %b)
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmplo_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmplo_wide_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmplo p0.s, p0/z, z0.s, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmplo.wide.nxv4i32(<vscale x 4 x i1> %pg,
                                                                     <vscale x 4 x i32> %a,
                                                                     <vscale x 2 x i64> %b)
  ret <vscale x 4 x i1> %out
}

;
; CMPLS
;

define <vscale x 16 x i1> @cmpls_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpls_wide_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpls p0.b, p0/z, z0.b, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpls.wide.nxv16i8(<vscale x 16 x i1> %pg,
                                                                      <vscale x 16 x i8> %a,
                                                                      <vscale x 2 x i64> %b)
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmpls_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpls_wide_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpls p0.h, p0/z, z0.h, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpls.wide.nxv8i16(<vscale x 8 x i1> %pg,
                                                                     <vscale x 8 x i16> %a,
                                                                     <vscale x 2 x i64> %b)
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmpls_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpls_wide_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpls p0.s, p0/z, z0.s, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpls.wide.nxv4i32(<vscale x 4 x i1> %pg,
                                                                     <vscale x 4 x i32> %a,
                                                                     <vscale x 2 x i64> %b)
  ret <vscale x 4 x i1> %out
}

;
; CMPLT
;

define <vscale x 16 x i1> @cmplt_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmplt_wide_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmplt p0.b, p0/z, z0.b, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmplt.wide.nxv16i8(<vscale x 16 x i1> %pg,
                                                                      <vscale x 16 x i8> %a,
                                                                      <vscale x 2 x i64> %b)
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmplt_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmplt_wide_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmplt p0.h, p0/z, z0.h, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmplt.wide.nxv8i16(<vscale x 8 x i1> %pg,
                                                                     <vscale x 8 x i16> %a,
                                                                     <vscale x 2 x i64> %b)
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmplt_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmplt_wide_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmplt p0.s, p0/z, z0.s, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmplt.wide.nxv4i32(<vscale x 4 x i1> %pg,
                                                                     <vscale x 4 x i32> %a,
                                                                     <vscale x 2 x i64> %b)
  ret <vscale x 4 x i1> %out
}

;
; CMPNE
;

define <vscale x 16 x i1> @cmpne_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: cmpne_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpne p0.b, p0/z, z0.b, z1.b
; CHECK-NEXT:    ret
  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpne.nxv16i8(<vscale x 16 x i1> %pg,
                                                                 <vscale x 16 x i8> %a,
                                                                 <vscale x 16 x i8> %b)
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmpne_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
; CHECK-LABEL: cmpne_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpne p0.h, p0/z, z0.h, z1.h
; CHECK-NEXT:    ret
  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpne.nxv8i16(<vscale x 8 x i1> %pg,
                                                                <vscale x 8 x i16> %a,
                                                                <vscale x 8 x i16> %b)
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmpne_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: cmpne_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpne p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT:    ret
  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpne.nxv4i32(<vscale x 4 x i1> %pg,
                                                                <vscale x 4 x i32> %a,
                                                                <vscale x 4 x i32> %b)
  ret <vscale x 4 x i1> %out
}

define <vscale x 2 x i1> @cmpne_d(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpne_d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpne p0.d, p0/z, z0.d, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.cmpne.nxv2i64(<vscale x 2 x i1> %pg,
                                                                <vscale x 2 x i64> %a,
                                                                <vscale x 2 x i64> %b)
  ret <vscale x 2 x i1> %out
}

define <vscale x 16 x i1> @cmpne_wide_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpne_wide_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpne p0.b, p0/z, z0.b, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpne.wide.nxv16i8(<vscale x 16 x i1> %pg,
                                                                      <vscale x 16 x i8> %a,
                                                                      <vscale x 2 x i64> %b)
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmpne_wide_h(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpne_wide_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpne p0.h, p0/z, z0.h, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 8 x i1> @llvm.aarch64.sve.cmpne.wide.nxv8i16(<vscale x 8 x i1> %pg,
                                                                     <vscale x 8 x i16> %a,
                                                                     <vscale x 2 x i64> %b)
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmpne_wide_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpne_wide_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpne p0.s, p0/z, z0.s, z1.d
; CHECK-NEXT:    ret
  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpne.wide.nxv4i32(<vscale x 4 x i1> %pg,
                                                                     <vscale x 4 x i32> %a,
                                                                     <vscale x 2 x i64> %b)
  ret <vscale x 4 x i1> %out
}

define <vscale x 16 x i1> @cmpne_ir_b(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: cmpne_ir_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.b
; CHECK-NEXT:    cmpne p0.b, p0/z, z0.b, z1.b
; CHECK-NEXT:    ret
  %out = icmp ne <vscale x 16 x i8> %a, %b
  ret <vscale x 16 x i1> %out
}

define <vscale x 8 x i1> @cmpne_ir_h(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
; CHECK-LABEL: cmpne_ir_h:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.h
; CHECK-NEXT:    cmpne p0.h, p0/z, z0.h, z1.h
; CHECK-NEXT:    ret
  %out = icmp ne <vscale x 8 x i16> %a, %b
  ret <vscale x 8 x i1> %out
}

define <vscale x 4 x i1> @cmpne_ir_s(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: cmpne_ir_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    cmpne p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT:    ret
  %out = icmp ne <vscale x 4 x i32> %a, %b
  ret <vscale x 4 x i1> %out
}

define <vscale x 2 x i1> @cmpne_ir_d(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: cmpne_ir_d:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    cmpne p0.d, p0/z, z0.d, z1.d
; CHECK-NEXT:    ret
  %out = icmp ne <vscale x 2 x i64> %a, %b
  ret <vscale x 2 x i1> %out
}

define <vscale x 1 x i1> @cmpne_ir_q(<vscale x 1 x i64> %a, <vscale x 1 x i64> %b) {
; CHECK-LABEL: cmpne_ir_q:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.d
; CHECK-NEXT:    cmpne p0.d, p0/z, z0.d, z1.d
; CHECK-NEXT:    punpklo p0.h, p0.b
; CHECK-NEXT:    ret
  %out = icmp ne <vscale x 1 x i64> %a, %b
  ret <vscale x 1 x i1> %out
}


define <vscale x 16 x i1> @cmpgt_wide_splat_b(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, i64 %b) {
; CHECK-LABEL: cmpgt_wide_splat_b:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov z1.d, x0
; CHECK-NEXT:    cmpgt p0.b, p0/z, z0.b, z1.d
; CHECK-NEXT:    ret
  %splat = call <vscale x 2 x i64> @llvm.aarch64.sve.dup.x.nxv2i64(i64 %b)
  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv16i8(<vscale x 16 x i1> %pg,
                                                                      <vscale x 16 x i8> %a,
                                                                      <vscale x 2 x i64> %splat)
  ret <vscale x 16 x i1> %out
}

define <vscale x 4 x i1> @cmpls_wide_splat_s(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, i64 %b) {
; CHECK-LABEL: cmpls_wide_splat_s:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov z1.d, x0
; CHECK-NEXT:    cmpls p0.s, p0/z, z0.s, z1.d
; CHECK-NEXT:    ret
  %splat = call <vscale x 2 x i64> @llvm.aarch64.sve.dup.x.nxv2i64(i64 %b)
  %out = call <vscale x 4 x i1> @llvm.aarch64.sve.cmpls.wide.nxv4i32(<vscale x 4 x i1> %pg,
                                                                     <vscale x 4 x i32> %a,
                                                                     <vscale x 2 x i64> %splat)
  ret <vscale x 4 x i1> %out
}

; Verify general predicate is folded into the compare
define <vscale x 4 x i1> @predicated_icmp(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
; CHECK-LABEL: predicated_icmp:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p0.s
; CHECK-NEXT:    cmpgt p0.s, p0/z, z0.s, z1.s
; CHECK-NEXT:    cmpge p0.s, p0/z, z2.s, z1.s
; CHECK-NEXT:    ret
  %icmp1 = icmp sgt <vscale x 4 x i32> %a, %b
  %icmp2 = icmp sle <vscale x 4 x i32> %b, %c
  %and = and <vscale x 4 x i1> %icmp1, %icmp2
  ret <vscale x 4 x i1> %and
}

define <vscale x 4 x i1> @predicated_icmp_unknown_lhs(<vscale x 4 x i1> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
; CHECK-LABEL: predicated_icmp_unknown_lhs:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpge p0.s, p0/z, z1.s, z0.s
; CHECK-NEXT:    ret
  %icmp = icmp sle <vscale x 4 x i32> %b, %c
  %and = and <vscale x 4 x i1> %a, %icmp
  ret <vscale x 4 x i1> %and
}

define <vscale x 4 x i1> @predicated_icmp_unknown_rhs(<vscale x 4 x i1> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
; CHECK-LABEL: predicated_icmp_unknown_rhs:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpge p0.s, p0/z, z1.s, z0.s
; CHECK-NEXT:    ret
  %icmp = icmp sle <vscale x 4 x i32> %b, %c
  %and = and <vscale x 4 x i1> %icmp, %a
  ret <vscale x 4 x i1> %and
}

define <vscale x 16 x i1> @predicated_icmp_eq_imm(<vscale x 16 x i1> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: predicated_icmp_eq_imm:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpeq p0.b, p0/z, z0.b, #0
; CHECK-NEXT:    ret
  %imm = shufflevector <vscale x 16 x i8> insertelement (<vscale x 16 x i8> undef, i8 0, i64 0), <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
  %icmp = icmp eq <vscale x 16 x i8> %b, %imm
  %and = and <vscale x 16 x i1> %a, %icmp
  ret <vscale x 16 x i1> %and
}

define <vscale x 8 x i1> @predicated_icmp_ne_imm(<vscale x 8 x i1> %a, <vscale x 8 x i16> %b) {
; CHECK-LABEL: predicated_icmp_ne_imm:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpne p0.h, p0/z, z0.h, #-16
; CHECK-NEXT:    ret
  %imm = shufflevector <vscale x 8 x i16> insertelement (<vscale x 8 x i16> undef, i16 -16, i64 0), <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
  %icmp = icmp ne <vscale x 8 x i16> %b, %imm
  %and = and <vscale x 8 x i1> %a, %icmp
  ret <vscale x 8 x i1> %and
}

define <vscale x 4 x i1> @predicated_icmp_sge_imm(<vscale x 4 x i1> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: predicated_icmp_sge_imm:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpge p0.s, p0/z, z0.s, #1
; CHECK-NEXT:    ret
  %imm = shufflevector <vscale x 4 x i32> insertelement (<vscale x 4 x i32> undef, i32 1, i64 0), <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
  %icmp = icmp sge <vscale x 4 x i32> %b, %imm
  %and = and <vscale x 4 x i1> %a, %icmp
  ret <vscale x 4 x i1> %and
}

define <vscale x 2 x i1> @predicated_icmp_sgt_imm(<vscale x 2 x i1> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: predicated_icmp_sgt_imm:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpgt p0.d, p0/z, z0.d, #2
; CHECK-NEXT:    ret
  %imm = shufflevector <vscale x 2 x i64> insertelement (<vscale x 2 x i64> undef, i64 2, i64 0), <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
  %icmp = icmp sgt <vscale x 2 x i64> %b, %imm
  %and = and <vscale x 2 x i1> %a, %icmp
  ret <vscale x 2 x i1> %and
}

define <vscale x 16 x i1> @predicated_icmp_sle_imm(<vscale x 16 x i1> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: predicated_icmp_sle_imm:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmple p0.b, p0/z, z0.b, #-1
; CHECK-NEXT:    ret
  %imm = shufflevector <vscale x 16 x i8> insertelement (<vscale x 16 x i8> undef, i8 -1, i64 0), <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
  %icmp = icmp sle <vscale x 16 x i8> %b, %imm
  %and = and <vscale x 16 x i1> %a, %icmp
  ret <vscale x 16 x i1> %and
}

define <vscale x 8 x i1> @predicated_icmp_slt_imm(<vscale x 8 x i1> %a, <vscale x 8 x i16> %b) {
; CHECK-LABEL: predicated_icmp_slt_imm:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmplt p0.h, p0/z, z0.h, #-2
; CHECK-NEXT:    ret
  %imm = shufflevector <vscale x 8 x i16> insertelement (<vscale x 8 x i16> undef, i16 -2, i64 0), <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
  %icmp = icmp slt <vscale x 8 x i16> %b, %imm
  %and = and <vscale x 8 x i1> %a, %icmp
  ret <vscale x 8 x i1> %and
}

define <vscale x 4 x i1> @predicated_icmp_uge_imm(<vscale x 4 x i1> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: predicated_icmp_uge_imm:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmphs p0.s, p0/z, z0.s, #1
; CHECK-NEXT:    ret
  %imm = shufflevector <vscale x 4 x i32> insertelement (<vscale x 4 x i32> undef, i32 1, i64 0), <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
  %icmp = icmp uge <vscale x 4 x i32> %b, %imm
  %and = and <vscale x 4 x i1> %a, %icmp
  ret <vscale x 4 x i1> %and
}

define <vscale x 2 x i1> @predicated_icmp_ugt_imm(<vscale x 2 x i1> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: predicated_icmp_ugt_imm:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmphi p0.d, p0/z, z0.d, #2
; CHECK-NEXT:    ret
  %imm = shufflevector <vscale x 2 x i64> insertelement (<vscale x 2 x i64> undef, i64 2, i64 0), <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
  %icmp = icmp ugt <vscale x 2 x i64> %b, %imm
  %and = and <vscale x 2 x i1> %a, %icmp
  ret <vscale x 2 x i1> %and
}

define <vscale x 16 x i1> @predicated_icmp_ule_imm(<vscale x 16 x i1> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: predicated_icmp_ule_imm:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmpls p0.b, p0/z, z0.b, #3
; CHECK-NEXT:    ret
  %imm = shufflevector <vscale x 16 x i8> insertelement (<vscale x 16 x i8> undef, i8 3, i64 0), <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
  %icmp = icmp ule <vscale x 16 x i8> %b, %imm
  %and = and <vscale x 16 x i1> %a, %icmp
  ret <vscale x 16 x i1> %and
}

define <vscale x 8 x i1> @predicated_icmp_ult_imm(<vscale x 8 x i1> %a, <vscale x 8 x i16> %b) {
; CHECK-LABEL: predicated_icmp_ult_imm:
; CHECK:       // %bb.0:
; CHECK-NEXT:    cmplo p0.h, p0/z, z0.h, #127
; CHECK-NEXT:    ret
  %imm = shufflevector <vscale x 8 x i16> insertelement (<vscale x 8 x i16> undef, i16 127, i64 0), <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
  %icmp = icmp ult <vscale x 8 x i16> %b, %imm
  %and = and <vscale x 8 x i1> %a, %icmp
  ret <vscale x 8 x i1> %and
}

%svboolx2 = type { <vscale x 4 x i1>, <vscale x 4 x i1> }

define %svboolx2 @and_of_multiuse_icmp_sle(<vscale x 4 x i1> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
; CHECK-LABEL: and_of_multiuse_icmp_sle:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p1.s
; CHECK-NEXT:    cmpge p1.s, p1/z, z1.s, z0.s
; CHECK-NEXT:    and p0.b, p0/z, p0.b, p1.b
; CHECK-NEXT:    ret
  %cmp = icmp sle <vscale x 4 x i32> %b, %c
  %and = and <vscale x 4 x i1> %a, %cmp
  %ins.1 = insertvalue %svboolx2 poison, <vscale x 4 x i1> %and, 0
  %ins.2 = insertvalue %svboolx2 %ins.1, <vscale x 4 x i1> %cmp, 1
  ret %svboolx2 %ins.2
}

define %svboolx2 @and_of_multiuse_icmp_sle_imm(<vscale x 4 x i1> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: and_of_multiuse_icmp_sle_imm:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p1.s
; CHECK-NEXT:    cmple p1.s, p1/z, z0.s, #1
; CHECK-NEXT:    and p0.b, p0/z, p0.b, p1.b
; CHECK-NEXT:    ret
  %imm = shufflevector <vscale x 4 x i32> insertelement (<vscale x 4 x i32> undef, i32 1, i64 0), <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
  %cmp = icmp sle <vscale x 4 x i32> %b, %imm
  %and = and <vscale x 4 x i1> %a, %cmp
  %ins.1 = insertvalue %svboolx2 poison, <vscale x 4 x i1> %and, 0
  %ins.2 = insertvalue %svboolx2 %ins.1, <vscale x 4 x i1> %cmp, 1
  ret %svboolx2 %ins.2
}

define %svboolx2 @and_of_multiuse_icmp_ugt(<vscale x 4 x i1> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %c) {
; CHECK-LABEL: and_of_multiuse_icmp_ugt:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p1.s
; CHECK-NEXT:    cmphi p1.s, p1/z, z0.s, z1.s
; CHECK-NEXT:    and p0.b, p0/z, p0.b, p1.b
; CHECK-NEXT:    ret
  %cmp = icmp ugt <vscale x 4 x i32> %b, %c
  %and = and <vscale x 4 x i1> %a, %cmp
  %ins.1 = insertvalue %svboolx2 poison, <vscale x 4 x i1> %and, 0
  %ins.2 = insertvalue %svboolx2 %ins.1, <vscale x 4 x i1> %cmp, 1
  ret %svboolx2 %ins.2
}

define %svboolx2 @and_of_multiuse_icmp_ugt_imm(<vscale x 4 x i1> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: and_of_multiuse_icmp_ugt_imm:
; CHECK:       // %bb.0:
; CHECK-NEXT:    ptrue p1.s
; CHECK-NEXT:    cmphi p1.s, p1/z, z0.s, #1
; CHECK-NEXT:    and p0.b, p0/z, p0.b, p1.b
; CHECK-NEXT:    ret
  %imm = shufflevector <vscale x 4 x i32> insertelement (<vscale x 4 x i32> undef, i32 1, i64 0), <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
  %cmp = icmp ugt <vscale x 4 x i32> %b, %imm
  %and = and <vscale x 4 x i1> %a, %cmp
  %ins.1 = insertvalue %svboolx2 poison, <vscale x 4 x i1> %and, 0
  %ins.2 = insertvalue %svboolx2 %ins.1, <vscale x 4 x i1> %cmp, 1
  ret %svboolx2 %ins.2
}

declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpeq.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpeq.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpeq.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
declare <vscale x 2 x i1> @llvm.aarch64.sve.cmpeq.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpeq.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)

declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
declare <vscale x 2 x i1> @llvm.aarch64.sve.cmpge.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpge.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpge.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpge.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)

declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
declare <vscale x 2 x i1> @llvm.aarch64.sve.cmpgt.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpgt.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)

declare <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
declare <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
declare <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
declare <vscale x 2 x i1> @llvm.aarch64.sve.cmphi.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
declare <vscale x 16 x i1> @llvm.aarch64.sve.cmphi.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
declare <vscale x 8 x i1> @llvm.aarch64.sve.cmphi.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
declare <vscale x 4 x i1> @llvm.aarch64.sve.cmphi.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)

declare <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
declare <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
declare <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
declare <vscale x 2 x i1> @llvm.aarch64.sve.cmphs.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
declare <vscale x 16 x i1> @llvm.aarch64.sve.cmphs.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
declare <vscale x 8 x i1> @llvm.aarch64.sve.cmphs.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
declare <vscale x 4 x i1> @llvm.aarch64.sve.cmphs.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)

declare <vscale x 16 x i1> @llvm.aarch64.sve.cmple.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
declare <vscale x 8 x i1> @llvm.aarch64.sve.cmple.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
declare <vscale x 4 x i1> @llvm.aarch64.sve.cmple.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)

declare <vscale x 16 x i1> @llvm.aarch64.sve.cmplo.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
declare <vscale x 8 x i1> @llvm.aarch64.sve.cmplo.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
declare <vscale x 4 x i1> @llvm.aarch64.sve.cmplo.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)

declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpls.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpls.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpls.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)

declare <vscale x 16 x i1> @llvm.aarch64.sve.cmplt.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
declare <vscale x 8 x i1> @llvm.aarch64.sve.cmplt.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
declare <vscale x 4 x i1> @llvm.aarch64.sve.cmplt.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)

declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpne.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 16 x i8>)
declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpne.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>)
declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpne.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>)
declare <vscale x 2 x i1> @llvm.aarch64.sve.cmpne.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>)
declare <vscale x 16 x i1> @llvm.aarch64.sve.cmpne.wide.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>, <vscale x 2 x i64>)
declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpne.wide.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 2 x i64>)
declare <vscale x 4 x i1> @llvm.aarch64.sve.cmpne.wide.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 2 x i64>)

declare <vscale x 2 x i64> @llvm.aarch64.sve.dup.x.nxv2i64(i64)