2920 lines
116 KiB
C++
2920 lines
116 KiB
C++
|
//===- llvm/lib/Target/X86/X86ISelCallLowering.cpp - Call lowering --------===//
|
||
|
|
//
|
||
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
||
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||
|
|
//
|
||
|
|
//===----------------------------------------------------------------------===//
|
||
|
|
//
|
||
|
|
/// \file
|
||
|
|
/// This file implements the lowering of LLVM calls to DAG nodes.
|
||
|
|
//
|
||
|
|
//===----------------------------------------------------------------------===//
|
||
|
|
|
||
|
|
#include "X86.h"
|
||
|
|
#include "X86CallingConv.h"
|
||
|
|
#include "X86FrameLowering.h"
|
||
|
|
#include "X86ISelLowering.h"
|
||
|
|
#include "X86InstrBuilder.h"
|
||
|
|
#include "X86MachineFunctionInfo.h"
|
||
|
|
#include "X86TargetMachine.h"
|
||
|
|
#include "X86TargetObjectFile.h"
|
||
|
|
#include "llvm/ADT/Statistic.h"
|
||
|
|
#include "llvm/Analysis/ObjCARCUtil.h"
|
||
|
|
#include "llvm/CodeGen/MachineJumpTableInfo.h"
|
||
|
|
#include "llvm/CodeGen/MachineModuleInfo.h"
|
||
|
|
#include "llvm/CodeGen/WinEHFuncInfo.h"
|
||
|
|
#include "llvm/IR/DiagnosticInfo.h"
|
||
|
|
#include "llvm/IR/IRBuilder.h"
|
||
|
|
|
||
|
|
#define DEBUG_TYPE "x86-isel"
|
||
|
|
|
||
|
|
using namespace llvm;
|
||
|
|
|
||
|
|
STATISTIC(NumTailCalls, "Number of tail calls");
|
||
|
|
|
||
|
|
/// Call this when the user attempts to do something unsupported, like
|
||
|
|
/// returning a double without SSE2 enabled on x86_64. This is not fatal, unlike
|
||
|
|
/// report_fatal_error, so calling code should attempt to recover without
|
||
|
|
/// crashing.
|
||
|
|
static void errorUnsupported(SelectionDAG &DAG, const SDLoc &dl,
|
||
|
|
const char *Msg) {
|
||
|
|
MachineFunction &MF = DAG.getMachineFunction();
|
||
|
|
DAG.getContext()->diagnose(
|
||
|
|
DiagnosticInfoUnsupported(MF.getFunction(), Msg, dl.getDebugLoc()));
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Returns true if a CC can dynamically exclude a register from the list of
|
||
|
|
/// callee-saved-registers (TargetRegistryInfo::getCalleeSavedRegs()) based on
|
||
|
|
/// the return registers.
|
||
|
|
static bool shouldDisableRetRegFromCSR(CallingConv::ID CC) {
|
||
|
|
switch (CC) {
|
||
|
|
default:
|
||
|
|
return false;
|
||
|
|
case CallingConv::X86_RegCall:
|
||
|
|
case CallingConv::PreserveMost:
|
||
|
|
case CallingConv::PreserveAll:
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Returns true if a CC can dynamically exclude a register from the list of
|
||
|
|
/// callee-saved-registers (TargetRegistryInfo::getCalleeSavedRegs()) based on
|
||
|
|
/// the parameters.
|
||
|
|
static bool shouldDisableArgRegFromCSR(CallingConv::ID CC) {
|
||
|
|
return CC == CallingConv::X86_RegCall;
|
||
|
|
}
|
||
|
|
|
||
|
|
static std::pair<MVT, unsigned>
|
||
|
|
handleMaskRegisterForCallingConv(unsigned NumElts, CallingConv::ID CC,
|
||
|
|
const X86Subtarget &Subtarget) {
|
||
|
|
// v2i1/v4i1/v8i1/v16i1 all pass in xmm registers unless the calling
|
||
|
|
// convention is one that uses k registers.
|
||
|
|
if (NumElts == 2)
|
||
|
|
return {MVT::v2i64, 1};
|
||
|
|
if (NumElts == 4)
|
||
|
|
return {MVT::v4i32, 1};
|
||
|
|
if (NumElts == 8 && CC != CallingConv::X86_RegCall &&
|
||
|
|
CC != CallingConv::Intel_OCL_BI)
|
||
|
|
return {MVT::v8i16, 1};
|
||
|
|
if (NumElts == 16 && CC != CallingConv::X86_RegCall &&
|
||
|
|
CC != CallingConv::Intel_OCL_BI)
|
||
|
|
return {MVT::v16i8, 1};
|
||
|
|
// v32i1 passes in ymm unless we have BWI and the calling convention is
|
||
|
|
// regcall.
|
||
|
|
if (NumElts == 32 && (!Subtarget.hasBWI() || CC != CallingConv::X86_RegCall))
|
||
|
|
return {MVT::v32i8, 1};
|
||
|
|
// Split v64i1 vectors if we don't have v64i8 available.
|
||
|
|
if (NumElts == 64 && Subtarget.hasBWI() && CC != CallingConv::X86_RegCall) {
|
||
|
|
if (Subtarget.useAVX512Regs())
|
||
|
|
return {MVT::v64i8, 1};
|
||
|
|
return {MVT::v32i8, 2};
|
||
|
|
}
|
||
|
|
|
||
|
|
// Break wide or odd vXi1 vectors into scalars to match avx2 behavior.
|
||
|
|
if (!isPowerOf2_32(NumElts) || (NumElts == 64 && !Subtarget.hasBWI()) ||
|
||
|
|
NumElts > 64)
|
||
|
|
return {MVT::i8, NumElts};
|
||
|
|
|
||
|
|
return {MVT::INVALID_SIMPLE_VALUE_TYPE, 0};
|
||
|
|
}
|
||
|
|
|
||
|
|
MVT X86TargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
|
||
|
|
CallingConv::ID CC,
|
||
|
|
EVT VT) const {
|
||
|
|
if (VT.isVector()) {
|
||
|
|
if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512()) {
|
||
|
|
unsigned NumElts = VT.getVectorNumElements();
|
||
|
|
|
||
|
|
MVT RegisterVT;
|
||
|
|
unsigned NumRegisters;
|
||
|
|
std::tie(RegisterVT, NumRegisters) =
|
||
|
|
handleMaskRegisterForCallingConv(NumElts, CC, Subtarget);
|
||
|
|
if (RegisterVT != MVT::INVALID_SIMPLE_VALUE_TYPE)
|
||
|
|
return RegisterVT;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (VT.getVectorElementType() == MVT::f16 && VT.getVectorNumElements() < 8)
|
||
|
|
return MVT::v8f16;
|
||
|
|
}
|
||
|
|
|
||
|
|
// We will use more GPRs for f64 and f80 on 32 bits when x87 is disabled.
|
||
|
|
if ((VT == MVT::f64 || VT == MVT::f80) && !Subtarget.is64Bit() &&
|
||
|
|
!Subtarget.hasX87())
|
||
|
|
return MVT::i32;
|
||
|
|
|
||
|
|
if (VT.isVector() && VT.getVectorElementType() == MVT::bf16)
|
||
|
|
return getRegisterTypeForCallingConv(Context, CC,
|
||
|
|
VT.changeVectorElementType(MVT::f16));
|
||
|
|
|
||
|
|
if (VT == MVT::bf16)
|
||
|
|
return MVT::f16;
|
||
|
|
|
||
|
|
return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
|
||
|
|
}
|
||
|
|
|
||
|
|
unsigned X86TargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
|
||
|
|
CallingConv::ID CC,
|
||
|
|
EVT VT) const {
|
||
|
|
if (VT.isVector()) {
|
||
|
|
if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512()) {
|
||
|
|
unsigned NumElts = VT.getVectorNumElements();
|
||
|
|
|
||
|
|
MVT RegisterVT;
|
||
|
|
unsigned NumRegisters;
|
||
|
|
std::tie(RegisterVT, NumRegisters) =
|
||
|
|
handleMaskRegisterForCallingConv(NumElts, CC, Subtarget);
|
||
|
|
if (RegisterVT != MVT::INVALID_SIMPLE_VALUE_TYPE)
|
||
|
|
return NumRegisters;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (VT.getVectorElementType() == MVT::f16 && VT.getVectorNumElements() < 8)
|
||
|
|
return 1;
|
||
|
|
}
|
||
|
|
|
||
|
|
// We have to split f64 to 2 registers and f80 to 3 registers on 32 bits if
|
||
|
|
// x87 is disabled.
|
||
|
|
if (!Subtarget.is64Bit() && !Subtarget.hasX87()) {
|
||
|
|
if (VT == MVT::f64)
|
||
|
|
return 2;
|
||
|
|
if (VT == MVT::f80)
|
||
|
|
return 3;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (VT.isVector() && VT.getVectorElementType() == MVT::bf16)
|
||
|
|
return getNumRegistersForCallingConv(Context, CC,
|
||
|
|
VT.changeVectorElementType(MVT::f16));
|
||
|
|
|
||
|
|
return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
|
||
|
|
}
|
||
|
|
|
||
|
|
unsigned X86TargetLowering::getVectorTypeBreakdownForCallingConv(
|
||
|
|
LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
|
||
|
|
unsigned &NumIntermediates, MVT &RegisterVT) const {
|
||
|
|
// Break wide or odd vXi1 vectors into scalars to match avx2 behavior.
|
||
|
|
if (VT.isVector() && VT.getVectorElementType() == MVT::i1 &&
|
||
|
|
Subtarget.hasAVX512() &&
|
||
|
|
(!isPowerOf2_32(VT.getVectorNumElements()) ||
|
||
|
|
(VT.getVectorNumElements() == 64 && !Subtarget.hasBWI()) ||
|
||
|
|
VT.getVectorNumElements() > 64)) {
|
||
|
|
RegisterVT = MVT::i8;
|
||
|
|
IntermediateVT = MVT::i1;
|
||
|
|
NumIntermediates = VT.getVectorNumElements();
|
||
|
|
return NumIntermediates;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Split v64i1 vectors if we don't have v64i8 available.
|
||
|
|
if (VT == MVT::v64i1 && Subtarget.hasBWI() && !Subtarget.useAVX512Regs() &&
|
||
|
|
CC != CallingConv::X86_RegCall) {
|
||
|
|
RegisterVT = MVT::v32i8;
|
||
|
|
IntermediateVT = MVT::v32i1;
|
||
|
|
NumIntermediates = 2;
|
||
|
|
return 2;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Split vNbf16 vectors according to vNf16.
|
||
|
|
if (VT.isVector() && VT.getVectorElementType() == MVT::bf16)
|
||
|
|
VT = VT.changeVectorElementType(MVT::f16);
|
||
|
|
|
||
|
|
return TargetLowering::getVectorTypeBreakdownForCallingConv(Context, CC, VT, IntermediateVT,
|
||
|
|
NumIntermediates, RegisterVT);
|
||
|
|
}
|
||
|
|
|
||
|
|
EVT X86TargetLowering::getSetCCResultType(const DataLayout &DL,
|
||
|
|
LLVMContext& Context,
|
||
|
|
EVT VT) const {
|
||
|
|
if (!VT.isVector())
|
||
|
|
return MVT::i8;
|
||
|
|
|
||
|
|
if (Subtarget.hasAVX512()) {
|
||
|
|
// Figure out what this type will be legalized to.
|
||
|
|
EVT LegalVT = VT;
|
||
|
|
while (getTypeAction(Context, LegalVT) != TypeLegal)
|
||
|
|
LegalVT = getTypeToTransformTo(Context, LegalVT);
|
||
|
|
|
||
|
|
// If we got a 512-bit vector then we'll definitely have a vXi1 compare.
|
||
|
|
if (LegalVT.getSimpleVT().is512BitVector())
|
||
|
|
return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());
|
||
|
|
|
||
|
|
if (LegalVT.getSimpleVT().isVector() && Subtarget.hasVLX()) {
|
||
|
|
// If we legalized to less than a 512-bit vector, then we will use a vXi1
|
||
|
|
// compare for vXi32/vXi64 for sure. If we have BWI we will also support
|
||
|
|
// vXi16/vXi8.
|
||
|
|
MVT EltVT = LegalVT.getSimpleVT().getVectorElementType();
|
||
|
|
if (Subtarget.hasBWI() || EltVT.getSizeInBits() >= 32)
|
||
|
|
return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return VT.changeVectorElementTypeToInteger();
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Helper for getByValTypeAlignment to determine
|
||
|
|
/// the desired ByVal argument alignment.
|
||
|
|
static void getMaxByValAlign(Type *Ty, Align &MaxAlign) {
|
||
|
|
if (MaxAlign == 16)
|
||
|
|
return;
|
||
|
|
if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {
|
||
|
|
if (VTy->getPrimitiveSizeInBits().getFixedValue() == 128)
|
||
|
|
MaxAlign = Align(16);
|
||
|
|
} else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
|
||
|
|
Align EltAlign;
|
||
|
|
getMaxByValAlign(ATy->getElementType(), EltAlign);
|
||
|
|
if (EltAlign > MaxAlign)
|
||
|
|
MaxAlign = EltAlign;
|
||
|
|
} else if (StructType *STy = dyn_cast<StructType>(Ty)) {
|
||
|
|
for (auto *EltTy : STy->elements()) {
|
||
|
|
Align EltAlign;
|
||
|
|
getMaxByValAlign(EltTy, EltAlign);
|
||
|
|
if (EltAlign > MaxAlign)
|
||
|
|
MaxAlign = EltAlign;
|
||
|
|
if (MaxAlign == 16)
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Return the desired alignment for ByVal aggregate
|
||
|
|
/// function arguments in the caller parameter area. For X86, aggregates
|
||
|
|
/// that contain SSE vectors are placed at 16-byte boundaries while the rest
|
||
|
|
/// are at 4-byte boundaries.
|
||
|
|
uint64_t X86TargetLowering::getByValTypeAlignment(Type *Ty,
|
||
|
|
const DataLayout &DL) const {
|
||
|
|
if (Subtarget.is64Bit()) {
|
||
|
|
// Max of 8 and alignment of type.
|
||
|
|
Align TyAlign = DL.getABITypeAlign(Ty);
|
||
|
|
if (TyAlign > 8)
|
||
|
|
return TyAlign.value();
|
||
|
|
return 8;
|
||
|
|
}
|
||
|
|
|
||
|
|
Align Alignment(4);
|
||
|
|
if (Subtarget.hasSSE1())
|
||
|
|
getMaxByValAlign(Ty, Alignment);
|
||
|
|
return Alignment.value();
|
||
|
|
}
|
||
|
|
|
||
|
|
/// It returns EVT::Other if the type should be determined using generic
|
||
|
|
/// target-independent logic.
|
||
|
|
/// For vector ops we check that the overall size isn't larger than our
|
||
|
|
/// preferred vector width.
|
||
|
|
EVT X86TargetLowering::getOptimalMemOpType(
|
||
|
|
const MemOp &Op, const AttributeList &FuncAttributes) const {
|
||
|
|
if (!FuncAttributes.hasFnAttr(Attribute::NoImplicitFloat)) {
|
||
|
|
if (Op.size() >= 16 &&
|
||
|
|
(!Subtarget.isUnalignedMem16Slow() || Op.isAligned(Align(16)))) {
|
||
|
|
// FIXME: Check if unaligned 64-byte accesses are slow.
|
||
|
|
if (Op.size() >= 64 && Subtarget.hasAVX512() && Subtarget.hasEVEX512() &&
|
||
|
|
(Subtarget.getPreferVectorWidth() >= 512)) {
|
||
|
|
return Subtarget.hasBWI() ? MVT::v64i8 : MVT::v16i32;
|
||
|
|
}
|
||
|
|
// FIXME: Check if unaligned 32-byte accesses are slow.
|
||
|
|
if (Op.size() >= 32 && Subtarget.hasAVX() &&
|
||
|
|
Subtarget.useLight256BitInstructions()) {
|
||
|
|
// Although this isn't a well-supported type for AVX1, we'll let
|
||
|
|
// legalization and shuffle lowering produce the optimal codegen. If we
|
||
|
|
// choose an optimal type with a vector element larger than a byte,
|
||
|
|
// getMemsetStores() may create an intermediate splat (using an integer
|
||
|
|
// multiply) before we splat as a vector.
|
||
|
|
return MVT::v32i8;
|
||
|
|
}
|
||
|
|
if (Subtarget.hasSSE2() && (Subtarget.getPreferVectorWidth() >= 128))
|
||
|
|
return MVT::v16i8;
|
||
|
|
// TODO: Can SSE1 handle a byte vector?
|
||
|
|
// If we have SSE1 registers we should be able to use them.
|
||
|
|
if (Subtarget.hasSSE1() && (Subtarget.is64Bit() || Subtarget.hasX87()) &&
|
||
|
|
(Subtarget.getPreferVectorWidth() >= 128))
|
||
|
|
return MVT::v4f32;
|
||
|
|
} else if (((Op.isMemcpy() && !Op.isMemcpyStrSrc()) || Op.isZeroMemset()) &&
|
||
|
|
Op.size() >= 8 && !Subtarget.is64Bit() && Subtarget.hasSSE2()) {
|
||
|
|
// Do not use f64 to lower memcpy if source is string constant. It's
|
||
|
|
// better to use i32 to avoid the loads.
|
||
|
|
// Also, do not use f64 to lower memset unless this is a memset of zeros.
|
||
|
|
// The gymnastics of splatting a byte value into an XMM register and then
|
||
|
|
// only using 8-byte stores (because this is a CPU with slow unaligned
|
||
|
|
// 16-byte accesses) makes that a loser.
|
||
|
|
return MVT::f64;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// This is a compromise. If we reach here, unaligned accesses may be slow on
|
||
|
|
// this target. However, creating smaller, aligned accesses could be even
|
||
|
|
// slower and would certainly be a lot more code.
|
||
|
|
if (Subtarget.is64Bit() && Op.size() >= 8)
|
||
|
|
return MVT::i64;
|
||
|
|
return MVT::i32;
|
||
|
|
}
|
||
|
|
|
||
|
|
bool X86TargetLowering::isSafeMemOpType(MVT VT) const {
|
||
|
|
if (VT == MVT::f32)
|
||
|
|
return Subtarget.hasSSE1();
|
||
|
|
if (VT == MVT::f64)
|
||
|
|
return Subtarget.hasSSE2();
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
|
||
|
|
static bool isBitAligned(Align Alignment, uint64_t SizeInBits) {
|
||
|
|
return (8 * Alignment.value()) % SizeInBits == 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
bool X86TargetLowering::isMemoryAccessFast(EVT VT, Align Alignment) const {
|
||
|
|
if (isBitAligned(Alignment, VT.getSizeInBits()))
|
||
|
|
return true;
|
||
|
|
switch (VT.getSizeInBits()) {
|
||
|
|
default:
|
||
|
|
// 8-byte and under are always assumed to be fast.
|
||
|
|
return true;
|
||
|
|
case 128:
|
||
|
|
return !Subtarget.isUnalignedMem16Slow();
|
||
|
|
case 256:
|
||
|
|
return !Subtarget.isUnalignedMem32Slow();
|
||
|
|
// TODO: What about AVX-512 (512-bit) accesses?
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
bool X86TargetLowering::allowsMisalignedMemoryAccesses(
|
||
|
|
EVT VT, unsigned, Align Alignment, MachineMemOperand::Flags Flags,
|
||
|
|
unsigned *Fast) const {
|
||
|
|
if (Fast)
|
||
|
|
*Fast = isMemoryAccessFast(VT, Alignment);
|
||
|
|
// NonTemporal vector memory ops must be aligned.
|
||
|
|
if (!!(Flags & MachineMemOperand::MONonTemporal) && VT.isVector()) {
|
||
|
|
// NT loads can only be vector aligned, so if its less aligned than the
|
||
|
|
// minimum vector size (which we can split the vector down to), we might as
|
||
|
|
// well use a regular unaligned vector load.
|
||
|
|
// We don't have any NT loads pre-SSE41.
|
||
|
|
if (!!(Flags & MachineMemOperand::MOLoad))
|
||
|
|
return (Alignment < 16 || !Subtarget.hasSSE41());
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
// Misaligned accesses of any size are always allowed.
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
|
||
|
|
bool X86TargetLowering::allowsMemoryAccess(LLVMContext &Context,
|
||
|
|
const DataLayout &DL, EVT VT,
|
||
|
|
unsigned AddrSpace, Align Alignment,
|
||
|
|
MachineMemOperand::Flags Flags,
|
||
|
|
unsigned *Fast) const {
|
||
|
|
if (Fast)
|
||
|
|
*Fast = isMemoryAccessFast(VT, Alignment);
|
||
|
|
if (!!(Flags & MachineMemOperand::MONonTemporal) && VT.isVector()) {
|
||
|
|
if (allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment, Flags,
|
||
|
|
/*Fast=*/nullptr))
|
||
|
|
return true;
|
||
|
|
// NonTemporal vector memory ops are special, and must be aligned.
|
||
|
|
if (!isBitAligned(Alignment, VT.getSizeInBits()))
|
||
|
|
return false;
|
||
|
|
switch (VT.getSizeInBits()) {
|
||
|
|
case 128:
|
||
|
|
if (!!(Flags & MachineMemOperand::MOLoad) && Subtarget.hasSSE41())
|
||
|
|
return true;
|
||
|
|
if (!!(Flags & MachineMemOperand::MOStore) && Subtarget.hasSSE2())
|
||
|
|
return true;
|
||
|
|
return false;
|
||
|
|
case 256:
|
||
|
|
if (!!(Flags & MachineMemOperand::MOLoad) && Subtarget.hasAVX2())
|
||
|
|
return true;
|
||
|
|
if (!!(Flags & MachineMemOperand::MOStore) && Subtarget.hasAVX())
|
||
|
|
return true;
|
||
|
|
return false;
|
||
|
|
case 512:
|
||
|
|
if (Subtarget.hasAVX512() && Subtarget.hasEVEX512())
|
||
|
|
return true;
|
||
|
|
return false;
|
||
|
|
default:
|
||
|
|
return false; // Don't have NonTemporal vector memory ops of this size.
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Return the entry encoding for a jump table in the
|
||
|
|
/// current function. The returned value is a member of the
|
||
|
|
/// MachineJumpTableInfo::JTEntryKind enum.
|
||
|
|
unsigned X86TargetLowering::getJumpTableEncoding() const {
|
||
|
|
// In GOT pic mode, each entry in the jump table is emitted as a @GOTOFF
|
||
|
|
// symbol.
|
||
|
|
if (isPositionIndependent() && Subtarget.isPICStyleGOT())
|
||
|
|
return MachineJumpTableInfo::EK_Custom32;
|
||
|
|
if (isPositionIndependent() &&
|
||
|
|
getTargetMachine().getCodeModel() == CodeModel::Large)
|
||
|
|
return MachineJumpTableInfo::EK_LabelDifference64;
|
||
|
|
|
||
|
|
// Otherwise, use the normal jump table encoding heuristics.
|
||
|
|
return TargetLowering::getJumpTableEncoding();
|
||
|
|
}
|
||
|
|
|
||
|
|
bool X86TargetLowering::useSoftFloat() const {
|
||
|
|
return Subtarget.useSoftFloat();
|
||
|
|
}
|
||
|
|
|
||
|
|
void X86TargetLowering::markLibCallAttributes(MachineFunction *MF, unsigned CC,
|
||
|
|
ArgListTy &Args) const {
|
||
|
|
|
||
|
|
// Only relabel X86-32 for C / Stdcall CCs.
|
||
|
|
if (Subtarget.is64Bit())
|
||
|
|
return;
|
||
|
|
if (CC != CallingConv::C && CC != CallingConv::X86_StdCall)
|
||
|
|
return;
|
||
|
|
unsigned ParamRegs = 0;
|
||
|
|
if (auto *M = MF->getFunction().getParent())
|
||
|
|
ParamRegs = M->getNumberRegisterParameters();
|
||
|
|
|
||
|
|
// Mark the first N int arguments as having reg
|
||
|
|
for (auto &Arg : Args) {
|
||
|
|
Type *T = Arg.Ty;
|
||
|
|
if (T->isIntOrPtrTy())
|
||
|
|
if (MF->getDataLayout().getTypeAllocSize(T) <= 8) {
|
||
|
|
unsigned numRegs = 1;
|
||
|
|
if (MF->getDataLayout().getTypeAllocSize(T) > 4)
|
||
|
|
numRegs = 2;
|
||
|
|
if (ParamRegs < numRegs)
|
||
|
|
return;
|
||
|
|
ParamRegs -= numRegs;
|
||
|
|
Arg.IsInReg = true;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
const MCExpr *
|
||
|
|
X86TargetLowering::LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI,
|
||
|
|
const MachineBasicBlock *MBB,
|
||
|
|
unsigned uid,MCContext &Ctx) const{
|
||
|
|
assert(isPositionIndependent() && Subtarget.isPICStyleGOT());
|
||
|
|
// In 32-bit ELF systems, our jump table entries are formed with @GOTOFF
|
||
|
|
// entries.
|
||
|
|
return MCSymbolRefExpr::create(MBB->getSymbol(),
|
||
|
|
MCSymbolRefExpr::VK_GOTOFF, Ctx);
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Returns relocation base for the given PIC jumptable.
|
||
|
|
SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table,
|
||
|
|
SelectionDAG &DAG) const {
|
||
|
|
if (!Subtarget.is64Bit())
|
||
|
|
// This doesn't have SDLoc associated with it, but is not really the
|
||
|
|
// same as a Register.
|
||
|
|
return DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(),
|
||
|
|
getPointerTy(DAG.getDataLayout()));
|
||
|
|
return Table;
|
||
|
|
}
|
||
|
|
|
||
|
|
/// This returns the relocation base for the given PIC jumptable,
|
||
|
|
/// the same as getPICJumpTableRelocBase, but as an MCExpr.
|
||
|
|
const MCExpr *X86TargetLowering::
|
||
|
|
getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI,
|
||
|
|
MCContext &Ctx) const {
|
||
|
|
// X86-64 uses RIP relative addressing based on the jump table label.
|
||
|
|
if (Subtarget.isPICStyleRIPRel() ||
|
||
|
|
(Subtarget.is64Bit() &&
|
||
|
|
getTargetMachine().getCodeModel() == CodeModel::Large))
|
||
|
|
return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx);
|
||
|
|
|
||
|
|
// Otherwise, the reference is relative to the PIC base.
|
||
|
|
return MCSymbolRefExpr::create(MF->getPICBaseSymbol(), Ctx);
|
||
|
|
}
|
||
|
|
|
||
|
|
std::pair<const TargetRegisterClass *, uint8_t>
|
||
|
|
X86TargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
|
||
|
|
MVT VT) const {
|
||
|
|
const TargetRegisterClass *RRC = nullptr;
|
||
|
|
uint8_t Cost = 1;
|
||
|
|
switch (VT.SimpleTy) {
|
||
|
|
default:
|
||
|
|
return TargetLowering::findRepresentativeClass(TRI, VT);
|
||
|
|
case MVT::i8: case MVT::i16: case MVT::i32: case MVT::i64:
|
||
|
|
RRC = Subtarget.is64Bit() ? &X86::GR64RegClass : &X86::GR32RegClass;
|
||
|
|
break;
|
||
|
|
case MVT::x86mmx:
|
||
|
|
RRC = &X86::VR64RegClass;
|
||
|
|
break;
|
||
|
|
case MVT::f32: case MVT::f64:
|
||
|
|
case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64:
|
||
|
|
case MVT::v4f32: case MVT::v2f64:
|
||
|
|
case MVT::v32i8: case MVT::v16i16: case MVT::v8i32: case MVT::v4i64:
|
||
|
|
case MVT::v8f32: case MVT::v4f64:
|
||
|
|
case MVT::v64i8: case MVT::v32i16: case MVT::v16i32: case MVT::v8i64:
|
||
|
|
case MVT::v16f32: case MVT::v8f64:
|
||
|
|
RRC = &X86::VR128XRegClass;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
return std::make_pair(RRC, Cost);
|
||
|
|
}
|
||
|
|
|
||
|
|
unsigned X86TargetLowering::getAddressSpace() const {
|
||
|
|
if (Subtarget.is64Bit())
|
||
|
|
return (getTargetMachine().getCodeModel() == CodeModel::Kernel) ? 256 : 257;
|
||
|
|
return 256;
|
||
|
|
}
|
||
|
|
|
||
|
|
static bool hasStackGuardSlotTLS(const Triple &TargetTriple) {
|
||
|
|
return TargetTriple.isOSGlibc() || TargetTriple.isOSFuchsia() ||
|
||
|
|
(TargetTriple.isAndroid() && !TargetTriple.isAndroidVersionLT(17));
|
||
|
|
}
|
||
|
|
|
||
|
|
static Constant* SegmentOffset(IRBuilderBase &IRB,
|
||
|
|
int Offset, unsigned AddressSpace) {
|
||
|
|
return ConstantExpr::getIntToPtr(
|
||
|
|
ConstantInt::get(Type::getInt32Ty(IRB.getContext()), Offset),
|
||
|
|
IRB.getPtrTy(AddressSpace));
|
||
|
|
}
|
||
|
|
|
||
|
|
Value *X86TargetLowering::getIRStackGuard(IRBuilderBase &IRB) const {
|
||
|
|
// glibc, bionic, and Fuchsia have a special slot for the stack guard in
|
||
|
|
// tcbhead_t; use it instead of the usual global variable (see
|
||
|
|
// sysdeps/{i386,x86_64}/nptl/tls.h)
|
||
|
|
if (hasStackGuardSlotTLS(Subtarget.getTargetTriple())) {
|
||
|
|
unsigned AddressSpace = getAddressSpace();
|
||
|
|
|
||
|
|
// <zircon/tls.h> defines ZX_TLS_STACK_GUARD_OFFSET with this value.
|
||
|
|
if (Subtarget.isTargetFuchsia())
|
||
|
|
return SegmentOffset(IRB, 0x10, AddressSpace);
|
||
|
|
|
||
|
|
Module *M = IRB.GetInsertBlock()->getParent()->getParent();
|
||
|
|
// Specially, some users may customize the base reg and offset.
|
||
|
|
int Offset = M->getStackProtectorGuardOffset();
|
||
|
|
// If we don't set -stack-protector-guard-offset value:
|
||
|
|
// %fs:0x28, unless we're using a Kernel code model, in which case
|
||
|
|
// it's %gs:0x28. gs:0x14 on i386.
|
||
|
|
if (Offset == INT_MAX)
|
||
|
|
Offset = (Subtarget.is64Bit()) ? 0x28 : 0x14;
|
||
|
|
|
||
|
|
StringRef GuardReg = M->getStackProtectorGuardReg();
|
||
|
|
if (GuardReg == "fs")
|
||
|
|
AddressSpace = X86AS::FS;
|
||
|
|
else if (GuardReg == "gs")
|
||
|
|
AddressSpace = X86AS::GS;
|
||
|
|
|
||
|
|
// Use symbol guard if user specify.
|
||
|
|
StringRef GuardSymb = M->getStackProtectorGuardSymbol();
|
||
|
|
if (!GuardSymb.empty()) {
|
||
|
|
GlobalVariable *GV = M->getGlobalVariable(GuardSymb);
|
||
|
|
if (!GV) {
|
||
|
|
Type *Ty = Subtarget.is64Bit() ? Type::getInt64Ty(M->getContext())
|
||
|
|
: Type::getInt32Ty(M->getContext());
|
||
|
|
GV = new GlobalVariable(*M, Ty, false, GlobalValue::ExternalLinkage,
|
||
|
|
nullptr, GuardSymb, nullptr,
|
||
|
|
GlobalValue::NotThreadLocal, AddressSpace);
|
||
|
|
if (!Subtarget.isTargetDarwin())
|
||
|
|
GV->setDSOLocal(M->getDirectAccessExternalData());
|
||
|
|
}
|
||
|
|
return GV;
|
||
|
|
}
|
||
|
|
|
||
|
|
return SegmentOffset(IRB, Offset, AddressSpace);
|
||
|
|
}
|
||
|
|
return TargetLowering::getIRStackGuard(IRB);
|
||
|
|
}
|
||
|
|
|
||
|
|
void X86TargetLowering::insertSSPDeclarations(Module &M) const {
|
||
|
|
// MSVC CRT provides functionalities for stack protection.
|
||
|
|
if (Subtarget.getTargetTriple().isWindowsMSVCEnvironment() ||
|
||
|
|
Subtarget.getTargetTriple().isWindowsItaniumEnvironment()) {
|
||
|
|
// MSVC CRT has a global variable holding security cookie.
|
||
|
|
M.getOrInsertGlobal("__security_cookie",
|
||
|
|
PointerType::getUnqual(M.getContext()));
|
||
|
|
|
||
|
|
// MSVC CRT has a function to validate security cookie.
|
||
|
|
FunctionCallee SecurityCheckCookie = M.getOrInsertFunction(
|
||
|
|
"__security_check_cookie", Type::getVoidTy(M.getContext()),
|
||
|
|
PointerType::getUnqual(M.getContext()));
|
||
|
|
if (Function *F = dyn_cast<Function>(SecurityCheckCookie.getCallee())) {
|
||
|
|
F->setCallingConv(CallingConv::X86_FastCall);
|
||
|
|
F->addParamAttr(0, Attribute::AttrKind::InReg);
|
||
|
|
}
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
StringRef GuardMode = M.getStackProtectorGuard();
|
||
|
|
|
||
|
|
// glibc, bionic, and Fuchsia have a special slot for the stack guard.
|
||
|
|
if ((GuardMode == "tls" || GuardMode.empty()) &&
|
||
|
|
hasStackGuardSlotTLS(Subtarget.getTargetTriple()))
|
||
|
|
return;
|
||
|
|
TargetLowering::insertSSPDeclarations(M);
|
||
|
|
}
|
||
|
|
|
||
|
|
Value *X86TargetLowering::getSDagStackGuard(const Module &M) const {
|
||
|
|
// MSVC CRT has a global variable holding security cookie.
|
||
|
|
if (Subtarget.getTargetTriple().isWindowsMSVCEnvironment() ||
|
||
|
|
Subtarget.getTargetTriple().isWindowsItaniumEnvironment()) {
|
||
|
|
return M.getGlobalVariable("__security_cookie");
|
||
|
|
}
|
||
|
|
return TargetLowering::getSDagStackGuard(M);
|
||
|
|
}
|
||
|
|
|
||
|
|
Function *X86TargetLowering::getSSPStackGuardCheck(const Module &M) const {
|
||
|
|
// MSVC CRT has a function to validate security cookie.
|
||
|
|
if (Subtarget.getTargetTriple().isWindowsMSVCEnvironment() ||
|
||
|
|
Subtarget.getTargetTriple().isWindowsItaniumEnvironment()) {
|
||
|
|
return M.getFunction("__security_check_cookie");
|
||
|
|
}
|
||
|
|
return TargetLowering::getSSPStackGuardCheck(M);
|
||
|
|
}
|
||
|
|
|
||
|
|
Value *
|
||
|
|
X86TargetLowering::getSafeStackPointerLocation(IRBuilderBase &IRB) const {
|
||
|
|
// Android provides a fixed TLS slot for the SafeStack pointer. See the
|
||
|
|
// definition of TLS_SLOT_SAFESTACK in
|
||
|
|
// https://android.googlesource.com/platform/bionic/+/master/libc/private/bionic_tls.h
|
||
|
|
if (Subtarget.isTargetAndroid()) {
|
||
|
|
// %fs:0x48, unless we're using a Kernel code model, in which case it's %gs:
|
||
|
|
// %gs:0x24 on i386
|
||
|
|
int Offset = (Subtarget.is64Bit()) ? 0x48 : 0x24;
|
||
|
|
return SegmentOffset(IRB, Offset, getAddressSpace());
|
||
|
|
}
|
||
|
|
|
||
|
|
// Fuchsia is similar.
|
||
|
|
if (Subtarget.isTargetFuchsia()) {
|
||
|
|
// <zircon/tls.h> defines ZX_TLS_UNSAFE_SP_OFFSET with this value.
|
||
|
|
return SegmentOffset(IRB, 0x18, getAddressSpace());
|
||
|
|
}
|
||
|
|
|
||
|
|
return TargetLowering::getSafeStackPointerLocation(IRB);
|
||
|
|
}
|
||
|
|
|
||
|
|
//===----------------------------------------------------------------------===//
|
||
|
|
// Return Value Calling Convention Implementation
|
||
|
|
//===----------------------------------------------------------------------===//
|
||
|
|
|
||
|
|
bool X86TargetLowering::CanLowerReturn(
|
||
|
|
CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg,
|
||
|
|
const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
|
||
|
|
SmallVector<CCValAssign, 16> RVLocs;
|
||
|
|
CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
|
||
|
|
return CCInfo.CheckReturn(Outs, RetCC_X86);
|
||
|
|
}
|
||
|
|
|
||
|
|
const MCPhysReg *X86TargetLowering::getScratchRegisters(CallingConv::ID) const {
|
||
|
|
static const MCPhysReg ScratchRegs[] = { X86::R11, 0 };
|
||
|
|
return ScratchRegs;
|
||
|
|
}
|
||
|
|
|
||
|
|
ArrayRef<MCPhysReg> X86TargetLowering::getRoundingControlRegisters() const {
|
||
|
|
// FIXME: We should def X86::FPCW for x87 as well. But it affects a lot of lit
|
||
|
|
// tests at the moment, which is not what we expected.
|
||
|
|
static const MCPhysReg RCRegs[] = {X86::MXCSR};
|
||
|
|
return RCRegs;
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Lowers masks values (v*i1) to the local register values
|
||
|
|
/// \returns DAG node after lowering to register type
|
||
|
|
static SDValue lowerMasksToReg(const SDValue &ValArg, const EVT &ValLoc,
|
||
|
|
const SDLoc &DL, SelectionDAG &DAG) {
|
||
|
|
EVT ValVT = ValArg.getValueType();
|
||
|
|
|
||
|
|
if (ValVT == MVT::v1i1)
|
||
|
|
return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ValLoc, ValArg,
|
||
|
|
DAG.getIntPtrConstant(0, DL));
|
||
|
|
|
||
|
|
if ((ValVT == MVT::v8i1 && (ValLoc == MVT::i8 || ValLoc == MVT::i32)) ||
|
||
|
|
(ValVT == MVT::v16i1 && (ValLoc == MVT::i16 || ValLoc == MVT::i32))) {
|
||
|
|
// Two stage lowering might be required
|
||
|
|
// bitcast: v8i1 -> i8 / v16i1 -> i16
|
||
|
|
// anyextend: i8 -> i32 / i16 -> i32
|
||
|
|
EVT TempValLoc = ValVT == MVT::v8i1 ? MVT::i8 : MVT::i16;
|
||
|
|
SDValue ValToCopy = DAG.getBitcast(TempValLoc, ValArg);
|
||
|
|
if (ValLoc == MVT::i32)
|
||
|
|
ValToCopy = DAG.getNode(ISD::ANY_EXTEND, DL, ValLoc, ValToCopy);
|
||
|
|
return ValToCopy;
|
||
|
|
}
|
||
|
|
|
||
|
|
if ((ValVT == MVT::v32i1 && ValLoc == MVT::i32) ||
|
||
|
|
(ValVT == MVT::v64i1 && ValLoc == MVT::i64)) {
|
||
|
|
// One stage lowering is required
|
||
|
|
// bitcast: v32i1 -> i32 / v64i1 -> i64
|
||
|
|
return DAG.getBitcast(ValLoc, ValArg);
|
||
|
|
}
|
||
|
|
|
||
|
|
return DAG.getNode(ISD::ANY_EXTEND, DL, ValLoc, ValArg);
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Breaks v64i1 value into two registers and adds the new node to the DAG
|
||
|
|
static void Passv64i1ArgInRegs(
|
||
|
|
const SDLoc &DL, SelectionDAG &DAG, SDValue &Arg,
|
||
|
|
SmallVectorImpl<std::pair<Register, SDValue>> &RegsToPass, CCValAssign &VA,
|
||
|
|
CCValAssign &NextVA, const X86Subtarget &Subtarget) {
|
||
|
|
assert(Subtarget.hasBWI() && "Expected AVX512BW target!");
|
||
|
|
assert(Subtarget.is32Bit() && "Expecting 32 bit target");
|
||
|
|
assert(Arg.getValueType() == MVT::i64 && "Expecting 64 bit value");
|
||
|
|
assert(VA.isRegLoc() && NextVA.isRegLoc() &&
|
||
|
|
"The value should reside in two registers");
|
||
|
|
|
||
|
|
// Before splitting the value we cast it to i64
|
||
|
|
Arg = DAG.getBitcast(MVT::i64, Arg);
|
||
|
|
|
||
|
|
// Splitting the value into two i32 types
|
||
|
|
SDValue Lo, Hi;
|
||
|
|
std::tie(Lo, Hi) = DAG.SplitScalar(Arg, DL, MVT::i32, MVT::i32);
|
||
|
|
|
||
|
|
// Attach the two i32 types into corresponding registers
|
||
|
|
RegsToPass.push_back(std::make_pair(VA.getLocReg(), Lo));
|
||
|
|
RegsToPass.push_back(std::make_pair(NextVA.getLocReg(), Hi));
|
||
|
|
}
|
||
|
|
|
||
|
|
SDValue
|
||
|
|
X86TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
|
||
|
|
bool isVarArg,
|
||
|
|
const SmallVectorImpl<ISD::OutputArg> &Outs,
|
||
|
|
const SmallVectorImpl<SDValue> &OutVals,
|
||
|
|
const SDLoc &dl, SelectionDAG &DAG) const {
|
||
|
|
MachineFunction &MF = DAG.getMachineFunction();
|
||
|
|
X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
|
||
|
|
|
||
|
|
// In some cases we need to disable registers from the default CSR list.
|
||
|
|
// For example, when they are used as return registers (preserve_* and X86's
|
||
|
|
// regcall) or for argument passing (X86's regcall).
|
||
|
|
bool ShouldDisableCalleeSavedRegister =
|
||
|
|
shouldDisableRetRegFromCSR(CallConv) ||
|
||
|
|
MF.getFunction().hasFnAttribute("no_caller_saved_registers");
|
||
|
|
|
||
|
|
if (CallConv == CallingConv::X86_INTR && !Outs.empty())
|
||
|
|
report_fatal_error("X86 interrupts may not return any value");
|
||
|
|
|
||
|
|
SmallVector<CCValAssign, 16> RVLocs;
|
||
|
|
CCState CCInfo(CallConv, isVarArg, MF, RVLocs, *DAG.getContext());
|
||
|
|
CCInfo.AnalyzeReturn(Outs, RetCC_X86);
|
||
|
|
|
||
|
|
SmallVector<std::pair<Register, SDValue>, 4> RetVals;
|
||
|
|
for (unsigned I = 0, OutsIndex = 0, E = RVLocs.size(); I != E;
|
||
|
|
++I, ++OutsIndex) {
|
||
|
|
CCValAssign &VA = RVLocs[I];
|
||
|
|
assert(VA.isRegLoc() && "Can only return in registers!");
|
||
|
|
|
||
|
|
// Add the register to the CalleeSaveDisableRegs list.
|
||
|
|
if (ShouldDisableCalleeSavedRegister)
|
||
|
|
MF.getRegInfo().disableCalleeSavedRegister(VA.getLocReg());
|
||
|
|
|
||
|
|
SDValue ValToCopy = OutVals[OutsIndex];
|
||
|
|
EVT ValVT = ValToCopy.getValueType();
|
||
|
|
|
||
|
|
// Promote values to the appropriate types.
|
||
|
|
if (VA.getLocInfo() == CCValAssign::SExt)
|
||
|
|
ValToCopy = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), ValToCopy);
|
||
|
|
else if (VA.getLocInfo() == CCValAssign::ZExt)
|
||
|
|
ValToCopy = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), ValToCopy);
|
||
|
|
else if (VA.getLocInfo() == CCValAssign::AExt) {
|
||
|
|
if (ValVT.isVector() && ValVT.getVectorElementType() == MVT::i1)
|
||
|
|
ValToCopy = lowerMasksToReg(ValToCopy, VA.getLocVT(), dl, DAG);
|
||
|
|
else
|
||
|
|
ValToCopy = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), ValToCopy);
|
||
|
|
}
|
||
|
|
else if (VA.getLocInfo() == CCValAssign::BCvt)
|
||
|
|
ValToCopy = DAG.getBitcast(VA.getLocVT(), ValToCopy);
|
||
|
|
|
||
|
|
assert(VA.getLocInfo() != CCValAssign::FPExt &&
|
||
|
|
"Unexpected FP-extend for return value.");
|
||
|
|
|
||
|
|
// Report an error if we have attempted to return a value via an XMM
|
||
|
|
// register and SSE was disabled.
|
||
|
|
if (!Subtarget.hasSSE1() && X86::FR32XRegClass.contains(VA.getLocReg())) {
|
||
|
|
errorUnsupported(DAG, dl, "SSE register return with SSE disabled");
|
||
|
|
VA.convertToReg(X86::FP0); // Set reg to FP0, avoid hitting asserts.
|
||
|
|
} else if (!Subtarget.hasSSE2() &&
|
||
|
|
X86::FR64XRegClass.contains(VA.getLocReg()) &&
|
||
|
|
ValVT == MVT::f64) {
|
||
|
|
// When returning a double via an XMM register, report an error if SSE2 is
|
||
|
|
// not enabled.
|
||
|
|
errorUnsupported(DAG, dl, "SSE2 register return with SSE2 disabled");
|
||
|
|
VA.convertToReg(X86::FP0); // Set reg to FP0, avoid hitting asserts.
|
||
|
|
}
|
||
|
|
|
||
|
|
// Returns in ST0/ST1 are handled specially: these are pushed as operands to
|
||
|
|
// the RET instruction and handled by the FP Stackifier.
|
||
|
|
if (VA.getLocReg() == X86::FP0 ||
|
||
|
|
VA.getLocReg() == X86::FP1) {
|
||
|
|
// If this is a copy from an xmm register to ST(0), use an FPExtend to
|
||
|
|
// change the value to the FP stack register class.
|
||
|
|
if (isScalarFPTypeInSSEReg(VA.getValVT()))
|
||
|
|
ValToCopy = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f80, ValToCopy);
|
||
|
|
RetVals.push_back(std::make_pair(VA.getLocReg(), ValToCopy));
|
||
|
|
// Don't emit a copytoreg.
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
|
||
|
|
// 64-bit vector (MMX) values are returned in XMM0 / XMM1 except for v1i64
|
||
|
|
// which is returned in RAX / RDX.
|
||
|
|
if (Subtarget.is64Bit()) {
|
||
|
|
if (ValVT == MVT::x86mmx) {
|
||
|
|
if (VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) {
|
||
|
|
ValToCopy = DAG.getBitcast(MVT::i64, ValToCopy);
|
||
|
|
ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64,
|
||
|
|
ValToCopy);
|
||
|
|
// If we don't have SSE2 available, convert to v4f32 so the generated
|
||
|
|
// register is legal.
|
||
|
|
if (!Subtarget.hasSSE2())
|
||
|
|
ValToCopy = DAG.getBitcast(MVT::v4f32, ValToCopy);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (VA.needsCustom()) {
|
||
|
|
assert(VA.getValVT() == MVT::v64i1 &&
|
||
|
|
"Currently the only custom case is when we split v64i1 to 2 regs");
|
||
|
|
|
||
|
|
Passv64i1ArgInRegs(dl, DAG, ValToCopy, RetVals, VA, RVLocs[++I],
|
||
|
|
Subtarget);
|
||
|
|
|
||
|
|
// Add the second register to the CalleeSaveDisableRegs list.
|
||
|
|
if (ShouldDisableCalleeSavedRegister)
|
||
|
|
MF.getRegInfo().disableCalleeSavedRegister(RVLocs[I].getLocReg());
|
||
|
|
} else {
|
||
|
|
RetVals.push_back(std::make_pair(VA.getLocReg(), ValToCopy));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
SDValue Glue;
|
||
|
|
SmallVector<SDValue, 6> RetOps;
|
||
|
|
RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
|
||
|
|
// Operand #1 = Bytes To Pop
|
||
|
|
RetOps.push_back(DAG.getTargetConstant(FuncInfo->getBytesToPopOnReturn(), dl,
|
||
|
|
MVT::i32));
|
||
|
|
|
||
|
|
// Copy the result values into the output registers.
|
||
|
|
for (auto &RetVal : RetVals) {
|
||
|
|
if (RetVal.first == X86::FP0 || RetVal.first == X86::FP1) {
|
||
|
|
RetOps.push_back(RetVal.second);
|
||
|
|
continue; // Don't emit a copytoreg.
|
||
|
|
}
|
||
|
|
|
||
|
|
Chain = DAG.getCopyToReg(Chain, dl, RetVal.first, RetVal.second, Glue);
|
||
|
|
Glue = Chain.getValue(1);
|
||
|
|
RetOps.push_back(
|
||
|
|
DAG.getRegister(RetVal.first, RetVal.second.getValueType()));
|
||
|
|
}
|
||
|
|
|
||
|
|
// Swift calling convention does not require we copy the sret argument
|
||
|
|
// into %rax/%eax for the return, and SRetReturnReg is not set for Swift.
|
||
|
|
|
||
|
|
// All x86 ABIs require that for returning structs by value we copy
|
||
|
|
// the sret argument into %rax/%eax (depending on ABI) for the return.
|
||
|
|
// We saved the argument into a virtual register in the entry block,
|
||
|
|
// so now we copy the value out and into %rax/%eax.
|
||
|
|
//
|
||
|
|
// Checking Function.hasStructRetAttr() here is insufficient because the IR
|
||
|
|
// may not have an explicit sret argument. If FuncInfo.CanLowerReturn is
|
||
|
|
// false, then an sret argument may be implicitly inserted in the SelDAG. In
|
||
|
|
// either case FuncInfo->setSRetReturnReg() will have been called.
|
||
|
|
if (Register SRetReg = FuncInfo->getSRetReturnReg()) {
|
||
|
|
// When we have both sret and another return value, we should use the
|
||
|
|
// original Chain stored in RetOps[0], instead of the current Chain updated
|
||
|
|
// in the above loop. If we only have sret, RetOps[0] equals to Chain.
|
||
|
|
|
||
|
|
// For the case of sret and another return value, we have
|
||
|
|
// Chain_0 at the function entry
|
||
|
|
// Chain_1 = getCopyToReg(Chain_0) in the above loop
|
||
|
|
// If we use Chain_1 in getCopyFromReg, we will have
|
||
|
|
// Val = getCopyFromReg(Chain_1)
|
||
|
|
// Chain_2 = getCopyToReg(Chain_1, Val) from below
|
||
|
|
|
||
|
|
// getCopyToReg(Chain_0) will be glued together with
|
||
|
|
// getCopyToReg(Chain_1, Val) into Unit A, getCopyFromReg(Chain_1) will be
|
||
|
|
// in Unit B, and we will have cyclic dependency between Unit A and Unit B:
|
||
|
|
// Data dependency from Unit B to Unit A due to usage of Val in
|
||
|
|
// getCopyToReg(Chain_1, Val)
|
||
|
|
// Chain dependency from Unit A to Unit B
|
||
|
|
|
||
|
|
// So here, we use RetOps[0] (i.e Chain_0) for getCopyFromReg.
|
||
|
|
SDValue Val = DAG.getCopyFromReg(RetOps[0], dl, SRetReg,
|
||
|
|
getPointerTy(MF.getDataLayout()));
|
||
|
|
|
||
|
|
Register RetValReg
|
||
|
|
= (Subtarget.is64Bit() && !Subtarget.isTarget64BitILP32()) ?
|
||
|
|
X86::RAX : X86::EAX;
|
||
|
|
Chain = DAG.getCopyToReg(Chain, dl, RetValReg, Val, Glue);
|
||
|
|
Glue = Chain.getValue(1);
|
||
|
|
|
||
|
|
// RAX/EAX now acts like a return value.
|
||
|
|
RetOps.push_back(
|
||
|
|
DAG.getRegister(RetValReg, getPointerTy(DAG.getDataLayout())));
|
||
|
|
|
||
|
|
// Add the returned register to the CalleeSaveDisableRegs list. Don't do
|
||
|
|
// this however for preserve_most/preserve_all to minimize the number of
|
||
|
|
// callee-saved registers for these CCs.
|
||
|
|
if (ShouldDisableCalleeSavedRegister &&
|
||
|
|
CallConv != CallingConv::PreserveAll &&
|
||
|
|
CallConv != CallingConv::PreserveMost)
|
||
|
|
MF.getRegInfo().disableCalleeSavedRegister(RetValReg);
|
||
|
|
}
|
||
|
|
|
||
|
|
const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
|
||
|
|
const MCPhysReg *I =
|
||
|
|
TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction());
|
||
|
|
if (I) {
|
||
|
|
for (; *I; ++I) {
|
||
|
|
if (X86::GR64RegClass.contains(*I))
|
||
|
|
RetOps.push_back(DAG.getRegister(*I, MVT::i64));
|
||
|
|
else
|
||
|
|
llvm_unreachable("Unexpected register class in CSRsViaCopy!");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
RetOps[0] = Chain; // Update chain.
|
||
|
|
|
||
|
|
// Add the glue if we have it.
|
||
|
|
if (Glue.getNode())
|
||
|
|
RetOps.push_back(Glue);
|
||
|
|
|
||
|
|
X86ISD::NodeType opcode = X86ISD::RET_GLUE;
|
||
|
|
if (CallConv == CallingConv::X86_INTR)
|
||
|
|
opcode = X86ISD::IRET;
|
||
|
|
return DAG.getNode(opcode, dl, MVT::Other, RetOps);
|
||
|
|
}
|
||
|
|
|
||
|
|
bool X86TargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
|
||
|
|
if (N->getNumValues() != 1 || !N->hasNUsesOfValue(1, 0))
|
||
|
|
return false;
|
||
|
|
|
||
|
|
SDValue TCChain = Chain;
|
||
|
|
SDNode *Copy = *N->use_begin();
|
||
|
|
if (Copy->getOpcode() == ISD::CopyToReg) {
|
||
|
|
// If the copy has a glue operand, we conservatively assume it isn't safe to
|
||
|
|
// perform a tail call.
|
||
|
|
if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue)
|
||
|
|
return false;
|
||
|
|
TCChain = Copy->getOperand(0);
|
||
|
|
} else if (Copy->getOpcode() != ISD::FP_EXTEND)
|
||
|
|
return false;
|
||
|
|
|
||
|
|
bool HasRet = false;
|
||
|
|
for (const SDNode *U : Copy->uses()) {
|
||
|
|
if (U->getOpcode() != X86ISD::RET_GLUE)
|
||
|
|
return false;
|
||
|
|
// If we are returning more than one value, we can definitely
|
||
|
|
// not make a tail call see PR19530
|
||
|
|
if (U->getNumOperands() > 4)
|
||
|
|
return false;
|
||
|
|
if (U->getNumOperands() == 4 &&
|
||
|
|
U->getOperand(U->getNumOperands() - 1).getValueType() != MVT::Glue)
|
||
|
|
return false;
|
||
|
|
HasRet = true;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!HasRet)
|
||
|
|
return false;
|
||
|
|
|
||
|
|
Chain = TCChain;
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
|
||
|
|
EVT X86TargetLowering::getTypeForExtReturn(LLVMContext &Context, EVT VT,
|
||
|
|
ISD::NodeType ExtendKind) const {
|
||
|
|
MVT ReturnMVT = MVT::i32;
|
||
|
|
|
||
|
|
bool Darwin = Subtarget.getTargetTriple().isOSDarwin();
|
||
|
|
if (VT == MVT::i1 || (!Darwin && (VT == MVT::i8 || VT == MVT::i16))) {
|
||
|
|
// The ABI does not require i1, i8 or i16 to be extended.
|
||
|
|
//
|
||
|
|
// On Darwin, there is code in the wild relying on Clang's old behaviour of
|
||
|
|
// always extending i8/i16 return values, so keep doing that for now.
|
||
|
|
// (PR26665).
|
||
|
|
ReturnMVT = MVT::i8;
|
||
|
|
}
|
||
|
|
|
||
|
|
EVT MinVT = getRegisterType(Context, ReturnMVT);
|
||
|
|
return VT.bitsLT(MinVT) ? MinVT : VT;
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Reads two 32 bit registers and creates a 64 bit mask value.
|
||
|
|
/// \param VA The current 32 bit value that need to be assigned.
|
||
|
|
/// \param NextVA The next 32 bit value that need to be assigned.
|
||
|
|
/// \param Root The parent DAG node.
|
||
|
|
/// \param [in,out] InGlue Represents SDvalue in the parent DAG node for
|
||
|
|
/// glue purposes. In the case the DAG is already using
|
||
|
|
/// physical register instead of virtual, we should glue
|
||
|
|
/// our new SDValue to InGlue SDvalue.
|
||
|
|
/// \return a new SDvalue of size 64bit.
|
||
|
|
static SDValue getv64i1Argument(CCValAssign &VA, CCValAssign &NextVA,
|
||
|
|
SDValue &Root, SelectionDAG &DAG,
|
||
|
|
const SDLoc &DL, const X86Subtarget &Subtarget,
|
||
|
|
SDValue *InGlue = nullptr) {
|
||
|
|
assert((Subtarget.hasBWI()) && "Expected AVX512BW target!");
|
||
|
|
assert(Subtarget.is32Bit() && "Expecting 32 bit target");
|
||
|
|
assert(VA.getValVT() == MVT::v64i1 &&
|
||
|
|
"Expecting first location of 64 bit width type");
|
||
|
|
assert(NextVA.getValVT() == VA.getValVT() &&
|
||
|
|
"The locations should have the same type");
|
||
|
|
assert(VA.isRegLoc() && NextVA.isRegLoc() &&
|
||
|
|
"The values should reside in two registers");
|
||
|
|
|
||
|
|
SDValue Lo, Hi;
|
||
|
|
SDValue ArgValueLo, ArgValueHi;
|
||
|
|
|
||
|
|
MachineFunction &MF = DAG.getMachineFunction();
|
||
|
|
const TargetRegisterClass *RC = &X86::GR32RegClass;
|
||
|
|
|
||
|
|
// Read a 32 bit value from the registers.
|
||
|
|
if (nullptr == InGlue) {
|
||
|
|
// When no physical register is present,
|
||
|
|
// create an intermediate virtual register.
|
||
|
|
Register Reg = MF.addLiveIn(VA.getLocReg(), RC);
|
||
|
|
ArgValueLo = DAG.getCopyFromReg(Root, DL, Reg, MVT::i32);
|
||
|
|
Reg = MF.addLiveIn(NextVA.getLocReg(), RC);
|
||
|
|
ArgValueHi = DAG.getCopyFromReg(Root, DL, Reg, MVT::i32);
|
||
|
|
} else {
|
||
|
|
// When a physical register is available read the value from it and glue
|
||
|
|
// the reads together.
|
||
|
|
ArgValueLo =
|
||
|
|
DAG.getCopyFromReg(Root, DL, VA.getLocReg(), MVT::i32, *InGlue);
|
||
|
|
*InGlue = ArgValueLo.getValue(2);
|
||
|
|
ArgValueHi =
|
||
|
|
DAG.getCopyFromReg(Root, DL, NextVA.getLocReg(), MVT::i32, *InGlue);
|
||
|
|
*InGlue = ArgValueHi.getValue(2);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Convert the i32 type into v32i1 type.
|
||
|
|
Lo = DAG.getBitcast(MVT::v32i1, ArgValueLo);
|
||
|
|
|
||
|
|
// Convert the i32 type into v32i1 type.
|
||
|
|
Hi = DAG.getBitcast(MVT::v32i1, ArgValueHi);
|
||
|
|
|
||
|
|
// Concatenate the two values together.
|
||
|
|
return DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v64i1, Lo, Hi);
|
||
|
|
}
|
||
|
|
|
||
|
|
/// The function will lower a register of various sizes (8/16/32/64)
|
||
|
|
/// to a mask value of the expected size (v8i1/v16i1/v32i1/v64i1)
|
||
|
|
/// \returns a DAG node contains the operand after lowering to mask type.
|
||
|
|
static SDValue lowerRegToMasks(const SDValue &ValArg, const EVT &ValVT,
|
||
|
|
const EVT &ValLoc, const SDLoc &DL,
|
||
|
|
SelectionDAG &DAG) {
|
||
|
|
SDValue ValReturned = ValArg;
|
||
|
|
|
||
|
|
if (ValVT == MVT::v1i1)
|
||
|
|
return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, ValReturned);
|
||
|
|
|
||
|
|
if (ValVT == MVT::v64i1) {
|
||
|
|
// In 32 bit machine, this case is handled by getv64i1Argument
|
||
|
|
assert(ValLoc == MVT::i64 && "Expecting only i64 locations");
|
||
|
|
// In 64 bit machine, There is no need to truncate the value only bitcast
|
||
|
|
} else {
|
||
|
|
MVT MaskLenVT;
|
||
|
|
switch (ValVT.getSimpleVT().SimpleTy) {
|
||
|
|
case MVT::v8i1:
|
||
|
|
MaskLenVT = MVT::i8;
|
||
|
|
break;
|
||
|
|
case MVT::v16i1:
|
||
|
|
MaskLenVT = MVT::i16;
|
||
|
|
break;
|
||
|
|
case MVT::v32i1:
|
||
|
|
MaskLenVT = MVT::i32;
|
||
|
|
break;
|
||
|
|
default:
|
||
|
|
llvm_unreachable("Expecting a vector of i1 types");
|
||
|
|
}
|
||
|
|
|
||
|
|
ValReturned = DAG.getNode(ISD::TRUNCATE, DL, MaskLenVT, ValReturned);
|
||
|
|
}
|
||
|
|
return DAG.getBitcast(ValVT, ValReturned);
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Lower the result values of a call into the
|
||
|
|
/// appropriate copies out of appropriate physical registers.
|
||
|
|
///
|
||
|
|
SDValue X86TargetLowering::LowerCallResult(
|
||
|
|
SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool isVarArg,
|
||
|
|
const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
|
||
|
|
SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
|
||
|
|
uint32_t *RegMask) const {
|
||
|
|
|
||
|
|
const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
|
||
|
|
// Assign locations to each value returned by this call.
|
||
|
|
SmallVector<CCValAssign, 16> RVLocs;
|
||
|
|
CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
|
||
|
|
*DAG.getContext());
|
||
|
|
CCInfo.AnalyzeCallResult(Ins, RetCC_X86);
|
||
|
|
|
||
|
|
// Copy all of the result registers out of their specified physreg.
|
||
|
|
for (unsigned I = 0, InsIndex = 0, E = RVLocs.size(); I != E;
|
||
|
|
++I, ++InsIndex) {
|
||
|
|
CCValAssign &VA = RVLocs[I];
|
||
|
|
EVT CopyVT = VA.getLocVT();
|
||
|
|
|
||
|
|
// In some calling conventions we need to remove the used registers
|
||
|
|
// from the register mask.
|
||
|
|
if (RegMask) {
|
||
|
|
for (MCPhysReg SubReg : TRI->subregs_inclusive(VA.getLocReg()))
|
||
|
|
RegMask[SubReg / 32] &= ~(1u << (SubReg % 32));
|
||
|
|
}
|
||
|
|
|
||
|
|
// Report an error if there was an attempt to return FP values via XMM
|
||
|
|
// registers.
|
||
|
|
if (!Subtarget.hasSSE1() && X86::FR32XRegClass.contains(VA.getLocReg())) {
|
||
|
|
errorUnsupported(DAG, dl, "SSE register return with SSE disabled");
|
||
|
|
if (VA.getLocReg() == X86::XMM1)
|
||
|
|
VA.convertToReg(X86::FP1); // Set reg to FP1, avoid hitting asserts.
|
||
|
|
else
|
||
|
|
VA.convertToReg(X86::FP0); // Set reg to FP0, avoid hitting asserts.
|
||
|
|
} else if (!Subtarget.hasSSE2() &&
|
||
|
|
X86::FR64XRegClass.contains(VA.getLocReg()) &&
|
||
|
|
CopyVT == MVT::f64) {
|
||
|
|
errorUnsupported(DAG, dl, "SSE2 register return with SSE2 disabled");
|
||
|
|
if (VA.getLocReg() == X86::XMM1)
|
||
|
|
VA.convertToReg(X86::FP1); // Set reg to FP1, avoid hitting asserts.
|
||
|
|
else
|
||
|
|
VA.convertToReg(X86::FP0); // Set reg to FP0, avoid hitting asserts.
|
||
|
|
}
|
||
|
|
|
||
|
|
// If we prefer to use the value in xmm registers, copy it out as f80 and
|
||
|
|
// use a truncate to move it from fp stack reg to xmm reg.
|
||
|
|
bool RoundAfterCopy = false;
|
||
|
|
if ((VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1) &&
|
||
|
|
isScalarFPTypeInSSEReg(VA.getValVT())) {
|
||
|
|
if (!Subtarget.hasX87())
|
||
|
|
report_fatal_error("X87 register return with X87 disabled");
|
||
|
|
CopyVT = MVT::f80;
|
||
|
|
RoundAfterCopy = (CopyVT != VA.getLocVT());
|
||
|
|
}
|
||
|
|
|
||
|
|
SDValue Val;
|
||
|
|
if (VA.needsCustom()) {
|
||
|
|
assert(VA.getValVT() == MVT::v64i1 &&
|
||
|
|
"Currently the only custom case is when we split v64i1 to 2 regs");
|
||
|
|
Val =
|
||
|
|
getv64i1Argument(VA, RVLocs[++I], Chain, DAG, dl, Subtarget, &InGlue);
|
||
|
|
} else {
|
||
|
|
Chain = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), CopyVT, InGlue)
|
||
|
|
.getValue(1);
|
||
|
|
Val = Chain.getValue(0);
|
||
|
|
InGlue = Chain.getValue(2);
|
||
|
|
}
|
||
|
|
|
||
|
|
if (RoundAfterCopy)
|
||
|
|
Val = DAG.getNode(ISD::FP_ROUND, dl, VA.getValVT(), Val,
|
||
|
|
// This truncation won't change the value.
|
||
|
|
DAG.getIntPtrConstant(1, dl, /*isTarget=*/true));
|
||
|
|
|
||
|
|
if (VA.isExtInLoc()) {
|
||
|
|
if (VA.getValVT().isVector() &&
|
||
|
|
VA.getValVT().getScalarType() == MVT::i1 &&
|
||
|
|
((VA.getLocVT() == MVT::i64) || (VA.getLocVT() == MVT::i32) ||
|
||
|
|
(VA.getLocVT() == MVT::i16) || (VA.getLocVT() == MVT::i8))) {
|
||
|
|
// promoting a mask type (v*i1) into a register of type i64/i32/i16/i8
|
||
|
|
Val = lowerRegToMasks(Val, VA.getValVT(), VA.getLocVT(), dl, DAG);
|
||
|
|
} else
|
||
|
|
Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val);
|
||
|
|
}
|
||
|
|
|
||
|
|
if (VA.getLocInfo() == CCValAssign::BCvt)
|
||
|
|
Val = DAG.getBitcast(VA.getValVT(), Val);
|
||
|
|
|
||
|
|
InVals.push_back(Val);
|
||
|
|
}
|
||
|
|
|
||
|
|
return Chain;
|
||
|
|
}
|
||
|
|
|
||
|
|
//===----------------------------------------------------------------------===//
|
||
|
|
// C & StdCall & Fast Calling Convention implementation
|
||
|
|
//===----------------------------------------------------------------------===//
|
||
|
|
// StdCall calling convention seems to be standard for many Windows' API
|
||
|
|
// routines and around. It differs from C calling convention just a little:
|
||
|
|
// callee should clean up the stack, not caller. Symbols should be also
|
||
|
|
// decorated in some fancy way :) It doesn't support any vector arguments.
|
||
|
|
// For info on fast calling convention see Fast Calling Convention (tail call)
|
||
|
|
// implementation LowerX86_32FastCCCallTo.
|
||
|
|
|
||
|
|
/// Determines whether Args, either a set of outgoing arguments to a call, or a
|
||
|
|
/// set of incoming args of a call, contains an sret pointer that the callee
|
||
|
|
/// pops
|
||
|
|
template <typename T>
|
||
|
|
static bool hasCalleePopSRet(const SmallVectorImpl<T> &Args,
|
||
|
|
const X86Subtarget &Subtarget) {
|
||
|
|
// Not C++20 (yet), so no concepts available.
|
||
|
|
static_assert(std::is_same_v<T, ISD::OutputArg> ||
|
||
|
|
std::is_same_v<T, ISD::InputArg>,
|
||
|
|
"requires ISD::OutputArg or ISD::InputArg");
|
||
|
|
|
||
|
|
// Only 32-bit pops the sret. It's a 64-bit world these days, so early-out
|
||
|
|
// for most compilations.
|
||
|
|
if (!Subtarget.is32Bit())
|
||
|
|
return false;
|
||
|
|
|
||
|
|
if (Args.empty())
|
||
|
|
return false;
|
||
|
|
|
||
|
|
// Most calls do not have an sret argument, check the arg next.
|
||
|
|
const ISD::ArgFlagsTy &Flags = Args[0].Flags;
|
||
|
|
if (!Flags.isSRet() || Flags.isInReg())
|
||
|
|
return false;
|
||
|
|
|
||
|
|
// The MSVCabi does not pop the sret.
|
||
|
|
if (Subtarget.getTargetTriple().isOSMSVCRT())
|
||
|
|
return false;
|
||
|
|
|
||
|
|
// MCUs don't pop the sret
|
||
|
|
if (Subtarget.isTargetMCU())
|
||
|
|
return false;
|
||
|
|
|
||
|
|
// Callee pops argument
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Make a copy of an aggregate at address specified by "Src" to address
|
||
|
|
/// "Dst" with size and alignment information specified by the specific
|
||
|
|
/// parameter attribute. The copy will be passed as a byval function parameter.
|
||
|
|
static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
|
||
|
|
SDValue Chain, ISD::ArgFlagsTy Flags,
|
||
|
|
SelectionDAG &DAG, const SDLoc &dl) {
|
||
|
|
SDValue SizeNode = DAG.getIntPtrConstant(Flags.getByValSize(), dl);
|
||
|
|
|
||
|
|
return DAG.getMemcpy(
|
||
|
|
Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(),
|
||
|
|
/*isVolatile*/ false, /*AlwaysInline=*/true,
|
||
|
|
/*isTailCall*/ false, MachinePointerInfo(), MachinePointerInfo());
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Return true if the calling convention is one that we can guarantee TCO for.
|
||
|
|
static bool canGuaranteeTCO(CallingConv::ID CC) {
|
||
|
|
return (CC == CallingConv::Fast || CC == CallingConv::GHC ||
|
||
|
|
CC == CallingConv::X86_RegCall || CC == CallingConv::HiPE ||
|
||
|
|
CC == CallingConv::Tail || CC == CallingConv::SwiftTail);
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Return true if we might ever do TCO for calls with this calling convention.
|
||
|
|
static bool mayTailCallThisCC(CallingConv::ID CC) {
|
||
|
|
switch (CC) {
|
||
|
|
// C calling conventions:
|
||
|
|
case CallingConv::C:
|
||
|
|
case CallingConv::Win64:
|
||
|
|
case CallingConv::X86_64_SysV:
|
||
|
|
// Callee pop conventions:
|
||
|
|
case CallingConv::X86_ThisCall:
|
||
|
|
case CallingConv::X86_StdCall:
|
||
|
|
case CallingConv::X86_VectorCall:
|
||
|
|
case CallingConv::X86_FastCall:
|
||
|
|
// Swift:
|
||
|
|
case CallingConv::Swift:
|
||
|
|
return true;
|
||
|
|
default:
|
||
|
|
return canGuaranteeTCO(CC);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Return true if the function is being made into a tailcall target by
|
||
|
|
/// changing its ABI.
|
||
|
|
static bool shouldGuaranteeTCO(CallingConv::ID CC, bool GuaranteedTailCallOpt) {
|
||
|
|
return (GuaranteedTailCallOpt && canGuaranteeTCO(CC)) ||
|
||
|
|
CC == CallingConv::Tail || CC == CallingConv::SwiftTail;
|
||
|
|
}
|
||
|
|
|
||
|
|
bool X86TargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
|
||
|
|
if (!CI->isTailCall())
|
||
|
|
return false;
|
||
|
|
|
||
|
|
CallingConv::ID CalleeCC = CI->getCallingConv();
|
||
|
|
if (!mayTailCallThisCC(CalleeCC))
|
||
|
|
return false;
|
||
|
|
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
|
||
|
|
SDValue
|
||
|
|
X86TargetLowering::LowerMemArgument(SDValue Chain, CallingConv::ID CallConv,
|
||
|
|
const SmallVectorImpl<ISD::InputArg> &Ins,
|
||
|
|
const SDLoc &dl, SelectionDAG &DAG,
|
||
|
|
const CCValAssign &VA,
|
||
|
|
MachineFrameInfo &MFI, unsigned i) const {
|
||
|
|
// Create the nodes corresponding to a load from this parameter slot.
|
||
|
|
ISD::ArgFlagsTy Flags = Ins[i].Flags;
|
||
|
|
bool AlwaysUseMutable = shouldGuaranteeTCO(
|
||
|
|
CallConv, DAG.getTarget().Options.GuaranteedTailCallOpt);
|
||
|
|
bool isImmutable = !AlwaysUseMutable && !Flags.isByVal();
|
||
|
|
EVT ValVT;
|
||
|
|
MVT PtrVT = getPointerTy(DAG.getDataLayout());
|
||
|
|
|
||
|
|
// If value is passed by pointer we have address passed instead of the value
|
||
|
|
// itself. No need to extend if the mask value and location share the same
|
||
|
|
// absolute size.
|
||
|
|
bool ExtendedInMem =
|
||
|
|
VA.isExtInLoc() && VA.getValVT().getScalarType() == MVT::i1 &&
|
||
|
|
VA.getValVT().getSizeInBits() != VA.getLocVT().getSizeInBits();
|
||
|
|
|
||
|
|
if (VA.getLocInfo() == CCValAssign::Indirect || ExtendedInMem)
|
||
|
|
ValVT = VA.getLocVT();
|
||
|
|
else
|
||
|
|
ValVT = VA.getValVT();
|
||
|
|
|
||
|
|
// FIXME: For now, all byval parameter objects are marked mutable. This can be
|
||
|
|
// changed with more analysis.
|
||
|
|
// In case of tail call optimization mark all arguments mutable. Since they
|
||
|
|
// could be overwritten by lowering of arguments in case of a tail call.
|
||
|
|
if (Flags.isByVal()) {
|
||
|
|
unsigned Bytes = Flags.getByValSize();
|
||
|
|
if (Bytes == 0) Bytes = 1; // Don't create zero-sized stack objects.
|
||
|
|
|
||
|
|
// FIXME: For now, all byval parameter objects are marked as aliasing. This
|
||
|
|
// can be improved with deeper analysis.
|
||
|
|
int FI = MFI.CreateFixedObject(Bytes, VA.getLocMemOffset(), isImmutable,
|
||
|
|
/*isAliased=*/true);
|
||
|
|
return DAG.getFrameIndex(FI, PtrVT);
|
||
|
|
}
|
||
|
|
|
||
|
|
EVT ArgVT = Ins[i].ArgVT;
|
||
|
|
|
||
|
|
// If this is a vector that has been split into multiple parts, don't elide
|
||
|
|
// the copy. The layout on the stack may not match the packed in-memory
|
||
|
|
// layout.
|
||
|
|
bool ScalarizedVector = ArgVT.isVector() && !VA.getLocVT().isVector();
|
||
|
|
|
||
|
|
// This is an argument in memory. We might be able to perform copy elision.
|
||
|
|
// If the argument is passed directly in memory without any extension, then we
|
||
|
|
// can perform copy elision. Large vector types, for example, may be passed
|
||
|
|
// indirectly by pointer.
|
||
|
|
if (Flags.isCopyElisionCandidate() &&
|
||
|
|
VA.getLocInfo() != CCValAssign::Indirect && !ExtendedInMem &&
|
||
|
|
!ScalarizedVector) {
|
||
|
|
SDValue PartAddr;
|
||
|
|
if (Ins[i].PartOffset == 0) {
|
||
|
|
// If this is a one-part value or the first part of a multi-part value,
|
||
|
|
// create a stack object for the entire argument value type and return a
|
||
|
|
// load from our portion of it. This assumes that if the first part of an
|
||
|
|
// argument is in memory, the rest will also be in memory.
|
||
|
|
int FI = MFI.CreateFixedObject(ArgVT.getStoreSize(), VA.getLocMemOffset(),
|
||
|
|
/*IsImmutable=*/false);
|
||
|
|
PartAddr = DAG.getFrameIndex(FI, PtrVT);
|
||
|
|
return DAG.getLoad(
|
||
|
|
ValVT, dl, Chain, PartAddr,
|
||
|
|
MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI));
|
||
|
|
}
|
||
|
|
|
||
|
|
// This is not the first piece of an argument in memory. See if there is
|
||
|
|
// already a fixed stack object including this offset. If so, assume it
|
||
|
|
// was created by the PartOffset == 0 branch above and create a load from
|
||
|
|
// the appropriate offset into it.
|
||
|
|
int64_t PartBegin = VA.getLocMemOffset();
|
||
|
|
int64_t PartEnd = PartBegin + ValVT.getSizeInBits() / 8;
|
||
|
|
int FI = MFI.getObjectIndexBegin();
|
||
|
|
for (; MFI.isFixedObjectIndex(FI); ++FI) {
|
||
|
|
int64_t ObjBegin = MFI.getObjectOffset(FI);
|
||
|
|
int64_t ObjEnd = ObjBegin + MFI.getObjectSize(FI);
|
||
|
|
if (ObjBegin <= PartBegin && PartEnd <= ObjEnd)
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
if (MFI.isFixedObjectIndex(FI)) {
|
||
|
|
SDValue Addr =
|
||
|
|
DAG.getNode(ISD::ADD, dl, PtrVT, DAG.getFrameIndex(FI, PtrVT),
|
||
|
|
DAG.getIntPtrConstant(Ins[i].PartOffset, dl));
|
||
|
|
return DAG.getLoad(ValVT, dl, Chain, Addr,
|
||
|
|
MachinePointerInfo::getFixedStack(
|
||
|
|
DAG.getMachineFunction(), FI, Ins[i].PartOffset));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
int FI = MFI.CreateFixedObject(ValVT.getSizeInBits() / 8,
|
||
|
|
VA.getLocMemOffset(), isImmutable);
|
||
|
|
|
||
|
|
// Set SExt or ZExt flag.
|
||
|
|
if (VA.getLocInfo() == CCValAssign::ZExt) {
|
||
|
|
MFI.setObjectZExt(FI, true);
|
||
|
|
} else if (VA.getLocInfo() == CCValAssign::SExt) {
|
||
|
|
MFI.setObjectSExt(FI, true);
|
||
|
|
}
|
||
|
|
|
||
|
|
MaybeAlign Alignment;
|
||
|
|
if (Subtarget.isTargetWindowsMSVC() && !Subtarget.is64Bit() &&
|
||
|
|
ValVT != MVT::f80)
|
||
|
|
Alignment = MaybeAlign(4);
|
||
|
|
SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
|
||
|
|
SDValue Val = DAG.getLoad(
|
||
|
|
ValVT, dl, Chain, FIN,
|
||
|
|
MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
|
||
|
|
Alignment);
|
||
|
|
return ExtendedInMem
|
||
|
|
? (VA.getValVT().isVector()
|
||
|
|
? DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VA.getValVT(), Val)
|
||
|
|
: DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val))
|
||
|
|
: Val;
|
||
|
|
}
|
||
|
|
|
||
|
|
// FIXME: Get this from tablegen.
|
||
|
|
static ArrayRef<MCPhysReg> get64BitArgumentGPRs(CallingConv::ID CallConv,
|
||
|
|
const X86Subtarget &Subtarget) {
|
||
|
|
assert(Subtarget.is64Bit());
|
||
|
|
|
||
|
|
if (Subtarget.isCallingConvWin64(CallConv)) {
|
||
|
|
static const MCPhysReg GPR64ArgRegsWin64[] = {
|
||
|
|
X86::RCX, X86::RDX, X86::R8, X86::R9
|
||
|
|
};
|
||
|
|
return ArrayRef(std::begin(GPR64ArgRegsWin64), std::end(GPR64ArgRegsWin64));
|
||
|
|
}
|
||
|
|
|
||
|
|
static const MCPhysReg GPR64ArgRegs64Bit[] = {
|
||
|
|
X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9
|
||
|
|
};
|
||
|
|
return ArrayRef(std::begin(GPR64ArgRegs64Bit), std::end(GPR64ArgRegs64Bit));
|
||
|
|
}
|
||
|
|
|
||
|
|
// FIXME: Get this from tablegen.
|
||
|
|
static ArrayRef<MCPhysReg> get64BitArgumentXMMs(MachineFunction &MF,
|
||
|
|
CallingConv::ID CallConv,
|
||
|
|
const X86Subtarget &Subtarget) {
|
||
|
|
assert(Subtarget.is64Bit());
|
||
|
|
if (Subtarget.isCallingConvWin64(CallConv)) {
|
||
|
|
// The XMM registers which might contain var arg parameters are shadowed
|
||
|
|
// in their paired GPR. So we only need to save the GPR to their home
|
||
|
|
// slots.
|
||
|
|
// TODO: __vectorcall will change this.
|
||
|
|
return std::nullopt;
|
||
|
|
}
|
||
|
|
|
||
|
|
bool isSoftFloat = Subtarget.useSoftFloat();
|
||
|
|
if (isSoftFloat || !Subtarget.hasSSE1())
|
||
|
|
// Kernel mode asks for SSE to be disabled, so there are no XMM argument
|
||
|
|
// registers.
|
||
|
|
return std::nullopt;
|
||
|
|
|
||
|
|
static const MCPhysReg XMMArgRegs64Bit[] = {
|
||
|
|
X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
|
||
|
|
X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
|
||
|
|
};
|
||
|
|
return ArrayRef(std::begin(XMMArgRegs64Bit), std::end(XMMArgRegs64Bit));
|
||
|
|
}
|
||
|
|
|
||
|
|
#ifndef NDEBUG
|
||
|
|
static bool isSortedByValueNo(ArrayRef<CCValAssign> ArgLocs) {
|
||
|
|
return llvm::is_sorted(
|
||
|
|
ArgLocs, [](const CCValAssign &A, const CCValAssign &B) -> bool {
|
||
|
|
return A.getValNo() < B.getValNo();
|
||
|
|
});
|
||
|
|
}
|
||
|
|
#endif
|
||
|
|
|
||
|
|
namespace {
|
||
|
|
/// This is a helper class for lowering variable arguments parameters.
|
||
|
|
class VarArgsLoweringHelper {
|
||
|
|
public:
|
||
|
|
VarArgsLoweringHelper(X86MachineFunctionInfo *FuncInfo, const SDLoc &Loc,
|
||
|
|
SelectionDAG &DAG, const X86Subtarget &Subtarget,
|
||
|
|
CallingConv::ID CallConv, CCState &CCInfo)
|
||
|
|
: FuncInfo(FuncInfo), DL(Loc), DAG(DAG), Subtarget(Subtarget),
|
||
|
|
TheMachineFunction(DAG.getMachineFunction()),
|
||
|
|
TheFunction(TheMachineFunction.getFunction()),
|
||
|
|
FrameInfo(TheMachineFunction.getFrameInfo()),
|
||
|
|
FrameLowering(*Subtarget.getFrameLowering()),
|
||
|
|
TargLowering(DAG.getTargetLoweringInfo()), CallConv(CallConv),
|
||
|
|
CCInfo(CCInfo) {}
|
||
|
|
|
||
|
|
// Lower variable arguments parameters.
|
||
|
|
void lowerVarArgsParameters(SDValue &Chain, unsigned StackSize);
|
||
|
|
|
||
|
|
private:
|
||
|
|
void createVarArgAreaAndStoreRegisters(SDValue &Chain, unsigned StackSize);
|
||
|
|
|
||
|
|
void forwardMustTailParameters(SDValue &Chain);
|
||
|
|
|
||
|
|
bool is64Bit() const { return Subtarget.is64Bit(); }
|
||
|
|
bool isWin64() const { return Subtarget.isCallingConvWin64(CallConv); }
|
||
|
|
|
||
|
|
X86MachineFunctionInfo *FuncInfo;
|
||
|
|
const SDLoc &DL;
|
||
|
|
SelectionDAG &DAG;
|
||
|
|
const X86Subtarget &Subtarget;
|
||
|
|
MachineFunction &TheMachineFunction;
|
||
|
|
const Function &TheFunction;
|
||
|
|
MachineFrameInfo &FrameInfo;
|
||
|
|
const TargetFrameLowering &FrameLowering;
|
||
|
|
const TargetLowering &TargLowering;
|
||
|
|
CallingConv::ID CallConv;
|
||
|
|
CCState &CCInfo;
|
||
|
|
};
|
||
|
|
} // namespace
|
||
|
|
|
||
|
|
void VarArgsLoweringHelper::createVarArgAreaAndStoreRegisters(
|
||
|
|
SDValue &Chain, unsigned StackSize) {
|
||
|
|
// If the function takes variable number of arguments, make a frame index for
|
||
|
|
// the start of the first vararg value... for expansion of llvm.va_start. We
|
||
|
|
// can skip this if there are no va_start calls.
|
||
|
|
if (is64Bit() || (CallConv != CallingConv::X86_FastCall &&
|
||
|
|
CallConv != CallingConv::X86_ThisCall)) {
|
||
|
|
FuncInfo->setVarArgsFrameIndex(
|
||
|
|
FrameInfo.CreateFixedObject(1, StackSize, true));
|
||
|
|
}
|
||
|
|
|
||
|
|
// 64-bit calling conventions support varargs and register parameters, so we
|
||
|
|
// have to do extra work to spill them in the prologue.
|
||
|
|
if (is64Bit()) {
|
||
|
|
// Find the first unallocated argument registers.
|
||
|
|
ArrayRef<MCPhysReg> ArgGPRs = get64BitArgumentGPRs(CallConv, Subtarget);
|
||
|
|
ArrayRef<MCPhysReg> ArgXMMs =
|
||
|
|
get64BitArgumentXMMs(TheMachineFunction, CallConv, Subtarget);
|
||
|
|
unsigned NumIntRegs = CCInfo.getFirstUnallocated(ArgGPRs);
|
||
|
|
unsigned NumXMMRegs = CCInfo.getFirstUnallocated(ArgXMMs);
|
||
|
|
|
||
|
|
assert(!(NumXMMRegs && !Subtarget.hasSSE1()) &&
|
||
|
|
"SSE register cannot be used when SSE is disabled!");
|
||
|
|
|
||
|
|
if (isWin64()) {
|
||
|
|
// Get to the caller-allocated home save location. Add 8 to account
|
||
|
|
// for the return address.
|
||
|
|
int HomeOffset = FrameLowering.getOffsetOfLocalArea() + 8;
|
||
|
|
FuncInfo->setRegSaveFrameIndex(
|
||
|
|
FrameInfo.CreateFixedObject(1, NumIntRegs * 8 + HomeOffset, false));
|
||
|
|
// Fixup to set vararg frame on shadow area (4 x i64).
|
||
|
|
if (NumIntRegs < 4)
|
||
|
|
FuncInfo->setVarArgsFrameIndex(FuncInfo->getRegSaveFrameIndex());
|
||
|
|
} else {
|
||
|
|
// For X86-64, if there are vararg parameters that are passed via
|
||
|
|
// registers, then we must store them to their spots on the stack so
|
||
|
|
// they may be loaded by dereferencing the result of va_next.
|
||
|
|
FuncInfo->setVarArgsGPOffset(NumIntRegs * 8);
|
||
|
|
FuncInfo->setVarArgsFPOffset(ArgGPRs.size() * 8 + NumXMMRegs * 16);
|
||
|
|
FuncInfo->setRegSaveFrameIndex(FrameInfo.CreateStackObject(
|
||
|
|
ArgGPRs.size() * 8 + ArgXMMs.size() * 16, Align(16), false));
|
||
|
|
}
|
||
|
|
|
||
|
|
SmallVector<SDValue, 6>
|
||
|
|
LiveGPRs; // list of SDValue for GPR registers keeping live input value
|
||
|
|
SmallVector<SDValue, 8> LiveXMMRegs; // list of SDValue for XMM registers
|
||
|
|
// keeping live input value
|
||
|
|
SDValue ALVal; // if applicable keeps SDValue for %al register
|
||
|
|
|
||
|
|
// Gather all the live in physical registers.
|
||
|
|
for (MCPhysReg Reg : ArgGPRs.slice(NumIntRegs)) {
|
||
|
|
Register GPR = TheMachineFunction.addLiveIn(Reg, &X86::GR64RegClass);
|
||
|
|
LiveGPRs.push_back(DAG.getCopyFromReg(Chain, DL, GPR, MVT::i64));
|
||
|
|
}
|
||
|
|
const auto &AvailableXmms = ArgXMMs.slice(NumXMMRegs);
|
||
|
|
if (!AvailableXmms.empty()) {
|
||
|
|
Register AL = TheMachineFunction.addLiveIn(X86::AL, &X86::GR8RegClass);
|
||
|
|
ALVal = DAG.getCopyFromReg(Chain, DL, AL, MVT::i8);
|
||
|
|
for (MCPhysReg Reg : AvailableXmms) {
|
||
|
|
// FastRegisterAllocator spills virtual registers at basic
|
||
|
|
// block boundary. That leads to usages of xmm registers
|
||
|
|
// outside of check for %al. Pass physical registers to
|
||
|
|
// VASTART_SAVE_XMM_REGS to avoid unneccessary spilling.
|
||
|
|
TheMachineFunction.getRegInfo().addLiveIn(Reg);
|
||
|
|
LiveXMMRegs.push_back(DAG.getRegister(Reg, MVT::v4f32));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Store the integer parameter registers.
|
||
|
|
SmallVector<SDValue, 8> MemOps;
|
||
|
|
SDValue RSFIN =
|
||
|
|
DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(),
|
||
|
|
TargLowering.getPointerTy(DAG.getDataLayout()));
|
||
|
|
unsigned Offset = FuncInfo->getVarArgsGPOffset();
|
||
|
|
for (SDValue Val : LiveGPRs) {
|
||
|
|
SDValue FIN = DAG.getNode(ISD::ADD, DL,
|
||
|
|
TargLowering.getPointerTy(DAG.getDataLayout()),
|
||
|
|
RSFIN, DAG.getIntPtrConstant(Offset, DL));
|
||
|
|
SDValue Store =
|
||
|
|
DAG.getStore(Val.getValue(1), DL, Val, FIN,
|
||
|
|
MachinePointerInfo::getFixedStack(
|
||
|
|
DAG.getMachineFunction(),
|
||
|
|
FuncInfo->getRegSaveFrameIndex(), Offset));
|
||
|
|
MemOps.push_back(Store);
|
||
|
|
Offset += 8;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Now store the XMM (fp + vector) parameter registers.
|
||
|
|
if (!LiveXMMRegs.empty()) {
|
||
|
|
SmallVector<SDValue, 12> SaveXMMOps;
|
||
|
|
SaveXMMOps.push_back(Chain);
|
||
|
|
SaveXMMOps.push_back(ALVal);
|
||
|
|
SaveXMMOps.push_back(RSFIN);
|
||
|
|
SaveXMMOps.push_back(
|
||
|
|
DAG.getTargetConstant(FuncInfo->getVarArgsFPOffset(), DL, MVT::i32));
|
||
|
|
llvm::append_range(SaveXMMOps, LiveXMMRegs);
|
||
|
|
MachineMemOperand *StoreMMO =
|
||
|
|
DAG.getMachineFunction().getMachineMemOperand(
|
||
|
|
MachinePointerInfo::getFixedStack(
|
||
|
|
DAG.getMachineFunction(), FuncInfo->getRegSaveFrameIndex(),
|
||
|
|
Offset),
|
||
|
|
MachineMemOperand::MOStore, 128, Align(16));
|
||
|
|
MemOps.push_back(DAG.getMemIntrinsicNode(X86ISD::VASTART_SAVE_XMM_REGS,
|
||
|
|
DL, DAG.getVTList(MVT::Other),
|
||
|
|
SaveXMMOps, MVT::i8, StoreMMO));
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!MemOps.empty())
|
||
|
|
Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
void VarArgsLoweringHelper::forwardMustTailParameters(SDValue &Chain) {
|
||
|
|
// Find the largest legal vector type.
|
||
|
|
MVT VecVT = MVT::Other;
|
||
|
|
// FIXME: Only some x86_32 calling conventions support AVX512.
|
||
|
|
if (Subtarget.useAVX512Regs() &&
|
||
|
|
(is64Bit() || (CallConv == CallingConv::X86_VectorCall ||
|
||
|
|
CallConv == CallingConv::Intel_OCL_BI)))
|
||
|
|
VecVT = MVT::v16f32;
|
||
|
|
else if (Subtarget.hasAVX())
|
||
|
|
VecVT = MVT::v8f32;
|
||
|
|
else if (Subtarget.hasSSE2())
|
||
|
|
VecVT = MVT::v4f32;
|
||
|
|
|
||
|
|
// We forward some GPRs and some vector types.
|
||
|
|
SmallVector<MVT, 2> RegParmTypes;
|
||
|
|
MVT IntVT = is64Bit() ? MVT::i64 : MVT::i32;
|
||
|
|
RegParmTypes.push_back(IntVT);
|
||
|
|
if (VecVT != MVT::Other)
|
||
|
|
RegParmTypes.push_back(VecVT);
|
||
|
|
|
||
|
|
// Compute the set of forwarded registers. The rest are scratch.
|
||
|
|
SmallVectorImpl<ForwardedRegister> &Forwards =
|
||
|
|
FuncInfo->getForwardedMustTailRegParms();
|
||
|
|
CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes, CC_X86);
|
||
|
|
|
||
|
|
// Forward AL for SysV x86_64 targets, since it is used for varargs.
|
||
|
|
if (is64Bit() && !isWin64() && !CCInfo.isAllocated(X86::AL)) {
|
||
|
|
Register ALVReg = TheMachineFunction.addLiveIn(X86::AL, &X86::GR8RegClass);
|
||
|
|
Forwards.push_back(ForwardedRegister(ALVReg, X86::AL, MVT::i8));
|
||
|
|
}
|
||
|
|
|
||
|
|
// Copy all forwards from physical to virtual registers.
|
||
|
|
for (ForwardedRegister &FR : Forwards) {
|
||
|
|
// FIXME: Can we use a less constrained schedule?
|
||
|
|
SDValue RegVal = DAG.getCopyFromReg(Chain, DL, FR.VReg, FR.VT);
|
||
|
|
FR.VReg = TheMachineFunction.getRegInfo().createVirtualRegister(
|
||
|
|
TargLowering.getRegClassFor(FR.VT));
|
||
|
|
Chain = DAG.getCopyToReg(Chain, DL, FR.VReg, RegVal);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
void VarArgsLoweringHelper::lowerVarArgsParameters(SDValue &Chain,
|
||
|
|
unsigned StackSize) {
|
||
|
|
// Set FrameIndex to the 0xAAAAAAA value to mark unset state.
|
||
|
|
// If necessary, it would be set into the correct value later.
|
||
|
|
FuncInfo->setVarArgsFrameIndex(0xAAAAAAA);
|
||
|
|
FuncInfo->setRegSaveFrameIndex(0xAAAAAAA);
|
||
|
|
|
||
|
|
if (FrameInfo.hasVAStart())
|
||
|
|
createVarArgAreaAndStoreRegisters(Chain, StackSize);
|
||
|
|
|
||
|
|
if (FrameInfo.hasMustTailInVarArgFunc())
|
||
|
|
forwardMustTailParameters(Chain);
|
||
|
|
}
|
||
|
|
|
||
|
|
SDValue X86TargetLowering::LowerFormalArguments(
|
||
|
|
SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
|
||
|
|
const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
|
||
|
|
SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
|
||
|
|
MachineFunction &MF = DAG.getMachineFunction();
|
||
|
|
X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
|
||
|
|
|
||
|
|
const Function &F = MF.getFunction();
|
||
|
|
if (F.hasExternalLinkage() && Subtarget.isTargetCygMing() &&
|
||
|
|
F.getName() == "main")
|
||
|
|
FuncInfo->setForceFramePointer(true);
|
||
|
|
|
||
|
|
MachineFrameInfo &MFI = MF.getFrameInfo();
|
||
|
|
bool Is64Bit = Subtarget.is64Bit();
|
||
|
|
bool IsWin64 = Subtarget.isCallingConvWin64(CallConv);
|
||
|
|
|
||
|
|
assert(
|
||
|
|
!(IsVarArg && canGuaranteeTCO(CallConv)) &&
|
||
|
|
"Var args not supported with calling conv' regcall, fastcc, ghc or hipe");
|
||
|
|
|
||
|
|
// Assign locations to all of the incoming arguments.
|
||
|
|
SmallVector<CCValAssign, 16> ArgLocs;
|
||
|
|
CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
|
||
|
|
|
||
|
|
// Allocate shadow area for Win64.
|
||
|
|
if (IsWin64)
|
||
|
|
CCInfo.AllocateStack(32, Align(8));
|
||
|
|
|
||
|
|
CCInfo.AnalyzeArguments(Ins, CC_X86);
|
||
|
|
|
||
|
|
// In vectorcall calling convention a second pass is required for the HVA
|
||
|
|
// types.
|
||
|
|
if (CallingConv::X86_VectorCall == CallConv) {
|
||
|
|
CCInfo.AnalyzeArgumentsSecondPass(Ins, CC_X86);
|
||
|
|
}
|
||
|
|
|
||
|
|
// The next loop assumes that the locations are in the same order of the
|
||
|
|
// input arguments.
|
||
|
|
assert(isSortedByValueNo(ArgLocs) &&
|
||
|
|
"Argument Location list must be sorted before lowering");
|
||
|
|
|
||
|
|
SDValue ArgValue;
|
||
|
|
for (unsigned I = 0, InsIndex = 0, E = ArgLocs.size(); I != E;
|
||
|
|
++I, ++InsIndex) {
|
||
|
|
assert(InsIndex < Ins.size() && "Invalid Ins index");
|
||
|
|
CCValAssign &VA = ArgLocs[I];
|
||
|
|
|
||
|
|
if (VA.isRegLoc()) {
|
||
|
|
EVT RegVT = VA.getLocVT();
|
||
|
|
if (VA.needsCustom()) {
|
||
|
|
assert(
|
||
|
|
VA.getValVT() == MVT::v64i1 &&
|
||
|
|
"Currently the only custom case is when we split v64i1 to 2 regs");
|
||
|
|
|
||
|
|
// v64i1 values, in regcall calling convention, that are
|
||
|
|
// compiled to 32 bit arch, are split up into two registers.
|
||
|
|
ArgValue =
|
||
|
|
getv64i1Argument(VA, ArgLocs[++I], Chain, DAG, dl, Subtarget);
|
||
|
|
} else {
|
||
|
|
const TargetRegisterClass *RC;
|
||
|
|
if (RegVT == MVT::i8)
|
||
|
|
RC = &X86::GR8RegClass;
|
||
|
|
else if (RegVT == MVT::i16)
|
||
|
|
RC = &X86::GR16RegClass;
|
||
|
|
else if (RegVT == MVT::i32)
|
||
|
|
RC = &X86::GR32RegClass;
|
||
|
|
else if (Is64Bit && RegVT == MVT::i64)
|
||
|
|
RC = &X86::GR64RegClass;
|
||
|
|
else if (RegVT == MVT::f16)
|
||
|
|
RC = Subtarget.hasAVX512() ? &X86::FR16XRegClass : &X86::FR16RegClass;
|
||
|
|
else if (RegVT == MVT::f32)
|
||
|
|
RC = Subtarget.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass;
|
||
|
|
else if (RegVT == MVT::f64)
|
||
|
|
RC = Subtarget.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass;
|
||
|
|
else if (RegVT == MVT::f80)
|
||
|
|
RC = &X86::RFP80RegClass;
|
||
|
|
else if (RegVT == MVT::f128)
|
||
|
|
RC = &X86::VR128RegClass;
|
||
|
|
else if (RegVT.is512BitVector())
|
||
|
|
RC = &X86::VR512RegClass;
|
||
|
|
else if (RegVT.is256BitVector())
|
||
|
|
RC = Subtarget.hasVLX() ? &X86::VR256XRegClass : &X86::VR256RegClass;
|
||
|
|
else if (RegVT.is128BitVector())
|
||
|
|
RC = Subtarget.hasVLX() ? &X86::VR128XRegClass : &X86::VR128RegClass;
|
||
|
|
else if (RegVT == MVT::x86mmx)
|
||
|
|
RC = &X86::VR64RegClass;
|
||
|
|
else if (RegVT == MVT::v1i1)
|
||
|
|
RC = &X86::VK1RegClass;
|
||
|
|
else if (RegVT == MVT::v8i1)
|
||
|
|
RC = &X86::VK8RegClass;
|
||
|
|
else if (RegVT == MVT::v16i1)
|
||
|
|
RC = &X86::VK16RegClass;
|
||
|
|
else if (RegVT == MVT::v32i1)
|
||
|
|
RC = &X86::VK32RegClass;
|
||
|
|
else if (RegVT == MVT::v64i1)
|
||
|
|
RC = &X86::VK64RegClass;
|
||
|
|
else
|
||
|
|
llvm_unreachable("Unknown argument type!");
|
||
|
|
|
||
|
|
Register Reg = MF.addLiveIn(VA.getLocReg(), RC);
|
||
|
|
ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT);
|
||
|
|
}
|
||
|
|
|
||
|
|
// If this is an 8 or 16-bit value, it is really passed promoted to 32
|
||
|
|
// bits. Insert an assert[sz]ext to capture this, then truncate to the
|
||
|
|
// right size.
|
||
|
|
if (VA.getLocInfo() == CCValAssign::SExt)
|
||
|
|
ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue,
|
||
|
|
DAG.getValueType(VA.getValVT()));
|
||
|
|
else if (VA.getLocInfo() == CCValAssign::ZExt)
|
||
|
|
ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue,
|
||
|
|
DAG.getValueType(VA.getValVT()));
|
||
|
|
else if (VA.getLocInfo() == CCValAssign::BCvt)
|
||
|
|
ArgValue = DAG.getBitcast(VA.getValVT(), ArgValue);
|
||
|
|
|
||
|
|
if (VA.isExtInLoc()) {
|
||
|
|
// Handle MMX values passed in XMM regs.
|
||
|
|
if (RegVT.isVector() && VA.getValVT().getScalarType() != MVT::i1)
|
||
|
|
ArgValue = DAG.getNode(X86ISD::MOVDQ2Q, dl, VA.getValVT(), ArgValue);
|
||
|
|
else if (VA.getValVT().isVector() &&
|
||
|
|
VA.getValVT().getScalarType() == MVT::i1 &&
|
||
|
|
((VA.getLocVT() == MVT::i64) || (VA.getLocVT() == MVT::i32) ||
|
||
|
|
(VA.getLocVT() == MVT::i16) || (VA.getLocVT() == MVT::i8))) {
|
||
|
|
// Promoting a mask type (v*i1) into a register of type i64/i32/i16/i8
|
||
|
|
ArgValue = lowerRegToMasks(ArgValue, VA.getValVT(), RegVT, dl, DAG);
|
||
|
|
} else
|
||
|
|
ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue);
|
||
|
|
}
|
||
|
|
} else {
|
||
|
|
assert(VA.isMemLoc());
|
||
|
|
ArgValue =
|
||
|
|
LowerMemArgument(Chain, CallConv, Ins, dl, DAG, VA, MFI, InsIndex);
|
||
|
|
}
|
||
|
|
|
||
|
|
// If value is passed via pointer - do a load.
|
||
|
|
if (VA.getLocInfo() == CCValAssign::Indirect &&
|
||
|
|
!(Ins[I].Flags.isByVal() && VA.isRegLoc())) {
|
||
|
|
ArgValue =
|
||
|
|
DAG.getLoad(VA.getValVT(), dl, Chain, ArgValue, MachinePointerInfo());
|
||
|
|
}
|
||
|
|
|
||
|
|
InVals.push_back(ArgValue);
|
||
|
|
}
|
||
|
|
|
||
|
|
for (unsigned I = 0, E = Ins.size(); I != E; ++I) {
|
||
|
|
if (Ins[I].Flags.isSwiftAsync()) {
|
||
|
|
auto X86FI = MF.getInfo<X86MachineFunctionInfo>();
|
||
|
|
if (Subtarget.is64Bit())
|
||
|
|
X86FI->setHasSwiftAsyncContext(true);
|
||
|
|
else {
|
||
|
|
int FI = MF.getFrameInfo().CreateStackObject(4, Align(4), false);
|
||
|
|
X86FI->setSwiftAsyncContextFrameIdx(FI);
|
||
|
|
SDValue St = DAG.getStore(DAG.getEntryNode(), dl, InVals[I],
|
||
|
|
DAG.getFrameIndex(FI, MVT::i32),
|
||
|
|
MachinePointerInfo::getFixedStack(MF, FI));
|
||
|
|
Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, St, Chain);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Swift calling convention does not require we copy the sret argument
|
||
|
|
// into %rax/%eax for the return. We don't set SRetReturnReg for Swift.
|
||
|
|
if (CallConv == CallingConv::Swift || CallConv == CallingConv::SwiftTail)
|
||
|
|
continue;
|
||
|
|
|
||
|
|
// All x86 ABIs require that for returning structs by value we copy the
|
||
|
|
// sret argument into %rax/%eax (depending on ABI) for the return. Save
|
||
|
|
// the argument into a virtual register so that we can access it from the
|
||
|
|
// return points.
|
||
|
|
if (Ins[I].Flags.isSRet()) {
|
||
|
|
assert(!FuncInfo->getSRetReturnReg() &&
|
||
|
|
"SRet return has already been set");
|
||
|
|
MVT PtrTy = getPointerTy(DAG.getDataLayout());
|
||
|
|
Register Reg =
|
||
|
|
MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrTy));
|
||
|
|
FuncInfo->setSRetReturnReg(Reg);
|
||
|
|
SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), dl, Reg, InVals[I]);
|
||
|
|
Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain);
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
unsigned StackSize = CCInfo.getStackSize();
|
||
|
|
// Align stack specially for tail calls.
|
||
|
|
if (shouldGuaranteeTCO(CallConv,
|
||
|
|
MF.getTarget().Options.GuaranteedTailCallOpt))
|
||
|
|
StackSize = GetAlignedArgumentStackSize(StackSize, DAG);
|
||
|
|
|
||
|
|
if (IsVarArg)
|
||
|
|
VarArgsLoweringHelper(FuncInfo, dl, DAG, Subtarget, CallConv, CCInfo)
|
||
|
|
.lowerVarArgsParameters(Chain, StackSize);
|
||
|
|
|
||
|
|
// Some CCs need callee pop.
|
||
|
|
if (X86::isCalleePop(CallConv, Is64Bit, IsVarArg,
|
||
|
|
MF.getTarget().Options.GuaranteedTailCallOpt)) {
|
||
|
|
FuncInfo->setBytesToPopOnReturn(StackSize); // Callee pops everything.
|
||
|
|
} else if (CallConv == CallingConv::X86_INTR && Ins.size() == 2) {
|
||
|
|
// X86 interrupts must pop the error code (and the alignment padding) if
|
||
|
|
// present.
|
||
|
|
FuncInfo->setBytesToPopOnReturn(Is64Bit ? 16 : 4);
|
||
|
|
} else {
|
||
|
|
FuncInfo->setBytesToPopOnReturn(0); // Callee pops nothing.
|
||
|
|
// If this is an sret function, the return should pop the hidden pointer.
|
||
|
|
if (!canGuaranteeTCO(CallConv) && hasCalleePopSRet(Ins, Subtarget))
|
||
|
|
FuncInfo->setBytesToPopOnReturn(4);
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!Is64Bit) {
|
||
|
|
// RegSaveFrameIndex is X86-64 only.
|
||
|
|
FuncInfo->setRegSaveFrameIndex(0xAAAAAAA);
|
||
|
|
}
|
||
|
|
|
||
|
|
FuncInfo->setArgumentStackSize(StackSize);
|
||
|
|
|
||
|
|
if (WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo()) {
|
||
|
|
EHPersonality Personality = classifyEHPersonality(F.getPersonalityFn());
|
||
|
|
if (Personality == EHPersonality::CoreCLR) {
|
||
|
|
assert(Is64Bit);
|
||
|
|
// TODO: Add a mechanism to frame lowering that will allow us to indicate
|
||
|
|
// that we'd prefer this slot be allocated towards the bottom of the frame
|
||
|
|
// (i.e. near the stack pointer after allocating the frame). Every
|
||
|
|
// funclet needs a copy of this slot in its (mostly empty) frame, and the
|
||
|
|
// offset from the bottom of this and each funclet's frame must be the
|
||
|
|
// same, so the size of funclets' (mostly empty) frames is dictated by
|
||
|
|
// how far this slot is from the bottom (since they allocate just enough
|
||
|
|
// space to accommodate holding this slot at the correct offset).
|
||
|
|
int PSPSymFI = MFI.CreateStackObject(8, Align(8), /*isSpillSlot=*/false);
|
||
|
|
EHInfo->PSPSymFrameIdx = PSPSymFI;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (shouldDisableArgRegFromCSR(CallConv) ||
|
||
|
|
F.hasFnAttribute("no_caller_saved_registers")) {
|
||
|
|
MachineRegisterInfo &MRI = MF.getRegInfo();
|
||
|
|
for (std::pair<Register, Register> Pair : MRI.liveins())
|
||
|
|
MRI.disableCalleeSavedRegister(Pair.first);
|
||
|
|
}
|
||
|
|
|
||
|
|
return Chain;
|
||
|
|
}
|
||
|
|
|
||
|
|
SDValue X86TargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr,
|
||
|
|
SDValue Arg, const SDLoc &dl,
|
||
|
|
SelectionDAG &DAG,
|
||
|
|
const CCValAssign &VA,
|
||
|
|
ISD::ArgFlagsTy Flags,
|
||
|
|
bool isByVal) const {
|
||
|
|
unsigned LocMemOffset = VA.getLocMemOffset();
|
||
|
|
SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
|
||
|
|
PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
|
||
|
|
StackPtr, PtrOff);
|
||
|
|
if (isByVal)
|
||
|
|
return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG, dl);
|
||
|
|
|
||
|
|
MaybeAlign Alignment;
|
||
|
|
if (Subtarget.isTargetWindowsMSVC() && !Subtarget.is64Bit() &&
|
||
|
|
Arg.getSimpleValueType() != MVT::f80)
|
||
|
|
Alignment = MaybeAlign(4);
|
||
|
|
return DAG.getStore(
|
||
|
|
Chain, dl, Arg, PtrOff,
|
||
|
|
MachinePointerInfo::getStack(DAG.getMachineFunction(), LocMemOffset),
|
||
|
|
Alignment);
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Emit a load of return address if tail call
|
||
|
|
/// optimization is performed and it is required.
|
||
|
|
SDValue X86TargetLowering::EmitTailCallLoadRetAddr(
|
||
|
|
SelectionDAG &DAG, SDValue &OutRetAddr, SDValue Chain, bool IsTailCall,
|
||
|
|
bool Is64Bit, int FPDiff, const SDLoc &dl) const {
|
||
|
|
// Adjust the Return address stack slot.
|
||
|
|
EVT VT = getPointerTy(DAG.getDataLayout());
|
||
|
|
OutRetAddr = getReturnAddressFrameIndex(DAG);
|
||
|
|
|
||
|
|
// Load the "old" Return address.
|
||
|
|
OutRetAddr = DAG.getLoad(VT, dl, Chain, OutRetAddr, MachinePointerInfo());
|
||
|
|
return SDValue(OutRetAddr.getNode(), 1);
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Emit a store of the return address if tail call
|
||
|
|
/// optimization is performed and it is required (FPDiff!=0).
|
||
|
|
static SDValue EmitTailCallStoreRetAddr(SelectionDAG &DAG, MachineFunction &MF,
|
||
|
|
SDValue Chain, SDValue RetAddrFrIdx,
|
||
|
|
EVT PtrVT, unsigned SlotSize,
|
||
|
|
int FPDiff, const SDLoc &dl) {
|
||
|
|
// Store the return address to the appropriate stack slot.
|
||
|
|
if (!FPDiff) return Chain;
|
||
|
|
// Calculate the new stack slot for the return address.
|
||
|
|
int NewReturnAddrFI =
|
||
|
|
MF.getFrameInfo().CreateFixedObject(SlotSize, (int64_t)FPDiff - SlotSize,
|
||
|
|
false);
|
||
|
|
SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, PtrVT);
|
||
|
|
Chain = DAG.getStore(Chain, dl, RetAddrFrIdx, NewRetAddrFrIdx,
|
||
|
|
MachinePointerInfo::getFixedStack(
|
||
|
|
DAG.getMachineFunction(), NewReturnAddrFI));
|
||
|
|
return Chain;
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Returns a vector_shuffle mask for an movs{s|d}, movd
|
||
|
|
/// operation of specified width.
|
||
|
|
SDValue X86TargetLowering::getMOVL(SelectionDAG &DAG, const SDLoc &dl, MVT VT,
|
||
|
|
SDValue V1, SDValue V2) const {
|
||
|
|
unsigned NumElems = VT.getVectorNumElements();
|
||
|
|
SmallVector<int, 8> Mask;
|
||
|
|
Mask.push_back(NumElems);
|
||
|
|
for (unsigned i = 1; i != NumElems; ++i)
|
||
|
|
Mask.push_back(i);
|
||
|
|
return DAG.getVectorShuffle(VT, dl, V1, V2, Mask);
|
||
|
|
}
|
||
|
|
|
||
|
|
SDValue
|
||
|
|
X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
|
||
|
|
SmallVectorImpl<SDValue> &InVals) const {
|
||
|
|
SelectionDAG &DAG = CLI.DAG;
|
||
|
|
SDLoc &dl = CLI.DL;
|
||
|
|
SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
|
||
|
|
SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
|
||
|
|
SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
|
||
|
|
SDValue Chain = CLI.Chain;
|
||
|
|
SDValue Callee = CLI.Callee;
|
||
|
|
CallingConv::ID CallConv = CLI.CallConv;
|
||
|
|
bool &isTailCall = CLI.IsTailCall;
|
||
|
|
bool isVarArg = CLI.IsVarArg;
|
||
|
|
const auto *CB = CLI.CB;
|
||
|
|
|
||
|
|
MachineFunction &MF = DAG.getMachineFunction();
|
||
|
|
bool Is64Bit = Subtarget.is64Bit();
|
||
|
|
bool IsWin64 = Subtarget.isCallingConvWin64(CallConv);
|
||
|
|
bool IsSibcall = false;
|
||
|
|
bool IsGuaranteeTCO = MF.getTarget().Options.GuaranteedTailCallOpt ||
|
||
|
|
CallConv == CallingConv::Tail || CallConv == CallingConv::SwiftTail;
|
||
|
|
bool IsCalleePopSRet = !IsGuaranteeTCO && hasCalleePopSRet(Outs, Subtarget);
|
||
|
|
X86MachineFunctionInfo *X86Info = MF.getInfo<X86MachineFunctionInfo>();
|
||
|
|
bool HasNCSR = (CB && isa<CallInst>(CB) &&
|
||
|
|
CB->hasFnAttr("no_caller_saved_registers"));
|
||
|
|
bool HasNoCfCheck = (CB && CB->doesNoCfCheck());
|
||
|
|
bool IsIndirectCall = (CB && isa<CallInst>(CB) && CB->isIndirectCall());
|
||
|
|
bool IsCFICall = IsIndirectCall && CLI.CFIType;
|
||
|
|
const Module *M = MF.getMMI().getModule();
|
||
|
|
Metadata *IsCFProtectionSupported = M->getModuleFlag("cf-protection-branch");
|
||
|
|
|
||
|
|
MachineFunction::CallSiteInfo CSInfo;
|
||
|
|
if (CallConv == CallingConv::X86_INTR)
|
||
|
|
report_fatal_error("X86 interrupts may not be called directly");
|
||
|
|
|
||
|
|
bool IsMustTail = CLI.CB && CLI.CB->isMustTailCall();
|
||
|
|
if (Subtarget.isPICStyleGOT() && !IsGuaranteeTCO && !IsMustTail) {
|
||
|
|
// If we are using a GOT, disable tail calls to external symbols with
|
||
|
|
// default visibility. Tail calling such a symbol requires using a GOT
|
||
|
|
// relocation, which forces early binding of the symbol. This breaks code
|
||
|
|
// that require lazy function symbol resolution. Using musttail or
|
||
|
|
// GuaranteedTailCallOpt will override this.
|
||
|
|
GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee);
|
||
|
|
if (!G || (!G->getGlobal()->hasLocalLinkage() &&
|
||
|
|
G->getGlobal()->hasDefaultVisibility()))
|
||
|
|
isTailCall = false;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (isTailCall && !IsMustTail) {
|
||
|
|
// Check if it's really possible to do a tail call.
|
||
|
|
isTailCall = IsEligibleForTailCallOptimization(
|
||
|
|
Callee, CallConv, IsCalleePopSRet, isVarArg, CLI.RetTy, Outs, OutVals,
|
||
|
|
Ins, DAG);
|
||
|
|
|
||
|
|
// Sibcalls are automatically detected tailcalls which do not require
|
||
|
|
// ABI changes.
|
||
|
|
if (!IsGuaranteeTCO && isTailCall)
|
||
|
|
IsSibcall = true;
|
||
|
|
|
||
|
|
if (isTailCall)
|
||
|
|
++NumTailCalls;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (IsMustTail && !isTailCall)
|
||
|
|
report_fatal_error("failed to perform tail call elimination on a call "
|
||
|
|
"site marked musttail");
|
||
|
|
|
||
|
|
assert(!(isVarArg && canGuaranteeTCO(CallConv)) &&
|
||
|
|
"Var args not supported with calling convention fastcc, ghc or hipe");
|
||
|
|
|
||
|
|
// Analyze operands of the call, assigning locations to each operand.
|
||
|
|
SmallVector<CCValAssign, 16> ArgLocs;
|
||
|
|
CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());
|
||
|
|
|
||
|
|
// Allocate shadow area for Win64.
|
||
|
|
if (IsWin64)
|
||
|
|
CCInfo.AllocateStack(32, Align(8));
|
||
|
|
|
||
|
|
CCInfo.AnalyzeArguments(Outs, CC_X86);
|
||
|
|
|
||
|
|
// In vectorcall calling convention a second pass is required for the HVA
|
||
|
|
// types.
|
||
|
|
if (CallingConv::X86_VectorCall == CallConv) {
|
||
|
|
CCInfo.AnalyzeArgumentsSecondPass(Outs, CC_X86);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Get a count of how many bytes are to be pushed on the stack.
|
||
|
|
unsigned NumBytes = CCInfo.getAlignedCallFrameSize();
|
||
|
|
if (IsSibcall)
|
||
|
|
// This is a sibcall. The memory operands are available in caller's
|
||
|
|
// own caller's stack.
|
||
|
|
NumBytes = 0;
|
||
|
|
else if (IsGuaranteeTCO && canGuaranteeTCO(CallConv))
|
||
|
|
NumBytes = GetAlignedArgumentStackSize(NumBytes, DAG);
|
||
|
|
|
||
|
|
int FPDiff = 0;
|
||
|
|
if (isTailCall &&
|
||
|
|
shouldGuaranteeTCO(CallConv,
|
||
|
|
MF.getTarget().Options.GuaranteedTailCallOpt)) {
|
||
|
|
// Lower arguments at fp - stackoffset + fpdiff.
|
||
|
|
unsigned NumBytesCallerPushed = X86Info->getBytesToPopOnReturn();
|
||
|
|
|
||
|
|
FPDiff = NumBytesCallerPushed - NumBytes;
|
||
|
|
|
||
|
|
// Set the delta of movement of the returnaddr stackslot.
|
||
|
|
// But only set if delta is greater than previous delta.
|
||
|
|
if (FPDiff < X86Info->getTCReturnAddrDelta())
|
||
|
|
X86Info->setTCReturnAddrDelta(FPDiff);
|
||
|
|
}
|
||
|
|
|
||
|
|
unsigned NumBytesToPush = NumBytes;
|
||
|
|
unsigned NumBytesToPop = NumBytes;
|
||
|
|
|
||
|
|
// If we have an inalloca argument, all stack space has already been allocated
|
||
|
|
// for us and be right at the top of the stack. We don't support multiple
|
||
|
|
// arguments passed in memory when using inalloca.
|
||
|
|
if (!Outs.empty() && Outs.back().Flags.isInAlloca()) {
|
||
|
|
NumBytesToPush = 0;
|
||
|
|
if (!ArgLocs.back().isMemLoc())
|
||
|
|
report_fatal_error("cannot use inalloca attribute on a register "
|
||
|
|
"parameter");
|
||
|
|
if (ArgLocs.back().getLocMemOffset() != 0)
|
||
|
|
report_fatal_error("any parameter with the inalloca attribute must be "
|
||
|
|
"the only memory argument");
|
||
|
|
} else if (CLI.IsPreallocated) {
|
||
|
|
assert(ArgLocs.back().isMemLoc() &&
|
||
|
|
"cannot use preallocated attribute on a register "
|
||
|
|
"parameter");
|
||
|
|
SmallVector<size_t, 4> PreallocatedOffsets;
|
||
|
|
for (size_t i = 0; i < CLI.OutVals.size(); ++i) {
|
||
|
|
if (CLI.CB->paramHasAttr(i, Attribute::Preallocated)) {
|
||
|
|
PreallocatedOffsets.push_back(ArgLocs[i].getLocMemOffset());
|
||
|
|
}
|
||
|
|
}
|
||
|
|
auto *MFI = DAG.getMachineFunction().getInfo<X86MachineFunctionInfo>();
|
||
|
|
size_t PreallocatedId = MFI->getPreallocatedIdForCallSite(CLI.CB);
|
||
|
|
MFI->setPreallocatedStackSize(PreallocatedId, NumBytes);
|
||
|
|
MFI->setPreallocatedArgOffsets(PreallocatedId, PreallocatedOffsets);
|
||
|
|
NumBytesToPush = 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!IsSibcall && !IsMustTail)
|
||
|
|
Chain = DAG.getCALLSEQ_START(Chain, NumBytesToPush,
|
||
|
|
NumBytes - NumBytesToPush, dl);
|
||
|
|
|
||
|
|
SDValue RetAddrFrIdx;
|
||
|
|
// Load return address for tail calls.
|
||
|
|
if (isTailCall && FPDiff)
|
||
|
|
Chain = EmitTailCallLoadRetAddr(DAG, RetAddrFrIdx, Chain, isTailCall,
|
||
|
|
Is64Bit, FPDiff, dl);
|
||
|
|
|
||
|
|
SmallVector<std::pair<Register, SDValue>, 8> RegsToPass;
|
||
|
|
SmallVector<SDValue, 8> MemOpChains;
|
||
|
|
SDValue StackPtr;
|
||
|
|
|
||
|
|
// The next loop assumes that the locations are in the same order of the
|
||
|
|
// input arguments.
|
||
|
|
assert(isSortedByValueNo(ArgLocs) &&
|
||
|
|
"Argument Location list must be sorted before lowering");
|
||
|
|
|
||
|
|
// Walk the register/memloc assignments, inserting copies/loads. In the case
|
||
|
|
// of tail call optimization arguments are handle later.
|
||
|
|
const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
|
||
|
|
for (unsigned I = 0, OutIndex = 0, E = ArgLocs.size(); I != E;
|
||
|
|
++I, ++OutIndex) {
|
||
|
|
assert(OutIndex < Outs.size() && "Invalid Out index");
|
||
|
|
// Skip inalloca/preallocated arguments, they have already been written.
|
||
|
|
ISD::ArgFlagsTy Flags = Outs[OutIndex].Flags;
|
||
|
|
if (Flags.isInAlloca() || Flags.isPreallocated())
|
||
|
|
continue;
|
||
|
|
|
||
|
|
CCValAssign &VA = ArgLocs[I];
|
||
|
|
EVT RegVT = VA.getLocVT();
|
||
|
|
SDValue Arg = OutVals[OutIndex];
|
||
|
|
bool isByVal = Flags.isByVal();
|
||
|
|
|
||
|
|
// Promote the value if needed.
|
||
|
|
switch (VA.getLocInfo()) {
|
||
|
|
default: llvm_unreachable("Unknown loc info!");
|
||
|
|
case CCValAssign::Full: break;
|
||
|
|
case CCValAssign::SExt:
|
||
|
|
Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, RegVT, Arg);
|
||
|
|
break;
|
||
|
|
case CCValAssign::ZExt:
|
||
|
|
Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, RegVT, Arg);
|
||
|
|
break;
|
||
|
|
case CCValAssign::AExt:
|
||
|
|
if (Arg.getValueType().isVector() &&
|
||
|
|
Arg.getValueType().getVectorElementType() == MVT::i1)
|
||
|
|
Arg = lowerMasksToReg(Arg, RegVT, dl, DAG);
|
||
|
|
else if (RegVT.is128BitVector()) {
|
||
|
|
// Special case: passing MMX values in XMM registers.
|
||
|
|
Arg = DAG.getBitcast(MVT::i64, Arg);
|
||
|
|
Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Arg);
|
||
|
|
Arg = getMOVL(DAG, dl, MVT::v2i64, DAG.getUNDEF(MVT::v2i64), Arg);
|
||
|
|
} else
|
||
|
|
Arg = DAG.getNode(ISD::ANY_EXTEND, dl, RegVT, Arg);
|
||
|
|
break;
|
||
|
|
case CCValAssign::BCvt:
|
||
|
|
Arg = DAG.getBitcast(RegVT, Arg);
|
||
|
|
break;
|
||
|
|
case CCValAssign::Indirect: {
|
||
|
|
if (isByVal) {
|
||
|
|
// Memcpy the argument to a temporary stack slot to prevent
|
||
|
|
// the caller from seeing any modifications the callee may make
|
||
|
|
// as guaranteed by the `byval` attribute.
|
||
|
|
int FrameIdx = MF.getFrameInfo().CreateStackObject(
|
||
|
|
Flags.getByValSize(),
|
||
|
|
std::max(Align(16), Flags.getNonZeroByValAlign()), false);
|
||
|
|
SDValue StackSlot =
|
||
|
|
DAG.getFrameIndex(FrameIdx, getPointerTy(DAG.getDataLayout()));
|
||
|
|
Chain =
|
||
|
|
CreateCopyOfByValArgument(Arg, StackSlot, Chain, Flags, DAG, dl);
|
||
|
|
// From now on treat this as a regular pointer
|
||
|
|
Arg = StackSlot;
|
||
|
|
isByVal = false;
|
||
|
|
} else {
|
||
|
|
// Store the argument.
|
||
|
|
SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT());
|
||
|
|
int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
|
||
|
|
Chain = DAG.getStore(
|
||
|
|
Chain, dl, Arg, SpillSlot,
|
||
|
|
MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI));
|
||
|
|
Arg = SpillSlot;
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (VA.needsCustom()) {
|
||
|
|
assert(VA.getValVT() == MVT::v64i1 &&
|
||
|
|
"Currently the only custom case is when we split v64i1 to 2 regs");
|
||
|
|
// Split v64i1 value into two registers
|
||
|
|
Passv64i1ArgInRegs(dl, DAG, Arg, RegsToPass, VA, ArgLocs[++I], Subtarget);
|
||
|
|
} else if (VA.isRegLoc()) {
|
||
|
|
RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
|
||
|
|
const TargetOptions &Options = DAG.getTarget().Options;
|
||
|
|
if (Options.EmitCallSiteInfo)
|
||
|
|
CSInfo.emplace_back(VA.getLocReg(), I);
|
||
|
|
if (isVarArg && IsWin64) {
|
||
|
|
// Win64 ABI requires argument XMM reg to be copied to the corresponding
|
||
|
|
// shadow reg if callee is a varargs function.
|
||
|
|
Register ShadowReg;
|
||
|
|
switch (VA.getLocReg()) {
|
||
|
|
case X86::XMM0: ShadowReg = X86::RCX; break;
|
||
|
|
case X86::XMM1: ShadowReg = X86::RDX; break;
|
||
|
|
case X86::XMM2: ShadowReg = X86::R8; break;
|
||
|
|
case X86::XMM3: ShadowReg = X86::R9; break;
|
||
|
|
}
|
||
|
|
if (ShadowReg)
|
||
|
|
RegsToPass.push_back(std::make_pair(ShadowReg, Arg));
|
||
|
|
}
|
||
|
|
} else if (!IsSibcall && (!isTailCall || isByVal)) {
|
||
|
|
assert(VA.isMemLoc());
|
||
|
|
if (!StackPtr.getNode())
|
||
|
|
StackPtr = DAG.getCopyFromReg(Chain, dl, RegInfo->getStackRegister(),
|
||
|
|
getPointerTy(DAG.getDataLayout()));
|
||
|
|
MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg,
|
||
|
|
dl, DAG, VA, Flags, isByVal));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!MemOpChains.empty())
|
||
|
|
Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
|
||
|
|
|
||
|
|
if (Subtarget.isPICStyleGOT()) {
|
||
|
|
// ELF / PIC requires GOT in the EBX register before function calls via PLT
|
||
|
|
// GOT pointer (except regcall).
|
||
|
|
if (!isTailCall) {
|
||
|
|
// Indirect call with RegCall calling convertion may use up all the
|
||
|
|
// general registers, so it is not suitable to bind EBX reister for
|
||
|
|
// GOT address, just let register allocator handle it.
|
||
|
|
if (CallConv != CallingConv::X86_RegCall)
|
||
|
|
RegsToPass.push_back(std::make_pair(
|
||
|
|
Register(X86::EBX), DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(),
|
||
|
|
getPointerTy(DAG.getDataLayout()))));
|
||
|
|
} else {
|
||
|
|
// If we are tail calling and generating PIC/GOT style code load the
|
||
|
|
// address of the callee into ECX. The value in ecx is used as target of
|
||
|
|
// the tail jump. This is done to circumvent the ebx/callee-saved problem
|
||
|
|
// for tail calls on PIC/GOT architectures. Normally we would just put the
|
||
|
|
// address of GOT into ebx and then call target@PLT. But for tail calls
|
||
|
|
// ebx would be restored (since ebx is callee saved) before jumping to the
|
||
|
|
// target@PLT.
|
||
|
|
|
||
|
|
// Note: The actual moving to ECX is done further down.
|
||
|
|
GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee);
|
||
|
|
if (G && !G->getGlobal()->hasLocalLinkage() &&
|
||
|
|
G->getGlobal()->hasDefaultVisibility())
|
||
|
|
Callee = LowerGlobalAddress(Callee, DAG);
|
||
|
|
else if (isa<ExternalSymbolSDNode>(Callee))
|
||
|
|
Callee = LowerExternalSymbol(Callee, DAG);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (Is64Bit && isVarArg && !IsWin64 && !IsMustTail &&
|
||
|
|
(Subtarget.hasSSE1() || !M->getModuleFlag("SkipRaxSetup"))) {
|
||
|
|
// From AMD64 ABI document:
|
||
|
|
// For calls that may call functions that use varargs or stdargs
|
||
|
|
// (prototype-less calls or calls to functions containing ellipsis (...) in
|
||
|
|
// the declaration) %al is used as hidden argument to specify the number
|
||
|
|
// of SSE registers used. The contents of %al do not need to match exactly
|
||
|
|
// the number of registers, but must be an ubound on the number of SSE
|
||
|
|
// registers used and is in the range 0 - 8 inclusive.
|
||
|
|
|
||
|
|
// Count the number of XMM registers allocated.
|
||
|
|
static const MCPhysReg XMMArgRegs[] = {
|
||
|
|
X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
|
||
|
|
X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
|
||
|
|
};
|
||
|
|
unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs);
|
||
|
|
assert((Subtarget.hasSSE1() || !NumXMMRegs)
|
||
|
|
&& "SSE registers cannot be used when SSE is disabled");
|
||
|
|
RegsToPass.push_back(std::make_pair(Register(X86::AL),
|
||
|
|
DAG.getConstant(NumXMMRegs, dl,
|
||
|
|
MVT::i8)));
|
||
|
|
}
|
||
|
|
|
||
|
|
if (isVarArg && IsMustTail) {
|
||
|
|
const auto &Forwards = X86Info->getForwardedMustTailRegParms();
|
||
|
|
for (const auto &F : Forwards) {
|
||
|
|
SDValue Val = DAG.getCopyFromReg(Chain, dl, F.VReg, F.VT);
|
||
|
|
RegsToPass.push_back(std::make_pair(F.PReg, Val));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// For tail calls lower the arguments to the 'real' stack slots. Sibcalls
|
||
|
|
// don't need this because the eligibility check rejects calls that require
|
||
|
|
// shuffling arguments passed in memory.
|
||
|
|
if (!IsSibcall && isTailCall) {
|
||
|
|
// Force all the incoming stack arguments to be loaded from the stack
|
||
|
|
// before any new outgoing arguments are stored to the stack, because the
|
||
|
|
// outgoing stack slots may alias the incoming argument stack slots, and
|
||
|
|
// the alias isn't otherwise explicit. This is slightly more conservative
|
||
|
|
// than necessary, because it means that each store effectively depends
|
||
|
|
// on every argument instead of just those arguments it would clobber.
|
||
|
|
SDValue ArgChain = DAG.getStackArgumentTokenFactor(Chain);
|
||
|
|
|
||
|
|
SmallVector<SDValue, 8> MemOpChains2;
|
||
|
|
SDValue FIN;
|
||
|
|
int FI = 0;
|
||
|
|
for (unsigned I = 0, OutsIndex = 0, E = ArgLocs.size(); I != E;
|
||
|
|
++I, ++OutsIndex) {
|
||
|
|
CCValAssign &VA = ArgLocs[I];
|
||
|
|
|
||
|
|
if (VA.isRegLoc()) {
|
||
|
|
if (VA.needsCustom()) {
|
||
|
|
assert((CallConv == CallingConv::X86_RegCall) &&
|
||
|
|
"Expecting custom case only in regcall calling convention");
|
||
|
|
// This means that we are in special case where one argument was
|
||
|
|
// passed through two register locations - Skip the next location
|
||
|
|
++I;
|
||
|
|
}
|
||
|
|
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
|
||
|
|
assert(VA.isMemLoc());
|
||
|
|
SDValue Arg = OutVals[OutsIndex];
|
||
|
|
ISD::ArgFlagsTy Flags = Outs[OutsIndex].Flags;
|
||
|
|
// Skip inalloca/preallocated arguments. They don't require any work.
|
||
|
|
if (Flags.isInAlloca() || Flags.isPreallocated())
|
||
|
|
continue;
|
||
|
|
// Create frame index.
|
||
|
|
int32_t Offset = VA.getLocMemOffset()+FPDiff;
|
||
|
|
uint32_t OpSize = (VA.getLocVT().getSizeInBits()+7)/8;
|
||
|
|
FI = MF.getFrameInfo().CreateFixedObject(OpSize, Offset, true);
|
||
|
|
FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
|
||
|
|
|
||
|
|
if (Flags.isByVal()) {
|
||
|
|
// Copy relative to framepointer.
|
||
|
|
SDValue Source = DAG.getIntPtrConstant(VA.getLocMemOffset(), dl);
|
||
|
|
if (!StackPtr.getNode())
|
||
|
|
StackPtr = DAG.getCopyFromReg(Chain, dl, RegInfo->getStackRegister(),
|
||
|
|
getPointerTy(DAG.getDataLayout()));
|
||
|
|
Source = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
|
||
|
|
StackPtr, Source);
|
||
|
|
|
||
|
|
MemOpChains2.push_back(CreateCopyOfByValArgument(Source, FIN,
|
||
|
|
ArgChain,
|
||
|
|
Flags, DAG, dl));
|
||
|
|
} else {
|
||
|
|
// Store relative to framepointer.
|
||
|
|
MemOpChains2.push_back(DAG.getStore(
|
||
|
|
ArgChain, dl, Arg, FIN,
|
||
|
|
MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI)));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (!MemOpChains2.empty())
|
||
|
|
Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2);
|
||
|
|
|
||
|
|
// Store the return address to the appropriate stack slot.
|
||
|
|
Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx,
|
||
|
|
getPointerTy(DAG.getDataLayout()),
|
||
|
|
RegInfo->getSlotSize(), FPDiff, dl);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Build a sequence of copy-to-reg nodes chained together with token chain
|
||
|
|
// and glue operands which copy the outgoing args into registers.
|
||
|
|
SDValue InGlue;
|
||
|
|
for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
|
||
|
|
Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
|
||
|
|
RegsToPass[i].second, InGlue);
|
||
|
|
InGlue = Chain.getValue(1);
|
||
|
|
}
|
||
|
|
|
||
|
|
if (DAG.getTarget().getCodeModel() == CodeModel::Large) {
|
||
|
|
assert(Is64Bit && "Large code model is only legal in 64-bit mode.");
|
||
|
|
// In the 64-bit large code model, we have to make all calls
|
||
|
|
// through a register, since the call instruction's 32-bit
|
||
|
|
// pc-relative offset may not be large enough to hold the whole
|
||
|
|
// address.
|
||
|
|
} else if (Callee->getOpcode() == ISD::GlobalAddress ||
|
||
|
|
Callee->getOpcode() == ISD::ExternalSymbol) {
|
||
|
|
// Lower direct calls to global addresses and external symbols. Setting
|
||
|
|
// ForCall to true here has the effect of removing WrapperRIP when possible
|
||
|
|
// to allow direct calls to be selected without first materializing the
|
||
|
|
// address into a register.
|
||
|
|
Callee = LowerGlobalOrExternal(Callee, DAG, /*ForCall=*/true);
|
||
|
|
} else if (Subtarget.isTarget64BitILP32() &&
|
||
|
|
Callee.getValueType() == MVT::i32) {
|
||
|
|
// Zero-extend the 32-bit Callee address into a 64-bit according to x32 ABI
|
||
|
|
Callee = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Callee);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Returns a chain & a glue for retval copy to use.
|
||
|
|
SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
|
||
|
|
SmallVector<SDValue, 8> Ops;
|
||
|
|
|
||
|
|
if (!IsSibcall && isTailCall && !IsMustTail) {
|
||
|
|
Chain = DAG.getCALLSEQ_END(Chain, NumBytesToPop, 0, InGlue, dl);
|
||
|
|
InGlue = Chain.getValue(1);
|
||
|
|
}
|
||
|
|
|
||
|
|
Ops.push_back(Chain);
|
||
|
|
Ops.push_back(Callee);
|
||
|
|
|
||
|
|
if (isTailCall)
|
||
|
|
Ops.push_back(DAG.getTargetConstant(FPDiff, dl, MVT::i32));
|
||
|
|
|
||
|
|
// Add argument registers to the end of the list so that they are known live
|
||
|
|
// into the call.
|
||
|
|
for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
|
||
|
|
Ops.push_back(DAG.getRegister(RegsToPass[i].first,
|
||
|
|
RegsToPass[i].second.getValueType()));
|
||
|
|
|
||
|
|
// Add a register mask operand representing the call-preserved registers.
|
||
|
|
const uint32_t *Mask = [&]() {
|
||
|
|
auto AdaptedCC = CallConv;
|
||
|
|
// If HasNCSR is asserted (attribute NoCallerSavedRegisters exists),
|
||
|
|
// use X86_INTR calling convention because it has the same CSR mask
|
||
|
|
// (same preserved registers).
|
||
|
|
if (HasNCSR)
|
||
|
|
AdaptedCC = (CallingConv::ID)CallingConv::X86_INTR;
|
||
|
|
// If NoCalleeSavedRegisters is requested, than use GHC since it happens
|
||
|
|
// to use the CSR_NoRegs_RegMask.
|
||
|
|
if (CB && CB->hasFnAttr("no_callee_saved_registers"))
|
||
|
|
AdaptedCC = (CallingConv::ID)CallingConv::GHC;
|
||
|
|
return RegInfo->getCallPreservedMask(MF, AdaptedCC);
|
||
|
|
}();
|
||
|
|
assert(Mask && "Missing call preserved mask for calling convention");
|
||
|
|
|
||
|
|
// If this is an invoke in a 32-bit function using a funclet-based
|
||
|
|
// personality, assume the function clobbers all registers. If an exception
|
||
|
|
// is thrown, the runtime will not restore CSRs.
|
||
|
|
// FIXME: Model this more precisely so that we can register allocate across
|
||
|
|
// the normal edge and spill and fill across the exceptional edge.
|
||
|
|
if (!Is64Bit && CLI.CB && isa<InvokeInst>(CLI.CB)) {
|
||
|
|
const Function &CallerFn = MF.getFunction();
|
||
|
|
EHPersonality Pers =
|
||
|
|
CallerFn.hasPersonalityFn()
|
||
|
|
? classifyEHPersonality(CallerFn.getPersonalityFn())
|
||
|
|
: EHPersonality::Unknown;
|
||
|
|
if (isFuncletEHPersonality(Pers))
|
||
|
|
Mask = RegInfo->getNoPreservedMask();
|
||
|
|
}
|
||
|
|
|
||
|
|
// Define a new register mask from the existing mask.
|
||
|
|
uint32_t *RegMask = nullptr;
|
||
|
|
|
||
|
|
// In some calling conventions we need to remove the used physical registers
|
||
|
|
// from the reg mask. Create a new RegMask for such calling conventions.
|
||
|
|
// RegMask for calling conventions that disable only return registers (e.g.
|
||
|
|
// preserve_most) will be modified later in LowerCallResult.
|
||
|
|
bool ShouldDisableArgRegs = shouldDisableArgRegFromCSR(CallConv) || HasNCSR;
|
||
|
|
if (ShouldDisableArgRegs || shouldDisableRetRegFromCSR(CallConv)) {
|
||
|
|
const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
|
||
|
|
|
||
|
|
// Allocate a new Reg Mask and copy Mask.
|
||
|
|
RegMask = MF.allocateRegMask();
|
||
|
|
unsigned RegMaskSize = MachineOperand::getRegMaskSize(TRI->getNumRegs());
|
||
|
|
memcpy(RegMask, Mask, sizeof(RegMask[0]) * RegMaskSize);
|
||
|
|
|
||
|
|
// Make sure all sub registers of the argument registers are reset
|
||
|
|
// in the RegMask.
|
||
|
|
if (ShouldDisableArgRegs) {
|
||
|
|
for (auto const &RegPair : RegsToPass)
|
||
|
|
for (MCPhysReg SubReg : TRI->subregs_inclusive(RegPair.first))
|
||
|
|
RegMask[SubReg / 32] &= ~(1u << (SubReg % 32));
|
||
|
|
}
|
||
|
|
|
||
|
|
// Create the RegMask Operand according to our updated mask.
|
||
|
|
Ops.push_back(DAG.getRegisterMask(RegMask));
|
||
|
|
} else {
|
||
|
|
// Create the RegMask Operand according to the static mask.
|
||
|
|
Ops.push_back(DAG.getRegisterMask(Mask));
|
||
|
|
}
|
||
|
|
|
||
|
|
if (InGlue.getNode())
|
||
|
|
Ops.push_back(InGlue);
|
||
|
|
|
||
|
|
if (isTailCall) {
|
||
|
|
// We used to do:
|
||
|
|
//// If this is the first return lowered for this function, add the regs
|
||
|
|
//// to the liveout set for the function.
|
||
|
|
// This isn't right, although it's probably harmless on x86; liveouts
|
||
|
|
// should be computed from returns not tail calls. Consider a void
|
||
|
|
// function making a tail call to a function returning int.
|
||
|
|
MF.getFrameInfo().setHasTailCall();
|
||
|
|
SDValue Ret = DAG.getNode(X86ISD::TC_RETURN, dl, NodeTys, Ops);
|
||
|
|
|
||
|
|
if (IsCFICall)
|
||
|
|
Ret.getNode()->setCFIType(CLI.CFIType->getZExtValue());
|
||
|
|
|
||
|
|
DAG.addNoMergeSiteInfo(Ret.getNode(), CLI.NoMerge);
|
||
|
|
DAG.addCallSiteInfo(Ret.getNode(), std::move(CSInfo));
|
||
|
|
return Ret;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (HasNoCfCheck && IsCFProtectionSupported && IsIndirectCall) {
|
||
|
|
Chain = DAG.getNode(X86ISD::NT_CALL, dl, NodeTys, Ops);
|
||
|
|
} else if (CLI.CB && objcarc::hasAttachedCallOpBundle(CLI.CB)) {
|
||
|
|
// Calls with a "clang.arc.attachedcall" bundle are special. They should be
|
||
|
|
// expanded to the call, directly followed by a special marker sequence and
|
||
|
|
// a call to a ObjC library function. Use the CALL_RVMARKER to do that.
|
||
|
|
assert(!isTailCall &&
|
||
|
|
"tail calls cannot be marked with clang.arc.attachedcall");
|
||
|
|
assert(Is64Bit && "clang.arc.attachedcall is only supported in 64bit mode");
|
||
|
|
|
||
|
|
// Add a target global address for the retainRV/claimRV runtime function
|
||
|
|
// just before the call target.
|
||
|
|
Function *ARCFn = *objcarc::getAttachedARCFunction(CLI.CB);
|
||
|
|
auto PtrVT = getPointerTy(DAG.getDataLayout());
|
||
|
|
auto GA = DAG.getTargetGlobalAddress(ARCFn, dl, PtrVT);
|
||
|
|
Ops.insert(Ops.begin() + 1, GA);
|
||
|
|
Chain = DAG.getNode(X86ISD::CALL_RVMARKER, dl, NodeTys, Ops);
|
||
|
|
} else {
|
||
|
|
Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, Ops);
|
||
|
|
}
|
||
|
|
|
||
|
|
if (IsCFICall)
|
||
|
|
Chain.getNode()->setCFIType(CLI.CFIType->getZExtValue());
|
||
|
|
|
||
|
|
InGlue = Chain.getValue(1);
|
||
|
|
DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
|
||
|
|
DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo));
|
||
|
|
|
||
|
|
// Save heapallocsite metadata.
|
||
|
|
if (CLI.CB)
|
||
|
|
if (MDNode *HeapAlloc = CLI.CB->getMetadata("heapallocsite"))
|
||
|
|
DAG.addHeapAllocSite(Chain.getNode(), HeapAlloc);
|
||
|
|
|
||
|
|
// Create the CALLSEQ_END node.
|
||
|
|
unsigned NumBytesForCalleeToPop = 0; // Callee pops nothing.
|
||
|
|
if (X86::isCalleePop(CallConv, Is64Bit, isVarArg,
|
||
|
|
DAG.getTarget().Options.GuaranteedTailCallOpt))
|
||
|
|
NumBytesForCalleeToPop = NumBytes; // Callee pops everything
|
||
|
|
else if (!canGuaranteeTCO(CallConv) && IsCalleePopSRet)
|
||
|
|
// If this call passes a struct-return pointer, the callee
|
||
|
|
// pops that struct pointer.
|
||
|
|
NumBytesForCalleeToPop = 4;
|
||
|
|
|
||
|
|
// Returns a glue for retval copy to use.
|
||
|
|
if (!IsSibcall) {
|
||
|
|
Chain = DAG.getCALLSEQ_END(Chain, NumBytesToPop, NumBytesForCalleeToPop,
|
||
|
|
InGlue, dl);
|
||
|
|
InGlue = Chain.getValue(1);
|
||
|
|
}
|
||
|
|
|
||
|
|
// Handle result values, copying them out of physregs into vregs that we
|
||
|
|
// return.
|
||
|
|
return LowerCallResult(Chain, InGlue, CallConv, isVarArg, Ins, dl, DAG,
|
||
|
|
InVals, RegMask);
|
||
|
|
}
|
||
|
|
|
||
|
|
//===----------------------------------------------------------------------===//
|
||
|
|
// Fast Calling Convention (tail call) implementation
|
||
|
|
//===----------------------------------------------------------------------===//
|
||
|
|
|
||
|
|
// Like std call, callee cleans arguments, convention except that ECX is
|
||
|
|
// reserved for storing the tail called function address. Only 2 registers are
|
||
|
|
// free for argument passing (inreg). Tail call optimization is performed
|
||
|
|
// provided:
|
||
|
|
// * tailcallopt is enabled
|
||
|
|
// * caller/callee are fastcc
|
||
|
|
// On X86_64 architecture with GOT-style position independent code only local
|
||
|
|
// (within module) calls are supported at the moment.
|
||
|
|
// To keep the stack aligned according to platform abi the function
|
||
|
|
// GetAlignedArgumentStackSize ensures that argument delta is always multiples
|
||
|
|
// of stack alignment. (Dynamic linkers need this - Darwin's dyld for example)
|
||
|
|
// If a tail called function callee has more arguments than the caller the
|
||
|
|
// caller needs to make sure that there is room to move the RETADDR to. This is
|
||
|
|
// achieved by reserving an area the size of the argument delta right after the
|
||
|
|
// original RETADDR, but before the saved framepointer or the spilled registers
|
||
|
|
// e.g. caller(arg1, arg2) calls callee(arg1, arg2,arg3,arg4)
|
||
|
|
// stack layout:
|
||
|
|
// arg1
|
||
|
|
// arg2
|
||
|
|
// RETADDR
|
||
|
|
// [ new RETADDR
|
||
|
|
// move area ]
|
||
|
|
// (possible EBP)
|
||
|
|
// ESI
|
||
|
|
// EDI
|
||
|
|
// local1 ..
|
||
|
|
|
||
|
|
/// Make the stack size align e.g 16n + 12 aligned for a 16-byte align
|
||
|
|
/// requirement.
|
||
|
|
unsigned
|
||
|
|
X86TargetLowering::GetAlignedArgumentStackSize(const unsigned StackSize,
|
||
|
|
SelectionDAG &DAG) const {
|
||
|
|
const Align StackAlignment = Subtarget.getFrameLowering()->getStackAlign();
|
||
|
|
const uint64_t SlotSize = Subtarget.getRegisterInfo()->getSlotSize();
|
||
|
|
assert(StackSize % SlotSize == 0 &&
|
||
|
|
"StackSize must be a multiple of SlotSize");
|
||
|
|
return alignTo(StackSize + SlotSize, StackAlignment) - SlotSize;
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Return true if the given stack call argument is already available in the
|
||
|
|
/// same position (relatively) of the caller's incoming argument stack.
|
||
|
|
static
|
||
|
|
bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags,
|
||
|
|
MachineFrameInfo &MFI, const MachineRegisterInfo *MRI,
|
||
|
|
const X86InstrInfo *TII, const CCValAssign &VA) {
|
||
|
|
unsigned Bytes = Arg.getValueSizeInBits() / 8;
|
||
|
|
|
||
|
|
for (;;) {
|
||
|
|
// Look through nodes that don't alter the bits of the incoming value.
|
||
|
|
unsigned Op = Arg.getOpcode();
|
||
|
|
if (Op == ISD::ZERO_EXTEND || Op == ISD::ANY_EXTEND || Op == ISD::BITCAST ||
|
||
|
|
Op == ISD::AssertZext) {
|
||
|
|
Arg = Arg.getOperand(0);
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
if (Op == ISD::TRUNCATE) {
|
||
|
|
const SDValue &TruncInput = Arg.getOperand(0);
|
||
|
|
if (TruncInput.getOpcode() == ISD::AssertZext &&
|
||
|
|
cast<VTSDNode>(TruncInput.getOperand(1))->getVT() ==
|
||
|
|
Arg.getValueType()) {
|
||
|
|
Arg = TruncInput.getOperand(0);
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
|
||
|
|
int FI = INT_MAX;
|
||
|
|
if (Arg.getOpcode() == ISD::CopyFromReg) {
|
||
|
|
Register VR = cast<RegisterSDNode>(Arg.getOperand(1))->getReg();
|
||
|
|
if (!VR.isVirtual())
|
||
|
|
return false;
|
||
|
|
MachineInstr *Def = MRI->getVRegDef(VR);
|
||
|
|
if (!Def)
|
||
|
|
return false;
|
||
|
|
if (!Flags.isByVal()) {
|
||
|
|
if (!TII->isLoadFromStackSlot(*Def, FI))
|
||
|
|
return false;
|
||
|
|
} else {
|
||
|
|
unsigned Opcode = Def->getOpcode();
|
||
|
|
if ((Opcode == X86::LEA32r || Opcode == X86::LEA64r ||
|
||
|
|
Opcode == X86::LEA64_32r) &&
|
||
|
|
Def->getOperand(1).isFI()) {
|
||
|
|
FI = Def->getOperand(1).getIndex();
|
||
|
|
Bytes = Flags.getByValSize();
|
||
|
|
} else
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
} else if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) {
|
||
|
|
if (Flags.isByVal())
|
||
|
|
// ByVal argument is passed in as a pointer but it's now being
|
||
|
|
// dereferenced. e.g.
|
||
|
|
// define @foo(%struct.X* %A) {
|
||
|
|
// tail call @bar(%struct.X* byval %A)
|
||
|
|
// }
|
||
|
|
return false;
|
||
|
|
SDValue Ptr = Ld->getBasePtr();
|
||
|
|
FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr);
|
||
|
|
if (!FINode)
|
||
|
|
return false;
|
||
|
|
FI = FINode->getIndex();
|
||
|
|
} else if (Arg.getOpcode() == ISD::FrameIndex && Flags.isByVal()) {
|
||
|
|
FrameIndexSDNode *FINode = cast<FrameIndexSDNode>(Arg);
|
||
|
|
FI = FINode->getIndex();
|
||
|
|
Bytes = Flags.getByValSize();
|
||
|
|
} else
|
||
|
|
return false;
|
||
|
|
|
||
|
|
assert(FI != INT_MAX);
|
||
|
|
if (!MFI.isFixedObjectIndex(FI))
|
||
|
|
return false;
|
||
|
|
|
||
|
|
if (Offset != MFI.getObjectOffset(FI))
|
||
|
|
return false;
|
||
|
|
|
||
|
|
// If this is not byval, check that the argument stack object is immutable.
|
||
|
|
// inalloca and argument copy elision can create mutable argument stack
|
||
|
|
// objects. Byval objects can be mutated, but a byval call intends to pass the
|
||
|
|
// mutated memory.
|
||
|
|
if (!Flags.isByVal() && !MFI.isImmutableObjectIndex(FI))
|
||
|
|
return false;
|
||
|
|
|
||
|
|
if (VA.getLocVT().getFixedSizeInBits() >
|
||
|
|
Arg.getValueSizeInBits().getFixedValue()) {
|
||
|
|
// If the argument location is wider than the argument type, check that any
|
||
|
|
// extension flags match.
|
||
|
|
if (Flags.isZExt() != MFI.isObjectZExt(FI) ||
|
||
|
|
Flags.isSExt() != MFI.isObjectSExt(FI)) {
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return Bytes == MFI.getObjectSize(FI);
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Check whether the call is eligible for tail call optimization. Targets
|
||
|
|
/// that want to do tail call optimization should implement this function.
|
||
|
|
bool X86TargetLowering::IsEligibleForTailCallOptimization(
|
||
|
|
SDValue Callee, CallingConv::ID CalleeCC, bool IsCalleePopSRet,
|
||
|
|
bool isVarArg, Type *RetTy, const SmallVectorImpl<ISD::OutputArg> &Outs,
|
||
|
|
const SmallVectorImpl<SDValue> &OutVals,
|
||
|
|
const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
|
||
|
|
if (!mayTailCallThisCC(CalleeCC))
|
||
|
|
return false;
|
||
|
|
|
||
|
|
// If -tailcallopt is specified, make fastcc functions tail-callable.
|
||
|
|
MachineFunction &MF = DAG.getMachineFunction();
|
||
|
|
const Function &CallerF = MF.getFunction();
|
||
|
|
|
||
|
|
// If the function return type is x86_fp80 and the callee return type is not,
|
||
|
|
// then the FP_EXTEND of the call result is not a nop. It's not safe to
|
||
|
|
// perform a tailcall optimization here.
|
||
|
|
if (CallerF.getReturnType()->isX86_FP80Ty() && !RetTy->isX86_FP80Ty())
|
||
|
|
return false;
|
||
|
|
|
||
|
|
CallingConv::ID CallerCC = CallerF.getCallingConv();
|
||
|
|
bool CCMatch = CallerCC == CalleeCC;
|
||
|
|
bool IsCalleeWin64 = Subtarget.isCallingConvWin64(CalleeCC);
|
||
|
|
bool IsCallerWin64 = Subtarget.isCallingConvWin64(CallerCC);
|
||
|
|
bool IsGuaranteeTCO = DAG.getTarget().Options.GuaranteedTailCallOpt ||
|
||
|
|
CalleeCC == CallingConv::Tail || CalleeCC == CallingConv::SwiftTail;
|
||
|
|
|
||
|
|
// Win64 functions have extra shadow space for argument homing. Don't do the
|
||
|
|
// sibcall if the caller and callee have mismatched expectations for this
|
||
|
|
// space.
|
||
|
|
if (IsCalleeWin64 != IsCallerWin64)
|
||
|
|
return false;
|
||
|
|
|
||
|
|
if (IsGuaranteeTCO) {
|
||
|
|
if (canGuaranteeTCO(CalleeCC) && CCMatch)
|
||
|
|
return true;
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Look for obvious safe cases to perform tail call optimization that do not
|
||
|
|
// require ABI changes. This is what gcc calls sibcall.
|
||
|
|
|
||
|
|
// Can't do sibcall if stack needs to be dynamically re-aligned. PEI needs to
|
||
|
|
// emit a special epilogue.
|
||
|
|
const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
|
||
|
|
if (RegInfo->hasStackRealignment(MF))
|
||
|
|
return false;
|
||
|
|
|
||
|
|
// Also avoid sibcall optimization if we're an sret return fn and the callee
|
||
|
|
// is incompatible. See comment in LowerReturn about why hasStructRetAttr is
|
||
|
|
// insufficient.
|
||
|
|
if (MF.getInfo<X86MachineFunctionInfo>()->getSRetReturnReg()) {
|
||
|
|
// For a compatible tail call the callee must return our sret pointer. So it
|
||
|
|
// needs to be (a) an sret function itself and (b) we pass our sret as its
|
||
|
|
// sret. Condition #b is harder to determine.
|
||
|
|
return false;
|
||
|
|
} else if (IsCalleePopSRet)
|
||
|
|
// The callee pops an sret, so we cannot tail-call, as our caller doesn't
|
||
|
|
// expect that.
|
||
|
|
return false;
|
||
|
|
|
||
|
|
// Do not sibcall optimize vararg calls unless all arguments are passed via
|
||
|
|
// registers.
|
||
|
|
LLVMContext &C = *DAG.getContext();
|
||
|
|
if (isVarArg && !Outs.empty()) {
|
||
|
|
// Optimizing for varargs on Win64 is unlikely to be safe without
|
||
|
|
// additional testing.
|
||
|
|
if (IsCalleeWin64 || IsCallerWin64)
|
||
|
|
return false;
|
||
|
|
|
||
|
|
SmallVector<CCValAssign, 16> ArgLocs;
|
||
|
|
CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C);
|
||
|
|
CCInfo.AnalyzeCallOperands(Outs, CC_X86);
|
||
|
|
for (const auto &VA : ArgLocs)
|
||
|
|
if (!VA.isRegLoc())
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
|
||
|
|
// If the call result is in ST0 / ST1, it needs to be popped off the x87
|
||
|
|
// stack. Therefore, if it's not used by the call it is not safe to optimize
|
||
|
|
// this into a sibcall.
|
||
|
|
bool Unused = false;
|
||
|
|
for (const auto &In : Ins) {
|
||
|
|
if (!In.Used) {
|
||
|
|
Unused = true;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (Unused) {
|
||
|
|
SmallVector<CCValAssign, 16> RVLocs;
|
||
|
|
CCState CCInfo(CalleeCC, false, MF, RVLocs, C);
|
||
|
|
CCInfo.AnalyzeCallResult(Ins, RetCC_X86);
|
||
|
|
for (const auto &VA : RVLocs) {
|
||
|
|
if (VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1)
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Check that the call results are passed in the same way.
|
||
|
|
if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, C, Ins,
|
||
|
|
RetCC_X86, RetCC_X86))
|
||
|
|
return false;
|
||
|
|
// The callee has to preserve all registers the caller needs to preserve.
|
||
|
|
const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
|
||
|
|
const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
|
||
|
|
if (!CCMatch) {
|
||
|
|
const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
|
||
|
|
if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
|
||
|
|
unsigned StackArgsSize = 0;
|
||
|
|
|
||
|
|
// If the callee takes no arguments then go on to check the results of the
|
||
|
|
// call.
|
||
|
|
if (!Outs.empty()) {
|
||
|
|
// Check if stack adjustment is needed. For now, do not do this if any
|
||
|
|
// argument is passed on the stack.
|
||
|
|
SmallVector<CCValAssign, 16> ArgLocs;
|
||
|
|
CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C);
|
||
|
|
|
||
|
|
// Allocate shadow area for Win64
|
||
|
|
if (IsCalleeWin64)
|
||
|
|
CCInfo.AllocateStack(32, Align(8));
|
||
|
|
|
||
|
|
CCInfo.AnalyzeCallOperands(Outs, CC_X86);
|
||
|
|
StackArgsSize = CCInfo.getStackSize();
|
||
|
|
|
||
|
|
if (CCInfo.getStackSize()) {
|
||
|
|
// Check if the arguments are already laid out in the right way as
|
||
|
|
// the caller's fixed stack objects.
|
||
|
|
MachineFrameInfo &MFI = MF.getFrameInfo();
|
||
|
|
const MachineRegisterInfo *MRI = &MF.getRegInfo();
|
||
|
|
const X86InstrInfo *TII = Subtarget.getInstrInfo();
|
||
|
|
for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
|
||
|
|
const CCValAssign &VA = ArgLocs[I];
|
||
|
|
SDValue Arg = OutVals[I];
|
||
|
|
ISD::ArgFlagsTy Flags = Outs[I].Flags;
|
||
|
|
if (VA.getLocInfo() == CCValAssign::Indirect)
|
||
|
|
return false;
|
||
|
|
if (!VA.isRegLoc()) {
|
||
|
|
if (!MatchingStackOffset(Arg, VA.getLocMemOffset(), Flags, MFI, MRI,
|
||
|
|
TII, VA))
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
bool PositionIndependent = isPositionIndependent();
|
||
|
|
// If the tailcall address may be in a register, then make sure it's
|
||
|
|
// possible to register allocate for it. In 32-bit, the call address can
|
||
|
|
// only target EAX, EDX, or ECX since the tail call must be scheduled after
|
||
|
|
// callee-saved registers are restored. These happen to be the same
|
||
|
|
// registers used to pass 'inreg' arguments so watch out for those.
|
||
|
|
if (!Subtarget.is64Bit() && ((!isa<GlobalAddressSDNode>(Callee) &&
|
||
|
|
!isa<ExternalSymbolSDNode>(Callee)) ||
|
||
|
|
PositionIndependent)) {
|
||
|
|
unsigned NumInRegs = 0;
|
||
|
|
// In PIC we need an extra register to formulate the address computation
|
||
|
|
// for the callee.
|
||
|
|
unsigned MaxInRegs = PositionIndependent ? 2 : 3;
|
||
|
|
|
||
|
|
for (const auto &VA : ArgLocs) {
|
||
|
|
if (!VA.isRegLoc())
|
||
|
|
continue;
|
||
|
|
Register Reg = VA.getLocReg();
|
||
|
|
switch (Reg) {
|
||
|
|
default: break;
|
||
|
|
case X86::EAX: case X86::EDX: case X86::ECX:
|
||
|
|
if (++NumInRegs == MaxInRegs)
|
||
|
|
return false;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
const MachineRegisterInfo &MRI = MF.getRegInfo();
|
||
|
|
if (!parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals))
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
|
||
|
|
bool CalleeWillPop =
|
||
|
|
X86::isCalleePop(CalleeCC, Subtarget.is64Bit(), isVarArg,
|
||
|
|
MF.getTarget().Options.GuaranteedTailCallOpt);
|
||
|
|
|
||
|
|
if (unsigned BytesToPop =
|
||
|
|
MF.getInfo<X86MachineFunctionInfo>()->getBytesToPopOnReturn()) {
|
||
|
|
// If we have bytes to pop, the callee must pop them.
|
||
|
|
bool CalleePopMatches = CalleeWillPop && BytesToPop == StackArgsSize;
|
||
|
|
if (!CalleePopMatches)
|
||
|
|
return false;
|
||
|
|
} else if (CalleeWillPop && StackArgsSize > 0) {
|
||
|
|
// If we don't have bytes to pop, make sure the callee doesn't pop any.
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Determines whether the callee is required to pop its own arguments.
|
||
|
|
/// Callee pop is necessary to support tail calls.
|
||
|
|
bool X86::isCalleePop(CallingConv::ID CallingConv,
|
||
|
|
bool is64Bit, bool IsVarArg, bool GuaranteeTCO) {
|
||
|
|
// If GuaranteeTCO is true, we force some calls to be callee pop so that we
|
||
|
|
// can guarantee TCO.
|
||
|
|
if (!IsVarArg && shouldGuaranteeTCO(CallingConv, GuaranteeTCO))
|
||
|
|
return true;
|
||
|
|
|
||
|
|
switch (CallingConv) {
|
||
|
|
default:
|
||
|
|
return false;
|
||
|
|
case CallingConv::X86_StdCall:
|
||
|
|
case CallingConv::X86_FastCall:
|
||
|
|
case CallingConv::X86_ThisCall:
|
||
|
|
case CallingConv::X86_VectorCall:
|
||
|
|
return !is64Bit;
|
||
|
|
}
|
||
|
|
}
|