samvv/bolt
1
0
Fork 0
Code Issues 24 Pull requests Projects Releases Packages Wiki Activity Actions
bolt/deps/llvm-18.1.8/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
Sam Vervaeck 174b6f4d4e
Embed LLVM 18.1.8
2025-02-14 19:21:04 +01:00

1332 lines
48 KiB
C++
Raw Blame History

//===-- RISCVInstructionSelector.cpp -----------------------------*- C++ -*-==//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
/// \file
/// This file implements the targeting of the InstructionSelector class for
/// RISC-V.
/// \todo This should be generated by TableGen.
//===----------------------------------------------------------------------===//
#include "MCTargetDesc/RISCVMatInt.h"
#include "RISCVRegisterBankInfo.h"
#include "RISCVSubtarget.h"
#include "RISCVTargetMachine.h"
#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/IR/IntrinsicsRISCV.h"
#include "llvm/Support/Debug.h"
#define DEBUG_TYPE "riscv-isel"
using namespace llvm;
using namespace MIPatternMatch;
#define GET_GLOBALISEL_PREDICATE_BITSET
#include "RISCVGenGlobalISel.inc"
#undef GET_GLOBALISEL_PREDICATE_BITSET
namespace {
class RISCVInstructionSelector : public InstructionSelector {
public:
RISCVInstructionSelector(const RISCVTargetMachine &TM,
const RISCVSubtarget &STI,
const RISCVRegisterBankInfo &RBI);
bool select(MachineInstr &MI) override;
static const char *getName() { return DEBUG_TYPE; }
private:
const TargetRegisterClass *
getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB) const;
bool isRegInGprb(Register Reg, MachineRegisterInfo &MRI) const;
bool isRegInFprb(Register Reg, MachineRegisterInfo &MRI) const;
// tblgen-erated 'select' implementation, used as the initial selector for
// the patterns that don't require complex C++.
bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
// A lowering phase that runs before any selection attempts.
// Returns true if the instruction was modified.
void preISelLower(MachineInstr &MI, MachineIRBuilder &MIB,
MachineRegisterInfo &MRI);
bool replacePtrWithInt(MachineOperand &Op, MachineIRBuilder &MIB,
MachineRegisterInfo &MRI);
// Custom selection methods
bool selectCopy(MachineInstr &MI, MachineRegisterInfo &MRI) const;
bool selectImplicitDef(MachineInstr &MI, MachineIRBuilder &MIB,
MachineRegisterInfo &MRI) const;
bool materializeImm(Register Reg, int64_t Imm, MachineIRBuilder &MIB) const;
bool selectAddr(MachineInstr &MI, MachineIRBuilder &MIB,
MachineRegisterInfo &MRI, bool IsLocal = true,
bool IsExternWeak = false) const;
bool selectSExtInreg(MachineInstr &MI, MachineIRBuilder &MIB) const;
bool selectSelect(MachineInstr &MI, MachineIRBuilder &MIB,
MachineRegisterInfo &MRI) const;
bool selectFPCompare(MachineInstr &MI, MachineIRBuilder &MIB,
MachineRegisterInfo &MRI) const;
bool selectIntrinsicWithSideEffects(MachineInstr &MI, MachineIRBuilder &MIB,
MachineRegisterInfo &MRI) const;
void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
MachineIRBuilder &MIB) const;
bool selectMergeValues(MachineInstr &MI, MachineIRBuilder &MIB,
MachineRegisterInfo &MRI) const;
bool selectUnmergeValues(MachineInstr &MI, MachineIRBuilder &MIB,
MachineRegisterInfo &MRI) const;
ComplexRendererFns selectShiftMask(MachineOperand &Root) const;
ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const;
template <unsigned ShAmt>
ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const {
return selectSHXADDOp(Root, ShAmt);
}
ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root,
unsigned ShAmt) const;
template <unsigned ShAmt>
ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root) const {
return selectSHXADD_UWOp(Root, ShAmt);
}
// Custom renderers for tablegen
void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
int OpIdx) const;
void renderImmSubFromXLen(MachineInstrBuilder &MIB, const MachineInstr &MI,
int OpIdx) const;
void renderImmSubFrom32(MachineInstrBuilder &MIB, const MachineInstr &MI,
int OpIdx) const;
void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,
int OpIdx) const;
void renderImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
int OpIdx) const;
void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI,
int OpIdx) const;
const RISCVSubtarget &STI;
const RISCVInstrInfo &TII;
const RISCVRegisterInfo &TRI;
const RISCVRegisterBankInfo &RBI;
const RISCVTargetMachine &TM;
// FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel
// uses "STI." in the code generated by TableGen. We need to unify the name of
// Subtarget variable.
const RISCVSubtarget *Subtarget = &STI;
#define GET_GLOBALISEL_PREDICATES_DECL
#include "RISCVGenGlobalISel.inc"
#undef GET_GLOBALISEL_PREDICATES_DECL
#define GET_GLOBALISEL_TEMPORARIES_DECL
#include "RISCVGenGlobalISel.inc"
#undef GET_GLOBALISEL_TEMPORARIES_DECL
};
} // end anonymous namespace
#define GET_GLOBALISEL_IMPL
#include "RISCVGenGlobalISel.inc"
#undef GET_GLOBALISEL_IMPL
RISCVInstructionSelector::RISCVInstructionSelector(
const RISCVTargetMachine &TM, const RISCVSubtarget &STI,
const RISCVRegisterBankInfo &RBI)
: STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI),
TM(TM),
#define GET_GLOBALISEL_PREDICATES_INIT
#include "RISCVGenGlobalISel.inc"
#undef GET_GLOBALISEL_PREDICATES_INIT
#define GET_GLOBALISEL_TEMPORARIES_INIT
#include "RISCVGenGlobalISel.inc"
#undef GET_GLOBALISEL_TEMPORARIES_INIT
{
}
InstructionSelector::ComplexRendererFns
RISCVInstructionSelector::selectShiftMask(MachineOperand &Root) const {
if (!Root.isReg())
return std::nullopt;
using namespace llvm::MIPatternMatch;
MachineRegisterInfo &MRI = MF->getRegInfo();
Register RootReg = Root.getReg();
Register ShAmtReg = RootReg;
const LLT ShiftLLT = MRI.getType(RootReg);
unsigned ShiftWidth = ShiftLLT.getSizeInBits();
assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
// Peek through zext.
Register ZExtSrcReg;
if (mi_match(ShAmtReg, MRI, m_GZExt(m_Reg(ZExtSrcReg)))) {
ShAmtReg = ZExtSrcReg;
}
APInt AndMask;
Register AndSrcReg;
if (mi_match(ShAmtReg, MRI, m_GAnd(m_Reg(AndSrcReg), m_ICst(AndMask)))) {
APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
if (ShMask.isSubsetOf(AndMask)) {
ShAmtReg = AndSrcReg;
} else {
// SimplifyDemandedBits may have optimized the mask so try restoring any
// bits that are known zero.
KnownBits Known = KB->getKnownBits(ShAmtReg);
if (ShMask.isSubsetOf(AndMask | Known.Zero))
ShAmtReg = AndSrcReg;
}
}
APInt Imm;
Register Reg;
if (mi_match(ShAmtReg, MRI, m_GAdd(m_Reg(Reg), m_ICst(Imm)))) {
if (Imm != 0 && Imm.urem(ShiftWidth) == 0)
// If we are shifting by X+N where N == 0 mod Size, then just shift by X
// to avoid the ADD.
ShAmtReg = Reg;
} else if (mi_match(ShAmtReg, MRI, m_GSub(m_ICst(Imm), m_Reg(Reg)))) {
if (Imm != 0 && Imm.urem(ShiftWidth) == 0) {
// If we are shifting by N-X where N == 0 mod Size, then just shift by -X
// to generate a NEG instead of a SUB of a constant.
ShAmtReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB;
return {{[=](MachineInstrBuilder &MIB) {
MachineIRBuilder(*MIB.getInstr())
.buildInstr(NegOpc, {ShAmtReg}, {Register(RISCV::X0), Reg});
MIB.addReg(ShAmtReg);
}}};
}
if (Imm.urem(ShiftWidth) == ShiftWidth - 1) {
// If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
// to generate a NOT instead of a SUB of a constant.
ShAmtReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
return {{[=](MachineInstrBuilder &MIB) {
MachineIRBuilder(*MIB.getInstr())
.buildInstr(RISCV::XORI, {ShAmtReg}, {Reg})
.addImm(-1);
MIB.addReg(ShAmtReg);
}}};
}
}
return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(ShAmtReg); }}};
}
InstructionSelector::ComplexRendererFns
RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
unsigned ShAmt) const {
using namespace llvm::MIPatternMatch;
MachineFunction &MF = *Root.getParent()->getParent()->getParent();
MachineRegisterInfo &MRI = MF.getRegInfo();
if (!Root.isReg())
return std::nullopt;
Register RootReg = Root.getReg();
const unsigned XLen = STI.getXLen();
APInt Mask, C2;
Register RegY;
std::optional<bool> LeftShift;
// (and (shl y, c2), mask)
if (mi_match(RootReg, MRI,
m_GAnd(m_GShl(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
LeftShift = true;
// (and (lshr y, c2), mask)
else if (mi_match(RootReg, MRI,
m_GAnd(m_GLShr(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
LeftShift = false;
if (LeftShift.has_value()) {
if (*LeftShift)
Mask &= maskTrailingZeros<uint64_t>(C2.getLimitedValue());
else
Mask &= maskTrailingOnes<uint64_t>(XLen - C2.getLimitedValue());
if (Mask.isShiftedMask()) {
unsigned Leading = XLen - Mask.getActiveBits();
unsigned Trailing = Mask.countr_zero();
// Given (and (shl y, c2), mask) in which mask has no leading zeros and
// c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.
if (*LeftShift && Leading == 0 && C2.ult(Trailing) && Trailing == ShAmt) {
Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
return {{[=](MachineInstrBuilder &MIB) {
MachineIRBuilder(*MIB.getInstr())
.buildInstr(RISCV::SRLI, {DstReg}, {RegY})
.addImm(Trailing - C2.getLimitedValue());
MIB.addReg(DstReg);
}}};
}
// Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and
// c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.
if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {
Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
return {{[=](MachineInstrBuilder &MIB) {
MachineIRBuilder(*MIB.getInstr())
.buildInstr(RISCV::SRLI, {DstReg}, {RegY})
.addImm(Leading + Trailing);
MIB.addReg(DstReg);
}}};
}
}
}
LeftShift.reset();
// (shl (and y, mask), c2)
if (mi_match(RootReg, MRI,
m_GShl(m_OneNonDBGUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),
m_ICst(C2))))
LeftShift = true;
// (lshr (and y, mask), c2)
else if (mi_match(RootReg, MRI,
m_GLShr(m_OneNonDBGUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),
m_ICst(C2))))
LeftShift = false;
if (LeftShift.has_value() && Mask.isShiftedMask()) {
unsigned Leading = XLen - Mask.getActiveBits();
unsigned Trailing = Mask.countr_zero();
// Given (shl (and y, mask), c2) in which mask has 32 leading zeros and
// c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD.
bool Cond = *LeftShift && Leading == 32 && Trailing > 0 &&
(Trailing + C2.getLimitedValue()) == ShAmt;
if (!Cond)
// Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and
// c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD.
Cond = !*LeftShift && Leading == 32 && C2.ult(Trailing) &&
(Trailing - C2.getLimitedValue()) == ShAmt;
if (Cond) {
Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
return {{[=](MachineInstrBuilder &MIB) {
MachineIRBuilder(*MIB.getInstr())
.buildInstr(RISCV::SRLIW, {DstReg}, {RegY})
.addImm(Trailing);
MIB.addReg(DstReg);
}}};
}
}
return std::nullopt;
}
InstructionSelector::ComplexRendererFns
RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
unsigned ShAmt) const {
using namespace llvm::MIPatternMatch;
MachineFunction &MF = *Root.getParent()->getParent()->getParent();
MachineRegisterInfo &MRI = MF.getRegInfo();
if (!Root.isReg())
return std::nullopt;
Register RootReg = Root.getReg();
// Given (and (shl x, c2), mask) in which mask is a shifted mask with
// 32 - ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
// c2 - ShAmt followed by SHXADD_UW with ShAmt for x amount.
APInt Mask, C2;
Register RegX;
if (mi_match(
RootReg, MRI,
m_OneNonDBGUse(m_GAnd(m_OneNonDBGUse(m_GShl(m_Reg(RegX), m_ICst(C2))),
m_ICst(Mask))))) {
Mask &= maskTrailingZeros<uint64_t>(C2.getLimitedValue());
if (Mask.isShiftedMask()) {
unsigned Leading = Mask.countl_zero();
unsigned Trailing = Mask.countr_zero();
if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) {
Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
return {{[=](MachineInstrBuilder &MIB) {
MachineIRBuilder(*MIB.getInstr())
.buildInstr(RISCV::SLLI, {DstReg}, {RegX})
.addImm(C2.getLimitedValue() - ShAmt);
MIB.addReg(DstReg);
}}};
}
}
}
return std::nullopt;
}
InstructionSelector::ComplexRendererFns
RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
MachineFunction &MF = *Root.getParent()->getParent()->getParent();
MachineRegisterInfo &MRI = MF.getRegInfo();
if (!Root.isReg())
return std::nullopt;
MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
return {{
[=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
[=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
}};
}
if (isBaseWithConstantOffset(Root, MRI)) {
MachineOperand &LHS = RootDef->getOperand(1);
MachineOperand &RHS = RootDef->getOperand(2);
MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
int64_t RHSC = RHSDef->getOperand(1).getCImm()->getSExtValue();
if (isInt<12>(RHSC)) {
if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
return {{
[=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },
[=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
}};
return {{[=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
[=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); }}};
}
}
// TODO: Need to get the immediate from a G_PTR_ADD. Should this be done in
// the combiner?
return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); },
[=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};
}
/// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.
/// CC Must be an ICMP Predicate.
static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {
switch (CC) {
default:
llvm_unreachable("Expected ICMP CmpInst::Predicate.");
case CmpInst::Predicate::ICMP_EQ:
return RISCVCC::COND_EQ;
case CmpInst::Predicate::ICMP_NE:
return RISCVCC::COND_NE;
case CmpInst::Predicate::ICMP_ULT:
return RISCVCC::COND_LTU;
case CmpInst::Predicate::ICMP_SLT:
return RISCVCC::COND_LT;
case CmpInst::Predicate::ICMP_UGE:
return RISCVCC::COND_GEU;
case CmpInst::Predicate::ICMP_SGE:
return RISCVCC::COND_GE;
}
}
static void getOperandsForBranch(Register CondReg, MachineRegisterInfo &MRI,
RISCVCC::CondCode &CC, Register &LHS,
Register &RHS) {
// Try to fold an ICmp. If that fails, use a NE compare with X0.
CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
if (!mi_match(CondReg, MRI, m_GICmp(m_Pred(Pred), m_Reg(LHS), m_Reg(RHS)))) {
LHS = CondReg;
RHS = RISCV::X0;
CC = RISCVCC::COND_NE;
return;
}
// We found an ICmp, do some canonicalizations.
// Adjust comparisons to use comparison with 0 if possible.
if (auto Constant = getIConstantVRegSExtVal(RHS, MRI)) {
switch (Pred) {
case CmpInst::Predicate::ICMP_SGT:
// Convert X > -1 to X >= 0
if (*Constant == -1) {
CC = RISCVCC::COND_GE;
RHS = RISCV::X0;
return;
}
break;
case CmpInst::Predicate::ICMP_SLT:
// Convert X < 1 to 0 >= X
if (*Constant == 1) {
CC = RISCVCC::COND_GE;
RHS = LHS;
LHS = RISCV::X0;
return;
}
break;
default:
break;
}
}
switch (Pred) {
default:
llvm_unreachable("Expected ICMP CmpInst::Predicate.");
case CmpInst::Predicate::ICMP_EQ:
case CmpInst::Predicate::ICMP_NE:
case CmpInst::Predicate::ICMP_ULT:
case CmpInst::Predicate::ICMP_SLT:
case CmpInst::Predicate::ICMP_UGE:
case CmpInst::Predicate::ICMP_SGE:
// These CCs are supported directly by RISC-V branches.
break;
case CmpInst::Predicate::ICMP_SGT:
case CmpInst::Predicate::ICMP_SLE:
case CmpInst::Predicate::ICMP_UGT:
case CmpInst::Predicate::ICMP_ULE:
// These CCs are not supported directly by RISC-V branches, but changing the
// direction of the CC and swapping LHS and RHS are.
Pred = CmpInst::getSwappedPredicate(Pred);
std::swap(LHS, RHS);
break;
}
CC = getRISCVCCFromICmp(Pred);
return;
}
bool RISCVInstructionSelector::select(MachineInstr &MI) {
MachineBasicBlock &MBB = *MI.getParent();
MachineFunction &MF = *MBB.getParent();
MachineRegisterInfo &MRI = MF.getRegInfo();
MachineIRBuilder MIB(MI);
preISelLower(MI, MIB, MRI);
const unsigned Opc = MI.getOpcode();
if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) {
if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) {
const Register DefReg = MI.getOperand(0).getReg();
const LLT DefTy = MRI.getType(DefReg);
const RegClassOrRegBank &RegClassOrBank =
MRI.getRegClassOrRegBank(DefReg);
const TargetRegisterClass *DefRC =
RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
if (!DefRC) {
if (!DefTy.isValid()) {
LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
return false;
}
const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>();
DefRC = getRegClassForTypeOnBank(DefTy, RB);
if (!DefRC) {
LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
return false;
}
}
MI.setDesc(TII.get(TargetOpcode::PHI));
return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
}
// Certain non-generic instructions also need some special handling.
if (MI.isCopy())
return selectCopy(MI, MRI);
return true;
}
if (selectImpl(MI, *CoverageInfo))
return true;
switch (Opc) {
case TargetOpcode::G_ANYEXT:
case TargetOpcode::G_PTRTOINT:
case TargetOpcode::G_INTTOPTR:
case TargetOpcode::G_TRUNC:
return selectCopy(MI, MRI);
case TargetOpcode::G_CONSTANT: {
Register DstReg = MI.getOperand(0).getReg();
int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue();
if (!materializeImm(DstReg, Imm, MIB))
return false;
MI.eraseFromParent();
return true;
}
case TargetOpcode::G_FCONSTANT: {
// TODO: Use constant pool for complext constants.
// TODO: Optimize +0.0 to use fcvt.d.w for s64 on rv32.
Register DstReg = MI.getOperand(0).getReg();
const APFloat &FPimm = MI.getOperand(1).getFPImm()->getValueAPF();
APInt Imm = FPimm.bitcastToAPInt();
unsigned Size = MRI.getType(DstReg).getSizeInBits();
if (Size == 32 || (Size == 64 && Subtarget->is64Bit())) {
Register GPRReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
if (!materializeImm(GPRReg, Imm.getSExtValue(), MIB))
return false;
unsigned Opcode = Size == 64 ? RISCV::FMV_D_X : RISCV::FMV_W_X;
auto FMV = MIB.buildInstr(Opcode, {DstReg}, {GPRReg});
if (!FMV.constrainAllUses(TII, TRI, RBI))
return false;
} else {
assert(Size == 64 && !Subtarget->is64Bit() &&
"Unexpected size or subtarget");
// Split into two pieces and build through the stack.
Register GPRRegHigh = MRI.createVirtualRegister(&RISCV::GPRRegClass);
Register GPRRegLow = MRI.createVirtualRegister(&RISCV::GPRRegClass);
if (!materializeImm(GPRRegHigh, Imm.extractBits(32, 32).getSExtValue(),
MIB))
return false;
if (!materializeImm(GPRRegLow, Imm.trunc(32).getSExtValue(), MIB))
return false;
MachineInstrBuilder PairF64 = MIB.buildInstr(
RISCV::BuildPairF64Pseudo, {DstReg}, {GPRRegLow, GPRRegHigh});
if (!PairF64.constrainAllUses(TII, TRI, RBI))
return false;
}
MI.eraseFromParent();
return true;
}
case TargetOpcode::G_GLOBAL_VALUE: {
auto *GV = MI.getOperand(1).getGlobal();
if (GV->isThreadLocal()) {
// TODO: implement this case.
return false;
}
return selectAddr(MI, MIB, MRI, GV->isDSOLocal(),
GV->hasExternalWeakLinkage());
}
case TargetOpcode::G_JUMP_TABLE:
case TargetOpcode::G_CONSTANT_POOL:
return selectAddr(MI, MIB, MRI);
case TargetOpcode::G_BRCOND: {
Register LHS, RHS;
RISCVCC::CondCode CC;
getOperandsForBranch(MI.getOperand(0).getReg(), MRI, CC, LHS, RHS);
auto Bcc = MIB.buildInstr(RISCVCC::getBrCond(CC), {}, {LHS, RHS})
.addMBB(MI.getOperand(1).getMBB());
MI.eraseFromParent();
return constrainSelectedInstRegOperands(*Bcc, TII, TRI, RBI);
}
case TargetOpcode::G_BRJT: {
// FIXME: Move to legalization?
const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();
unsigned EntrySize = MJTI->getEntrySize(MF.getDataLayout());
assert((EntrySize == 4 || (Subtarget->is64Bit() && EntrySize == 8)) &&
"Unsupported jump-table entry size");
assert(
(MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32 ||
MJTI->getEntryKind() == MachineJumpTableInfo::EK_Custom32 ||
MJTI->getEntryKind() == MachineJumpTableInfo::EK_BlockAddress) &&
"Unexpected jump-table entry kind");
auto SLL =
MIB.buildInstr(RISCV::SLLI, {&RISCV::GPRRegClass}, {MI.getOperand(2)})
.addImm(Log2_32(EntrySize));
if (!SLL.constrainAllUses(TII, TRI, RBI))
return false;
// TODO: Use SHXADD. Moving to legalization would fix this automatically.
auto ADD = MIB.buildInstr(RISCV::ADD, {&RISCV::GPRRegClass},
{MI.getOperand(0), SLL.getReg(0)});
if (!ADD.constrainAllUses(TII, TRI, RBI))
return false;
unsigned LdOpc = EntrySize == 8 ? RISCV::LD : RISCV::LW;
auto Dest =
MIB.buildInstr(LdOpc, {&RISCV::GPRRegClass}, {ADD.getReg(0)})
.addImm(0)
.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getJumpTable(MF), MachineMemOperand::MOLoad,
EntrySize, Align(MJTI->getEntryAlignment(MF.getDataLayout()))));
if (!Dest.constrainAllUses(TII, TRI, RBI))
return false;
// If the Kind is EK_LabelDifference32, the table stores an offset from
// the location of the table. Add the table address to get an absolute
// address.
if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32) {
Dest = MIB.buildInstr(RISCV::ADD, {&RISCV::GPRRegClass},
{Dest.getReg(0), MI.getOperand(0)});
if (!Dest.constrainAllUses(TII, TRI, RBI))
return false;
}
auto Branch =
MIB.buildInstr(RISCV::PseudoBRIND, {}, {Dest.getReg(0)}).addImm(0);
if (!Branch.constrainAllUses(TII, TRI, RBI))
return false;
MI.eraseFromParent();
return true;
}
case TargetOpcode::G_BRINDIRECT:
MI.setDesc(TII.get(RISCV::PseudoBRIND));
MI.addOperand(MachineOperand::CreateImm(0));
return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
case TargetOpcode::G_SEXT_INREG:
return selectSExtInreg(MI, MIB);
case TargetOpcode::G_FRAME_INDEX: {
// TODO: We may want to replace this code with the SelectionDAG patterns,
// which fail to get imported because it uses FrameAddrRegImm, which is a
// ComplexPattern
MI.setDesc(TII.get(RISCV::ADDI));
MI.addOperand(MachineOperand::CreateImm(0));
return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
}
case TargetOpcode::G_SELECT:
return selectSelect(MI, MIB, MRI);
case TargetOpcode::G_FCMP:
return selectFPCompare(MI, MIB, MRI);
case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
return selectIntrinsicWithSideEffects(MI, MIB, MRI);
case TargetOpcode::G_FENCE: {
AtomicOrdering FenceOrdering =
static_cast<AtomicOrdering>(MI.getOperand(0).getImm());
SyncScope::ID FenceSSID =
static_cast<SyncScope::ID>(MI.getOperand(1).getImm());
emitFence(FenceOrdering, FenceSSID, MIB);
MI.eraseFromParent();
return true;
}
case TargetOpcode::G_IMPLICIT_DEF:
return selectImplicitDef(MI, MIB, MRI);
case TargetOpcode::G_MERGE_VALUES:
return selectMergeValues(MI, MIB, MRI);
case TargetOpcode::G_UNMERGE_VALUES:
return selectUnmergeValues(MI, MIB, MRI);
default:
return false;
}
}
bool RISCVInstructionSelector::selectMergeValues(
MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) const {
assert(MI.getOpcode() == TargetOpcode::G_MERGE_VALUES);
// Build a F64 Pair from operands
if (MI.getNumOperands() != 3)
return false;
Register Dst = MI.getOperand(0).getReg();
Register Lo = MI.getOperand(1).getReg();
Register Hi = MI.getOperand(2).getReg();
if (!isRegInFprb(Dst, MRI) || !isRegInGprb(Lo, MRI) || !isRegInGprb(Hi, MRI))
return false;
MI.setDesc(TII.get(RISCV::BuildPairF64Pseudo));
return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
}
bool RISCVInstructionSelector::selectUnmergeValues(
MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) const {
assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
// Split F64 Src into two s32 parts
if (MI.getNumOperands() != 3)
return false;
Register Src = MI.getOperand(2).getReg();
Register Lo = MI.getOperand(0).getReg();
Register Hi = MI.getOperand(1).getReg();
if (!isRegInFprb(Src, MRI) || !isRegInGprb(Lo, MRI) || !isRegInGprb(Hi, MRI))
return false;
MI.setDesc(TII.get(RISCV::SplitF64Pseudo));
return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
}
bool RISCVInstructionSelector::replacePtrWithInt(MachineOperand &Op,
MachineIRBuilder &MIB,
MachineRegisterInfo &MRI) {
Register PtrReg = Op.getReg();
assert(MRI.getType(PtrReg).isPointer() && "Operand is not a pointer!");
const LLT sXLen = LLT::scalar(STI.getXLen());
auto PtrToInt = MIB.buildPtrToInt(sXLen, PtrReg);
MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(RISCV::GPRBRegBankID));
Op.setReg(PtrToInt.getReg(0));
return select(*PtrToInt);
}
void RISCVInstructionSelector::preISelLower(MachineInstr &MI,
MachineIRBuilder &MIB,
MachineRegisterInfo &MRI) {
switch (MI.getOpcode()) {
case TargetOpcode::G_PTR_ADD: {
Register DstReg = MI.getOperand(0).getReg();
const LLT sXLen = LLT::scalar(STI.getXLen());
replacePtrWithInt(MI.getOperand(1), MIB, MRI);
MI.setDesc(TII.get(TargetOpcode::G_ADD));
MRI.setType(DstReg, sXLen);
break;
}
case TargetOpcode::G_PTRMASK: {
Register DstReg = MI.getOperand(0).getReg();
const LLT sXLen = LLT::scalar(STI.getXLen());
replacePtrWithInt(MI.getOperand(1), MIB, MRI);
MI.setDesc(TII.get(TargetOpcode::G_AND));
MRI.setType(DstReg, sXLen);
}
}
}
void RISCVInstructionSelector::renderNegImm(MachineInstrBuilder &MIB,
const MachineInstr &MI,
int OpIdx) const {
assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
"Expected G_CONSTANT");
int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
MIB.addImm(-CstVal);
}
void RISCVInstructionSelector::renderImmSubFromXLen(MachineInstrBuilder &MIB,
const MachineInstr &MI,
int OpIdx) const {
assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
"Expected G_CONSTANT");
uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();
MIB.addImm(STI.getXLen() - CstVal);
}
void RISCVInstructionSelector::renderImmSubFrom32(MachineInstrBuilder &MIB,
const MachineInstr &MI,
int OpIdx) const {
assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
"Expected G_CONSTANT");
uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();
MIB.addImm(32 - CstVal);
}
void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,
const MachineInstr &MI,
int OpIdx) const {
assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
"Expected G_CONSTANT");
int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
MIB.addImm(CstVal + 1);
}
void RISCVInstructionSelector::renderImm(MachineInstrBuilder &MIB,
const MachineInstr &MI,
int OpIdx) const {
assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
"Expected G_CONSTANT");
int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
MIB.addImm(CstVal);
}
void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB,
const MachineInstr &MI,
int OpIdx) const {
assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
"Expected G_CONSTANT");
uint64_t C = MI.getOperand(1).getCImm()->getZExtValue();
MIB.addImm(llvm::countr_zero(C));
}
const TargetRegisterClass *RISCVInstructionSelector::getRegClassForTypeOnBank(
LLT Ty, const RegisterBank &RB) const {
if (RB.getID() == RISCV::GPRBRegBankID) {
if (Ty.getSizeInBits() <= 32 || (STI.is64Bit() && Ty.getSizeInBits() == 64))
return &RISCV::GPRRegClass;
}
if (RB.getID() == RISCV::FPRBRegBankID) {
if (Ty.getSizeInBits() == 32)
return &RISCV::FPR32RegClass;
if (Ty.getSizeInBits() == 64)
return &RISCV::FPR64RegClass;
}
// TODO: Non-GPR register classes.
return nullptr;
}
bool RISCVInstructionSelector::isRegInGprb(Register Reg,
MachineRegisterInfo &MRI) const {
return RBI.getRegBank(Reg, MRI, TRI)->getID() == RISCV::GPRBRegBankID;
}
bool RISCVInstructionSelector::isRegInFprb(Register Reg,
MachineRegisterInfo &MRI) const {
return RBI.getRegBank(Reg, MRI, TRI)->getID() == RISCV::FPRBRegBankID;
}
bool RISCVInstructionSelector::selectCopy(MachineInstr &MI,
MachineRegisterInfo &MRI) const {
Register DstReg = MI.getOperand(0).getReg();
if (DstReg.isPhysical())
return true;
const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
MRI.getType(DstReg), *RBI.getRegBank(DstReg, MRI, TRI));
assert(DstRC &&
"Register class not available for LLT, register bank combination");
// No need to constrain SrcReg. It will get constrained when
// we hit another of its uses or its defs.
// Copies do not have constraints.
if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
<< " operand\n");
return false;
}
MI.setDesc(TII.get(RISCV::COPY));
return true;
}
bool RISCVInstructionSelector::selectImplicitDef(
MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) const {
assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
const Register DstReg = MI.getOperand(0).getReg();
const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
MRI.getType(DstReg), *RBI.getRegBank(DstReg, MRI, TRI));
assert(DstRC &&
"Register class not available for LLT, register bank combination");
if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
<< " operand\n");
}
MI.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
return true;
}
bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
MachineIRBuilder &MIB) const {
MachineRegisterInfo &MRI = *MIB.getMRI();
if (Imm == 0) {
MIB.buildCopy(DstReg, Register(RISCV::X0));
RBI.constrainGenericRegister(DstReg, RISCV::GPRRegClass, MRI);
return true;
}
RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Imm, *Subtarget);
unsigned NumInsts = Seq.size();
Register SrcReg = RISCV::X0;
for (unsigned i = 0; i < NumInsts; i++) {
Register TmpReg = i < NumInsts - 1
? MRI.createVirtualRegister(&RISCV::GPRRegClass)
: DstReg;
const RISCVMatInt::Inst &I = Seq[i];
MachineInstr *Result;
switch (I.getOpndKind()) {
case RISCVMatInt::Imm:
// clang-format off
Result = MIB.buildInstr(I.getOpcode(), {TmpReg}, {})
.addImm(I.getImm());
// clang-format on
break;
case RISCVMatInt::RegX0:
Result = MIB.buildInstr(I.getOpcode(), {TmpReg},
{SrcReg, Register(RISCV::X0)});
break;
case RISCVMatInt::RegReg:
Result = MIB.buildInstr(I.getOpcode(), {TmpReg}, {SrcReg, SrcReg});
break;
case RISCVMatInt::RegImm:
Result =
MIB.buildInstr(I.getOpcode(), {TmpReg}, {SrcReg}).addImm(I.getImm());
break;
}
if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
return false;
SrcReg = TmpReg;
}
return true;
}
bool RISCVInstructionSelector::selectAddr(MachineInstr &MI,
MachineIRBuilder &MIB,
MachineRegisterInfo &MRI,
bool IsLocal,
bool IsExternWeak) const {
assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
MI.getOpcode() == TargetOpcode::G_JUMP_TABLE ||
MI.getOpcode() == TargetOpcode::G_CONSTANT_POOL) &&
"Unexpected opcode");
const MachineOperand &DispMO = MI.getOperand(1);
Register DefReg = MI.getOperand(0).getReg();
const LLT DefTy = MRI.getType(DefReg);
// When HWASAN is used and tagging of global variables is enabled
// they should be accessed via the GOT, since the tagged address of a global
// is incompatible with existing code models. This also applies to non-pic
// mode.
if (TM.isPositionIndependent() || Subtarget->allowTaggedGlobals()) {
if (IsLocal && !Subtarget->allowTaggedGlobals()) {
// Use PC-relative addressing to access the symbol. This generates the
// pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
// %pcrel_lo(auipc)).
MI.setDesc(TII.get(RISCV::PseudoLLA));
return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
}
// Use PC-relative addressing to access the GOT for this symbol, then
// load the address from the GOT. This generates the pattern (PseudoLGA
// sym), which expands to (ld (addi (auipc %got_pcrel_hi(sym))
// %pcrel_lo(auipc))).
MachineFunction &MF = *MI.getParent()->getParent();
MachineMemOperand *MemOp = MF.getMachineMemOperand(
MachinePointerInfo::getGOT(MF),
MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
MachineMemOperand::MOInvariant,
DefTy, Align(DefTy.getSizeInBits() / 8));
auto Result = MIB.buildInstr(RISCV::PseudoLGA, {DefReg}, {})
.addDisp(DispMO, 0)
.addMemOperand(MemOp);
if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
return false;
MI.eraseFromParent();
return true;
}
switch (TM.getCodeModel()) {
default: {
reportGISelFailure(const_cast<MachineFunction &>(*MF), *TPC, *MORE,
getName(), "Unsupported code model for lowering", MI);
return false;
}
case CodeModel::Small: {
// Must lie within a single 2 GiB address range and must lie between
// absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi
// (lui %hi(sym)) %lo(sym)).
Register AddrHiDest = MRI.createVirtualRegister(&RISCV::GPRRegClass);
MachineInstr *AddrHi = MIB.buildInstr(RISCV::LUI, {AddrHiDest}, {})
.addDisp(DispMO, 0, RISCVII::MO_HI);
if (!constrainSelectedInstRegOperands(*AddrHi, TII, TRI, RBI))
return false;
auto Result = MIB.buildInstr(RISCV::ADDI, {DefReg}, {AddrHiDest})
.addDisp(DispMO, 0, RISCVII::MO_LO);
if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
return false;
MI.eraseFromParent();
return true;
}
case CodeModel::Medium:
// Emit LGA/LLA instead of the sequence it expands to because the pcrel_lo
// relocation needs to reference a label that points to the auipc
// instruction itself, not the global. This cannot be done inside the
// instruction selector.
if (IsExternWeak) {
// An extern weak symbol may be undefined, i.e. have value 0, which may
// not be within 2GiB of PC, so use GOT-indirect addressing to access the
// symbol. This generates the pattern (PseudoLGA sym), which expands to
// (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
MachineFunction &MF = *MI.getParent()->getParent();
MachineMemOperand *MemOp = MF.getMachineMemOperand(
MachinePointerInfo::getGOT(MF),
MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
MachineMemOperand::MOInvariant,
DefTy, Align(DefTy.getSizeInBits() / 8));
auto Result = MIB.buildInstr(RISCV::PseudoLGA, {DefReg}, {})
.addDisp(DispMO, 0)
.addMemOperand(MemOp);
if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
return false;
MI.eraseFromParent();
return true;
}
// Generate a sequence for accessing addresses within any 2GiB range
// within the address space. This generates the pattern (PseudoLLA sym),
// which expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
MI.setDesc(TII.get(RISCV::PseudoLLA));
return constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
}
return false;
}
bool RISCVInstructionSelector::selectSExtInreg(MachineInstr &MI,
MachineIRBuilder &MIB) const {
if (!STI.isRV64())
return false;
const MachineOperand &Size = MI.getOperand(2);
// Only Size == 32 (i.e. shift by 32 bits) is acceptable at this point.
if (!Size.isImm() || Size.getImm() != 32)
return false;
const MachineOperand &Src = MI.getOperand(1);
const MachineOperand &Dst = MI.getOperand(0);
// addiw rd, rs, 0 (i.e. sext.w rd, rs)
MachineInstr *NewMI =
MIB.buildInstr(RISCV::ADDIW, {Dst.getReg()}, {Src.getReg()}).addImm(0U);
if (!constrainSelectedInstRegOperands(*NewMI, TII, TRI, RBI))
return false;
MI.eraseFromParent();
return true;
}
bool RISCVInstructionSelector::selectSelect(MachineInstr &MI,
MachineIRBuilder &MIB,
MachineRegisterInfo &MRI) const {
auto &SelectMI = cast<GSelect>(MI);
Register LHS, RHS;
RISCVCC::CondCode CC;
getOperandsForBranch(SelectMI.getCondReg(), MRI, CC, LHS, RHS);
Register DstReg = SelectMI.getReg(0);
unsigned Opc = RISCV::Select_GPR_Using_CC_GPR;
if (RBI.getRegBank(DstReg, MRI, TRI)->getID() == RISCV::FPRBRegBankID) {
unsigned Size = MRI.getType(DstReg).getSizeInBits();
Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR
: RISCV::Select_FPR64_Using_CC_GPR;
}
MachineInstr *Result = MIB.buildInstr(Opc)
.addDef(DstReg)
.addReg(LHS)
.addReg(RHS)
.addImm(CC)
.addReg(SelectMI.getTrueReg())
.addReg(SelectMI.getFalseReg());
MI.eraseFromParent();
return constrainSelectedInstRegOperands(*Result, TII, TRI, RBI);
}
// Convert an FCMP predicate to one of the supported F or D instructions.
static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size) {
assert((Size == 32 || Size == 64) && "Unsupported size");
switch (Pred) {
default:
llvm_unreachable("Unsupported predicate");
case CmpInst::FCMP_OLT:
return Size == 32 ? RISCV::FLT_S : RISCV::FLT_D;
case CmpInst::FCMP_OLE:
return Size == 32 ? RISCV::FLE_S : RISCV::FLE_D;
case CmpInst::FCMP_OEQ:
return Size == 32 ? RISCV::FEQ_S : RISCV::FEQ_D;
}
}
// Try legalizing an FCMP by swapping or inverting the predicate to one that
// is supported.
static bool legalizeFCmpPredicate(Register &LHS, Register &RHS,
CmpInst::Predicate &Pred, bool &NeedInvert) {
auto isLegalFCmpPredicate = [](CmpInst::Predicate Pred) {
return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE ||
Pred == CmpInst::FCMP_OEQ;
};
assert(!isLegalFCmpPredicate(Pred) && "Predicate already legal?");
CmpInst::Predicate InvPred = CmpInst::getSwappedPredicate(Pred);
if (isLegalFCmpPredicate(InvPred)) {
Pred = InvPred;
std::swap(LHS, RHS);
return true;
}
InvPred = CmpInst::getInversePredicate(Pred);
NeedInvert = true;
if (isLegalFCmpPredicate(InvPred)) {
Pred = InvPred;
return true;
}
InvPred = CmpInst::getSwappedPredicate(InvPred);
if (isLegalFCmpPredicate(InvPred)) {
Pred = InvPred;
std::swap(LHS, RHS);
return true;
}
return false;
}
// Emit a sequence of instructions to compare LHS and RHS using Pred. Return
// the result in DstReg.
// FIXME: Maybe we should expand this earlier.
bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI,
MachineIRBuilder &MIB,
MachineRegisterInfo &MRI) const {
auto &CmpMI = cast<GFCmp>(MI);
CmpInst::Predicate Pred = CmpMI.getCond();
Register DstReg = CmpMI.getReg(0);
Register LHS = CmpMI.getLHSReg();
Register RHS = CmpMI.getRHSReg();
unsigned Size = MRI.getType(LHS).getSizeInBits();
assert((Size == 32 || Size == 64) && "Unexpected size");
Register TmpReg = DstReg;
bool NeedInvert = false;
// First try swapping operands or inverting.
if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) {
if (NeedInvert)
TmpReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
auto Cmp = MIB.buildInstr(getFCmpOpcode(Pred, Size), {TmpReg}, {LHS, RHS});
if (!Cmp.constrainAllUses(TII, TRI, RBI))
return false;
} else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UEQ) {
// fcmp one LHS, RHS => (OR (FLT LHS, RHS), (FLT RHS, LHS))
NeedInvert = Pred == CmpInst::FCMP_UEQ;
auto Cmp1 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OLT, Size),
{&RISCV::GPRRegClass}, {LHS, RHS});
if (!Cmp1.constrainAllUses(TII, TRI, RBI))
return false;
auto Cmp2 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OLT, Size),
{&RISCV::GPRRegClass}, {RHS, LHS});
if (!Cmp2.constrainAllUses(TII, TRI, RBI))
return false;
if (NeedInvert)
TmpReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
auto Or =
MIB.buildInstr(RISCV::OR, {TmpReg}, {Cmp1.getReg(0), Cmp2.getReg(0)});
if (!Or.constrainAllUses(TII, TRI, RBI))
return false;
} else if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
// fcmp ord LHS, RHS => (AND (FEQ LHS, LHS), (FEQ RHS, RHS))
// FIXME: If LHS and RHS are the same we can use a single FEQ.
NeedInvert = Pred == CmpInst::FCMP_UNO;
auto Cmp1 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OEQ, Size),
{&RISCV::GPRRegClass}, {LHS, LHS});
if (!Cmp1.constrainAllUses(TII, TRI, RBI))
return false;
auto Cmp2 = MIB.buildInstr(getFCmpOpcode(CmpInst::FCMP_OEQ, Size),
{&RISCV::GPRRegClass}, {RHS, RHS});
if (!Cmp2.constrainAllUses(TII, TRI, RBI))
return false;
if (NeedInvert)
TmpReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
auto And =
MIB.buildInstr(RISCV::AND, {TmpReg}, {Cmp1.getReg(0), Cmp2.getReg(0)});
if (!And.constrainAllUses(TII, TRI, RBI))
return false;
} else
llvm_unreachable("Unhandled predicate");
// Emit an XORI to invert the result if needed.
if (NeedInvert) {
auto Xor = MIB.buildInstr(RISCV::XORI, {DstReg}, {TmpReg}).addImm(1);
if (!Xor.constrainAllUses(TII, TRI, RBI))
return false;
}
MI.eraseFromParent();
return true;
}
bool RISCVInstructionSelector::selectIntrinsicWithSideEffects(
MachineInstr &MI, MachineIRBuilder &MIB, MachineRegisterInfo &MRI) const {
assert(MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS &&
"Unexpected opcode");
// Find the intrinsic ID.
unsigned IntrinID = cast<GIntrinsic>(MI).getIntrinsicID();
// Select the instruction.
switch (IntrinID) {
default:
return false;
case Intrinsic::trap:
MIB.buildInstr(RISCV::UNIMP, {}, {});
break;
case Intrinsic::debugtrap:
MIB.buildInstr(RISCV::EBREAK, {}, {});
break;
}
MI.eraseFromParent();
return true;
}
void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
SyncScope::ID FenceSSID,
MachineIRBuilder &MIB) const {
if (STI.hasStdExtZtso()) {
// The only fence that needs an instruction is a sequentially-consistent
// cross-thread fence.
if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
FenceSSID == SyncScope::System) {
// fence rw, rw
MIB.buildInstr(RISCV::FENCE, {}, {})
.addImm(RISCVFenceField::R | RISCVFenceField::W)
.addImm(RISCVFenceField::R | RISCVFenceField::W);
return;
}
// MEMBARRIER is a compiler barrier; it codegens to a no-op.
MIB.buildInstr(TargetOpcode::MEMBARRIER, {}, {});
return;
}
// singlethread fences only synchronize with signal handlers on the same
// thread and thus only need to preserve instruction order, not actually
// enforce memory ordering.
if (FenceSSID == SyncScope::SingleThread) {
MIB.buildInstr(TargetOpcode::MEMBARRIER, {}, {});
return;
}
// Refer to Table A.6 in the version 2.3 draft of the RISC-V Instruction Set
// Manual: Volume I.
unsigned Pred, Succ;
switch (FenceOrdering) {
default:
llvm_unreachable("Unexpected ordering");
case AtomicOrdering::AcquireRelease:
// fence acq_rel -> fence.tso
MIB.buildInstr(RISCV::FENCE_TSO, {}, {});
return;
case AtomicOrdering::Acquire:
// fence acquire -> fence r, rw
Pred = RISCVFenceField::R;
Succ = RISCVFenceField::R | RISCVFenceField::W;
break;
case AtomicOrdering::Release:
// fence release -> fence rw, w
Pred = RISCVFenceField::R | RISCVFenceField::W;
Succ = RISCVFenceField::W;
break;
case AtomicOrdering::SequentiallyConsistent:
// fence seq_cst -> fence rw, rw
Pred = RISCVFenceField::R | RISCVFenceField::W;
Succ = RISCVFenceField::R | RISCVFenceField::W;
break;
}
MIB.buildInstr(RISCV::FENCE, {}, {}).addImm(Pred).addImm(Succ);
}
namespace llvm {
InstructionSelector *
createRISCVInstructionSelector(const RISCVTargetMachine &TM,
RISCVSubtarget &Subtarget,
RISCVRegisterBankInfo &RBI) {
return new RISCVInstructionSelector(TM, Subtarget, RBI);
}
} // end namespace llvm
Reference in a new issue View git blame Copy permalink