802 lines
33 KiB
C++
802 lines
33 KiB
C++
//===- LLVMInlining.cpp - LLVM inlining interface and logic -----*- C++ -*-===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// Logic for inlining LLVM functions and the definition of the
|
|
// LLVMInliningInterface.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "LLVMInlining.h"
|
|
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
|
|
#include "mlir/IR/Matchers.h"
|
|
#include "mlir/Interfaces/DataLayoutInterfaces.h"
|
|
#include "mlir/Transforms/InliningUtils.h"
|
|
#include "llvm/ADT/ScopeExit.h"
|
|
#include "llvm/Support/Debug.h"
|
|
|
|
#define DEBUG_TYPE "llvm-inliner"
|
|
|
|
using namespace mlir;
|
|
|
|
/// Check whether the given alloca is an input to a lifetime intrinsic,
|
|
/// optionally passing through one or more casts on the way. This is not
|
|
/// transitive through block arguments.
|
|
static bool hasLifetimeMarkers(LLVM::AllocaOp allocaOp) {
|
|
SmallVector<Operation *> stack(allocaOp->getUsers().begin(),
|
|
allocaOp->getUsers().end());
|
|
while (!stack.empty()) {
|
|
Operation *op = stack.pop_back_val();
|
|
if (isa<LLVM::LifetimeStartOp, LLVM::LifetimeEndOp>(op))
|
|
return true;
|
|
if (isa<LLVM::BitcastOp>(op))
|
|
stack.append(op->getUsers().begin(), op->getUsers().end());
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/// Handles alloca operations in the inlined blocks:
|
|
/// - Moves all alloca operations with a constant size in the former entry block
|
|
/// of the callee into the entry block of the caller, so they become part of
|
|
/// the function prologue/epilogue during code generation.
|
|
/// - Inserts lifetime intrinsics that limit the scope of inlined static allocas
|
|
/// to the inlined blocks.
|
|
/// - Inserts StackSave and StackRestore operations if dynamic allocas were
|
|
/// inlined.
|
|
static void
|
|
handleInlinedAllocas(Operation *call,
|
|
iterator_range<Region::iterator> inlinedBlocks) {
|
|
// Locate the entry block of the closest callsite ancestor that has either the
|
|
// IsolatedFromAbove or AutomaticAllocationScope trait. In pure LLVM dialect
|
|
// programs, this is the LLVMFuncOp containing the call site. However, in
|
|
// mixed-dialect programs, the callsite might be nested in another operation
|
|
// that carries one of these traits. In such scenarios, this traversal stops
|
|
// at the closest ancestor with either trait, ensuring visibility post
|
|
// relocation and respecting allocation scopes.
|
|
Block *callerEntryBlock = nullptr;
|
|
Operation *currentOp = call;
|
|
while (Operation *parentOp = currentOp->getParentOp()) {
|
|
if (parentOp->mightHaveTrait<OpTrait::IsIsolatedFromAbove>() ||
|
|
parentOp->mightHaveTrait<OpTrait::AutomaticAllocationScope>()) {
|
|
callerEntryBlock = ¤tOp->getParentRegion()->front();
|
|
break;
|
|
}
|
|
currentOp = parentOp;
|
|
}
|
|
|
|
// Avoid relocating the alloca operations if the call has been inlined into
|
|
// the entry block already, which is typically the encompassing
|
|
// LLVM function, or if the relevant entry block cannot be identified.
|
|
Block *calleeEntryBlock = &(*inlinedBlocks.begin());
|
|
if (!callerEntryBlock || callerEntryBlock == calleeEntryBlock)
|
|
return;
|
|
|
|
SmallVector<std::tuple<LLVM::AllocaOp, IntegerAttr, bool>> allocasToMove;
|
|
bool shouldInsertLifetimes = false;
|
|
bool hasDynamicAlloca = false;
|
|
// Conservatively only move static alloca operations that are part of the
|
|
// entry block and do not inspect nested regions, since they may execute
|
|
// conditionally or have other unknown semantics.
|
|
for (auto allocaOp : calleeEntryBlock->getOps<LLVM::AllocaOp>()) {
|
|
IntegerAttr arraySize;
|
|
if (!matchPattern(allocaOp.getArraySize(), m_Constant(&arraySize))) {
|
|
hasDynamicAlloca = true;
|
|
continue;
|
|
}
|
|
bool shouldInsertLifetime =
|
|
arraySize.getValue() != 0 && !hasLifetimeMarkers(allocaOp);
|
|
shouldInsertLifetimes |= shouldInsertLifetime;
|
|
allocasToMove.emplace_back(allocaOp, arraySize, shouldInsertLifetime);
|
|
}
|
|
// Check the remaining inlined blocks for dynamic allocas as well.
|
|
for (Block &block : llvm::drop_begin(inlinedBlocks)) {
|
|
if (hasDynamicAlloca)
|
|
break;
|
|
hasDynamicAlloca =
|
|
llvm::any_of(block.getOps<LLVM::AllocaOp>(), [](auto allocaOp) {
|
|
return !matchPattern(allocaOp.getArraySize(), m_Constant());
|
|
});
|
|
}
|
|
if (allocasToMove.empty() && !hasDynamicAlloca)
|
|
return;
|
|
OpBuilder builder(calleeEntryBlock, calleeEntryBlock->begin());
|
|
Value stackPtr;
|
|
if (hasDynamicAlloca) {
|
|
// This may result in multiple stacksave/stackrestore intrinsics in the same
|
|
// scope if some are already present in the body of the caller. This is not
|
|
// invalid IR, but LLVM cleans these up in InstCombineCalls.cpp, along with
|
|
// other cases where the stacksave/stackrestore is redundant.
|
|
stackPtr = builder.create<LLVM::StackSaveOp>(
|
|
call->getLoc(), LLVM::LLVMPointerType::get(call->getContext()));
|
|
}
|
|
builder.setInsertionPoint(callerEntryBlock, callerEntryBlock->begin());
|
|
for (auto &[allocaOp, arraySize, shouldInsertLifetime] : allocasToMove) {
|
|
auto newConstant = builder.create<LLVM::ConstantOp>(
|
|
allocaOp->getLoc(), allocaOp.getArraySize().getType(), arraySize);
|
|
// Insert a lifetime start intrinsic where the alloca was before moving it.
|
|
if (shouldInsertLifetime) {
|
|
OpBuilder::InsertionGuard insertionGuard(builder);
|
|
builder.setInsertionPoint(allocaOp);
|
|
builder.create<LLVM::LifetimeStartOp>(
|
|
allocaOp.getLoc(), arraySize.getValue().getLimitedValue(),
|
|
allocaOp.getResult());
|
|
}
|
|
allocaOp->moveAfter(newConstant);
|
|
allocaOp.getArraySizeMutable().assign(newConstant.getResult());
|
|
}
|
|
if (!shouldInsertLifetimes && !hasDynamicAlloca)
|
|
return;
|
|
// Insert a lifetime end intrinsic before each return in the callee function.
|
|
for (Block &block : inlinedBlocks) {
|
|
if (!block.getTerminator()->hasTrait<OpTrait::ReturnLike>())
|
|
continue;
|
|
builder.setInsertionPoint(block.getTerminator());
|
|
if (hasDynamicAlloca)
|
|
builder.create<LLVM::StackRestoreOp>(call->getLoc(), stackPtr);
|
|
for (auto &[allocaOp, arraySize, shouldInsertLifetime] : allocasToMove) {
|
|
if (shouldInsertLifetime)
|
|
builder.create<LLVM::LifetimeEndOp>(
|
|
allocaOp.getLoc(), arraySize.getValue().getLimitedValue(),
|
|
allocaOp.getResult());
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Maps all alias scopes in the inlined operations to deep clones of the scopes
|
|
/// and domain. This is required for code such as `foo(a, b); foo(a2, b2);` to
|
|
/// not incorrectly return `noalias` for e.g. operations on `a` and `a2`.
|
|
static void
|
|
deepCloneAliasScopes(iterator_range<Region::iterator> inlinedBlocks) {
|
|
DenseMap<Attribute, Attribute> mapping;
|
|
|
|
// Register handles in the walker to create the deep clones.
|
|
// The walker ensures that an attribute is only ever walked once and does a
|
|
// post-order walk, ensuring the domain is visited prior to the scope.
|
|
AttrTypeWalker walker;
|
|
|
|
// Perform the deep clones while visiting. Builders create a distinct
|
|
// attribute to make sure that new instances are always created by the
|
|
// uniquer.
|
|
walker.addWalk([&](LLVM::AliasScopeDomainAttr domainAttr) {
|
|
mapping[domainAttr] = LLVM::AliasScopeDomainAttr::get(
|
|
domainAttr.getContext(), domainAttr.getDescription());
|
|
});
|
|
|
|
walker.addWalk([&](LLVM::AliasScopeAttr scopeAttr) {
|
|
mapping[scopeAttr] = LLVM::AliasScopeAttr::get(
|
|
cast<LLVM::AliasScopeDomainAttr>(mapping.lookup(scopeAttr.getDomain())),
|
|
scopeAttr.getDescription());
|
|
});
|
|
|
|
// Map an array of scopes to an array of deep clones.
|
|
auto convertScopeList = [&](ArrayAttr arrayAttr) -> ArrayAttr {
|
|
if (!arrayAttr)
|
|
return nullptr;
|
|
|
|
// Create the deep clones if necessary.
|
|
walker.walk(arrayAttr);
|
|
|
|
return ArrayAttr::get(arrayAttr.getContext(),
|
|
llvm::map_to_vector(arrayAttr, [&](Attribute attr) {
|
|
return mapping.lookup(attr);
|
|
}));
|
|
};
|
|
|
|
for (Block &block : inlinedBlocks) {
|
|
for (Operation &op : block) {
|
|
if (auto aliasInterface = dyn_cast<LLVM::AliasAnalysisOpInterface>(op)) {
|
|
aliasInterface.setAliasScopes(
|
|
convertScopeList(aliasInterface.getAliasScopesOrNull()));
|
|
aliasInterface.setNoAliasScopes(
|
|
convertScopeList(aliasInterface.getNoAliasScopesOrNull()));
|
|
}
|
|
|
|
if (auto noAliasScope = dyn_cast<LLVM::NoAliasScopeDeclOp>(op)) {
|
|
// Create the deep clones if necessary.
|
|
walker.walk(noAliasScope.getScopeAttr());
|
|
|
|
noAliasScope.setScopeAttr(cast<LLVM::AliasScopeAttr>(
|
|
mapping.lookup(noAliasScope.getScopeAttr())));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Creates a new ArrayAttr by concatenating `lhs` with `rhs`.
|
|
/// Returns null if both parameters are null. If only one attribute is null,
|
|
/// return the other.
|
|
static ArrayAttr concatArrayAttr(ArrayAttr lhs, ArrayAttr rhs) {
|
|
if (!lhs)
|
|
return rhs;
|
|
if (!rhs)
|
|
return lhs;
|
|
|
|
SmallVector<Attribute> result;
|
|
llvm::append_range(result, lhs);
|
|
llvm::append_range(result, rhs);
|
|
return ArrayAttr::get(lhs.getContext(), result);
|
|
}
|
|
|
|
/// Attempts to return the underlying pointer value that `pointerValue` is based
|
|
/// on. This traverses down the chain of operations to the last operation
|
|
/// producing the base pointer and returns it. If it encounters an operation it
|
|
/// cannot further traverse through, returns the operation's result.
|
|
static Value getUnderlyingObject(Value pointerValue) {
|
|
while (true) {
|
|
if (auto gepOp = pointerValue.getDefiningOp<LLVM::GEPOp>()) {
|
|
pointerValue = gepOp.getBase();
|
|
continue;
|
|
}
|
|
|
|
if (auto addrCast = pointerValue.getDefiningOp<LLVM::AddrSpaceCastOp>()) {
|
|
pointerValue = addrCast.getOperand();
|
|
continue;
|
|
}
|
|
|
|
break;
|
|
}
|
|
|
|
return pointerValue;
|
|
}
|
|
|
|
/// Attempts to return the set of all underlying pointer values that
|
|
/// `pointerValue` is based on. This function traverses through select
|
|
/// operations and block arguments unlike getUnderlyingObject.
|
|
static SmallVector<Value> getUnderlyingObjectSet(Value pointerValue) {
|
|
SmallVector<Value> result;
|
|
|
|
SmallVector<Value> workList{pointerValue};
|
|
// Avoid dataflow loops.
|
|
SmallPtrSet<Value, 4> seen;
|
|
do {
|
|
Value current = workList.pop_back_val();
|
|
current = getUnderlyingObject(current);
|
|
|
|
if (!seen.insert(current).second)
|
|
continue;
|
|
|
|
if (auto selectOp = current.getDefiningOp<LLVM::SelectOp>()) {
|
|
workList.push_back(selectOp.getTrueValue());
|
|
workList.push_back(selectOp.getFalseValue());
|
|
continue;
|
|
}
|
|
|
|
if (auto blockArg = dyn_cast<BlockArgument>(current)) {
|
|
Block *parentBlock = blockArg.getParentBlock();
|
|
|
|
// Attempt to find all block argument operands for every predecessor.
|
|
// If any operand to the block argument wasn't found in a predecessor,
|
|
// conservatively add the block argument to the result set.
|
|
SmallVector<Value> operands;
|
|
bool anyUnknown = false;
|
|
for (auto iter = parentBlock->pred_begin();
|
|
iter != parentBlock->pred_end(); iter++) {
|
|
auto branch = dyn_cast<BranchOpInterface>((*iter)->getTerminator());
|
|
if (!branch) {
|
|
result.push_back(blockArg);
|
|
anyUnknown = true;
|
|
break;
|
|
}
|
|
|
|
Value operand = branch.getSuccessorOperands(
|
|
iter.getSuccessorIndex())[blockArg.getArgNumber()];
|
|
if (!operand) {
|
|
result.push_back(blockArg);
|
|
anyUnknown = true;
|
|
break;
|
|
}
|
|
|
|
operands.push_back(operand);
|
|
}
|
|
|
|
if (!anyUnknown)
|
|
llvm::append_range(workList, operands);
|
|
|
|
continue;
|
|
}
|
|
|
|
result.push_back(current);
|
|
} while (!workList.empty());
|
|
|
|
return result;
|
|
}
|
|
|
|
/// Creates a new AliasScopeAttr for every noalias parameter and attaches it to
|
|
/// the appropriate inlined memory operations in an attempt to preserve the
|
|
/// original semantics of the parameter attribute.
|
|
static void createNewAliasScopesFromNoAliasParameter(
|
|
Operation *call, iterator_range<Region::iterator> inlinedBlocks) {
|
|
|
|
// First collect all noalias parameters. These have been specially marked by
|
|
// the `handleArgument` implementation by using the `ssa.copy` intrinsic and
|
|
// attaching a `noalias` attribute to it.
|
|
// These are only meant to be temporary and should therefore be deleted after
|
|
// we're done using them here.
|
|
SetVector<LLVM::SSACopyOp> noAliasParams;
|
|
for (Value argument : cast<LLVM::CallOp>(call).getArgOperands()) {
|
|
for (Operation *user : argument.getUsers()) {
|
|
auto ssaCopy = llvm::dyn_cast<LLVM::SSACopyOp>(user);
|
|
if (!ssaCopy)
|
|
continue;
|
|
if (!ssaCopy->hasAttr(LLVM::LLVMDialect::getNoAliasAttrName()))
|
|
continue;
|
|
|
|
noAliasParams.insert(ssaCopy);
|
|
}
|
|
}
|
|
|
|
// If there were none, we have nothing to do here.
|
|
if (noAliasParams.empty())
|
|
return;
|
|
|
|
// Scope exit block to make it impossible to forget to get rid of the
|
|
// intrinsics.
|
|
auto exit = llvm::make_scope_exit([&] {
|
|
for (LLVM::SSACopyOp ssaCopyOp : noAliasParams) {
|
|
ssaCopyOp.replaceAllUsesWith(ssaCopyOp.getOperand());
|
|
ssaCopyOp->erase();
|
|
}
|
|
});
|
|
|
|
// Create a new domain for this specific inlining and a new scope for every
|
|
// noalias parameter.
|
|
auto functionDomain = LLVM::AliasScopeDomainAttr::get(
|
|
call->getContext(), cast<LLVM::CallOp>(call).getCalleeAttr().getAttr());
|
|
DenseMap<Value, LLVM::AliasScopeAttr> pointerScopes;
|
|
for (LLVM::SSACopyOp copyOp : noAliasParams) {
|
|
auto scope = LLVM::AliasScopeAttr::get(functionDomain);
|
|
pointerScopes[copyOp] = scope;
|
|
|
|
OpBuilder(call).create<LLVM::NoAliasScopeDeclOp>(call->getLoc(), scope);
|
|
}
|
|
|
|
// Go through every instruction and attempt to find which noalias parameters
|
|
// it is definitely based on and definitely not based on.
|
|
for (Block &inlinedBlock : inlinedBlocks) {
|
|
for (auto aliasInterface :
|
|
inlinedBlock.getOps<LLVM::AliasAnalysisOpInterface>()) {
|
|
|
|
// Collect the pointer arguments affected by the alias scopes.
|
|
SmallVector<Value> pointerArgs = aliasInterface.getAccessedOperands();
|
|
|
|
// Find the set of underlying pointers that this pointer is based on.
|
|
SmallPtrSet<Value, 4> basedOnPointers;
|
|
for (Value pointer : pointerArgs)
|
|
llvm::copy(getUnderlyingObjectSet(pointer),
|
|
std::inserter(basedOnPointers, basedOnPointers.begin()));
|
|
|
|
bool aliasesOtherKnownObject = false;
|
|
// Go through the based on pointers and check that they are either:
|
|
// * Constants that can be ignored (undef, poison, null pointer).
|
|
// * Based on a noalias parameter.
|
|
// * Other pointers that we know can't alias with our noalias parameter.
|
|
//
|
|
// Any other value might be a pointer based on any noalias parameter that
|
|
// hasn't been identified. In that case conservatively don't add any
|
|
// scopes to this operation indicating either aliasing or not aliasing
|
|
// with any parameter.
|
|
if (llvm::any_of(basedOnPointers, [&](Value object) {
|
|
if (matchPattern(object, m_Constant()))
|
|
return false;
|
|
|
|
if (noAliasParams.contains(object.getDefiningOp<LLVM::SSACopyOp>()))
|
|
return false;
|
|
|
|
// TODO: This should include other arguments from the inlined
|
|
// callable.
|
|
if (isa_and_nonnull<LLVM::AllocaOp, LLVM::AddressOfOp>(
|
|
object.getDefiningOp())) {
|
|
aliasesOtherKnownObject = true;
|
|
return false;
|
|
}
|
|
return true;
|
|
}))
|
|
continue;
|
|
|
|
// Add all noalias parameter scopes to the noalias scope list that we are
|
|
// not based on.
|
|
SmallVector<Attribute> noAliasScopes;
|
|
for (LLVM::SSACopyOp noAlias : noAliasParams) {
|
|
if (basedOnPointers.contains(noAlias))
|
|
continue;
|
|
|
|
noAliasScopes.push_back(pointerScopes[noAlias]);
|
|
}
|
|
|
|
if (!noAliasScopes.empty())
|
|
aliasInterface.setNoAliasScopes(
|
|
concatArrayAttr(aliasInterface.getNoAliasScopesOrNull(),
|
|
ArrayAttr::get(call->getContext(), noAliasScopes)));
|
|
|
|
// Don't add alias scopes to call operations or operations that might
|
|
// operate on pointers not based on any noalias parameter.
|
|
// Since we add all scopes to an operation's noalias list that it
|
|
// definitely doesn't alias, we mustn't do the same for the alias.scope
|
|
// list if other objects are involved.
|
|
//
|
|
// Consider the following case:
|
|
// %0 = llvm.alloca
|
|
// %1 = select %magic, %0, %noalias_param
|
|
// store 5, %1 (1) noalias=[scope(...)]
|
|
// ...
|
|
// store 3, %0 (2) noalias=[scope(noalias_param), scope(...)]
|
|
//
|
|
// We can add the scopes of any noalias parameters that aren't
|
|
// noalias_param's scope to (1) and add all of them to (2). We mustn't add
|
|
// the scope of noalias_param to the alias.scope list of (1) since
|
|
// that would mean (2) cannot alias with (1) which is wrong since both may
|
|
// store to %0.
|
|
//
|
|
// In conclusion, only add scopes to the alias.scope list if all pointers
|
|
// have a corresponding scope.
|
|
// Call operations are included in this list since we do not know whether
|
|
// the callee accesses any memory besides the ones passed as its
|
|
// arguments.
|
|
if (aliasesOtherKnownObject ||
|
|
isa<LLVM::CallOp>(aliasInterface.getOperation()))
|
|
continue;
|
|
|
|
SmallVector<Attribute> aliasScopes;
|
|
for (LLVM::SSACopyOp noAlias : noAliasParams)
|
|
if (basedOnPointers.contains(noAlias))
|
|
aliasScopes.push_back(pointerScopes[noAlias]);
|
|
|
|
if (!aliasScopes.empty())
|
|
aliasInterface.setAliasScopes(
|
|
concatArrayAttr(aliasInterface.getAliasScopesOrNull(),
|
|
ArrayAttr::get(call->getContext(), aliasScopes)));
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Appends any alias scopes of the call operation to any inlined memory
|
|
/// operation.
|
|
static void
|
|
appendCallOpAliasScopes(Operation *call,
|
|
iterator_range<Region::iterator> inlinedBlocks) {
|
|
auto callAliasInterface = dyn_cast<LLVM::AliasAnalysisOpInterface>(call);
|
|
if (!callAliasInterface)
|
|
return;
|
|
|
|
ArrayAttr aliasScopes = callAliasInterface.getAliasScopesOrNull();
|
|
ArrayAttr noAliasScopes = callAliasInterface.getNoAliasScopesOrNull();
|
|
// If the call has neither alias scopes or noalias scopes we have nothing to
|
|
// do here.
|
|
if (!aliasScopes && !noAliasScopes)
|
|
return;
|
|
|
|
// Simply append the call op's alias and noalias scopes to any operation
|
|
// implementing AliasAnalysisOpInterface.
|
|
for (Block &block : inlinedBlocks) {
|
|
for (auto aliasInterface : block.getOps<LLVM::AliasAnalysisOpInterface>()) {
|
|
if (aliasScopes)
|
|
aliasInterface.setAliasScopes(concatArrayAttr(
|
|
aliasInterface.getAliasScopesOrNull(), aliasScopes));
|
|
|
|
if (noAliasScopes)
|
|
aliasInterface.setNoAliasScopes(concatArrayAttr(
|
|
aliasInterface.getNoAliasScopesOrNull(), noAliasScopes));
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Handles all interactions with alias scopes during inlining.
|
|
static void handleAliasScopes(Operation *call,
|
|
iterator_range<Region::iterator> inlinedBlocks) {
|
|
deepCloneAliasScopes(inlinedBlocks);
|
|
createNewAliasScopesFromNoAliasParameter(call, inlinedBlocks);
|
|
appendCallOpAliasScopes(call, inlinedBlocks);
|
|
}
|
|
|
|
/// Appends any access groups of the call operation to any inlined memory
|
|
/// operation.
|
|
static void handleAccessGroups(Operation *call,
|
|
iterator_range<Region::iterator> inlinedBlocks) {
|
|
auto callAccessGroupInterface = dyn_cast<LLVM::AccessGroupOpInterface>(call);
|
|
if (!callAccessGroupInterface)
|
|
return;
|
|
|
|
auto accessGroups = callAccessGroupInterface.getAccessGroupsOrNull();
|
|
if (!accessGroups)
|
|
return;
|
|
|
|
// Simply append the call op's access groups to any operation implementing
|
|
// AccessGroupOpInterface.
|
|
for (Block &block : inlinedBlocks)
|
|
for (auto accessGroupOpInterface :
|
|
block.getOps<LLVM::AccessGroupOpInterface>())
|
|
accessGroupOpInterface.setAccessGroups(concatArrayAttr(
|
|
accessGroupOpInterface.getAccessGroupsOrNull(), accessGroups));
|
|
}
|
|
|
|
/// If `requestedAlignment` is higher than the alignment specified on `alloca`,
|
|
/// realigns `alloca` if this does not exceed the natural stack alignment.
|
|
/// Returns the post-alignment of `alloca`, whether it was realigned or not.
|
|
static uint64_t tryToEnforceAllocaAlignment(LLVM::AllocaOp alloca,
|
|
uint64_t requestedAlignment,
|
|
DataLayout const &dataLayout) {
|
|
uint64_t allocaAlignment = alloca.getAlignment().value_or(1);
|
|
if (requestedAlignment <= allocaAlignment)
|
|
// No realignment necessary.
|
|
return allocaAlignment;
|
|
uint64_t naturalStackAlignmentBits = dataLayout.getStackAlignment();
|
|
// If the natural stack alignment is not specified, the data layout returns
|
|
// zero. Optimistically allow realignment in this case.
|
|
if (naturalStackAlignmentBits == 0 ||
|
|
// If the requested alignment exceeds the natural stack alignment, this
|
|
// will trigger a dynamic stack realignment, so we prefer to copy...
|
|
8 * requestedAlignment <= naturalStackAlignmentBits ||
|
|
// ...unless the alloca already triggers dynamic stack realignment. Then
|
|
// we might as well further increase the alignment to avoid a copy.
|
|
8 * allocaAlignment > naturalStackAlignmentBits) {
|
|
alloca.setAlignment(requestedAlignment);
|
|
allocaAlignment = requestedAlignment;
|
|
}
|
|
return allocaAlignment;
|
|
}
|
|
|
|
/// Tries to find and return the alignment of the pointer `value` by looking for
|
|
/// an alignment attribute on the defining allocation op or function argument.
|
|
/// If the found alignment is lower than `requestedAlignment`, tries to realign
|
|
/// the pointer, then returns the resulting post-alignment, regardless of
|
|
/// whether it was realigned or not. If no existing alignment attribute is
|
|
/// found, returns 1 (i.e., assume that no alignment is guaranteed).
|
|
static uint64_t tryToEnforceAlignment(Value value, uint64_t requestedAlignment,
|
|
DataLayout const &dataLayout) {
|
|
if (Operation *definingOp = value.getDefiningOp()) {
|
|
if (auto alloca = dyn_cast<LLVM::AllocaOp>(definingOp))
|
|
return tryToEnforceAllocaAlignment(alloca, requestedAlignment,
|
|
dataLayout);
|
|
if (auto addressOf = dyn_cast<LLVM::AddressOfOp>(definingOp))
|
|
if (auto global = SymbolTable::lookupNearestSymbolFrom<LLVM::GlobalOp>(
|
|
definingOp, addressOf.getGlobalNameAttr()))
|
|
return global.getAlignment().value_or(1);
|
|
// We don't currently handle this operation; assume no alignment.
|
|
return 1;
|
|
}
|
|
// Since there is no defining op, this is a block argument. Probably this
|
|
// comes directly from a function argument, so check that this is the case.
|
|
Operation *parentOp = value.getParentBlock()->getParentOp();
|
|
if (auto func = dyn_cast<LLVM::LLVMFuncOp>(parentOp)) {
|
|
// Use the alignment attribute set for this argument in the parent function
|
|
// if it has been set.
|
|
auto blockArg = llvm::cast<BlockArgument>(value);
|
|
if (Attribute alignAttr = func.getArgAttr(
|
|
blockArg.getArgNumber(), LLVM::LLVMDialect::getAlignAttrName()))
|
|
return cast<IntegerAttr>(alignAttr).getValue().getLimitedValue();
|
|
}
|
|
// We didn't find anything useful; assume no alignment.
|
|
return 1;
|
|
}
|
|
|
|
/// Introduces a new alloca and copies the memory pointed to by `argument` to
|
|
/// the address of the new alloca, then returns the value of the new alloca.
|
|
static Value handleByValArgumentInit(OpBuilder &builder, Location loc,
|
|
Value argument, Type elementType,
|
|
uint64_t elementTypeSize,
|
|
uint64_t targetAlignment) {
|
|
// Allocate the new value on the stack.
|
|
Value allocaOp;
|
|
{
|
|
// Since this is a static alloca, we can put it directly in the entry block,
|
|
// so they can be absorbed into the prologue/epilogue at code generation.
|
|
OpBuilder::InsertionGuard insertionGuard(builder);
|
|
Block *entryBlock = &(*argument.getParentRegion()->begin());
|
|
builder.setInsertionPointToStart(entryBlock);
|
|
Value one = builder.create<LLVM::ConstantOp>(loc, builder.getI64Type(),
|
|
builder.getI64IntegerAttr(1));
|
|
allocaOp = builder.create<LLVM::AllocaOp>(
|
|
loc, argument.getType(), elementType, one, targetAlignment);
|
|
}
|
|
// Copy the pointee to the newly allocated value.
|
|
Value copySize = builder.create<LLVM::ConstantOp>(
|
|
loc, builder.getI64Type(), builder.getI64IntegerAttr(elementTypeSize));
|
|
builder.create<LLVM::MemcpyOp>(loc, allocaOp, argument, copySize,
|
|
/*isVolatile=*/false);
|
|
return allocaOp;
|
|
}
|
|
|
|
/// Handles a function argument marked with the byval attribute by introducing a
|
|
/// memcpy or realigning the defining operation, if required either due to the
|
|
/// pointee being writeable in the callee, and/or due to an alignment mismatch.
|
|
/// `requestedAlignment` specifies the alignment set in the "align" argument
|
|
/// attribute (or 1 if no align attribute was set).
|
|
static Value handleByValArgument(OpBuilder &builder, Operation *callable,
|
|
Value argument, Type elementType,
|
|
uint64_t requestedAlignment) {
|
|
auto func = cast<LLVM::LLVMFuncOp>(callable);
|
|
LLVM::MemoryEffectsAttr memoryEffects = func.getMemoryAttr();
|
|
// If there is no memory effects attribute, assume that the function is
|
|
// not read-only.
|
|
bool isReadOnly = memoryEffects &&
|
|
memoryEffects.getArgMem() != LLVM::ModRefInfo::ModRef &&
|
|
memoryEffects.getArgMem() != LLVM::ModRefInfo::Mod;
|
|
// Check if there's an alignment mismatch requiring us to copy.
|
|
DataLayout dataLayout = DataLayout::closest(callable);
|
|
uint64_t minimumAlignment = dataLayout.getTypeABIAlignment(elementType);
|
|
if (isReadOnly) {
|
|
if (requestedAlignment <= minimumAlignment)
|
|
return argument;
|
|
uint64_t currentAlignment =
|
|
tryToEnforceAlignment(argument, requestedAlignment, dataLayout);
|
|
if (currentAlignment >= requestedAlignment)
|
|
return argument;
|
|
}
|
|
uint64_t targetAlignment = std::max(requestedAlignment, minimumAlignment);
|
|
return handleByValArgumentInit(builder, func.getLoc(), argument, elementType,
|
|
dataLayout.getTypeSize(elementType),
|
|
targetAlignment);
|
|
}
|
|
|
|
namespace {
|
|
struct LLVMInlinerInterface : public DialectInlinerInterface {
|
|
using DialectInlinerInterface::DialectInlinerInterface;
|
|
|
|
LLVMInlinerInterface(Dialect *dialect)
|
|
: DialectInlinerInterface(dialect),
|
|
// Cache set of StringAttrs for fast lookup in `isLegalToInline`.
|
|
disallowedFunctionAttrs({
|
|
StringAttr::get(dialect->getContext(), "noduplicate"),
|
|
StringAttr::get(dialect->getContext(), "noinline"),
|
|
StringAttr::get(dialect->getContext(), "optnone"),
|
|
StringAttr::get(dialect->getContext(), "presplitcoroutine"),
|
|
StringAttr::get(dialect->getContext(), "returns_twice"),
|
|
StringAttr::get(dialect->getContext(), "strictfp"),
|
|
}) {}
|
|
|
|
bool isLegalToInline(Operation *call, Operation *callable,
|
|
bool wouldBeCloned) const final {
|
|
if (!wouldBeCloned)
|
|
return false;
|
|
if (!isa<LLVM::CallOp>(call)) {
|
|
LLVM_DEBUG(llvm::dbgs()
|
|
<< "Cannot inline: call is not an LLVM::CallOp\n");
|
|
return false;
|
|
}
|
|
auto funcOp = dyn_cast<LLVM::LLVMFuncOp>(callable);
|
|
if (!funcOp) {
|
|
LLVM_DEBUG(llvm::dbgs()
|
|
<< "Cannot inline: callable is not an LLVM::LLVMFuncOp\n");
|
|
return false;
|
|
}
|
|
if (funcOp.isVarArg()) {
|
|
LLVM_DEBUG(llvm::dbgs() << "Cannot inline: callable is variadic\n");
|
|
return false;
|
|
}
|
|
// TODO: Generate aliasing metadata from noalias argument/result attributes.
|
|
if (auto attrs = funcOp.getArgAttrs()) {
|
|
for (DictionaryAttr attrDict : attrs->getAsRange<DictionaryAttr>()) {
|
|
if (attrDict.contains(LLVM::LLVMDialect::getInAllocaAttrName())) {
|
|
LLVM_DEBUG(llvm::dbgs() << "Cannot inline " << funcOp.getSymName()
|
|
<< ": inalloca arguments not supported\n");
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
// TODO: Handle exceptions.
|
|
if (funcOp.getPersonality()) {
|
|
LLVM_DEBUG(llvm::dbgs() << "Cannot inline " << funcOp.getSymName()
|
|
<< ": unhandled function personality\n");
|
|
return false;
|
|
}
|
|
if (funcOp.getPassthrough()) {
|
|
// TODO: Used attributes should not be passthrough.
|
|
if (llvm::any_of(*funcOp.getPassthrough(), [&](Attribute attr) {
|
|
auto stringAttr = dyn_cast<StringAttr>(attr);
|
|
if (!stringAttr)
|
|
return false;
|
|
if (disallowedFunctionAttrs.contains(stringAttr)) {
|
|
LLVM_DEBUG(llvm::dbgs()
|
|
<< "Cannot inline " << funcOp.getSymName()
|
|
<< ": found disallowed function attribute "
|
|
<< stringAttr << "\n");
|
|
return true;
|
|
}
|
|
return false;
|
|
}))
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool isLegalToInline(Region *, Region *, bool, IRMapping &) const final {
|
|
return true;
|
|
}
|
|
|
|
bool isLegalToInline(Operation *op, Region *, bool, IRMapping &) const final {
|
|
// The inliner cannot handle variadic function arguments.
|
|
return !isa<LLVM::VaStartOp>(op);
|
|
}
|
|
|
|
/// Handle the given inlined return by replacing it with a branch. This
|
|
/// overload is called when the inlined region has more than one block.
|
|
void handleTerminator(Operation *op, Block *newDest) const final {
|
|
// Only return needs to be handled here.
|
|
auto returnOp = dyn_cast<LLVM::ReturnOp>(op);
|
|
if (!returnOp)
|
|
return;
|
|
|
|
// Replace the return with a branch to the dest.
|
|
OpBuilder builder(op);
|
|
builder.create<LLVM::BrOp>(op->getLoc(), returnOp.getOperands(), newDest);
|
|
op->erase();
|
|
}
|
|
|
|
/// Handle the given inlined return by replacing the uses of the call with the
|
|
/// operands of the return. This overload is called when the inlined region
|
|
/// only contains one block.
|
|
void handleTerminator(Operation *op, ValueRange valuesToRepl) const final {
|
|
// Return will be the only terminator present.
|
|
auto returnOp = cast<LLVM::ReturnOp>(op);
|
|
|
|
// Replace the values directly with the return operands.
|
|
assert(returnOp.getNumOperands() == valuesToRepl.size());
|
|
for (auto [dst, src] : llvm::zip(valuesToRepl, returnOp.getOperands()))
|
|
dst.replaceAllUsesWith(src);
|
|
}
|
|
|
|
Value handleArgument(OpBuilder &builder, Operation *call, Operation *callable,
|
|
Value argument,
|
|
DictionaryAttr argumentAttrs) const final {
|
|
if (std::optional<NamedAttribute> attr =
|
|
argumentAttrs.getNamed(LLVM::LLVMDialect::getByValAttrName())) {
|
|
Type elementType = cast<TypeAttr>(attr->getValue()).getValue();
|
|
uint64_t requestedAlignment = 1;
|
|
if (std::optional<NamedAttribute> alignAttr =
|
|
argumentAttrs.getNamed(LLVM::LLVMDialect::getAlignAttrName())) {
|
|
requestedAlignment = cast<IntegerAttr>(alignAttr->getValue())
|
|
.getValue()
|
|
.getLimitedValue();
|
|
}
|
|
return handleByValArgument(builder, callable, argument, elementType,
|
|
requestedAlignment);
|
|
}
|
|
if ([[maybe_unused]] std::optional<NamedAttribute> attr =
|
|
argumentAttrs.getNamed(LLVM::LLVMDialect::getNoAliasAttrName())) {
|
|
if (argument.use_empty())
|
|
return argument;
|
|
|
|
// This code is essentially a workaround for deficiencies in the
|
|
// inliner interface: We need to transform operations *after* inlined
|
|
// based on the argument attributes of the parameters *before* inlining.
|
|
// This method runs prior to actual inlining and thus cannot transform the
|
|
// post-inlining code, while `processInlinedCallBlocks` does not have
|
|
// access to pre-inlining function arguments. Additionally, it is required
|
|
// to distinguish which parameter an SSA value originally came from.
|
|
// As a workaround until this is changed: Create an ssa.copy intrinsic
|
|
// with the noalias attribute that can easily be found, and is extremely
|
|
// unlikely to exist in the code prior to inlining, using this to
|
|
// communicate between this method and `processInlinedCallBlocks`.
|
|
// TODO: Fix this by refactoring the inliner interface.
|
|
auto copyOp = builder.create<LLVM::SSACopyOp>(call->getLoc(), argument);
|
|
copyOp->setDiscardableAttr(
|
|
builder.getStringAttr(LLVM::LLVMDialect::getNoAliasAttrName()),
|
|
builder.getUnitAttr());
|
|
return copyOp;
|
|
}
|
|
return argument;
|
|
}
|
|
|
|
void processInlinedCallBlocks(
|
|
Operation *call,
|
|
iterator_range<Region::iterator> inlinedBlocks) const override {
|
|
handleInlinedAllocas(call, inlinedBlocks);
|
|
handleAliasScopes(call, inlinedBlocks);
|
|
handleAccessGroups(call, inlinedBlocks);
|
|
}
|
|
|
|
// Keeping this (immutable) state on the interface allows us to look up
|
|
// StringAttrs instead of looking up strings, since StringAttrs are bound to
|
|
// the current context and thus cannot be initialized as static fields.
|
|
const DenseSet<StringAttr> disallowedFunctionAttrs;
|
|
};
|
|
|
|
} // end anonymous namespace
|
|
|
|
void LLVM::detail::addLLVMInlinerInterface(LLVM::LLVMDialect *dialect) {
|
|
dialect->addInterfaces<LLVMInlinerInterface>();
|
|
}
|