Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions include/PTO/Transforms/VMIControlFlowSupport.h
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@
#ifndef PTO_TRANSFORMS_VMICONTROLFLOWSUPPORT_H
#define PTO_TRANSFORMS_VMICONTROLFLOWSUPPORT_H

#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/SCF/IR/SCF.h"
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/Value.h"
#include "mlir/Support/LLVM.h"

Expand All @@ -37,6 +39,14 @@ class VMIControlFlowSupport {
scf::WhileOp whileOp, EquivalenceCallback addEquivalent);
};

/// Return the result types agreed on by every call to \p func. A null type
/// marks a result position whose callers currently disagree.
SmallVector<Type> getConsistentCallResultTypes(ModuleOp module,
func::FuncOp func);

/// Collect the current argument types used to rebuild a function signature.
SmallVector<Type> getFunctionInputTypes(func::FuncOp func);

} // namespace pto
} // namespace mlir

Expand Down
4 changes: 2 additions & 2 deletions include/PTO/Transforms/VMILayoutPropagation.h
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ class VMILayoutPropagator {
void enqueue(Value value, VMILayoutAttr layout);
LogicalResult addUseConflict(OpOperand &operand,
VMIValueLayoutAssignment &assignment,
VMILayoutAttr layout);
VMILayoutAttr layout) const;
LogicalResult propagateFact(Value value, VMILayoutAttr layout);
LogicalResult propagateOperandFact(OpOperand &operand, VMILayoutAttr layout);
LogicalResult propagateThrough(Operation *op, Value changedValue,
Expand All @@ -81,7 +81,7 @@ class VMILayoutPropagator {
RewriterBase &rewriter,
DenseMap<Value, Value> &assignedValues);
FailureOr<Value> materializeAt(Value source, VMILayoutAttr layout,
RewriterBase &rewriter, Location loc);
RewriterBase &rewriter, Location loc) const;
LogicalResult materializeUseConflict(Value assignedValue,
VMILayoutConflict conflict,
RewriterBase &rewriter);
Expand Down
16 changes: 4 additions & 12 deletions lib/PTO/Transforms/PTOInferVPTOVecScope.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
//===----------------------------------------------------------------------===//

#include "PTO/Transforms/Passes.h"
#include "Utils.h"

#include "PTO/Support/CodeConstants.h"

Expand Down Expand Up @@ -306,15 +307,6 @@ computeMovedOpsForResultlessScope(ArrayRef<Operation *> ops) {
return movedOps;
}

static Operation *getAncestorInBlock(Operation *op, Block &block) {
for (Operation *cur = op; cur; cur = cur->getParentOp()) {
if (cur->getBlock() == &block) {
return cur;
}
}
return nullptr;
}

static FailureOr<Operation *>
cloneVecScopeProducerForUse(
Value value, Operation *user, Operation *logicalScopeAnchor,
Expand Down Expand Up @@ -481,7 +473,7 @@ static LogicalResult rematerializeEscapingValueForUserSegments(
continue;
}

Operation *ancestor = getAncestorInBlock(user, block);
Operation *ancestor = pto::getAncestorInBlock(user, &block);
if (!ancestor) {
return failure();
}
Expand Down Expand Up @@ -804,7 +796,7 @@ static LogicalResult repairEscapingSubclusters(Block &block,
ops.push_back(&op);
}

auto flush = [&]() -> FailureOr<bool> {
auto flush = [&pending, &cache, context]() -> FailureOr<bool> {
FailureOr<bool> changed =
fixOneEscapingSubcluster(pending, cache, context);
pending.clear();
Expand Down Expand Up @@ -854,7 +846,7 @@ static LogicalResult inferVecScopesInBlock(Block &block, MLIRContext *context) {

SmallVector<Operation *, mlir::pto::kValue16> pending;

auto flush = [&]() -> LogicalResult {
auto flush = [&pending, context]() -> LogicalResult {
if (failed(wrapGreedySubclusters(pending, context))) {
return failure();
}
Expand Down
16 changes: 4 additions & 12 deletions lib/PTO/Transforms/PTORemoveIdentityTMov.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include "PTO/Transforms/InsertSync/PTOIRTranslator.h"
#include "PTO/Transforms/InsertSync/SyncCommon.h"
#include "PTO/Transforms/Passes.h"
#include "Utils.h"

#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
Expand Down Expand Up @@ -258,23 +259,14 @@ static bool hasSameConcreteAddressRange(const BaseMemInfo *srcInfo,
return srcRootAddr && dstRootAddr && *srcRootAddr == *dstRootAddr;
}

static Operation *getAncestorInBlock(Operation *op, Block *block) {
for (Operation *cur = op; cur; cur = cur->getParentOp()) {
if (cur->getBlock() == block) {
return cur;
}
}
return nullptr;
}

static bool hasUseAfterOp(Value value, Operation *currentOp) {
Block *block = currentOp->getBlock();
for (OpOperand &use : value.getUses()) {
Operation *owner = use.getOwner();
if (owner == currentOp) {
continue;
}
Operation *ancestor = getAncestorInBlock(owner, block);
Operation *ancestor = pto::getAncestorInBlock(owner, block);
if (!ancestor) {
return true;
}
Expand All @@ -293,7 +285,7 @@ static bool hasLaterUseOfSameAddressRange(
if (entry.first == op.getSrc()) {
continue;
}
bool sameRange = llvm::any_of(entry.second, [&](const auto &info) {
bool sameRange = llvm::any_of(entry.second, [dstInfo](const auto &info) {
return hasSameConcreteAddressRange(info.get(), dstInfo);
});
if (sameRange && hasUseAfterOp(entry.first, op)) {
Expand Down Expand Up @@ -371,7 +363,7 @@ struct PTORemoveIdentityTMovPass
SmallVector<TMovOp, mlir::pto::kValue16> identityMoves;
SmallVector<TMovOp, mlir::pto::kValue16> memInfoCandidates;

func.walk([&](TMovOp op) {
func.walk([&identityMoves, &memInfoCandidates](TMovOp op) {
if (!hasPlainTMovSemantics(op) || !hasCompatibleIdentityTypes(op) ||
touchesLowPrecisionElement(op)) {
return;
Expand Down
19 changes: 4 additions & 15 deletions lib/PTO/Transforms/PTOVerifyTFreePass.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

#include "PTO/IR/PTO.h"
#include "PTO/Transforms/Passes.h"
#include "Utils.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Pass/Pass.h"

Expand Down Expand Up @@ -43,22 +44,10 @@ static TFreeOp findMatchingTFree(TPopOp tpopOp) {
return {};
}

static Operation *getTopLevelAncestorInBlock(Operation *op, Block *block) {
Operation *current = op;
while (current && current->getBlock() != block) {
Region *parentRegion = current->getParentRegion();
if (!parentRegion) {
return nullptr;
}
current = parentRegion->getParentOp();
}
return current;
}

static bool hasSamePipeTPopInRegion(Operation *op, Value pipeHandle,
TPopOp current) {
bool found = false;
op->walk([&](TPopOp nestedTpop) {
op->walk([current, pipeHandle, &found](TPopOp nestedTpop) {
if (nestedTpop == current) {
return WalkResult::advance();
}
Expand Down Expand Up @@ -100,7 +89,7 @@ static LogicalResult verifyNoTileUsesAfterTFree(TPopOp tpopOp,
Block *block = tpopOp->getBlock();

for (OpOperand &use : tile.getUses()) {
Operation *topLevelOwner = getTopLevelAncestorInBlock(use.getOwner(), block);
Operation *topLevelOwner = pto::getAncestorInBlock(use.getOwner(), block);
if (!topLevelOwner) {
return tpopOp.emitOpError(
"borrowed tile uses must stay in the same parent block as the producing tpop");
Expand Down Expand Up @@ -129,7 +118,7 @@ struct PTOVerifyTFreePass
func::FuncOp funcOp = getOperation();

SmallVector<TPopOp> tpops;
funcOp.walk([&](TPopOp op) { tpops.push_back(op); });
funcOp.walk([&tpops](TPopOp op) { tpops.push_back(op); });

for (TPopOp tpopOp : tpops) {
if (!isInsideSectionOrAttributedKernel(tpopOp, funcOp)) {
Expand Down
11 changes: 6 additions & 5 deletions lib/PTO/Transforms/TileFusion/FusionAnalysis.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -835,10 +835,11 @@ static void recordLastLocalConsumer(std::optional<unsigned> &lastLocalConsumer,
}
}

static void finalizeBlockLiveness(
Block &block, DenseMap<Operation *, FusionOpKind> &kindByOp,
DenseMap<Operation *, unsigned> &computeNodeByOp,
SmallVectorImpl<MutableLiveness> &mutableLiveness) {
static void
finalizeBlockLiveness(const Block &block,
DenseMap<Operation *, FusionOpKind> &kindByOp,
DenseMap<Operation *, unsigned> &computeNodeByOp,
SmallVectorImpl<MutableLiveness> &mutableLiveness) {
for (MutableLiveness &state : mutableLiveness) {
for (OpOperand &use : state.live.value.getUses()) {
Operation *user = use.getOwner();
Expand Down Expand Up @@ -947,7 +948,7 @@ static void recordWriteInstanceUse(Operation *user,
}
}

static void finalizeWriteInstanceUse(Block &block,
static void finalizeWriteInstanceUse(const Block &block,
const MutableLiveness &storageState,
OpOperand &use,
DFGConstructionState &state) {
Expand Down
35 changes: 15 additions & 20 deletions lib/PTO/Transforms/TileFusion/PTOFusionLoadStoreElision.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include "PTO/Support/CodeConstants.h"
#include "PTO/IR/PTO.h"
#include "PTO/Transforms/Passes.h"
#include "../Utils.h"

#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
Expand Down Expand Up @@ -218,16 +219,8 @@ static Value getCanonicalTrackedValue(Value value) {
return value;
}

static Operation *getTopLevelAncestorInBlock(Operation *op, Block *block) {
for (Operation *cur = op; cur; cur = cur->getParentOp()) {
if (cur->getBlock() == block) {
return cur;
}
}
return nullptr;
}

static Region *getDirectRegionUnderAncestor(Operation *op, Operation *ancestor) {
static Region *getDirectRegionUnderAncestor(Operation *op,
const Operation *ancestor) {
for (Operation *cur = op; cur; cur = cur->getParentOp()) {
Operation *parent = cur->getParentOp();
if (parent == ancestor) {
Expand Down Expand Up @@ -402,7 +395,7 @@ static void pruneTrackedStoresForLoadBase(SmallVectorImpl<TrackedStore> &stores,
stores.clear();
return;
}
llvm::erase_if(stores, [&](const TrackedStore &store) {
llvm::erase_if(stores, [base](const TrackedStore &store) {
return areEquivalentValues(store.base, base);
});
}
Expand All @@ -415,7 +408,7 @@ static bool isTailStoreUseCompatible(
return true;
}
if (context.regionOp->isProperAncestor(owner)) {
Operation *topLevelUser = getTopLevelAncestorInBlock(owner, context.body);
Operation *topLevelUser = pto::getAncestorInBlock(owner, context.body);
if (!topLevelUser) {
return false;
}
Expand All @@ -429,7 +422,7 @@ static bool isTailStoreUseCompatible(
}

Operation *topLevelUser =
getTopLevelAncestorInBlock(owner, context.parentBlock);
pto::getAncestorInBlock(owner, context.parentBlock);
if (!topLevelUser) {
return areMutuallyExclusiveByIfRegion(localScopeOp, owner);
}
Expand Down Expand Up @@ -557,7 +550,7 @@ using RegionContextMap =

static RegionContextMap buildRegionContexts(func::FuncOp func) {
RegionContextMap contexts;
func.walk([&](pto::FusionRegionOp fusionRegion) {
func.walk([&contexts](pto::FusionRegionOp fusionRegion) {
std::optional<FusionRegionStoreContext> context =
buildFusionRegionStoreContext(fusionRegion);
if (context) {
Expand All @@ -570,15 +563,16 @@ static RegionContextMap buildRegionContexts(func::FuncOp func) {
static void elideFusionRegionBodies(func::FuncOp func,
RegionContextMap &contexts,
bool &changed) {
func.walk([&](pto::FusionRegionOp fusionRegion) {
func.walk([&contexts, &changed](pto::FusionRegionOp fusionRegion) {
auto it = contexts.find(fusionRegion.getOperation());
if (it == contexts.end()) {
return;
}
Block &body = fusionRegion.getBody().front();
if (isSupportedStraightLineBlock(body)) {
changed |=
elideLoadStoreRoundTripsInLeafBody(body, &it->second, nullptr);
changed =
elideLoadStoreRoundTripsInLeafBody(body, &it->second, nullptr) ||
changed;
}
});
}
Expand All @@ -591,14 +585,15 @@ static void runElisionForLeafBody(Block *body, Operation *scopeOp,
}
auto it = contexts.find(fusionRegion.getOperation());
if (it != contexts.end()) {
changed |= elideLoadStoreRoundTripsInLeafBody(*body, &it->second, scopeOp);
changed = elideLoadStoreRoundTripsInLeafBody(*body, &it->second, scopeOp) ||
changed;
}
}

template <typename ScopeOp>
static void elideVectorScopeBodies(func::FuncOp func, RegionContextMap &contexts,
bool &changed) {
func.walk([&](ScopeOp scope) {
func.walk([&contexts, &changed](ScopeOp scope) {
auto fusionRegion = scope->template getParentOfType<pto::FusionRegionOp>();
if (fusionRegion && isSupportedStraightLineBlock(scope.getBody().front())) {
runElisionForLeafBody(&scope.getBody().front(), scope, fusionRegion,
Expand All @@ -609,7 +604,7 @@ static void elideVectorScopeBodies(func::FuncOp func, RegionContextMap &contexts

static void elideLoopBodies(func::FuncOp func, RegionContextMap &contexts,
bool &changed) {
func.walk([&](scf::ForOp loop) {
func.walk([&contexts, &changed](scf::ForOp loop) {
if (!isSupportedLoopRoot(loop)) {
return;
}
Expand Down
28 changes: 16 additions & 12 deletions lib/PTO/Transforms/TileFusion/PTOFusionPredicateElision.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -326,16 +326,19 @@ buildFusionRegionPredicateContext(pto::FusionRegionOp fusionRegion,
FusionRegionPredicateContext context;
context.fusionRegion = fusionRegion;

fusionRegion.walk([&](Operation *op) -> WalkResult {
if (op != fusionRegion.getOperation() && isa<pto::FusionRegionOp>(op)) {
return WalkResult::skip();
}

if (std::optional<PltCandidate> candidate = buildPltCandidate(op)) {
context.pltCandidates.push_back(std::move(*candidate));
}
return WalkResult::advance();
});
fusionRegion.walk(
[fusionRegion, &context](Operation *op) mutable -> WalkResult {
const bool isNestedFusionRegion =
op != fusionRegion.getOperation() && isa<pto::FusionRegionOp>(op);
if (isNestedFusionRegion) {
return WalkResult::skip();
}

if (std::optional<PltCandidate> candidate = buildPltCandidate(op)) {
context.pltCandidates.push_back(std::move(*candidate));
}
return WalkResult::advance();
});

populateDominatingCandidateIndices(context.pltCandidates, dominanceInfo);
return context;
Expand Down Expand Up @@ -417,7 +420,8 @@ struct PTOFusionPredicateElisionPass

DominanceInfo &dominanceInfo = getAnalysis<DominanceInfo>();
SmallVector<FusionRegionPredicateContext, mlir::pto::kValue4> fusionContexts;
func.walk([&](pto::FusionRegionOp fusionRegion) {
func.walk([&dominanceInfo,
&fusionContexts](pto::FusionRegionOp fusionRegion) {
FusionRegionPredicateContext context =
buildFusionRegionPredicateContext(fusionRegion, dominanceInfo);
if (!context.pltCandidates.empty()) {
Expand All @@ -427,7 +431,7 @@ struct PTOFusionPredicateElisionPass

bool changed = false;
for (FusionRegionPredicateContext &context : fusionContexts) {
changed |= elideEquivalentPltCandidates(context);
changed = elideEquivalentPltCandidates(context) || changed;
}

if (!changed) {
Expand Down
Loading
Loading