Skip to content

Commit

Permalink
[LV] Assert that there's a resume phi for epilogue loops (NFC).
Browse files Browse the repository at this point in the history
This patch adds an assert to createAndCollectMergePhiForReduction to
make sure there is a resume phi when vectorizing the epilogue loop. This
is needed to set the resume value from the main vector loop.

This assertion guards against the issue caused the revert of
llvm#78304.
  • Loading branch information
fhahn committed May 2, 2024
1 parent 86ab10c commit c7209cb
Showing 1 changed file with 8 additions and 4 deletions.
12 changes: 8 additions & 4 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7450,7 +7450,8 @@ static void AddRuntimeUnrollDisableMetaData(Loop *L) {
static void createAndCollectMergePhiForReduction(
VPInstruction *RedResult,
DenseMap<const RecurrenceDescriptor *, Value *> &ReductionResumeValues,
VPTransformState &State, Loop *OrigLoop, BasicBlock *LoopMiddleBlock) {
VPTransformState &State, Loop *OrigLoop, BasicBlock *LoopMiddleBlock,
bool VectorizingEpilogue) {
if (!RedResult ||
RedResult->getOpcode() != VPInstruction::ComputeReductionResult)
return;
Expand All @@ -7463,6 +7464,9 @@ static void createAndCollectMergePhiForReduction(
State.get(RedResult, VPIteration(State.UF - 1, VPLane::getFirstLane()));
auto *ResumePhi =
dyn_cast<PHINode>(PhiR->getStartValue()->getUnderlyingValue());
assert((!VectorizingEpilogue || ResumePhi) &&
"when vectorizing the epilogue loop, we need a resume phi from main "
"vector loop");

// TODO: bc.merge.rdx should not be created here, instead it should be
// modeled in VPlan.
Expand Down Expand Up @@ -7588,9 +7592,9 @@ LoopVectorizationPlanner::executePlan(
auto *ExitVPBB =
cast<VPBasicBlock>(BestVPlan.getVectorLoopRegion()->getSingleSuccessor());
for (VPRecipeBase &R : *ExitVPBB) {
createAndCollectMergePhiForReduction(dyn_cast<VPInstruction>(&R),
ReductionResumeValues, State, OrigLoop,
State.CFG.VPBB2IRBB[ExitVPBB]);
createAndCollectMergePhiForReduction(
dyn_cast<VPInstruction>(&R), ReductionResumeValues, State, OrigLoop,
State.CFG.VPBB2IRBB[ExitVPBB], ExpandedSCEVs);
}

// 2.6. Maintain Loop Hints
Expand Down

0 comments on commit c7209cb

Please sign in to comment.