From 816068e462888f31f4bcf71f2f728eb77ff2ea15 Mon Sep 17 00:00:00 2001 From: tcwzxx Date: Mon, 19 Aug 2024 20:16:51 +0800 Subject: [PATCH] [NFC][SLP] Remove useless code of the schedule (#104697) Currently, the SLP schedule has two containers of `ScheduleData`: `ExtraScheduleDataMap` and `ScheduleDataMap`. However, the `ScheduleData` in `ExtraScheduleDataMap` is only used to indicate whether the instruction is processed or not and does not participate in the schedule, which is useless. `ScheduleDataMap` is sufficient for this purpose. The `OpValue` member is used only in `ExtraScheduleDataMap`, which is also useless. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 155 +++++------------- 1 file changed, 38 insertions(+), 117 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 9ecd8160a97891..dee6d688b1b903 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -795,16 +795,6 @@ struct InstructionsState { } // end anonymous namespace -/// Chooses the correct key for scheduling data. If \p Op has the same (or -/// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p -/// OpValue. -static Value *isOneOf(const InstructionsState &S, Value *Op) { - auto *I = dyn_cast(Op); - if (I && S.isOpcodeOrAlt(I)) - return Op; - return S.OpValue; -} - /// \returns true if \p Opcode is allowed as part of the main/alternate /// instruction for SLP vectorization. /// @@ -3583,14 +3573,14 @@ class BoUpSLP { ScheduleData() = default; - void init(int BlockSchedulingRegionID, Value *OpVal) { + void init(int BlockSchedulingRegionID, Instruction *I) { FirstInBundle = this; NextInBundle = nullptr; NextLoadStore = nullptr; IsScheduled = false; SchedulingRegionID = BlockSchedulingRegionID; clearDependencies(); - OpValue = OpVal; + Inst = I; TE = nullptr; } @@ -3696,9 +3686,6 @@ class BoUpSLP { Instruction *Inst = nullptr; - /// Opcode of the current instruction in the schedule data. - Value *OpValue = nullptr; - /// The TreeEntry that this instruction corresponds to. TreeEntry *TE = nullptr; @@ -3815,18 +3802,6 @@ class BoUpSLP { return nullptr; } - ScheduleData *getScheduleData(Value *V, Value *Key) { - if (V == Key) - return getScheduleData(V); - auto I = ExtraScheduleDataMap.find(V); - if (I != ExtraScheduleDataMap.end()) { - ScheduleData *SD = I->second.lookup(Key); - if (SD && isInSchedulingRegion(SD)) - return SD; - } - return nullptr; - } - bool isInSchedulingRegion(ScheduleData *SD) const { return SD->SchedulingRegionID == SchedulingRegionID; } @@ -3840,27 +3815,24 @@ class BoUpSLP { for (ScheduleData *BundleMember = SD; BundleMember; BundleMember = BundleMember->NextInBundle) { - if (BundleMember->Inst != BundleMember->OpValue) - continue; // Handle the def-use chain dependencies. // Decrement the unscheduled counter and insert to ready list if ready. auto &&DecrUnsched = [this, &ReadyList](Instruction *I) { - doForAllOpcodes(I, [&ReadyList](ScheduleData *OpDef) { - if (OpDef && OpDef->hasValidDependencies() && - OpDef->incrementUnscheduledDeps(-1) == 0) { - // There are no more unscheduled dependencies after - // decrementing, so we can put the dependent instruction - // into the ready list. - ScheduleData *DepBundle = OpDef->FirstInBundle; - assert(!DepBundle->IsScheduled && - "already scheduled bundle gets ready"); - ReadyList.insert(DepBundle); - LLVM_DEBUG(dbgs() - << "SLP: gets ready (def): " << *DepBundle << "\n"); - } - }); + ScheduleData *OpDef = getScheduleData(I); + if (OpDef && OpDef->hasValidDependencies() && + OpDef->incrementUnscheduledDeps(-1) == 0) { + // There are no more unscheduled dependencies after + // decrementing, so we can put the dependent instruction + // into the ready list. + ScheduleData *DepBundle = OpDef->FirstInBundle; + assert(!DepBundle->IsScheduled && + "already scheduled bundle gets ready"); + ReadyList.insert(DepBundle); + LLVM_DEBUG(dbgs() + << "SLP: gets ready (def): " << *DepBundle << "\n"); + } }; // If BundleMember is a vector bundle, its operands may have been @@ -3944,8 +3916,7 @@ class BoUpSLP { "primary schedule data not in window?"); assert(isInSchedulingRegion(SD->FirstInBundle) && "entire bundle in window!"); - (void)SD; - doForAllOpcodes(I, [](ScheduleData *SD) { SD->verify(); }); + SD->verify(); } for (auto *SD : ReadyInsts) { @@ -3955,29 +3926,17 @@ class BoUpSLP { } } - void doForAllOpcodes(Value *V, - function_ref Action) { - if (ScheduleData *SD = getScheduleData(V)) - Action(SD); - auto I = ExtraScheduleDataMap.find(V); - if (I != ExtraScheduleDataMap.end()) - for (auto &P : I->second) - if (isInSchedulingRegion(P.second)) - Action(P.second); - } - /// Put all instructions into the ReadyList which are ready for scheduling. template void initialFillReadyList(ReadyListType &ReadyList) { for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) { - doForAllOpcodes(I, [&](ScheduleData *SD) { - if (SD->isSchedulingEntity() && SD->hasValidDependencies() && - SD->isReady()) { - ReadyList.insert(SD); - LLVM_DEBUG(dbgs() - << "SLP: initially in ready list: " << *SD << "\n"); - } - }); + ScheduleData *SD = getScheduleData(I); + if (SD && SD->isSchedulingEntity() && SD->hasValidDependencies() && + SD->isReady()) { + ReadyList.insert(SD); + LLVM_DEBUG(dbgs() + << "SLP: initially in ready list: " << *SD << "\n"); + } } } @@ -4035,10 +3994,6 @@ class BoUpSLP { /// ScheduleData structures are recycled. DenseMap ScheduleDataMap; - /// Attaches ScheduleData to Instruction with the leading key. - DenseMap> - ExtraScheduleDataMap; - /// The ready-list for scheduling (only used for the dry-run). SetVector ReadyInsts; @@ -11898,8 +11853,7 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) { auto *Bundle = BlocksSchedules[BB]->getScheduleData(V); if (Bundle && Bundle->isPartOfBundle()) for (; Bundle; Bundle = Bundle->NextInBundle) - if (Bundle->OpValue == Bundle->Inst) - Res.second = Bundle->Inst; + Res.second = Bundle->Inst; } // LastInst can still be null at this point if there's either not an entry @@ -14966,7 +14920,8 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef VL, BoUpSLP *SLP, // initial bundle to the region. if (ScheduleEnd != OldScheduleEnd) { for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) - doForAllOpcodes(I, [](ScheduleData *SD) { SD->clearDependencies(); }); + if (ScheduleData *SD = getScheduleData(I)) + SD->clearDependencies(); ReSchedule = true; } if (Bundle) { @@ -15085,37 +15040,21 @@ BoUpSLP::ScheduleData *BoUpSLP::BlockScheduling::allocateScheduleDataChunks() { return &(ScheduleDataChunks.back()[ChunkPos++]); } -bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V, - const InstructionsState &S) { - if (getScheduleData(V, isOneOf(S, V))) - return true; +bool BoUpSLP::BlockScheduling::extendSchedulingRegion( + Value *V, const InstructionsState &S) { Instruction *I = dyn_cast(V); assert(I && "bundle member must be an instruction"); assert(!isa(I) && !isVectorLikeInstWithConstOps(I) && !doesNotNeedToBeScheduled(I) && "phi nodes/insertelements/extractelements/extractvalues don't need to " "be scheduled"); - auto &&CheckScheduleForI = [this, &S](Instruction *I) -> bool { - ScheduleData *ISD = getScheduleData(I); - if (!ISD) - return false; - assert(isInSchedulingRegion(ISD) && - "ScheduleData not in scheduling region"); - ScheduleData *SD = allocateScheduleDataChunks(); - SD->Inst = I; - SD->init(SchedulingRegionID, S.OpValue); - ExtraScheduleDataMap[I][S.OpValue] = SD; - return true; - }; - if (CheckScheduleForI(I)) + if (getScheduleData(I)) return true; if (!ScheduleStart) { // It's the first instruction in the new region. initScheduleData(I, I->getNextNode(), nullptr, nullptr); ScheduleStart = I; ScheduleEnd = I->getNextNode(); - if (isOneOf(S, I) != I) - CheckScheduleForI(I); assert(ScheduleEnd && "tried to vectorize a terminator?"); LLVM_DEBUG(dbgs() << "SLP: initialize schedule region to " << *I << "\n"); return true; @@ -15154,8 +15093,6 @@ bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V, "Instruction is in wrong basic block."); initScheduleData(I, ScheduleStart, nullptr, FirstLoadStoreInRegion); ScheduleStart = I; - if (isOneOf(S, I) != I) - CheckScheduleForI(I); LLVM_DEBUG(dbgs() << "SLP: extend schedule region start to " << *I << "\n"); return true; @@ -15168,8 +15105,6 @@ bool BoUpSLP::BlockScheduling::extendSchedulingRegion(Value *V, initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion, nullptr); ScheduleEnd = I->getNextNode(); - if (isOneOf(S, I) != I) - CheckScheduleForI(I); assert(ScheduleEnd && "tried to vectorize a terminator?"); LLVM_DEBUG(dbgs() << "SLP: extend schedule region end to " << *I << "\n"); return true; @@ -15188,7 +15123,6 @@ void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI, if (!SD) { SD = allocateScheduleDataChunks(); ScheduleDataMap[I] = SD; - SD->Inst = I; } assert(!isInSchedulingRegion(SD) && "new ScheduleData already in scheduling region"); @@ -15242,8 +15176,8 @@ void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD, BundleMember->resetUnscheduledDeps(); // Handle def-use chain dependencies. - if (BundleMember->OpValue != BundleMember->Inst) { - if (ScheduleData *UseSD = getScheduleData(BundleMember->Inst)) { + for (User *U : BundleMember->Inst->users()) { + if (ScheduleData *UseSD = getScheduleData(cast(U))) { BundleMember->Dependencies++; ScheduleData *DestBundle = UseSD->FirstInBundle; if (!DestBundle->IsScheduled) @@ -15251,17 +15185,6 @@ void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD, if (!DestBundle->hasValidDependencies()) WorkList.push_back(DestBundle); } - } else { - for (User *U : BundleMember->Inst->users()) { - if (ScheduleData *UseSD = getScheduleData(cast(U))) { - BundleMember->Dependencies++; - ScheduleData *DestBundle = UseSD->FirstInBundle; - if (!DestBundle->IsScheduled) - BundleMember->incrementUnscheduledDeps(1); - if (!DestBundle->hasValidDependencies()) - WorkList.push_back(DestBundle); - } - } } auto MakeControlDependent = [&](Instruction *I) { @@ -15409,12 +15332,12 @@ void BoUpSLP::BlockScheduling::resetSchedule() { assert(ScheduleStart && "tried to reset schedule on block which has not been scheduled"); for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) { - doForAllOpcodes(I, [&](ScheduleData *SD) { + if (ScheduleData *SD = getScheduleData(I)) { assert(isInSchedulingRegion(SD) && "ScheduleData not in scheduling region"); SD->IsScheduled = false; SD->resetUnscheduledDeps(); - }); + } } ReadyInsts.clear(); } @@ -15449,7 +15372,7 @@ void BoUpSLP::scheduleBlock(BlockScheduling *BS) { int Idx = 0; for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd; I = I->getNextNode()) { - BS->doForAllOpcodes(I, [this, &Idx, BS](ScheduleData *SD) { + if (ScheduleData *SD = BS->getScheduleData(I)) { TreeEntry *SDTE = getTreeEntry(SD->Inst); (void)SDTE; assert((isVectorLikeInstWithConstOps(SD->Inst) || @@ -15460,7 +15383,7 @@ void BoUpSLP::scheduleBlock(BlockScheduling *BS) { if (SD->isSchedulingEntity() && SD->isPartOfBundle()) BS->calculateDependencies(SD, false, this); - }); + } } BS->initialFillReadyList(ReadyInsts); @@ -15492,11 +15415,9 @@ void BoUpSLP::scheduleBlock(BlockScheduling *BS) { #if !defined(NDEBUG) || defined(EXPENSIVE_CHECKS) // Check that all schedulable entities got scheduled for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd; I = I->getNextNode()) { - BS->doForAllOpcodes(I, [&](ScheduleData *SD) { - if (SD->isSchedulingEntity() && SD->hasValidDependencies()) { - assert(SD->IsScheduled && "must be scheduled at this point"); - } - }); + ScheduleData *SD = BS->getScheduleData(I); + if (SD && SD->isSchedulingEntity() && SD->hasValidDependencies()) + assert(SD->IsScheduled && "must be scheduled at this point"); } #endif