Skip to content

Commit aef8f42

Browse files
amanasifkhalidamanasifkhalid
authored
JIT: Check for profile consistency throughout JIT backend (#111684)
Part of #107749.
1 parent f652094 commit aef8f42

File tree

5 files changed

+93
-35
lines changed

5 files changed

+93
-35
lines changed

‎src/coreclr/jit/compiler.cpp

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4872,11 +4872,6 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl
48724872
DoPhase(this, PHASE_COMPUTE_DOMINATORS, &Compiler::fgComputeDominators);
48734873
}
48744874

4875-
// Drop back to just checking profile likelihoods.
4876-
//
4877-
activePhaseChecks &= ~PhaseChecks::CHECK_PROFILE;
4878-
activePhaseChecks |= PhaseChecks::CHECK_LIKELIHOODS;
4879-
48804875
#ifdef DEBUG
48814876
fgDebugCheckLinks();
48824877
#endif
@@ -5157,11 +5152,6 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl
51575152
DoPhase(this, PHASE_SWITCH_RECOGNITION, &Compiler::optSwitchRecognition);
51585153
}
51595154

5160-
// Drop back to just checking profile likelihoods.
5161-
//
5162-
activePhaseChecks &= ~PhaseChecks::CHECK_PROFILE;
5163-
activePhaseChecks |= PhaseChecks::CHECK_LIKELIHOODS;
5164-
51655155
#ifdef DEBUG
51665156
// Stash the current estimate of the function's size if necessary.
51675157
if (verbose && opts.OptimizationEnabled())

‎src/coreclr/jit/helperexpansion.cpp

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -744,7 +744,7 @@ bool Compiler::fgExpandThreadLocalAccessForCallNativeAOT(BasicBlock** pBlock, St
744744
fastPathBb->inheritWeight(prevBb);
745745

746746
// fallback will just execute first time
747-
fallbackBb->bbSetRunRarely();
747+
fallbackBb->inheritWeightPercentage(tlsRootNullCondBB, 0);
748748

749749
fgRedirectTargetEdge(prevBb, tlsRootNullCondBB);
750750

@@ -1180,7 +1180,7 @@ bool Compiler::fgExpandThreadLocalAccessForCall(BasicBlock** pBlock, Statement*
11801180
fastPathBb->inheritWeight(prevBb);
11811181

11821182
// fallback will just execute first time
1183-
fallbackBb->bbSetRunRarely();
1183+
fallbackBb->inheritWeightPercentage(prevBb, 0);
11841184

11851185
// All blocks are expected to be in the same EH region
11861186
assert(BasicBlock::sameEHRegion(prevBb, block));
@@ -1545,7 +1545,7 @@ bool Compiler::fgExpandStaticInitForCall(BasicBlock** pBlock, Statement* stmt, G
15451545

15461546
block->inheritWeight(prevBb);
15471547
isInitedBb->inheritWeight(prevBb);
1548-
helperCallBb->bbSetRunRarely();
1548+
helperCallBb->inheritWeightPercentage(isInitedBb, 0);
15491549

15501550
// All blocks are expected to be in the same EH region
15511551
assert(BasicBlock::sameEHRegion(prevBb, block));
@@ -1847,6 +1847,7 @@ bool Compiler::fgVNBasedIntrinsicExpansionForCall_ReadUtf8(BasicBlock** pBlock,
18471847
//
18481848
// Redirect prevBb to lengthCheckBb
18491849
fgRedirectTargetEdge(prevBb, lengthCheckBb);
1850+
lengthCheckBb->inheritWeight(prevBb);
18501851
assert(prevBb->JumpsToNext());
18511852

18521853
{
@@ -1859,6 +1860,11 @@ bool Compiler::fgVNBasedIntrinsicExpansionForCall_ReadUtf8(BasicBlock** pBlock,
18591860
// review: we assume length check always succeeds??
18601861
trueEdge->setLikelihood(1.0);
18611862
falseEdge->setLikelihood(0.0);
1863+
1864+
if (lengthCheckBb->hasProfileWeight())
1865+
{
1866+
fastpathBb->setBBProfileWeight(falseEdge->getLikelyWeight());
1867+
}
18621868
}
18631869

18641870
{
@@ -1869,10 +1875,8 @@ bool Compiler::fgVNBasedIntrinsicExpansionForCall_ReadUtf8(BasicBlock** pBlock,
18691875
}
18701876

18711877
//
1872-
// Re-distribute weights
1878+
// Ensure all flow out of prevBb converges into block
18731879
//
1874-
lengthCheckBb->inheritWeight(prevBb);
1875-
fastpathBb->inheritWeight(lengthCheckBb);
18761880
block->inheritWeight(prevBb);
18771881

18781882
// All blocks are expected to be in the same EH region
@@ -2551,11 +2555,18 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt,
25512555
trueEdge->setLikelihood(nullcheckTrueLikelihood);
25522556
}
25532557

2558+
// Set nullcheckBb's weight here, so we can propagate it to its successors below
2559+
nullcheckBb->inheritWeight(firstBb);
2560+
25542561
if (typeCheckNotNeeded)
25552562
{
25562563
FlowEdge* const falseEdge = fgAddRefPred(fallbackBb, nullcheckBb);
25572564
nullcheckBb->SetFalseEdge(falseEdge);
25582565
falseEdge->setLikelihood(nullcheckFalseLikelihood);
2566+
fallbackBb->inheritWeight(nullcheckBb);
2567+
fallbackBb->scaleBBWeight(nullcheckFalseLikelihood);
2568+
lastBb->inheritWeight(nullcheckBb);
2569+
lastBb->scaleBBWeight(nullcheckTrueLikelihood);
25592570

25602571
typeCheckSucceedBb = nullptr;
25612572
}
@@ -2631,7 +2642,6 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt,
26312642
// The same goes for inherited weights -- the block where we test for B will have
26322643
// the weight of A times the likelihood that A's test fails, etc.
26332644
//
2634-
nullcheckBb->inheritWeight(firstBb);
26352645
weight_t sumOfPreviousLikelihood = 0;
26362646
for (int candidateId = 0; candidateId < numOfCandidates; candidateId++)
26372647
{
@@ -2666,28 +2676,22 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt,
26662676
sumOfPreviousLikelihood += likelihood;
26672677
}
26682678

2669-
if (fallbackBb->KindIs(BBJ_THROW))
2670-
{
2671-
fallbackBb->bbSetRunRarely();
2672-
}
2673-
else
2679+
fallbackBb->inheritWeight(lastTypeCheckBb);
2680+
fallbackBb->scaleBBWeight(lastTypeCheckBb->GetFalseEdge()->getLikelihood());
2681+
2682+
if (fallbackBb->KindIs(BBJ_ALWAYS))
26742683
{
2675-
assert(fallbackBb->KindIs(BBJ_ALWAYS));
26762684
FlowEdge* const newEdge = fgAddRefPred(lastBb, fallbackBb);
26772685
fallbackBb->SetTargetEdge(newEdge);
2678-
fallbackBb->inheritWeight(lastTypeCheckBb);
2679-
weight_t lastTypeCheckFailedLikelihood = lastTypeCheckBb->GetFalseEdge()->getLikelihood();
2680-
fallbackBb->scaleBBWeight(lastTypeCheckFailedLikelihood);
26812686
}
26822687

26832688
if (!typeCheckNotNeeded)
26842689
{
26852690
typeCheckSucceedBb->inheritWeight(typeChecksBbs[0]);
26862691
typeCheckSucceedBb->scaleBBWeight(sumOfPreviousLikelihood);
2692+
lastBb->inheritWeight(firstBb);
26872693
}
26882694

2689-
lastBb->inheritWeight(firstBb);
2690-
26912695
//
26922696
// Validate EH regions
26932697
//

‎src/coreclr/jit/jiteh.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2660,7 +2660,7 @@ bool Compiler::fgCreateFiltersForGenericExceptions()
26602660
filterBb->bbCodeOffs = handlerBb->bbCodeOffs;
26612661
filterBb->bbHndIndex = handlerBb->bbHndIndex;
26622662
filterBb->bbTryIndex = handlerBb->bbTryIndex;
2663-
filterBb->bbSetRunRarely();
2663+
filterBb->inheritWeightPercentage(handlerBb, 0);
26642664
filterBb->SetFlags(BBF_INTERNAL | BBF_DONT_REMOVE);
26652665

26662666
handlerBb->bbCatchTyp = BBCT_FILTER_HANDLER;

‎src/coreclr/jit/lower.cpp

Lines changed: 59 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1071,11 +1071,11 @@ GenTree* Lowering::LowerSwitch(GenTree* node)
10711071
for (unsigned i = 0; i < jumpCnt - 1; ++i)
10721072
{
10731073
assert(currentBlock != nullptr);
1074-
BasicBlock* const targetBlock = jumpTab[i]->getDestinationBlock();
10751074

10761075
// Remove the switch from the predecessor list of this case target's block.
10771076
// We'll add the proper new predecessor edge later.
1078-
FlowEdge* const oldEdge = jumpTab[i];
1077+
FlowEdge* const oldEdge = jumpTab[i];
1078+
BasicBlock* const targetBlock = oldEdge->getDestinationBlock();
10791079

10801080
// Compute the likelihood that this test is successful.
10811081
// Divide by number of cases still sharing this edge (reduces likelihood)
@@ -1136,8 +1136,9 @@ GenTree* Lowering::LowerSwitch(GenTree* node)
11361136
{
11371137
BasicBlock* const newBlock = comp->fgNewBBafter(BBJ_ALWAYS, currentBlock, true);
11381138
FlowEdge* const newEdge = comp->fgAddRefPred(newBlock, currentBlock);
1139-
currentBlock = newBlock;
1140-
currentBBRange = &LIR::AsRange(currentBlock);
1139+
newBlock->inheritWeight(currentBlock);
1140+
currentBlock = newBlock;
1141+
currentBBRange = &LIR::AsRange(currentBlock);
11411142
afterDefaultCondBlock->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge);
11421143
}
11431144

@@ -1212,6 +1213,25 @@ GenTree* Lowering::LowerSwitch(GenTree* node)
12121213
currentBlock->RemoveFlags(BBF_DONT_REMOVE);
12131214
comp->fgRemoveBlock(currentBlock, /* unreachable */ false); // It's an empty block.
12141215
}
1216+
1217+
// Update flow into switch targets
1218+
if (afterDefaultCondBlock->hasProfileWeight())
1219+
{
1220+
bool profileInconsistent = false;
1221+
for (unsigned i = 0; i < jumpCnt - 1; i++)
1222+
{
1223+
BasicBlock* const targetBlock = jumpTab[i]->getDestinationBlock();
1224+
targetBlock->setBBProfileWeight(targetBlock->computeIncomingWeight());
1225+
profileInconsistent |= (targetBlock->NumSucc() > 0);
1226+
}
1227+
1228+
if (profileInconsistent)
1229+
{
1230+
JITDUMP("Switch lowering: Flow out of " FMT_BB " needs to be propagated. Data %s inconsistent.\n",
1231+
afterDefaultCondBlock->bbNum, comp->fgPgoConsistent ? "is now" : "was already");
1232+
comp->fgPgoConsistent = false;
1233+
}
1234+
}
12151235
}
12161236
else
12171237
{
@@ -1265,11 +1285,28 @@ GenTree* Lowering::LowerSwitch(GenTree* node)
12651285
JITDUMP("Zero weight switch block " FMT_BB ", distributing likelihoods equally per case\n",
12661286
afterDefaultCondBlock->bbNum);
12671287
// jumpCnt-1 here because we peeled the default after copying this value.
1268-
weight_t const newLikelihood = 1.0 / (jumpCnt - 1);
1288+
weight_t const newLikelihood = 1.0 / (jumpCnt - 1);
1289+
bool profileInconsistent = false;
12691290
for (unsigned i = 0; i < successors.numDistinctSuccs; i++)
12701291
{
1271-
FlowEdge* const edge = successors.nonDuplicates[i];
1292+
FlowEdge* const edge = successors.nonDuplicates[i];
1293+
weight_t const oldEdgeWeight = edge->getLikelyWeight();
12721294
edge->setLikelihood(newLikelihood * edge->getDupCount());
1295+
weight_t const newEdgeWeight = edge->getLikelyWeight();
1296+
1297+
if (afterDefaultCondBlock->hasProfileWeight())
1298+
{
1299+
BasicBlock* const targetBlock = edge->getDestinationBlock();
1300+
targetBlock->increaseBBProfileWeight(newEdgeWeight - oldEdgeWeight);
1301+
profileInconsistent |= (targetBlock->NumSucc() > 0);
1302+
}
1303+
}
1304+
1305+
if (profileInconsistent)
1306+
{
1307+
JITDUMP("Switch lowering: Flow out of " FMT_BB " needs to be propagated. Data %s inconsistent.\n",
1308+
afterDefaultCondBlock->bbNum, comp->fgPgoConsistent ? "is now" : "was already");
1309+
comp->fgPgoConsistent = false;
12731310
}
12741311
}
12751312
else
@@ -1452,6 +1489,22 @@ bool Lowering::TryLowerSwitchToBitTest(FlowEdge* jumpTable[],
14521489

14531490
bbSwitch->SetCond(case1Edge, case0Edge);
14541491

1492+
//
1493+
// Update profile
1494+
//
1495+
if (bbSwitch->hasProfileWeight())
1496+
{
1497+
bbCase0->setBBProfileWeight(bbCase0->computeIncomingWeight());
1498+
bbCase1->setBBProfileWeight(bbCase1->computeIncomingWeight());
1499+
1500+
if ((bbCase0->NumSucc() > 0) || (bbCase1->NumSucc() > 0))
1501+
{
1502+
JITDUMP("TryLowerSwitchToBitTest: Flow out of " FMT_BB " needs to be propagated. Data %s inconsistent.\n",
1503+
bbSwitch->bbNum, comp->fgPgoConsistent ? "is now" : "was already");
1504+
comp->fgPgoConsistent = false;
1505+
}
1506+
}
1507+
14551508
var_types bitTableType = (bitCount <= (genTypeSize(TYP_INT) * 8)) ? TYP_INT : TYP_LONG;
14561509
GenTree* bitTableIcon = comp->gtNewIconNode(bitTable, bitTableType);
14571510

‎src/coreclr/jit/optimizer.cpp

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2240,6 +2240,8 @@ bool Compiler::optInvertWhileLoop(BasicBlock* block)
22402240
//
22412241
bNewCond->inheritWeight(block);
22422242

2243+
const weight_t totalWeight = bTest->bbWeight;
2244+
22432245
if (haveProfileWeights)
22442246
{
22452247
bTest->decreaseBBProfileWeight(block->bbWeight);
@@ -2300,6 +2302,15 @@ bool Compiler::optInvertWhileLoop(BasicBlock* block)
23002302
}
23012303
}
23022304

2305+
const weight_t loopWeight = bTest->bbWeight;
2306+
const weight_t nonLoopWeight = bNewCond->bbWeight;
2307+
if (haveProfileWeights && !fgProfileWeightsConsistent(totalWeight, loopWeight + nonLoopWeight))
2308+
{
2309+
JITDUMP("Redirecting flow from " FMT_BB " to " FMT_BB " introduced inconsistency. Data %s inconsistent.\n",
2310+
bTest->bbNum, bNewCond->bbNum, fgPgoConsistent ? "is now" : "was already");
2311+
fgPgoConsistent = false;
2312+
}
2313+
23032314
#ifdef DEBUG
23042315
if (verbose)
23052316
{

0 commit comments

Comments
 (0)