-
-
Notifications
You must be signed in to change notification settings - Fork 261
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add missing ldc-profgen tool for LLVM 18
- Loading branch information
Showing
14 changed files
with
6,393 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
|
||
set(LLVM_LINK_COMPONENTS | ||
AllTargetsDescs | ||
AllTargetsDisassemblers | ||
AllTargetsInfos | ||
DebugInfoDWARF | ||
Core | ||
MC | ||
IPO | ||
MCDisassembler | ||
Object | ||
ProfileData | ||
Support | ||
Symbolize | ||
TargetParser | ||
) | ||
|
||
add_llvm_tool(llvm-profgen | ||
llvm-profgen.cpp | ||
PerfReader.cpp | ||
CSPreInliner.cpp | ||
ProfiledBinary.cpp | ||
ProfileGenerator.cpp | ||
MissingFrameInferrer.cpp | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,316 @@ | ||
//===-- CSPreInliner.cpp - Profile guided preinliner -------------- C++ -*-===// | ||
// | ||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||
// See https://llvm.org/LICENSE.txt for license information. | ||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||
// | ||
//===----------------------------------------------------------------------===// | ||
|
||
#include "CSPreInliner.h" | ||
#include "ProfiledBinary.h" | ||
#include "llvm/ADT/SCCIterator.h" | ||
#include "llvm/ADT/Statistic.h" | ||
#include "llvm/DebugInfo/Symbolize/SymbolizableModule.h" | ||
#include "llvm/Transforms/IPO/SampleProfile.h" | ||
#include <cstdint> | ||
#include <queue> | ||
|
||
#define DEBUG_TYPE "cs-preinliner" | ||
|
||
using namespace llvm; | ||
using namespace sampleprof; | ||
|
||
STATISTIC(PreInlNumCSInlined, | ||
"Number of functions inlined with context sensitive profile"); | ||
STATISTIC(PreInlNumCSNotInlined, | ||
"Number of functions not inlined with context sensitive profile"); | ||
STATISTIC(PreInlNumCSInlinedHitMinLimit, | ||
"Number of functions with FDO inline stopped due to min size limit"); | ||
STATISTIC(PreInlNumCSInlinedHitMaxLimit, | ||
"Number of functions with FDO inline stopped due to max size limit"); | ||
STATISTIC( | ||
PreInlNumCSInlinedHitGrowthLimit, | ||
"Number of functions with FDO inline stopped due to growth size limit"); | ||
|
||
// The switches specify inline thresholds used in SampleProfileLoader inlining. | ||
// TODO: the actual threshold to be tuned here because the size here is based | ||
// on machine code not LLVM IR. | ||
namespace llvm { | ||
cl::opt<bool> EnableCSPreInliner( | ||
"csspgo-preinliner", cl::Hidden, cl::init(true), | ||
cl::desc("Run a global pre-inliner to merge context profile based on " | ||
"estimated global top-down inline decisions")); | ||
|
||
cl::opt<bool> UseContextCostForPreInliner( | ||
"use-context-cost-for-preinliner", cl::Hidden, cl::init(true), | ||
cl::desc("Use context-sensitive byte size cost for preinliner decisions")); | ||
} // namespace llvm | ||
|
||
static cl::opt<bool> SamplePreInlineReplay( | ||
"csspgo-replay-preinline", cl::Hidden, cl::init(false), | ||
cl::desc( | ||
"Replay previous inlining and adjust context profile accordingly")); | ||
|
||
static cl::opt<int> CSPreinlMultiplierForPrevInl( | ||
"csspgo-preinliner-multiplier-for-previous-inlining", cl::Hidden, | ||
cl::init(100), | ||
cl::desc( | ||
"Multiplier to bump up callsite threshold for previous inlining.")); | ||
|
||
CSPreInliner::CSPreInliner(SampleContextTracker &Tracker, | ||
ProfiledBinary &Binary, ProfileSummary *Summary) | ||
: UseContextCost(UseContextCostForPreInliner), | ||
// TODO: Pass in a guid-to-name map in order for | ||
// ContextTracker.getFuncNameFor to work, if `Profiles` can have md5 codes | ||
// as their profile context. | ||
ContextTracker(Tracker), Binary(Binary), Summary(Summary) { | ||
// Set default preinliner hot/cold call site threshold tuned with CSSPGO. | ||
// for good performance with reasonable profile size. | ||
if (!SampleHotCallSiteThreshold.getNumOccurrences()) | ||
SampleHotCallSiteThreshold = 1500; | ||
if (!SampleColdCallSiteThreshold.getNumOccurrences()) | ||
SampleColdCallSiteThreshold = 0; | ||
if (!ProfileInlineLimitMax.getNumOccurrences()) | ||
ProfileInlineLimitMax = 50000; | ||
} | ||
|
||
std::vector<FunctionId> CSPreInliner::buildTopDownOrder() { | ||
std::vector<FunctionId> Order; | ||
// Trim cold edges to get a more stable call graph. This allows for a more | ||
// stable top-down order which in turns helps the stablity of the generated | ||
// profile from run to run. | ||
uint64_t ColdCountThreshold = ProfileSummaryBuilder::getColdCountThreshold( | ||
(Summary->getDetailedSummary())); | ||
ProfiledCallGraph ProfiledCG(ContextTracker, ColdCountThreshold); | ||
|
||
// Now that we have a profiled call graph, construct top-down order | ||
// by building up SCC and reversing SCC order. | ||
scc_iterator<ProfiledCallGraph *> I = scc_begin(&ProfiledCG); | ||
while (!I.isAtEnd()) { | ||
auto Range = *I; | ||
if (SortProfiledSCC) { | ||
// Sort nodes in one SCC based on callsite hotness. | ||
scc_member_iterator<ProfiledCallGraph *> SI(*I); | ||
Range = *SI; | ||
} | ||
for (auto *Node : Range) { | ||
if (Node != ProfiledCG.getEntryNode()) | ||
Order.push_back(Node->Name); | ||
} | ||
++I; | ||
} | ||
std::reverse(Order.begin(), Order.end()); | ||
|
||
return Order; | ||
} | ||
|
||
bool CSPreInliner::getInlineCandidates(ProfiledCandidateQueue &CQueue, | ||
const FunctionSamples *CallerSamples) { | ||
assert(CallerSamples && "Expect non-null caller samples"); | ||
|
||
// Ideally we want to consider everything a function calls, but as far as | ||
// context profile is concerned, only those frames that are children of | ||
// current one in the trie is relavent. So we walk the trie instead of call | ||
// targets from function profile. | ||
ContextTrieNode *CallerNode = | ||
ContextTracker.getContextNodeForProfile(CallerSamples); | ||
|
||
bool HasNewCandidate = false; | ||
for (auto &Child : CallerNode->getAllChildContext()) { | ||
ContextTrieNode *CalleeNode = &Child.second; | ||
FunctionSamples *CalleeSamples = CalleeNode->getFunctionSamples(); | ||
if (!CalleeSamples) | ||
continue; | ||
|
||
// Call site count is more reliable, so we look up the corresponding call | ||
// target profile in caller's context profile to retrieve call site count. | ||
uint64_t CalleeEntryCount = CalleeSamples->getHeadSamplesEstimate(); | ||
uint64_t CallsiteCount = 0; | ||
LineLocation Callsite = CalleeNode->getCallSiteLoc(); | ||
if (auto CallTargets = CallerSamples->findCallTargetMapAt(Callsite)) { | ||
auto It = CallTargets->find(CalleeSamples->getFunction()); | ||
if (It != CallTargets->end()) | ||
CallsiteCount = It->second; | ||
} | ||
|
||
// TODO: call site and callee entry count should be mostly consistent, add | ||
// check for that. | ||
HasNewCandidate = true; | ||
uint32_t CalleeSize = getFuncSize(CalleeNode); | ||
CQueue.emplace(CalleeSamples, std::max(CallsiteCount, CalleeEntryCount), | ||
CalleeSize); | ||
} | ||
|
||
return HasNewCandidate; | ||
} | ||
|
||
uint32_t CSPreInliner::getFuncSize(const ContextTrieNode *ContextNode) { | ||
if (UseContextCost) | ||
return Binary.getFuncSizeForContext(ContextNode); | ||
|
||
return ContextNode->getFunctionSamples()->getBodySamples().size(); | ||
} | ||
|
||
bool CSPreInliner::shouldInline(ProfiledInlineCandidate &Candidate) { | ||
bool WasInlined = | ||
Candidate.CalleeSamples->getContext().hasAttribute(ContextWasInlined); | ||
// If replay inline is requested, simply follow the inline decision of the | ||
// profiled binary. | ||
if (SamplePreInlineReplay) | ||
return WasInlined; | ||
|
||
unsigned int SampleThreshold = SampleColdCallSiteThreshold; | ||
uint64_t ColdCountThreshold = ProfileSummaryBuilder::getColdCountThreshold( | ||
(Summary->getDetailedSummary())); | ||
|
||
if (Candidate.CallsiteCount <= ColdCountThreshold) | ||
SampleThreshold = SampleColdCallSiteThreshold; | ||
else { | ||
// Linearly adjust threshold based on normalized hotness, i.e, a value in | ||
// [0,1]. Use 10% cutoff instead of the max count as the normalization | ||
// upperbound for stability. | ||
double NormalizationUpperBound = | ||
ProfileSummaryBuilder::getEntryForPercentile( | ||
Summary->getDetailedSummary(), 100000 /* 10% */) | ||
.MinCount; | ||
double NormalizationLowerBound = ColdCountThreshold; | ||
double NormalizedHotness = | ||
(Candidate.CallsiteCount - NormalizationLowerBound) / | ||
(NormalizationUpperBound - NormalizationLowerBound); | ||
if (NormalizedHotness > 1.0) | ||
NormalizedHotness = 1.0; | ||
// Add 1 to ensure hot callsites get a non-zero threshold, which could | ||
// happen when SampleColdCallSiteThreshold is 0. This is when we do not | ||
// want any inlining for cold callsites. | ||
SampleThreshold = SampleHotCallSiteThreshold * NormalizedHotness * 100 + | ||
SampleColdCallSiteThreshold + 1; | ||
// Bump up the threshold to favor previous compiler inline decision. The | ||
// compiler has more insight and knowledge about functions based on their IR | ||
// and attribures and should be able to make a more reasonable inline | ||
// decision. | ||
if (WasInlined) | ||
SampleThreshold *= CSPreinlMultiplierForPrevInl; | ||
} | ||
|
||
return (Candidate.SizeCost < SampleThreshold); | ||
} | ||
|
||
void CSPreInliner::processFunction(const FunctionId Name) { | ||
FunctionSamples *FSamples = ContextTracker.getBaseSamplesFor(Name); | ||
if (!FSamples) | ||
return; | ||
|
||
unsigned FuncSize = | ||
getFuncSize(ContextTracker.getContextNodeForProfile(FSamples)); | ||
unsigned FuncFinalSize = FuncSize; | ||
unsigned SizeLimit = FuncSize * ProfileInlineGrowthLimit; | ||
SizeLimit = std::min(SizeLimit, (unsigned)ProfileInlineLimitMax); | ||
SizeLimit = std::max(SizeLimit, (unsigned)ProfileInlineLimitMin); | ||
|
||
LLVM_DEBUG(dbgs() << "Process " << Name | ||
<< " for context-sensitive pre-inlining (pre-inline size: " | ||
<< FuncSize << ", size limit: " << SizeLimit << ")\n"); | ||
|
||
ProfiledCandidateQueue CQueue; | ||
getInlineCandidates(CQueue, FSamples); | ||
|
||
while (!CQueue.empty() && FuncFinalSize < SizeLimit) { | ||
ProfiledInlineCandidate Candidate = CQueue.top(); | ||
CQueue.pop(); | ||
bool ShouldInline = false; | ||
if ((ShouldInline = shouldInline(Candidate))) { | ||
// We mark context as inlined as the corresponding context profile | ||
// won't be merged into that function's base profile. | ||
++PreInlNumCSInlined; | ||
ContextTracker.markContextSamplesInlined(Candidate.CalleeSamples); | ||
Candidate.CalleeSamples->getContext().setAttribute( | ||
ContextShouldBeInlined); | ||
FuncFinalSize += Candidate.SizeCost; | ||
getInlineCandidates(CQueue, Candidate.CalleeSamples); | ||
} else { | ||
++PreInlNumCSNotInlined; | ||
} | ||
LLVM_DEBUG( | ||
dbgs() << (ShouldInline ? " Inlined" : " Outlined") | ||
<< " context profile for: " | ||
<< ContextTracker.getContextString(*Candidate.CalleeSamples) | ||
<< " (callee size: " << Candidate.SizeCost | ||
<< ", call count:" << Candidate.CallsiteCount << ")\n"); | ||
} | ||
|
||
if (!CQueue.empty()) { | ||
if (SizeLimit == (unsigned)ProfileInlineLimitMax) | ||
++PreInlNumCSInlinedHitMaxLimit; | ||
else if (SizeLimit == (unsigned)ProfileInlineLimitMin) | ||
++PreInlNumCSInlinedHitMinLimit; | ||
else | ||
++PreInlNumCSInlinedHitGrowthLimit; | ||
} | ||
|
||
LLVM_DEBUG({ | ||
if (!CQueue.empty()) | ||
dbgs() << " Inline candidates ignored due to size limit (inliner " | ||
"original size: " | ||
<< FuncSize << ", inliner final size: " << FuncFinalSize | ||
<< ", size limit: " << SizeLimit << ")\n"; | ||
|
||
while (!CQueue.empty()) { | ||
ProfiledInlineCandidate Candidate = CQueue.top(); | ||
CQueue.pop(); | ||
bool WasInlined = | ||
Candidate.CalleeSamples->getContext().hasAttribute(ContextWasInlined); | ||
dbgs() << " " | ||
<< ContextTracker.getContextString(*Candidate.CalleeSamples) | ||
<< " (candidate size:" << Candidate.SizeCost | ||
<< ", call count: " << Candidate.CallsiteCount << ", previously " | ||
<< (WasInlined ? "inlined)\n" : "not inlined)\n"); | ||
} | ||
}); | ||
} | ||
|
||
void CSPreInliner::run() { | ||
#ifndef NDEBUG | ||
auto printProfileNames = [](SampleContextTracker &ContextTracker, | ||
bool IsInput) { | ||
uint32_t Size = 0; | ||
for (auto *Node : ContextTracker) { | ||
FunctionSamples *FSamples = Node->getFunctionSamples(); | ||
if (FSamples) { | ||
Size++; | ||
dbgs() << " [" << ContextTracker.getContextString(Node) << "] " | ||
<< FSamples->getTotalSamples() << ":" | ||
<< FSamples->getHeadSamples() << "\n"; | ||
} | ||
} | ||
dbgs() << (IsInput ? "Input" : "Output") << " context-sensitive profiles (" | ||
<< Size << " total):\n"; | ||
}; | ||
#endif | ||
|
||
LLVM_DEBUG(printProfileNames(ContextTracker, true)); | ||
|
||
// Execute global pre-inliner to estimate a global top-down inline | ||
// decision and merge profiles accordingly. This helps with profile | ||
// merge for ThinLTO otherwise we won't be able to merge profiles back | ||
// to base profile across module/thin-backend boundaries. | ||
// It also helps better compress context profile to control profile | ||
// size, as we now only need context profile for functions going to | ||
// be inlined. | ||
for (FunctionId FuncName : buildTopDownOrder()) { | ||
processFunction(FuncName); | ||
} | ||
|
||
// Not inlined context profiles are merged into its base, so we can | ||
// trim out such profiles from the output. | ||
for (auto *Node : ContextTracker) { | ||
FunctionSamples *FProfile = Node->getFunctionSamples(); | ||
if (FProfile && | ||
(Node->getParentContext() != &ContextTracker.getRootContext() && | ||
!FProfile->getContext().hasState(InlinedContext))) { | ||
Node->setFunctionSamples(nullptr); | ||
} | ||
} | ||
FunctionSamples::ProfileIsPreInlined = true; | ||
|
||
LLVM_DEBUG(printProfileNames(ContextTracker, false)); | ||
} |
Oops, something went wrong.