diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h index 9f989aaf227796..4e6ef4c1a8c313 100644 --- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -307,7 +307,8 @@ enum GlobalValueSummarySymtabCodes { // [valueid, n x stackidindex] FS_PERMODULE_CALLSITE_INFO = 26, // Summary of per-module allocation memprof metadata. - // [n x (alloc type, nummib, nummib x stackidindex)] + // [nummib, nummib x (alloc type, nummib, nummib x stackidindex), + // [nummib x total size]?] FS_PERMODULE_ALLOC_INFO = 27, // Summary of combined index memprof callsite metadata. // [valueid, numstackindices, numver, @@ -316,19 +317,9 @@ enum GlobalValueSummarySymtabCodes { // Summary of combined index allocation memprof metadata. // [nummib, numver, // nummib x (alloc type, numstackids, numstackids x stackidindex), - // numver x version] + // numver x version, [nummib x total size]?] FS_COMBINED_ALLOC_INFO = 29, FS_STACK_IDS = 30, - // Summary of per-module allocation memprof metadata when we are tracking - // total sizes. - // [n x (alloc type, total size, nummib, nummib x stackidindex)] - FS_PERMODULE_ALLOC_INFO_TOTAL_SIZES = 31, - // Summary of combined index allocation memprof metadata when we are tracking - // total sizes. - // [nummib, numver, - // nummib x (alloc type, total size, numstackids, - // numstackids x stackidindex), numver x version] - FS_COMBINED_ALLOC_INFO_TOTAL_SIZES = 32, }; enum MetadataCodes { diff --git a/llvm/include/llvm/IR/ModuleSummaryIndex.h b/llvm/include/llvm/IR/ModuleSummaryIndex.h index 950c0ed824dbb3..949e60ffd1dbf5 100644 --- a/llvm/include/llvm/IR/ModuleSummaryIndex.h +++ b/llvm/include/llvm/IR/ModuleSummaryIndex.h @@ -404,8 +404,8 @@ struct AllocInfo { std::vector MIBs; // If requested, keep track of total profiled sizes for each MIB. This will be - // a vector of the same length and order as the MIBs vector. - std::unique_ptr> TotalSizes; + // a vector of the same length and order as the MIBs vector, if non-empty. + std::vector TotalSizes; AllocInfo(std::vector MIBs) : MIBs(std::move(MIBs)) { Versions.push_back(0); @@ -427,10 +427,10 @@ inline raw_ostream &operator<<(raw_ostream &OS, const AllocInfo &AE) { for (auto &M : AE.MIBs) { OS << "\t\t" << M << "\n"; } - if (AE.TotalSizes) { + if (!AE.TotalSizes.empty()) { OS << " TotalSizes per MIB:\n\t\t"; First = true; - for (auto &TS : *AE.TotalSizes) { + for (auto &TS : AE.TotalSizes) { if (!First) OS << ", "; First = false; @@ -1445,7 +1445,7 @@ class ModuleSummaryIndex { // in the way some record are interpreted, like flags for instance. // Note that incrementing this may require changes in both BitcodeReader.cpp // and BitcodeWriter.cpp. - static constexpr uint64_t BitcodeSummaryVersion = 9; + static constexpr uint64_t BitcodeSummaryVersion = 10; // Regular LTO module name for ASM writer static constexpr const char *getRegularLTOModuleName() { diff --git a/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h b/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h index 26bf08f9f7439c..b2747d24c5396d 100644 --- a/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h +++ b/llvm/include/llvm/IR/ModuleSummaryIndexYAML.h @@ -224,7 +224,6 @@ template <> struct CustomMappingTraits { V.emplace(RefGUID, /*IsAnalysis=*/false); Refs.push_back(ValueInfo(/*IsAnalysis=*/false, &*V.find(RefGUID))); } - std::vector EmptyAllocInfo; Elem.SummaryList.push_back(std::make_unique( GlobalValueSummary::GVFlags( static_cast(FSum.Linkage), @@ -239,7 +238,7 @@ template <> struct CustomMappingTraits { std::move(FSum.TypeTestAssumeConstVCalls), std::move(FSum.TypeCheckedLoadConstVCalls), ArrayRef{}, ArrayRef{}, - std::move(EmptyAllocInfo))); + ArrayRef{})); } } static void output(IO &io, GlobalValueSummaryMapTy &V) { diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp index c3b6fd8bc20af9..e9490ccba82157 100644 --- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp +++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp @@ -548,8 +548,7 @@ static void computeFunctionSummary( Allocs.push_back(AllocInfo(std::move(MIBs))); if (MemProfReportHintedSizes) { assert(Allocs.back().MIBs.size() == TotalSizes.size()); - Allocs.back().TotalSizes = - std::make_unique>(std::move(TotalSizes)); + Allocs.back().TotalSizes = std::move(TotalSizes); } } else if (!InstCallsite.empty()) { SmallVector StackIdIndices; @@ -948,7 +947,6 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex( CantBePromoted.insert(GV->getGUID()); // Create the appropriate summary type. if (Function *F = dyn_cast(GV)) { - std::vector EmptyAllocInfo; std::unique_ptr Summary = std::make_unique( GVFlags, /*InstCount=*/0, @@ -971,7 +969,7 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex( ArrayRef{}, ArrayRef{}, ArrayRef{}, - ArrayRef{}, std::move(EmptyAllocInfo)); + ArrayRef{}, ArrayRef{}); Index.addGlobalValueSummary(*GV, std::move(Summary)); } else { std::unique_ptr Summary = diff --git a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp index 82aa7b7e077a1a..b7ed9cdf631454 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeAnalyzer.cpp @@ -328,8 +328,6 @@ GetCodeName(unsigned CodeID, unsigned BlockID, STRINGIFY_CODE(FS, COMBINED_CALLSITE_INFO) STRINGIFY_CODE(FS, COMBINED_ALLOC_INFO) STRINGIFY_CODE(FS, STACK_IDS) - STRINGIFY_CODE(FS, PERMODULE_ALLOC_INFO_TOTAL_SIZES) - STRINGIFY_CODE(FS, COMBINED_ALLOC_INFO_TOTAL_SIZES) } case bitc::METADATA_ATTACHMENT_ID: switch (CodeID) { diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 1cba859a3faded..6203c6e5119d18 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -7991,19 +7991,17 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { break; } - case bitc::FS_PERMODULE_ALLOC_INFO: - case bitc::FS_PERMODULE_ALLOC_INFO_TOTAL_SIZES: { - bool HasTotalSizes = BitCode == bitc::FS_PERMODULE_ALLOC_INFO_TOTAL_SIZES; + case bitc::FS_PERMODULE_ALLOC_INFO: { unsigned I = 0; std::vector MIBs; - std::vector TotalSizes; - while (I < Record.size()) { - assert(Record.size() - I >= (HasTotalSizes ? 3 : 2)); + unsigned NumMIBs = 0; + if (Version >= 10) + NumMIBs = Record[I++]; + unsigned MIBsRead = 0; + while ((Version >= 10 && MIBsRead++ < NumMIBs) || + (Version < 10 && I < Record.size())) { + assert(Record.size() - I >= 2); AllocationType AllocType = (AllocationType)Record[I++]; - if (HasTotalSizes) { - TotalSizes.push_back(Record[I++]); - assert(TotalSizes.back()); - } unsigned NumStackEntries = Record[I++]; assert(Record.size() - I >= NumStackEntries); SmallVector StackIdList; @@ -8014,32 +8012,31 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { } MIBs.push_back(MIBInfo(AllocType, std::move(StackIdList))); } + std::vector TotalSizes; + // We either have no sizes or NumMIBs of them. + assert(I == Record.size() || Record.size() - I == NumMIBs); + if (I < Record.size()) { + MIBsRead = 0; + while (MIBsRead++ < NumMIBs) + TotalSizes.push_back(Record[I++]); + } PendingAllocs.push_back(AllocInfo(std::move(MIBs))); - assert(HasTotalSizes != TotalSizes.empty()); - if (HasTotalSizes) { + if (!TotalSizes.empty()) { assert(PendingAllocs.back().MIBs.size() == TotalSizes.size()); - PendingAllocs.back().TotalSizes = - std::make_unique>(std::move(TotalSizes)); + PendingAllocs.back().TotalSizes = std::move(TotalSizes); } break; } - case bitc::FS_COMBINED_ALLOC_INFO: - case bitc::FS_COMBINED_ALLOC_INFO_TOTAL_SIZES: { - bool HasTotalSizes = BitCode == bitc::FS_COMBINED_ALLOC_INFO_TOTAL_SIZES; + case bitc::FS_COMBINED_ALLOC_INFO: { unsigned I = 0; std::vector MIBs; - std::vector TotalSizes; unsigned NumMIBs = Record[I++]; unsigned NumVersions = Record[I++]; unsigned MIBsRead = 0; while (MIBsRead++ < NumMIBs) { - assert(Record.size() - I >= (HasTotalSizes ? 3 : 2)); + assert(Record.size() - I >= 2); AllocationType AllocType = (AllocationType)Record[I++]; - if (HasTotalSizes) { - TotalSizes.push_back(Record[I++]); - assert(TotalSizes.back()); - } unsigned NumStackEntries = Record[I++]; assert(Record.size() - I >= NumStackEntries); SmallVector StackIdList; @@ -8054,13 +8051,20 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { SmallVector Versions; for (unsigned J = 0; J < NumVersions; J++) Versions.push_back(Record[I++]); + std::vector TotalSizes; + // We either have no sizes or NumMIBs of them. + assert(I == Record.size() || Record.size() - I == NumMIBs); + if (I < Record.size()) { + MIBsRead = 0; + while (MIBsRead++ < NumMIBs) { + TotalSizes.push_back(Record[I++]); + } + } PendingAllocs.push_back( AllocInfo(std::move(Versions), std::move(MIBs))); - assert(HasTotalSizes != TotalSizes.empty()); - if (HasTotalSizes) { + if (!TotalSizes.empty()) { assert(PendingAllocs.back().MIBs.size() == TotalSizes.size()); - PendingAllocs.back().TotalSizes = - std::make_unique>(std::move(TotalSizes)); + PendingAllocs.back().TotalSizes = std::move(TotalSizes); } break; } diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index 4463eb48b9811c..b3ebe70e8c52fc 100644 --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -230,8 +230,7 @@ class ModuleBitcodeWriterBase : public BitcodeWriterBase { void writePerModuleFunctionSummaryRecord( SmallVector &NameVals, GlobalValueSummary *Summary, unsigned ValueID, unsigned FSCallsAbbrev, unsigned FSCallsProfileAbbrev, - unsigned CallsiteAbbrev, unsigned AllocAbbrev, - unsigned AllocTotalSizeAbbrev, const Function &F); + unsigned CallsiteAbbrev, unsigned AllocAbbrev, const Function &F); void writeModuleLevelReferences(const GlobalVariable &V, SmallVector &NameVals, unsigned FSModRefsAbbrev, @@ -4159,7 +4158,7 @@ static void writeTypeIdCompatibleVtableSummaryRecord( static void writeFunctionHeapProfileRecords( BitstreamWriter &Stream, FunctionSummary *FS, unsigned CallsiteAbbrev, - unsigned AllocAbbrev, unsigned AllocTotalSizeAbbrev, bool PerModule, + unsigned AllocAbbrev, bool PerModule, std::function GetValueID, std::function GetStackIndex) { SmallVector Record; @@ -4190,35 +4189,27 @@ static void writeFunctionHeapProfileRecords( // Per module alloc versions should always have a single entry of // value 0. assert(!PerModule || (AI.Versions.size() == 1 && AI.Versions[0] == 0)); - if (!PerModule) { - Record.push_back(AI.MIBs.size()); + Record.push_back(AI.MIBs.size()); + if (!PerModule) Record.push_back(AI.Versions.size()); - } - unsigned I = 0; - assert(!AI.TotalSizes || AI.TotalSizes->size() == AI.MIBs.size()); for (auto &MIB : AI.MIBs) { Record.push_back((uint8_t)MIB.AllocType); - if (AI.TotalSizes) { - Record.push_back((*AI.TotalSizes)[I]); - assert((*AI.TotalSizes)[I]); - } Record.push_back(MIB.StackIdIndices.size()); for (auto Id : MIB.StackIdIndices) Record.push_back(GetStackIndex(Id)); - I++; } if (!PerModule) { for (auto V : AI.Versions) Record.push_back(V); } - unsigned Code = - PerModule ? (AI.TotalSizes ? bitc::FS_PERMODULE_ALLOC_INFO_TOTAL_SIZES - : bitc::FS_PERMODULE_ALLOC_INFO) - : (AI.TotalSizes ? bitc::FS_COMBINED_ALLOC_INFO_TOTAL_SIZES - : bitc::FS_COMBINED_ALLOC_INFO); - unsigned AllocAbbrevToUse = - AI.TotalSizes ? AllocTotalSizeAbbrev : AllocAbbrev; - Stream.EmitRecord(Code, Record, AllocAbbrevToUse); + assert(AI.TotalSizes.empty() || AI.TotalSizes.size() == AI.MIBs.size()); + if (!AI.TotalSizes.empty()) { + for (auto Size : AI.TotalSizes) + Record.push_back(Size); + } + Stream.EmitRecord(PerModule ? bitc::FS_PERMODULE_ALLOC_INFO + : bitc::FS_COMBINED_ALLOC_INFO, + Record, AllocAbbrev); } } @@ -4227,7 +4218,7 @@ void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord( SmallVector &NameVals, GlobalValueSummary *Summary, unsigned ValueID, unsigned FSCallsRelBFAbbrev, unsigned FSCallsProfileAbbrev, unsigned CallsiteAbbrev, - unsigned AllocAbbrev, unsigned AllocTotalSizeAbbrev, const Function &F) { + unsigned AllocAbbrev, const Function &F) { NameVals.push_back(ValueID); FunctionSummary *FS = cast(Summary); @@ -4238,7 +4229,7 @@ void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord( }); writeFunctionHeapProfileRecords( - Stream, FS, CallsiteAbbrev, AllocAbbrev, AllocTotalSizeAbbrev, + Stream, FS, CallsiteAbbrev, AllocAbbrev, /*PerModule*/ true, /*GetValueId*/ [&](const ValueInfo &VI) { return getValueId(VI); }, /*GetStackIndex*/ [&](unsigned I) { return I; }); @@ -4445,18 +4436,13 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() { Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_ALLOC_INFO)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib // n x (alloc type, numstackids, numstackids x stackidindex) + // optional: nummib x total size Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); unsigned AllocAbbrev = Stream.EmitAbbrev(std::move(Abbv)); - Abbv = std::make_shared(); - Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_ALLOC_INFO_TOTAL_SIZES)); - // n x (alloc type, total size, numstackids, numstackids x stackidindex) - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - unsigned AllocTotalSizeAbbrev = Stream.EmitAbbrev(std::move(Abbv)); - SmallVector NameVals; // Iterate over the list of functions instead of the Index to // ensure the ordering is stable. @@ -4474,10 +4460,9 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() { continue; } auto *Summary = VI.getSummaryList()[0].get(); - writePerModuleFunctionSummaryRecord(NameVals, Summary, VE.getValueID(&F), - FSCallsRelBFAbbrev, - FSCallsProfileAbbrev, CallsiteAbbrev, - AllocAbbrev, AllocTotalSizeAbbrev, F); + writePerModuleFunctionSummaryRecord( + NameVals, Summary, VE.getValueID(&F), FSCallsRelBFAbbrev, + FSCallsProfileAbbrev, CallsiteAbbrev, AllocAbbrev, F); } // Capture references from GlobalVariable initializers, which are outside @@ -4597,6 +4582,7 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver // nummib x (alloc type, numstackids, numstackids x stackidindex), // numver x version + // optional: nummib x total size Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); unsigned AllocAbbrev = Stream.EmitAbbrev(std::move(Abbv)); @@ -4607,16 +4593,6 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { return DecSummaries->count(GVS); }; - Abbv = std::make_shared(); - Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_ALLOC_INFO_TOTAL_SIZES)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver - // nummib x (alloc type, total size, numstackids, numstackids x stackidindex), - // numver x version - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); - unsigned AllocTotalSizeAbbrev = Stream.EmitAbbrev(std::move(Abbv)); - // The aliases are emitted as a post-pass, and will point to the value // id of the aliasee. Save them in a vector for post-processing. SmallVector Aliases; @@ -4704,9 +4680,10 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { getReferencedTypeIds(FS, ReferencedTypeIds); writeFunctionHeapProfileRecords( - Stream, FS, CallsiteAbbrev, AllocAbbrev, AllocTotalSizeAbbrev, + Stream, FS, CallsiteAbbrev, AllocAbbrev, /*PerModule*/ false, - /*GetValueId*/ [&](const ValueInfo &VI) -> unsigned { + /*GetValueId*/ + [&](const ValueInfo &VI) -> unsigned { std::optional ValueID = GetValueId(VI); // This can happen in shared index files for distributed ThinLTO if // the callee function summary is not included. Record 0 which we @@ -4716,7 +4693,8 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { return 0; return *ValueID; }, - /*GetStackIndex*/ [&](unsigned I) { + /*GetStackIndex*/ + [&](unsigned I) { // Get the corresponding index into the list of StackIds actually // being written for this combined index (which may be a subset in // the case of distributed indexes). diff --git a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp index 01138555a8fbcc..ef9ddeaaab632c 100644 --- a/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp +++ b/llvm/lib/Transforms/IPO/MemProfContextDisambiguation.cpp @@ -1766,14 +1766,14 @@ IndexCallsiteContextGraph::IndexCallsiteContextGraph( EmptyContext; unsigned I = 0; assert(!MemProfReportHintedSizes || - (AN.TotalSizes && AN.TotalSizes->size() == AN.MIBs.size())); + AN.TotalSizes.size() == AN.MIBs.size()); // Now add all of the MIBs and their stack nodes. for (auto &MIB : AN.MIBs) { CallStack::const_iterator> StackContext(&MIB); uint64_t TotalSize = 0; if (MemProfReportHintedSizes) - TotalSize = (*AN.TotalSizes)[I]; + TotalSize = AN.TotalSizes[I]; addStackNodesForMIB::const_iterator>( AllocNode, StackContext, EmptyContext, MIB.AllocType, TotalSize); diff --git a/llvm/test/Bitcode/summary_version.ll b/llvm/test/Bitcode/summary_version.ll index 98feab6fe2f995..26c64f81a773f1 100644 --- a/llvm/test/Bitcode/summary_version.ll +++ b/llvm/test/Bitcode/summary_version.ll @@ -2,7 +2,7 @@ ; RUN: opt -module-summary %s -o - | llvm-bcanalyzer -dump | FileCheck %s ; CHECK: +; CHECK: diff --git a/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll b/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll index 19e228fd5355c7..b3f1e770810d2d 100644 --- a/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll +++ b/llvm/test/Bitcode/thinlto-func-summary-vtableref-pgo.ll @@ -11,7 +11,7 @@ ; RUN: llvm-dis -o - %t.o | llvm-as -o - | llvm-dis -o - | FileCheck %s --check-prefix=DIS ; CHECK: +; CHECK-NEXT: ; The `VALUE_GUID` below represents the "_ZTV4Base" referenced by the instruction ; that loads vtable pointers.