SecProfSummary = 1,
SecNameTable = 2,
SecProfileSymbolList = 3,
+ SecFuncOffsetTable = 4,
// marker for the first type of profile.
SecFuncProfileFirst = 32,
SecLBRProfile = SecFuncProfileFirst
return "NameTableSection";
case SecProfileSymbolList:
return "ProfileSymbolListSection";
+ case SecFuncOffsetTable:
+ return "FuncOffsetTableSection";
case SecLBRProfile:
return "LBRProfileSection";
}
/// Print the profile for \p FName on stream \p OS.
void dumpFunctionProfile(StringRef FName, raw_ostream &OS = dbgs());
- virtual void collectFuncsToUse(const Module &M) {}
+ virtual void collectFuncsFrom(const Module &M) {}
/// Print all the profiles on stream \p OS.
void dump(raw_ostream &OS = dbgs());
bool at_eof() const { return Data >= End; }
/// Read the next function profile instance.
- std::error_code readFuncProfile();
+ std::error_code readFuncProfile(const uint8_t *Start);
/// Read the contents of the given profile instance.
std::error_code readProfile(FunctionSamples &FProfile);
virtual std::error_code verifySPMagic(uint64_t Magic) override;
virtual std::error_code readOneSection(const uint8_t *Start, uint64_t Size,
SecType Type) override;
- std::error_code readProfileSymbolList(uint64_t Size);
+ std::error_code readProfileSymbolList();
+ std::error_code readFuncOffsetTable();
+ std::error_code readFuncProfiles();
+
+ /// The table mapping from function name to the offset of its FunctionSample
+ /// towards file start.
+ DenseMap<StringRef, uint64_t> FuncOffsetTable;
+ /// The set containing the functions to use when compiling a module.
+ DenseSet<StringRef> FuncsToUse;
+ /// Use all functions from the input profile.
+ bool UseAllFuncs = true;
public:
SampleProfileReaderExtBinary(std::unique_ptr<MemoryBuffer> B, LLVMContext &C,
virtual std::unique_ptr<ProfileSymbolList> getProfileSymbolList() override {
return std::move(ProfSymList);
};
+
+ /// Collect functions with definitions in Module \p M.
+ void collectFuncsFrom(const Module &M) override;
};
class SampleProfileReaderCompactBinary : public SampleProfileReaderBinary {
std::error_code read() override;
/// Collect functions to be used when compiling Module \p M.
- void collectFuncsToUse(const Module &M) override;
+ void collectFuncsFrom(const Module &M) override;
};
using InlineCallStack = SmallVector<FunctionSamples *, 10>;
protected:
uint64_t markSectionStart(SecType Type);
std::error_code addNewSection(SecType Sec, uint64_t SectionStart);
- virtual void initSectionLayout() = 0;
+ virtual void initSectionHdrLayout() = 0;
virtual std::error_code
writeSections(const StringMap<FunctionSamples> &ProfileMap) = 0;
- // Specifiy the section layout in the profile. Note that the order in
- // SecHdrTable (order to collect sections) may be different from the
- // order in SectionLayout (order to write out sections into profile).
- SmallVector<SecHdrTableEntry, 8> SectionLayout;
+ // Specifiy the order of sections in section header table. Note
+ // the order of sections in the profile may be different that the
+ // order in SectionHdrLayout. sample Reader will follow the order
+ // in SectionHdrLayout to read each section.
+ SmallVector<SecHdrTableEntry, 8> SectionHdrLayout;
private:
void allocSecHdrTable();
public:
SampleProfileWriterExtBinary(std::unique_ptr<raw_ostream> &OS)
: SampleProfileWriterExtBinaryBase(OS) {
- initSectionLayout();
+ initSectionHdrLayout();
}
+ virtual std::error_code writeSample(const FunctionSamples &S) override;
virtual void setProfileSymbolList(ProfileSymbolList *PSL) override {
ProfSymList = PSL;
};
private:
- virtual void initSectionLayout() override {
- SectionLayout = {{SecProfSummary, 0, 0, 0},
- {SecNameTable, 0, 0, 0},
- {SecLBRProfile, 0, 0, 0},
- {SecProfileSymbolList, 0, 0, 0}};
+ virtual void initSectionHdrLayout() override {
+ // Note that SecFuncOffsetTable section is written after SecLBRProfile
+ // in the profile, but is put before SecLBRProfile in SectionHdrLayout.
+ //
+ // This is because sample reader follows the order of SectionHdrLayout to
+ // read each section, to read function profiles on demand sample reader
+ // need to get the offset of each function profile first.
+ //
+ // SecFuncOffsetTable section is written after SecLBRProfile in the
+ // profile because FuncOffsetTable needs to be populated while section
+ // SecLBRProfile is written.
+ SectionHdrLayout = {{SecProfSummary, 0, 0, 0},
+ {SecNameTable, 0, 0, 0},
+ {SecFuncOffsetTable, 0, 0, 0},
+ {SecLBRProfile, 0, 0, 0},
+ {SecProfileSymbolList, 0, 0, 0}};
};
virtual std::error_code
writeSections(const StringMap<FunctionSamples> &ProfileMap) override;
ProfileSymbolList *ProfSymList = nullptr;
+
+ // Save the start of SecLBRProfile so we can compute the offset to the
+ // start of SecLBRProfile for each Function's Profile and will keep it
+ // in FuncOffsetTable.
+ uint64_t SecLBRProfileStart;
+ // FuncOffsetTable maps function name to its profile offset in SecLBRProfile
+ // section. It is used to load function profile on demand.
+ MapVector<StringRef, uint64_t> FuncOffsetTable;
+ std::error_code writeFuncOffsetTable();
};
// CompactBinary is a compact format of binary profile which both reduces
return sampleprof_error::success;
}
-std::error_code SampleProfileReaderBinary::readFuncProfile() {
+std::error_code
+SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) {
+ Data = Start;
auto NumHeadSamples = readNumber<uint64_t>();
if (std::error_code EC = NumHeadSamples.getError())
return EC;
std::error_code SampleProfileReaderBinary::read() {
while (!at_eof()) {
- if (std::error_code EC = readFuncProfile())
+ if (std::error_code EC = readFuncProfile(Data))
return EC;
}
return EC;
break;
case SecLBRProfile:
- while (Data < Start + Size) {
- if (std::error_code EC = readFuncProfile())
- return EC;
- }
+ if (std::error_code EC = readFuncProfiles())
+ return EC;
break;
case SecProfileSymbolList:
- if (std::error_code EC = readProfileSymbolList(Size))
+ if (std::error_code EC = readProfileSymbolList())
+ return EC;
+ break;
+ case SecFuncOffsetTable:
+ if (std::error_code EC = readFuncOffsetTable())
return EC;
break;
default:
return sampleprof_error::success;
}
-std::error_code
-SampleProfileReaderExtBinary::readProfileSymbolList(uint64_t Size) {
+void SampleProfileReaderExtBinary::collectFuncsFrom(const Module &M) {
+ UseAllFuncs = false;
+ FuncsToUse.clear();
+ for (auto &F : M)
+ FuncsToUse.insert(FunctionSamples::getCanonicalFnName(F));
+}
+
+std::error_code SampleProfileReaderExtBinary::readFuncOffsetTable() {
+ auto Size = readNumber<uint64_t>();
+ if (std::error_code EC = Size.getError())
+ return EC;
+
+ FuncOffsetTable.reserve(*Size);
+ for (uint32_t I = 0; I < *Size; ++I) {
+ auto FName(readStringFromTable());
+ if (std::error_code EC = FName.getError())
+ return EC;
+
+ auto Offset = readNumber<uint64_t>();
+ if (std::error_code EC = Offset.getError())
+ return EC;
+
+ FuncOffsetTable[*FName] = *Offset;
+ }
+ return sampleprof_error::success;
+}
+
+std::error_code SampleProfileReaderExtBinary::readFuncProfiles() {
+ const uint8_t *Start = Data;
+ if (UseAllFuncs) {
+ while (Data < End) {
+ if (std::error_code EC = readFuncProfile(Data))
+ return EC;
+ }
+ assert(Data == End && "More data is read than expected");
+ return sampleprof_error::success;
+ }
+
+ for (auto Name : FuncsToUse) {
+ auto iter = FuncOffsetTable.find(Name);
+ if (iter == FuncOffsetTable.end())
+ continue;
+ const uint8_t *FuncProfileAddr = Start + iter->second;
+ assert(FuncProfileAddr < End && "out of LBRProfile section");
+ if (std::error_code EC = readFuncProfile(FuncProfileAddr))
+ return EC;
+ }
+ Data = End;
+ return sampleprof_error::success;
+}
+
+std::error_code SampleProfileReaderExtBinary::readProfileSymbolList() {
if (!ProfSymList)
ProfSymList = std::make_unique<ProfileSymbolList>();
- if (std::error_code EC = ProfSymList->read(Data, Size))
+ if (std::error_code EC = ProfSymList->read(Data, End - Data))
return EC;
- Data = Data + Size;
+ Data = End;
return sampleprof_error::success;
}
for (auto Offset : OffsetsToUse) {
const uint8_t *SavedData = Data;
- Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()) +
- Offset;
- if (std::error_code EC = readFuncProfile())
+ if (std::error_code EC = readFuncProfile(
+ reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()) +
+ Offset))
return EC;
Data = SavedData;
}
}
uint64_t SampleProfileReaderExtBinaryBase::getFileSize() {
- auto &LastEntry = SecHdrTable.back();
- return LastEntry.Offset + LastEntry.Size;
+ // Sections in SecHdrTable is not necessarily in the same order as
+ // sections in the profile because section like FuncOffsetTable needs
+ // to be written after section LBRProfile but needs to be read before
+ // section LBRProfile, so we cannot simply use the last entry in
+ // SecHdrTable to calculate the file size.
+ uint64_t FileSize = 0;
+ for (auto &Entry : SecHdrTable) {
+ FileSize = std::max(Entry.Offset + Entry.Size, FileSize);
+ }
+ return FileSize;
}
bool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) {
return sampleprof_error::success;
}
-void SampleProfileReaderCompactBinary::collectFuncsToUse(const Module &M) {
+void SampleProfileReaderCompactBinary::collectFuncsFrom(const Module &M) {
UseAllFuncs = false;
FuncsToUse.clear();
- for (auto &F : M) {
- StringRef CanonName = FunctionSamples::getCanonicalFnName(F);
- FuncsToUse.insert(CanonName);
- }
+ for (auto &F : M)
+ FuncsToUse.insert(FunctionSamples::getCanonicalFnName(F));
}
std::error_code SampleProfileReaderBinary::readSummaryEntry(
SecHdrTableEntry &
SampleProfileWriterExtBinaryBase::getEntryInLayout(SecType Type) {
auto SecIt = std::find_if(
- SectionLayout.begin(), SectionLayout.end(),
+ SectionHdrLayout.begin(), SectionHdrLayout.end(),
[=](const auto &Entry) -> bool { return Entry.Type == Type; });
return *SecIt;
}
return sampleprof_error::success;
}
+std::error_code
+SampleProfileWriterExtBinary::writeSample(const FunctionSamples &S) {
+ uint64_t Offset = OutputStream->tell();
+ StringRef Name = S.getName();
+ FuncOffsetTable[Name] = Offset - SecLBRProfileStart;
+ encodeULEB128(S.getHeadSamples(), *OutputStream);
+ return writeBody(S);
+}
+
+std::error_code SampleProfileWriterExtBinary::writeFuncOffsetTable() {
+ auto &OS = *OutputStream;
+
+ // Write out the table size.
+ encodeULEB128(FuncOffsetTable.size(), OS);
+
+ // Write out FuncOffsetTable.
+ for (auto entry : FuncOffsetTable) {
+ writeNameIdx(entry.first);
+ encodeULEB128(entry.second, OS);
+ }
+ return sampleprof_error::success;
+}
+
std::error_code SampleProfileWriterExtBinary::writeSections(
const StringMap<FunctionSamples> &ProfileMap) {
uint64_t SectionStart = markSectionStart(SecProfSummary);
return EC;
SectionStart = markSectionStart(SecLBRProfile);
+ SecLBRProfileStart = OutputStream->tell();
if (std::error_code EC = writeFuncProfiles(ProfileMap))
return EC;
if (std::error_code EC = addNewSection(SecLBRProfile, SectionStart))
if (std::error_code EC = addNewSection(SecProfileSymbolList, SectionStart))
return EC;
+ SectionStart = markSectionStart(SecFuncOffsetTable);
+ if (std::error_code EC = writeFuncOffsetTable())
+ return EC;
+ if (std::error_code EC = addNewSection(SecFuncOffsetTable, SectionStart))
+ return EC;
+
return sampleprof_error::success;
}
}
void SampleProfileWriterExtBinaryBase::setToCompressAllSections() {
- for (auto &Entry : SectionLayout)
+ for (auto &Entry : SectionHdrLayout)
addSecFlags(Entry, SecFlagCompress);
}
void SampleProfileWriterExtBinaryBase::addSectionFlags(SecType Type,
SecFlags Flags) {
- for (auto &Entry : SectionLayout) {
+ for (auto &Entry : SectionHdrLayout) {
if (Entry.Type == Type)
addSecFlags(Entry, Flags);
}
void SampleProfileWriterExtBinaryBase::allocSecHdrTable() {
support::endian::Writer Writer(*OutputStream, support::little);
- Writer.write(static_cast<uint64_t>(SectionLayout.size()));
+ Writer.write(static_cast<uint64_t>(SectionHdrLayout.size()));
SecHdrTableOffset = OutputStream->tell();
- for (uint32_t i = 0; i < SectionLayout.size(); i++) {
+ for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) {
Writer.write(static_cast<uint64_t>(-1));
Writer.write(static_cast<uint64_t>(-1));
Writer.write(static_cast<uint64_t>(-1));
IndexMap.insert({static_cast<uint32_t>(SecHdrTable[i].Type), i});
}
- // Write the sections in the order specified in SectionLayout.
- // That is the sections order Reader will see. Note that the
- // sections order in which Reader expects to read may be different
- // from the order in which Writer is able to write, so we need
- // to adjust the order in SecHdrTable to be consistent with
- // SectionLayout when we write SecHdrTable to the memory.
- for (uint32_t i = 0; i < SectionLayout.size(); i++) {
- uint32_t idx = IndexMap[static_cast<uint32_t>(SectionLayout[i].Type)];
+ // Write the section header table in the order specified in
+ // SectionHdrLayout. That is the sections order Reader will see.
+ // Note that the sections order in which Reader expects to read
+ // may be different from the order in which Writer is able to
+ // write, so we need to adjust the order in SecHdrTable to be
+ // consistent with SectionHdrLayout when we write SecHdrTable
+ // to the memory.
+ for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) {
+ uint32_t idx = IndexMap[static_cast<uint32_t>(SectionHdrLayout[i].Type)];
Writer.write(static_cast<uint64_t>(SecHdrTable[idx].Type));
Writer.write(static_cast<uint64_t>(SecHdrTable[idx].Flags));
Writer.write(static_cast<uint64_t>(SecHdrTable[idx].Offset));
return false;
}
Reader = std::move(ReaderOrErr.get());
- Reader->collectFuncsToUse(M);
+ Reader->collectFuncsFrom(M);
ProfileIsValid = (Reader->read() == sampleprof_error::success);
PSL = Reader->getProfileSymbolList();
auto ReaderOrErr = SampleProfileReader::create(Profile, Context);
ASSERT_TRUE(NoError(ReaderOrErr.getError()));
Reader = std::move(ReaderOrErr.get());
- Reader->collectFuncsToUse(M);
+ Reader->collectFuncsFrom(M);
}
void testRoundTrip(SampleProfileFormat Format, bool Remap) {
BarSamples.addCalledTargetSamples(1, 0, MconstructName, 1000);
BarSamples.addCalledTargetSamples(1, 0, StringviewName, 437);
+ StringRef BazName("_Z3bazi");
+ FunctionSamples BazSamples;
+ BazSamples.setName(BazName);
+ BazSamples.addTotalSamples(12557);
+ BazSamples.addHeadSamples(1257);
+ BazSamples.addBodySamples(1, 0, 12557);
+
Module M("my_module", Context);
FunctionType *fn_type =
FunctionType::get(Type::getVoidTy(Context), {}, false);
StringMap<FunctionSamples> Profiles;
Profiles[FooName] = std::move(FooSamples);
Profiles[BarName] = std::move(BarSamples);
+ Profiles[BazName] = std::move(BazSamples);
ProfileSymbolList List;
if (Format == SampleProfileFormat::SPF_Ext_Binary) {
ASSERT_TRUE(NoError(EC));
}
- ASSERT_EQ(2u, Reader->getProfiles().size());
-
FunctionSamples *ReadFooSamples = Reader->getSamplesFor(FooName);
ASSERT_TRUE(ReadFooSamples != nullptr);
if (Format != SampleProfileFormat::SPF_Compact_Binary) {
ReadBarSamples->findCallTargetMapAt(1, 0);
ASSERT_FALSE(CTMap.getError());
+ // Because _Z3bazi is not defined in module M, expect _Z3bazi's profile
+ // is not loaded when the profile is ExtBinary or Compact format because
+ // these formats support loading function profiles on demand.
+ FunctionSamples *ReadBazSamples = Reader->getSamplesFor(BazName);
+ if (Format == SampleProfileFormat::SPF_Ext_Binary ||
+ Format == SampleProfileFormat::SPF_Compact_Binary) {
+ ASSERT_TRUE(ReadBazSamples == nullptr);
+ ASSERT_EQ(2u, Reader->getProfiles().size());
+ } else {
+ ASSERT_TRUE(ReadBazSamples != nullptr);
+ ASSERT_EQ(12557u, ReadBazSamples->getTotalSamples());
+ ASSERT_EQ(3u, Reader->getProfiles().size());
+ }
+
std::string MconstructGUID;
StringRef MconstructRep =
getRepInFormat(MconstructName, Format, MconstructGUID);
auto VerifySummary = [](ProfileSummary &Summary) mutable {
ASSERT_EQ(ProfileSummary::PSK_Sample, Summary.getKind());
- ASSERT_EQ(123603u, Summary.getTotalCount());
- ASSERT_EQ(6u, Summary.getNumCounts());
- ASSERT_EQ(2u, Summary.getNumFunctions());
+ ASSERT_EQ(136160u, Summary.getTotalCount());
+ ASSERT_EQ(7u, Summary.getNumCounts());
+ ASSERT_EQ(3u, Summary.getNumFunctions());
ASSERT_EQ(1437u, Summary.getMaxFunctionCount());
ASSERT_EQ(60351u, Summary.getMaxCount());
Cutoff = 990000;
auto NinetyNinePerc = find_if(Details, Predicate);
ASSERT_EQ(60000u, EightyPerc->MinCount);
- ASSERT_EQ(60000u, NinetyPerc->MinCount);
- ASSERT_EQ(60000u, NinetyFivePerc->MinCount);
+ ASSERT_EQ(12557u, NinetyPerc->MinCount);
+ ASSERT_EQ(12557u, NinetyFivePerc->MinCount);
ASSERT_EQ(610u, NinetyNinePerc->MinCount);
};
projects / llvm / commitdiff