/// \brief Determine the layout of the FPM stream, given the MSF layout. An FPM
/// stream spans 1 or more blocks, each at equally spaced intervals throughout
/// the file.
-MSFStreamLayout getFpmStreamLayout(const MSFLayout &Msf);
+MSFStreamLayout getFpmStreamLayout(const MSFLayout &Msf,
+ bool IncludeUnusedFpmData = false,
+ bool AltFpm = false);
inline bool isValidBlockSize(uint32_t Size) {
switch (Size) {
inline uint32_t getFirstUnreservedBlock() { return 3; }
inline uint64_t bytesToBlocks(uint64_t NumBytes, uint64_t BlockSize) {
- return alignTo(NumBytes, BlockSize) / BlockSize;
+ return divideCeil(NumBytes, BlockSize);
}
inline uint64_t blockToOffset(uint64_t BlockNumber, uint64_t BlockSize) {
return L.SB->BlockSize;
}
-inline uint32_t getNumFpmIntervals(const MSFLayout &L) {
- uint32_t Length = getFpmIntervalLength(L);
- return alignTo(L.SB->NumBlocks, Length) / Length;
-}
+inline uint32_t getNumFpmIntervals(const MSFLayout &L,
+ bool IncludeUnusedFpmData = false) {
+ if (IncludeUnusedFpmData)
+ return divideCeil(L.SB->NumBlocks, L.SB->BlockSize);
-inline uint32_t getFullFpmByteSize(const MSFLayout &L) {
- return alignTo(L.SB->NumBlocks, 8) / 8;
+ // We want the minimum number of intervals required, where each interval can
+ // represent BlockSize * 8 blocks.
+ return divideCeil(L.SB->NumBlocks, 8 * L.SB->BlockSize);
}
Error validateSuperBlock(const SuperBlock &SB);
static std::unique_ptr<WritableMappedBlockStream>
createFpmStream(const MSFLayout &Layout, WritableBinaryStreamRef MsfData,
- BumpPtrAllocator &Allocator);
+ BumpPtrAllocator &Allocator, bool AltFpm = false);
support::endianness getEndian() const override {
return support::little;
private:
Expected<msf::MSFLayout> finalizeMsfLayout();
+ void commitFpm(WritableBinaryStream &MsfBuffer, const msf::MSFLayout &Layout);
+
BumpPtrAllocator &Allocator;
std::unique_ptr<msf::MSFBuilder> Msf;
return (Value + Align - 1) / Align * Align;
}
+/// Returns the integer ceil(Numerator / Denominator).
+inline uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator) {
+ return alignTo(Numerator, Denominator) / Denominator;
+}
+
/// \c alignTo for contexts where a constant expression is required.
/// \sa alignTo
///
return make_error<MSFError>(msf_error_code::insufficient_buffer,
"There are no free Blocks in the file");
uint32_t AllocBlocks = NumBlocks - NumFreeBlocks;
- FreeBlocks.resize(AllocBlocks + FreeBlocks.size(), true);
+ uint32_t OldBlockCount = FreeBlocks.size();
+ uint32_t NewBlockCount = AllocBlocks + OldBlockCount;
+ uint32_t NextFpmBlock = alignTo(OldBlockCount, BlockSize) + 1;
+ FreeBlocks.resize(NewBlockCount, true);
+ // If we crossed over an fpm page, we actually need to allocate 2 extra
+ // blocks for each FPM group crossed and mark both blocks from the group as
+ // used. We may not actually use them since there are many more FPM blocks
+ // present than are required to represent all blocks in a given PDB, but we
+ // need to make sure they aren't allocated to a stream or something else.
+ // At the end when committing the PDB, we'll go through and mark the
+ // extraneous ones unused.
+ while (NextFpmBlock < NewBlockCount) {
+ NewBlockCount += 2;
+ FreeBlocks.resize(NewBlockCount, true);
+ FreeBlocks.reset(NextFpmBlock, NextFpmBlock + 2);
+ NextFpmBlock += BlockSize;
+ }
}
int I = 0;
return Size;
}
+static void finalizeFpmBlockStatus(uint32_t B, ArrayRef<ulittle32_t> &FpmBlocks,
+ BitVector &Fpm) {
+ if (FpmBlocks.empty() || FpmBlocks.front() != B) {
+ Fpm.set(B);
+ return;
+ }
+
+ // If the next block in the actual layout is this block, it should *not* be
+ // free.
+ assert(!Fpm.test(B));
+ FpmBlocks = FpmBlocks.drop_front();
+}
+
Expected<MSFLayout> MSFBuilder::build() {
SuperBlock *SB = Allocator.Allocate<SuperBlock>();
MSFLayout L;
}
}
+ // FPM blocks occur in pairs at every `BlockLength` interval. While blocks of
+ // this form are reserved for FPM blocks, not all blocks of this form will
+ // actually be needed for FPM data because there are more blocks of this form
+ // than are required to represent a PDB file with a given number of blocks.
+ // So we need to find out which blocks are *actually* going to be real FPM
+ // blocks, then mark the reset of the reserved blocks as unallocated.
+ MSFStreamLayout FpmLayout = msf::getFpmStreamLayout(L, true);
+ auto FpmBlocks = makeArrayRef(FpmLayout.Blocks);
+ for (uint32_t B = kFreePageMap0Block; B < SB->NumBlocks;
+ B += msf::getFpmIntervalLength(L)) {
+ finalizeFpmBlockStatus(B, FpmBlocks, FreeBlocks);
+ finalizeFpmBlockStatus(B + 1, FpmBlocks, FreeBlocks);
+ }
+ L.FreePageMap = FreeBlocks;
+
return L;
}
return Error::success();
}
-MSFStreamLayout llvm::msf::getFpmStreamLayout(const MSFLayout &Msf) {
+MSFStreamLayout llvm::msf::getFpmStreamLayout(const MSFLayout &Msf,
+ bool IncludeUnusedFpmData,
+ bool AltFpm) {
MSFStreamLayout FL;
- uint32_t NumFpmIntervals = getNumFpmIntervals(Msf);
+ uint32_t NumFpmIntervals = getNumFpmIntervals(Msf, IncludeUnusedFpmData);
support::ulittle32_t FpmBlock = Msf.SB->FreeBlockMapBlock;
assert(FpmBlock == 1 || FpmBlock == 2);
- while (NumFpmIntervals > 0) {
+ if (AltFpm) {
+ // If they requested the alternate FPM, then 2 becomes 1 and 1 becomes 2.
+ FpmBlock = 3U - FpmBlock;
+ }
+ for (uint32_t I = 0; I < NumFpmIntervals; ++I) {
FL.Blocks.push_back(FpmBlock);
FpmBlock += msf::getFpmIntervalLength(Msf);
- --NumFpmIntervals;
}
- FL.Length = getFullFpmByteSize(Msf);
+
+ if (IncludeUnusedFpmData)
+ FL.Length = NumFpmIntervals * Msf.SB->BlockSize;
+ else
+ FL.Length = divideCeil(Msf.SB->NumBlocks, 8);
return FL;
}
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/DebugInfo/MSF/MSFCommon.h"
+#include "llvm/Support/BinaryStreamWriter.h"
#include "llvm/Support/Endian.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/MathExtras.h"
std::unique_ptr<WritableMappedBlockStream>
WritableMappedBlockStream::createFpmStream(const MSFLayout &Layout,
WritableBinaryStreamRef MsfData,
- BumpPtrAllocator &Allocator) {
- MSFStreamLayout SL(getFpmStreamLayout(Layout));
- return createStream(Layout.SB->BlockSize, SL, MsfData, Allocator);
+ BumpPtrAllocator &Allocator,
+ bool AltFpm) {
+ // We only want to give the user a stream containing the bytes of the FPM that
+ // are actually valid, but we want to initialize all of the bytes, even those
+ // that come from reserved FPM blocks where the entire block is unused. To do
+ // this, we first create the full layout, which gives us a stream with all
+ // bytes and all blocks, and initialize everything to 0xFF (all blocks in the
+ // file are unused). Then we create the minimal layout (which contains only a
+ // subset of the bytes previously initialized), and return that to the user.
+ MSFStreamLayout MinLayout(getFpmStreamLayout(Layout, false, AltFpm));
+
+ MSFStreamLayout FullLayout(getFpmStreamLayout(Layout, true, AltFpm));
+ auto Result =
+ createStream(Layout.SB->BlockSize, FullLayout, MsfData, Allocator);
+ if (!Result)
+ return Result;
+ std::vector<uint8_t> InitData(Layout.SB->BlockSize, 0xFF);
+ BinaryStreamWriter Initializer(*Result);
+ while (Initializer.bytesRemaining() > 0)
+ cantFail(Initializer.writeBytes(InitData));
+ return createStream(Layout.SB->BlockSize, MinLayout, MsfData, Allocator);
}
Error WritableMappedBlockStream::readBytes(uint32_t Offset, uint32_t Size,
MappedBlockStream::createFpmStream(ContainerLayout, *Buffer, Allocator);
BinaryStreamReader FpmReader(*FpmStream);
ArrayRef<uint8_t> FpmBytes;
- if (auto EC = FpmReader.readBytes(FpmBytes,
- msf::getFullFpmByteSize(ContainerLayout)))
+ if (auto EC = FpmReader.readBytes(FpmBytes, FpmReader.bytesRemaining()))
return EC;
uint32_t BlocksRemaining = getBlockCount();
uint32_t BI = 0;
return SN;
}
+void PDBFileBuilder::commitFpm(WritableBinaryStream &MsfBuffer,
+ const MSFLayout &Layout) {
+ auto FpmStream =
+ WritableMappedBlockStream::createFpmStream(Layout, MsfBuffer, Allocator);
+
+ // We only need to create the alt fpm stream so that it gets initialized.
+ WritableMappedBlockStream::createFpmStream(Layout, MsfBuffer, Allocator,
+ true);
+
+ uint32_t BI = 0;
+ BinaryStreamWriter FpmWriter(*FpmStream);
+ while (BI < Layout.SB->NumBlocks) {
+ uint8_t ThisByte = 0;
+ for (uint32_t I = 0; I < 8; ++I) {
+ bool IsFree =
+ (BI < Layout.SB->NumBlocks) ? Layout.FreePageMap.test(BI) : true;
+ uint8_t Mask = uint8_t(IsFree) << I;
+ ThisByte |= Mask;
+ ++BI;
+ }
+ cantFail(FpmWriter.writeObject(ThisByte));
+ }
+ assert(FpmWriter.bytesRemaining() == 0);
+}
+
Error PDBFileBuilder::commit(StringRef Filename) {
assert(!Filename.empty());
auto ExpectedLayout = finalizeMsfLayout();
if (auto EC = Writer.writeObject(*Layout.SB))
return EC;
+
+ commitFpm(Buffer, Layout);
+
uint32_t BlockMapOffset =
msf::blockToOffset(Layout.SB->BlockMapAddr, Layout.SB->BlockSize);
Writer.setOffset(BlockMapOffset);
--- /dev/null
+; RUN: llvm-pdbutil yaml2pdb -pdb=%t.pdb %p/Inputs/one-symbol.yaml
+; RUN: llvm-pdbutil bytes -fpm %t.pdb | FileCheck %s
+
+
+CHECK: Free Page Map
+CHECK-NEXT: ============================================================
+CHECK-NEXT: Block 1 (
+CHECK-NEXT: 1000: 04F8FFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF |................................|
+CHECK-NEXT: 1020: FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF |................................|
+CHECK: 1FE0: FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF |................................|
+CHECK: )
set(DebugInfoMSFSources
MappedBlockStreamTest.cpp
MSFBuilderTest.cpp
+ MSFCommonTest.cpp
)
add_llvm_unittest(DebugInfoMSFTests
#include "llvm/DebugInfo/MSF/MSFCommon.h"
#include "llvm/Testing/Support/Error.h"
+#include "gmock/gmock-matchers.h"
+#include "gmock/gmock.h"
#include "gtest/gtest.h"
using namespace llvm;
using namespace llvm::msf;
+using namespace testing;
namespace {
class MSFBuilderTest : public testing::Test {
EXPECT_EQ(1U, L.DirectoryBlocks.size());
EXPECT_EQ(B + 1, L.DirectoryBlocks[0]);
}
+
+TEST_F(MSFBuilderTest, StreamDoesntUseFpmBlocks) {
+ Expected<MSFBuilder> ExpectedMsf = MSFBuilder::create(Allocator, 4096);
+ ASSERT_THAT_EXPECTED(ExpectedMsf, Succeeded());
+ auto &Msf = *ExpectedMsf;
+
+ // A block is 4096 bytes, and every 4096 blocks we have 2 reserved FPM blocks.
+ // By creating add a stream that spans 4096*4096*3 bytes, we ensure that we
+ // cross over a couple of reserved FPM blocks, and that none of them are
+ // allocated to the stream.
+ constexpr uint32_t StreamSize = 4096 * 4096 * 3;
+ Expected<uint32_t> SN = Msf.addStream(StreamSize);
+ ASSERT_THAT_EXPECTED(SN, Succeeded());
+
+ auto ExpectedLayout = Msf.build();
+ ASSERT_THAT_EXPECTED(ExpectedLayout, Succeeded());
+ MSFLayout &L = *ExpectedLayout;
+ auto BlocksRef = L.StreamMap[*SN];
+ std::vector<uint32_t> Blocks(BlocksRef.begin(), BlocksRef.end());
+ EXPECT_EQ(StreamSize, L.StreamSizes[*SN]);
+
+ for (uint32_t I = 0; I <= 3; ++I) {
+ // Pages from the regular FPM are allocated, while pages from the alt fpm
+ // are free.
+ EXPECT_FALSE(L.FreePageMap.test(1 + I * 4096));
+ EXPECT_TRUE(L.FreePageMap.test(2 + I * 4096));
+ }
+
+ for (uint32_t I = 1; I <= 3; ++I) {
+ EXPECT_THAT(Blocks, Not(Contains(1 + I * 4096)));
+ EXPECT_THAT(Blocks, Not(Contains(2 + I * 4096)));
+ }
+}
--- /dev/null
+//===- MSFBuilderTest.cpp Tests manipulation of MSF stream metadata ------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/DebugInfo/MSF/MSFCommon.h"
+#include "llvm/Testing/Support/Error.h"
+
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::msf;
+
+TEST(MSFCommonTest, BytesToBlocks) {
+ EXPECT_EQ(0ULL, bytesToBlocks(0, 4096));
+ EXPECT_EQ(1ULL, bytesToBlocks(12, 4096));
+ EXPECT_EQ(1ULL, bytesToBlocks(4096, 4096));
+ EXPECT_EQ(2ULL, bytesToBlocks(4097, 4096));
+ EXPECT_EQ(2ULL, bytesToBlocks(600, 512));
+}
+
+TEST(MSFCommonTest, FpmIntervals) {
+ SuperBlock SB;
+ SB.FreeBlockMapBlock = 1;
+ SB.BlockSize = 4096;
+
+ MSFLayout L;
+ L.SB = &SB;
+
+ SB.NumBlocks = 12;
+ EXPECT_EQ(1u, getNumFpmIntervals(L, false));
+ SB.NumBlocks = SB.BlockSize;
+ EXPECT_EQ(1u, getNumFpmIntervals(L, false));
+ SB.NumBlocks = SB.BlockSize + 1;
+ EXPECT_EQ(1u, getNumFpmIntervals(L, false));
+ SB.NumBlocks = SB.BlockSize * 8;
+ EXPECT_EQ(1u, getNumFpmIntervals(L, false));
+ SB.NumBlocks = SB.BlockSize * 8 + 1;
+ EXPECT_EQ(2u, getNumFpmIntervals(L, false));
+
+ SB.NumBlocks = 12;
+ EXPECT_EQ(1u, getNumFpmIntervals(L, true));
+ SB.NumBlocks = SB.BlockSize;
+ EXPECT_EQ(1u, getNumFpmIntervals(L, true));
+ SB.NumBlocks = SB.BlockSize + 1;
+ EXPECT_EQ(2u, getNumFpmIntervals(L, true));
+ SB.NumBlocks = SB.BlockSize * 8;
+ EXPECT_EQ(8u, getNumFpmIntervals(L, true));
+ SB.NumBlocks = SB.BlockSize * 8 + 1;
+ EXPECT_EQ(9u, getNumFpmIntervals(L, true));
+}
+
+TEST(MSFCommonTest, FpmStreamLayout) {
+ SuperBlock SB;
+ MSFLayout L;
+ L.SB = &SB;
+ SB.FreeBlockMapBlock = 1;
+
+ // Each FPM block has 4096 bytes for a maximum of 4096*8 allocation states.
+ SB.BlockSize = 4096;
+
+ // 1. When we're not including unused FPM data, the length of the FPM stream
+ // should be only long enough to contain 1 bit for each block.
+
+ // 1a. When the PDB has <= 4096*8 blocks, there should only be one FPM block.
+ SB.NumBlocks = 8000;
+ MSFStreamLayout SL = getFpmStreamLayout(L, false, false);
+ EXPECT_EQ(1000u, SL.Length);
+ EXPECT_EQ(1u, SL.Blocks.size());
+ EXPECT_EQ(SB.FreeBlockMapBlock, SL.Blocks.front());
+
+ SL = getFpmStreamLayout(L, false, true);
+ EXPECT_EQ(1000u, SL.Length);
+ EXPECT_EQ(1u, SL.Blocks.size());
+ EXPECT_EQ(3u - SB.FreeBlockMapBlock, SL.Blocks.front());
+
+ // 1b. When the PDB has > 4096*8 blocks, there should be multiple FPM blocks.
+ SB.NumBlocks = SB.BlockSize * 8 + 1;
+ SL = getFpmStreamLayout(L, false, false);
+ EXPECT_EQ(SB.BlockSize + 1, SL.Length);
+ EXPECT_EQ(2u, SL.Blocks.size());
+ EXPECT_EQ(SB.FreeBlockMapBlock, SL.Blocks[0]);
+ EXPECT_EQ(SB.FreeBlockMapBlock + SB.BlockSize, SL.Blocks[1]);
+
+ SL = getFpmStreamLayout(L, false, true);
+ EXPECT_EQ(SB.BlockSize + 1, SL.Length);
+ EXPECT_EQ(2u, SL.Blocks.size());
+ EXPECT_EQ(3u - SB.FreeBlockMapBlock, SL.Blocks[0]);
+ EXPECT_EQ(3u - SB.FreeBlockMapBlock + SB.BlockSize, SL.Blocks[1]);
+
+ // 2. When we are including unused FPM data, there should be one FPM block
+ // at every BlockSize interval in the file, even if entire FPM blocks are
+ // unused.
+ SB.NumBlocks = SB.BlockSize * 8 + 1;
+ SL = getFpmStreamLayout(L, true, false);
+ EXPECT_EQ(SB.BlockSize * 9, SL.Length);
+ EXPECT_EQ(9u, SL.Blocks.size());
+ for (int I = 0; I < 9; ++I)
+ EXPECT_EQ(I * SB.BlockSize + SB.FreeBlockMapBlock, SL.Blocks[I]);
+}
#include "llvm/Support/BinaryStreamWriter.h"
#include "llvm/Testing/Support/Error.h"
+#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include <unordered_map>
EXPECT_EQ(Str[0], Str[1]);
}
+} // namespace
+
+MATCHER_P3(BlockIsFilledWith, Layout, BlockIndex, Byte, "succeeded") {
+ uint64_t Offset = msf::blockToOffset(BlockIndex, Layout.SB->BlockSize);
+ ArrayRef<uint8_t> BufferRef = makeArrayRef(arg);
+ BufferRef = BufferRef.slice(Offset, Layout.SB->BlockSize);
+ return llvm::all_of(BufferRef, [this](uint8_t B) { return B == Byte; });
+}
+
+namespace {
+TEST(MappedBlockStreamTest, CreateFpmStream) {
+ BumpPtrAllocator Allocator;
+ SuperBlock SB;
+ MSFLayout L;
+ L.SB = &SB;
+
+ SB.FreeBlockMapBlock = 1;
+ SB.BlockSize = 4096;
+
+ constexpr uint32_t NumFileBlocks = 4096 * 4;
+
+ std::vector<uint8_t> MsfBuffer(NumFileBlocks * SB.BlockSize);
+ MutableBinaryByteStream MsfStream(MsfBuffer, llvm::support::little);
+
+ SB.NumBlocks = NumFileBlocks;
+ auto FpmStream =
+ WritableMappedBlockStream::createFpmStream(L, MsfStream, Allocator);
+ // 4096 * 4 / 8 = 2048 bytes of FPM data is needed to describe 4096 * 4
+ // blocks. This translates to 1 FPM block.
+ EXPECT_EQ(2048u, FpmStream->getLength());
+ EXPECT_EQ(1u, FpmStream->getStreamLayout().Blocks.size());
+ EXPECT_EQ(1u, FpmStream->getStreamLayout().Blocks[0]);
+ // All blocks from FPM1 should be 1 initialized, and all blocks from FPM2
+ // should be 0 initialized (since we requested the main FPM, not the alt FPM)
+ for (int I = 0; I < 4; ++I) {
+ EXPECT_THAT(MsfBuffer, BlockIsFilledWith(L, 1 + I * SB.BlockSize, 0xFF));
+ EXPECT_THAT(MsfBuffer, BlockIsFilledWith(L, 2 + I * SB.BlockSize, 0));
+ }
+
+ ::memset(MsfBuffer.data(), 0, MsfBuffer.size());
+ FpmStream =
+ WritableMappedBlockStream::createFpmStream(L, MsfStream, Allocator, true);
+ // 4096 * 4 / 8 = 2048 bytes of FPM data is needed to describe 4096 * 4
+ // blocks. This translates to 1 FPM block.
+ EXPECT_EQ(2048u, FpmStream->getLength());
+ EXPECT_EQ(1u, FpmStream->getStreamLayout().Blocks.size());
+ EXPECT_EQ(2u, FpmStream->getStreamLayout().Blocks[0]);
+ // All blocks from FPM2 should be 1 initialized, and all blocks from FPM1
+ // should be 0 initialized (since we requested the alt FPM, not the main FPM)
+ for (int I = 0; I < 4; ++I) {
+ EXPECT_THAT(MsfBuffer, BlockIsFilledWith(L, 1 + I * SB.BlockSize, 0));
+ EXPECT_THAT(MsfBuffer, BlockIsFilledWith(L, 2 + I * SB.BlockSize, 0xFF));
+ }
+}
} // end anonymous namespace
projects / llvm / commitdiff