[Obj2Yaml] Add support for minidump generation with 64b memory ranges. (llvm#101272)

This PR adds support for `obj2yaml` and `yaml2obj` to generate minidumps
that have a Memory64List stream. This is a prerequisite to llvm#101086.

Worth noting
- ~~const dropped on minidumps so we could cache a MemoryDescriptor_64
to it's actual offset, preventing the need to loop multiple times~~
- doesn't reuse the existing `ListStream` code in some places, because
the Memory64List has a different width size field (unsigned 64), and a
larger header than all the other streams. I determined refactoring the
existing code to support Mem64 would be worse than supporting the
special case.

Author: Jlalond

llvm/include/llvm/BinaryFormat/Minidump.h

@@ -74,6 +74,18 @@ struct MemoryDescriptor_64 {
   support::ulittle64_t StartOfMemoryRange;
   support::ulittle64_t DataSize;
 };
+static_assert(sizeof(MemoryDescriptor_64) == 16);
+
+struct MemoryListHeader {
+  support::ulittle32_t NumberOfMemoryRanges;
+};
+static_assert(sizeof(MemoryListHeader) == 4);
+
+struct Memory64ListHeader {
+  support::ulittle64_t NumberOfMemoryRanges;
+  support::ulittle64_t BaseRVA;
+};
+static_assert(sizeof(Memory64ListHeader) == 16);
 
 struct MemoryInfoListHeader {
   support::ulittle32_t SizeOfHeader;

llvm/include/llvm/Object/Minidump.h

@@ -11,6 +11,7 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/fallible_iterator.h"
 #include "llvm/ADT/iterator.h"
 #include "llvm/BinaryFormat/Minidump.h"
 #include "llvm/Object/Binary.h"
@@ -103,6 +104,13 @@ class MinidumpFile : public Binary {
         minidump::StreamType::MemoryList);
   }
 
+  /// Returns the header to the memory 64 list stream. An error is returned if
+  /// the file does not contain this stream.
+  Expected<minidump::Memory64ListHeader> getMemoryList64Header() const {
+    return getStream<minidump::Memory64ListHeader>(
+        minidump::StreamType::Memory64List);
+  }
+
   class MemoryInfoIterator
       : public iterator_facade_base<MemoryInfoIterator,
                                     std::forward_iterator_tag,
@@ -132,6 +140,90 @@ class MinidumpFile : public Binary {
     size_t Stride;
   };
 
+  /// Class the provides an iterator over the memory64 memory ranges. Only the
+  /// the first descriptor is validated as readable beforehand.
+  class Memory64Iterator {
+  public:
+    static Memory64Iterator
+    begin(ArrayRef<uint8_t> Storage,
+          ArrayRef<minidump::MemoryDescriptor_64> Descriptors) {
+      return Memory64Iterator(Storage, Descriptors);
+    }
+
+    static Memory64Iterator end() { return Memory64Iterator(); }
+
+    bool operator==(const Memory64Iterator &R) const {
+      return IsEnd == R.IsEnd;
+    }
+
+    bool operator!=(const Memory64Iterator &R) const { return !(*this == R); }
+
+    const std::pair<minidump::MemoryDescriptor_64, ArrayRef<uint8_t>> &
+    operator*() {
+      return Current;
+    }
+
+    const std::pair<minidump::MemoryDescriptor_64, ArrayRef<uint8_t>> *
+    operator->() {
+      return &Current;
+    }
+
+    Error inc() {
+      if (Descriptors.empty()) {
+        IsEnd = true;
+        return Error::success();
+      }
+
+      // Drop front gives us an array ref, so we need to call .front() as well.
+      const minidump::MemoryDescriptor_64 &Descriptor = Descriptors.front();
+      if (Descriptor.DataSize > Storage.size()) {
+        IsEnd = true;
+        return make_error<GenericBinaryError>(
+            "Memory64 Descriptor exceeds end of file.",
+            object_error::unexpected_eof);
+      }
+
+      ArrayRef<uint8_t> Content = Storage.take_front(Descriptor.DataSize);
+      Current = std::make_pair(Descriptor, Content);
+
+      Storage = Storage.drop_front(Descriptor.DataSize);
+      Descriptors = Descriptors.drop_front();
+
+      return Error::success();
+    }
+
+  private:
+    // This constructor expects that the first descriptor is readable.
+    Memory64Iterator(ArrayRef<uint8_t> Storage,
+                     ArrayRef<minidump::MemoryDescriptor_64> Descriptors)
+        : Storage(Storage), Descriptors(Descriptors), IsEnd(false) {
+      assert(!Descriptors.empty() &&
+             Storage.size() >= Descriptors.front().DataSize);
+      minidump::MemoryDescriptor_64 Descriptor = Descriptors.front();
+      ArrayRef<uint8_t> Content = Storage.take_front(Descriptor.DataSize);
+      Current = std::make_pair(Descriptor, Content);
+      this->Descriptors = Descriptors.drop_front();
+      this->Storage = Storage.drop_front(Descriptor.DataSize);
+    }
+
+    Memory64Iterator()
+        : Storage(ArrayRef<uint8_t>()),
+          Descriptors(ArrayRef<minidump::MemoryDescriptor_64>()), IsEnd(true) {}
+
+    std::pair<minidump::MemoryDescriptor_64, ArrayRef<uint8_t>> Current;
+    ArrayRef<uint8_t> Storage;
+    ArrayRef<minidump::MemoryDescriptor_64> Descriptors;
+    bool IsEnd;
+  };
+
+  using FallibleMemory64Iterator = llvm::fallible_iterator<Memory64Iterator>;
+
+  /// Returns an iterator that pairs each descriptor with it's respective
+  /// content from the Memory64List stream. An error is returned if the file
+  /// does not contain a Memory64List stream, or if the descriptor data is
+  /// unreadable.
+  iterator_range<FallibleMemory64Iterator> getMemory64List(Error &Err) const;
+
   /// Returns the list of descriptors embedded in the MemoryInfoList stream. The
   /// descriptors provide properties (e.g. permissions) of interesting regions
   /// of memory at the time the minidump was taken. An error is returned if the
@@ -152,15 +244,15 @@ class MinidumpFile : public Binary {
   }
 
   /// Return a slice of the given data array, with bounds checking.
-  static Expected<ArrayRef<uint8_t>> getDataSlice(ArrayRef<uint8_t> Data,
-                                                  size_t Offset, size_t Size);
+  static Expected<ArrayRef<uint8_t>>
+  getDataSlice(ArrayRef<uint8_t> Data, uint64_t Offset, uint64_t Size);
 
   /// Return the slice of the given data array as an array of objects of the
   /// given type. The function checks that the input array is large enough to
   /// contain the correct number of objects of the given type.
   template <typename T>
   static Expected<ArrayRef<T>> getDataSliceAs(ArrayRef<uint8_t> Data,
-                                              size_t Offset, size_t Count);
+                                              uint64_t Offset, uint64_t Count);
 
   MinidumpFile(MemoryBufferRef Source, const minidump::Header &Header,
                ArrayRef<minidump::Directory> Streams,
@@ -199,15 +291,16 @@ Expected<const T &> MinidumpFile::getStream(minidump::StreamType Type) const {
 
 template <typename T>
 Expected<ArrayRef<T>> MinidumpFile::getDataSliceAs(ArrayRef<uint8_t> Data,
-                                                   size_t Offset,
-                                                   size_t Count) {
+                                                   uint64_t Offset,
+                                                   uint64_t Count) {
   // Check for overflow.
-  if (Count > std::numeric_limits<size_t>::max() / sizeof(T))
+  if (Count > std::numeric_limits<uint64_t>::max() / sizeof(T))
     return createEOFError();
   Expected<ArrayRef<uint8_t>> Slice =
       getDataSlice(Data, Offset, sizeof(T) * Count);
   if (!Slice)
     return Slice.takeError();
+
   return ArrayRef<T>(reinterpret_cast<const T *>(Slice->data()), Count);
 }
 

llvm/include/llvm/ObjectYAML/MinidumpYAML.h

@@ -29,6 +29,7 @@ struct Stream {
     Exception,
     MemoryInfoList,
     MemoryList,
+    Memory64List,
     ModuleList,
     RawContent,
     SystemInfo,
@@ -98,12 +99,30 @@ struct ParsedMemoryDescriptor {
   minidump::MemoryDescriptor Entry;
   yaml::BinaryRef Content;
 };
+
+struct ParsedMemory64Descriptor {
+  static constexpr Stream::StreamKind Kind = Stream::StreamKind::Memory64List;
+  static constexpr minidump::StreamType Type =
+      minidump::StreamType::Memory64List;
+
+  minidump::MemoryDescriptor_64 Entry;
+  yaml::BinaryRef Content;
+};
 } // namespace detail
 
 using ModuleListStream = detail::ListStream<detail::ParsedModule>;
 using ThreadListStream = detail::ListStream<detail::ParsedThread>;
 using MemoryListStream = detail::ListStream<detail::ParsedMemoryDescriptor>;
 
+struct Memory64ListStream
+    : public detail::ListStream<detail::ParsedMemory64Descriptor> {
+  minidump::Memory64ListHeader Header;
+
+  explicit Memory64ListStream(
+      std::vector<detail::ParsedMemory64Descriptor> Entries = {})
+      : ListStream(Entries) {}
+};
+
 /// ExceptionStream minidump stream.
 struct ExceptionStream : public Stream {
   minidump::ExceptionStream MDExceptionStream;
@@ -244,6 +263,12 @@ template <> struct MappingContextTraits<minidump::MemoryDescriptor, BinaryRef> {
                       BinaryRef &Content);
 };
 
+template <>
+struct MappingContextTraits<minidump::MemoryDescriptor_64, BinaryRef> {
+  static void mapping(IO &IO, minidump::MemoryDescriptor_64 &Memory,
+                      BinaryRef &Content);
+};
+
 } // namespace yaml
 
 } // namespace llvm
@@ -269,11 +294,14 @@ LLVM_YAML_DECLARE_MAPPING_TRAITS(
     llvm::MinidumpYAML::ModuleListStream::entry_type)
 LLVM_YAML_DECLARE_MAPPING_TRAITS(
     llvm::MinidumpYAML::ThreadListStream::entry_type)
+LLVM_YAML_DECLARE_MAPPING_TRAITS(
+    llvm::MinidumpYAML::Memory64ListStream::entry_type)
 
 LLVM_YAML_IS_SEQUENCE_VECTOR(std::unique_ptr<llvm::MinidumpYAML::Stream>)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::MinidumpYAML::MemoryListStream::entry_type)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::MinidumpYAML::ModuleListStream::entry_type)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::MinidumpYAML::ThreadListStream::entry_type)
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::MinidumpYAML::Memory64ListStream::entry_type)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::minidump::MemoryInfo)
 
 LLVM_YAML_DECLARE_MAPPING_TRAITS(llvm::MinidumpYAML::Object)

llvm/lib/Object/Minidump.cpp

@@ -99,8 +99,9 @@ template Expected<ArrayRef<Thread>>
 template Expected<ArrayRef<MemoryDescriptor>>
     MinidumpFile::getListStream(StreamType) const;
 
-Expected<ArrayRef<uint8_t>>
-MinidumpFile::getDataSlice(ArrayRef<uint8_t> Data, size_t Offset, size_t Size) {
+Expected<ArrayRef<uint8_t>> MinidumpFile::getDataSlice(ArrayRef<uint8_t> Data,
+                                                       uint64_t Offset,
+                                                       uint64_t Size) {
   // Check for overflow.
   if (Offset + Size < Offset || Offset + Size < Size ||
       Offset + Size > Data.size())
@@ -154,3 +155,43 @@ MinidumpFile::create(MemoryBufferRef Source) {
   return std::unique_ptr<MinidumpFile>(
       new MinidumpFile(Source, Hdr, *ExpectedStreams, std::move(StreamMap)));
 }
+
+iterator_range<MinidumpFile::FallibleMemory64Iterator>
+MinidumpFile::getMemory64List(Error &Err) const {
+  ErrorAsOutParameter ErrAsOutParam(&Err);
+  auto end = FallibleMemory64Iterator::end(Memory64Iterator::end());
+  Expected<minidump::Memory64ListHeader> ListHeader = getMemoryList64Header();
+  if (!ListHeader) {
+    Err = ListHeader.takeError();
+    return make_range(end, end);
+  }
+
+  std::optional<ArrayRef<uint8_t>> Stream =
+      getRawStream(StreamType::Memory64List);
+  if (!Stream) {
+    Err = createError("No such stream");
+    return make_range(end, end);
+  }
+
+  Expected<ArrayRef<minidump::MemoryDescriptor_64>> Descriptors =
+      getDataSliceAs<minidump::MemoryDescriptor_64>(
+          *Stream, sizeof(Memory64ListHeader),
+          ListHeader->NumberOfMemoryRanges);
+
+  if (!Descriptors) {
+    Err = Descriptors.takeError();
+    return make_range(end, end);
+  }
+
+  if (!Descriptors->empty() &&
+      ListHeader->BaseRVA + Descriptors->front().DataSize > getData().size()) {
+    Err = createError("Memory64List header RVA out of range");
+    return make_range(end, end);
+  }
+
+  return make_range(FallibleMemory64Iterator::itr(
+                        Memory64Iterator::begin(
+                            getData().slice(ListHeader->BaseRVA), *Descriptors),
+                        Err),
+                    FallibleMemory64Iterator::end(Memory64Iterator::end()));
+}

llvm/lib/ObjectYAML/MinidumpEmitter.cpp

@@ -136,6 +136,30 @@ static size_t layout(BlobAllocator &File, MinidumpYAML::ExceptionStream &S) {
   return DataEnd;
 }
 
+static size_t layout(BlobAllocator &File, MinidumpYAML::Memory64ListStream &S) {
+  size_t BaseRVA = File.tell() + sizeof(minidump::Memory64ListHeader);
+  BaseRVA += S.Entries.size() * sizeof(minidump::MemoryDescriptor_64);
+  S.Header.BaseRVA = BaseRVA;
+  S.Header.NumberOfMemoryRanges = S.Entries.size();
+  File.allocateObject(S.Header);
+  for (auto &E : S.Entries)
+    File.allocateObject(E.Entry);
+
+  // Save the new offset for the stream size.
+  size_t DataEnd = File.tell();
+  for (auto &E : S.Entries) {
+    File.allocateBytes(E.Content);
+    if (E.Entry.DataSize > E.Content.binary_size()) {
+      size_t Padding = E.Entry.DataSize - E.Content.binary_size();
+      File.allocateCallback(Padding, [Padding](raw_ostream &OS) {
+        OS << std::string(Padding, '\0');
+      });
+    }
+  }
+
+  return DataEnd;
+}
+
 static void layout(BlobAllocator &File, MemoryListStream::entry_type &Range) {
   Range.Entry.Memory = layout(File, Range.Content);
 }
@@ -190,6 +214,9 @@ static Directory layout(BlobAllocator &File, Stream &S) {
   case Stream::StreamKind::MemoryList:
     DataEnd = layout(File, cast<MemoryListStream>(S));
     break;
+  case Stream::StreamKind::Memory64List:
+    DataEnd = layout(File, cast<Memory64ListStream>(S));
+    break;
   case Stream::StreamKind::ModuleList:
     DataEnd = layout(File, cast<ModuleListStream>(S));
     break;