[libFuzzer] add exeprimental -rss_limit_mb flag to fight against OOMs

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@268807 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/lib/Fuzzer/FuzzerDriver.cpp b/lib/Fuzzer/FuzzerDriver.cpp
index db5207908231ce13c268a1fbd01fdd4c92f7eddd..38e196899578511a31a5a2d07dafa5402ddf56b7 100644 (file)
--- a/lib/Fuzzer/FuzzerDriver.cpp
+++ b/lib/Fuzzer/FuzzerDriver.cpp
@@ -234,6 +234,21 @@ static int RunInMultipleProcesses(const std::vector<std::string> &Args,
   return HasErrors ? 1 : 0;
 }
 
+static void RssThread(Fuzzer *F, size_t RssLimitMb) {
+  while (true) {
+    std::this_thread::sleep_for(std::chrono::seconds(1));
+    size_t Peak = GetPeakRSSMb();
+    if (Peak > RssLimitMb)
+      F->RssLimitCallback(Peak, RssLimitMb);
+  }
+}
+
+static void StartRssThread(Fuzzer *F, size_t RssLimitMb) {
+  if (!RssLimitMb) return;
+  std::thread T(RssThread, F, RssLimitMb);
+  T.detach();
+}
+
 int RunOneTest(Fuzzer *F, const char *InputFilePath) {
   Unit U = FileToVector(InputFilePath);
   Unit PreciseSizedU(U);
@@ -331,6 +346,8 @@ static int FuzzerDriver(const std::vector<std::string> &Args,
     if (U.size() <= Word::GetMaxSize())
       MD.AddWordToManualDictionary(Word(U.data(), U.size()));
 
+  StartRssThread(&F, Flags.rss_limit_mb);
+
   // Timer
   if (Flags.timeout > 0)
     SetTimer(Flags.timeout / 2 + 1);

diff --git a/lib/Fuzzer/FuzzerFlags.def b/lib/Fuzzer/FuzzerFlags.def
index 86b203ce0ce345615ec10cc493d4d4ab3c7dcab5..86597cc6808dfe29011e20fa79ba84d4518e62b1 100644 (file)
--- a/lib/Fuzzer/FuzzerFlags.def
+++ b/lib/Fuzzer/FuzzerFlags.def
@@ -81,6 +81,8 @@ FUZZER_FLAG_INT(close_fd_mask, 0, "If 1, close stdout at startup; "
     "Be careful, this will also close e.g. asan's stderr/stdout.")
 FUZZER_FLAG_INT(detect_leaks, 1, "If 1, and if LeakSanitizer is enabled "
     "try to detect memory leaks during fuzzing (i.e. not only at shut down).")
+FUZZER_FLAG_INT(rss_limit_mb, 0, "If non-zero, the fuzzer will exit upon"
+    "reaching this limit of RSS memory usage.")
 
 FUZZER_DEPRECATED_FLAG(exit_on_first)
 FUZZER_DEPRECATED_FLAG(save_minimized_corpus)

diff --git a/lib/Fuzzer/FuzzerInternal.h b/lib/Fuzzer/FuzzerInternal.h
index a34bfec57c1435a800348a52fbd9920edd4782ed..75f058b7d1c38e84d6e81928fa97bf23f3dd2161 100644 (file)
--- a/lib/Fuzzer/FuzzerInternal.h
+++ b/lib/Fuzzer/FuzzerInternal.h
@@ -353,6 +353,7 @@ public:
   MutationDispatcher &GetMD() { return MD; }
   void PrintFinalStats();
   void SetMaxLen(size_t MaxLen);
+  void RssLimitCallback(size_t RssPeakMb, size_t RssLimitMb);
 
 private:
   void AlarmCallback();

diff --git a/lib/Fuzzer/FuzzerLoop.cpp b/lib/Fuzzer/FuzzerLoop.cpp
index 3b0b339bf9c701f30b7845eba1090809caa596a2..e02ebcf6b4afc9b652de8f38a2b4258a53b7bc31 100644 (file)
--- a/lib/Fuzzer/FuzzerLoop.cpp
+++ b/lib/Fuzzer/FuzzerLoop.cpp
@@ -176,6 +176,17 @@ void Fuzzer::AlarmCallback() {
   }
 }
 
+void Fuzzer::RssLimitCallback(size_t RssPeakMb, size_t RssLimitMb) {
+  Printf("==%d== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %zdMb)\n",
+         GetPid(), RssPeakMb, RssLimitMb);
+  Printf("*****************************************************************\n");
+  Printf("** Experimental! TODO: dump the stack trace and the reproducer **\n");
+  Printf("*****************************************************************\n");
+  Printf("SUMMARY: libFuzzer: out-of-memory\n");
+  PrintFinalStats();
+  _Exit(Options.ErrorExitCode); // Stop right now.
+}
+
 void Fuzzer::PrintStats(const char *Where, const char *End) {
   size_t ExecPerSec = execPerSec();
   if (Options.OutputCSV) {