Resubmit "[Tooling] Parse compilation database command lines on Windows."

This patch introduced the ability to decide at runtime whether to parse
JSON compilation database command lines using Gnu syntax or Windows
syntax.  However, there were many existing unit tests written that
hardcoded Gnu-specific paths.  These tests were now failing because
the auto-detection logic was choosing to parse them using Windows
rules.

This resubmission of the patch fixes this by introducing an enum
which defines the syntax mode, which defaults to auto-detect, but
for which the unit tests force Gnu style parsing.

Reviewed By: alexfh
Differential Revision: https://reviews.llvm.org/D23628

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@279120 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/include/clang/Tooling/JSONCompilationDatabase.h b/include/clang/Tooling/JSONCompilationDatabase.h
index 2a13fc155cead0e8382be23a9d2f3951d67fb476..ee64166c81b36ac68fc0e7e006c7b94195f1c83d 100644 (file)
--- a/include/clang/Tooling/JSONCompilationDatabase.h
+++ b/include/clang/Tooling/JSONCompilationDatabase.h
@@ -55,6 +55,7 @@ namespace tooling {
 ///
 /// JSON compilation databases can for example be generated in CMake projects
 /// by setting the flag -DCMAKE_EXPORT_COMPILE_COMMANDS.
+enum class JSONCommandLineSyntax { Windows, Gnu, AutoDetect };
 class JSONCompilationDatabase : public CompilationDatabase {
 public:
   /// \brief Loads a JSON compilation database from the specified file.
@@ -62,13 +63,15 @@ public:
   /// Returns NULL and sets ErrorMessage if the database could not be
   /// loaded from the given file.
   static std::unique_ptr<JSONCompilationDatabase>
-  loadFromFile(StringRef FilePath, std::string &ErrorMessage);
+  loadFromFile(StringRef FilePath, std::string &ErrorMessage,
+               JSONCommandLineSyntax Syntax);
 
   /// \brief Loads a JSON compilation database from a data buffer.
   ///
   /// Returns NULL and sets ErrorMessage if the database could not be loaded.
   static std::unique_ptr<JSONCompilationDatabase>
-  loadFromBuffer(StringRef DatabaseString, std::string &ErrorMessage);
+  loadFromBuffer(StringRef DatabaseString, std::string &ErrorMessage,
+                 JSONCommandLineSyntax Syntax);
 
   /// \brief Returns all compile comamnds in which the specified file was
   /// compiled.
@@ -89,8 +92,9 @@ public:
 
 private:
   /// \brief Constructs a JSON compilation database on a memory buffer.
-  JSONCompilationDatabase(std::unique_ptr<llvm::MemoryBuffer> Database)
-      : Database(std::move(Database)),
+  JSONCompilationDatabase(std::unique_ptr<llvm::MemoryBuffer> Database,
+                          JSONCommandLineSyntax Syntax)
+      : Database(std::move(Database)), Syntax(Syntax),
         YAMLStream(this->Database->getBuffer(), SM) {}
 
   /// \brief Parses the database file and creates the index.
@@ -123,6 +127,7 @@ private:
   FileMatchTrie MatchTrie;
 
   std::unique_ptr<llvm::MemoryBuffer> Database;
+  JSONCommandLineSyntax Syntax;
   llvm::SourceMgr SM;
   llvm::yaml::Stream YAMLStream;
 };

diff --git a/lib/Tooling/JSONCompilationDatabase.cpp b/lib/Tooling/JSONCompilationDatabase.cpp
index 299fbdc149bfb2fefe538ee64c9062b3913d282b..07292da6b3947aec21ab3eaf62a47bcb757027dd 100644 (file)
--- a/lib/Tooling/JSONCompilationDatabase.cpp
+++ b/lib/Tooling/JSONCompilationDatabase.cpp
@@ -16,7 +16,10 @@
 #include "clang/Tooling/CompilationDatabasePluginRegistry.h"
 #include "clang/Tooling/Tooling.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/StringSaver.h"
 #include <system_error>
 
 namespace clang {
@@ -111,8 +114,31 @@ class CommandLineArgumentParser {
   std::vector<std::string> CommandLine;
 };
 
-std::vector<std::string> unescapeCommandLine(
-    StringRef EscapedCommandLine) {
+std::vector<std::string> unescapeCommandLine(JSONCommandLineSyntax Syntax,
+                                             StringRef EscapedCommandLine) {
+  if (Syntax == JSONCommandLineSyntax::AutoDetect) {
+    llvm::Triple Triple(llvm::sys::getProcessTriple());
+    if (Triple.getOS() == llvm::Triple::OSType::Win32) {
+      // Assume Windows command line parsing on Win32 unless the triple
+      // explicitly
+      // tells us otherwise.
+      if (!Triple.hasEnvironment() ||
+          Triple.getEnvironment() == llvm::Triple::EnvironmentType::MSVC)
+        Syntax = JSONCommandLineSyntax::Windows;
+      else
+        Syntax = JSONCommandLineSyntax::Gnu;
+    }
+  }
+
+  if (Syntax == JSONCommandLineSyntax::Windows) {
+    llvm::BumpPtrAllocator Alloc;
+    llvm::StringSaver Saver(Alloc);
+    llvm::SmallVector<const char *, 64> T;
+    llvm::cl::TokenizeWindowsCommandLine(EscapedCommandLine, Saver, T);
+    std::vector<std::string> Result(T.begin(), T.end());
+    return Result;
+  }
+  assert(Syntax == JSONCommandLineSyntax::Gnu);
   CommandLineArgumentParser parser(EscapedCommandLine);
   return parser.parse();
 }
@@ -123,7 +149,8 @@ class JSONCompilationDatabasePlugin : public CompilationDatabasePlugin {
     SmallString<1024> JSONDatabasePath(Directory);
     llvm::sys::path::append(JSONDatabasePath, "compile_commands.json");
     std::unique_ptr<CompilationDatabase> Database(
-        JSONCompilationDatabase::loadFromFile(JSONDatabasePath, ErrorMessage));
+        JSONCompilationDatabase::loadFromFile(
+            JSONDatabasePath, ErrorMessage, JSONCommandLineSyntax::AutoDetect));
     if (!Database)
       return nullptr;
     return Database;
@@ -143,7 +170,8 @@ volatile int JSONAnchorSource = 0;
 
 std::unique_ptr<JSONCompilationDatabase>
 JSONCompilationDatabase::loadFromFile(StringRef FilePath,
-                                      std::string &ErrorMessage) {
+                                      std::string &ErrorMessage,
+                                      JSONCommandLineSyntax Syntax) {
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> DatabaseBuffer =
       llvm::MemoryBuffer::getFile(FilePath);
   if (std::error_code Result = DatabaseBuffer.getError()) {
@@ -151,7 +179,7 @@ JSONCompilationDatabase::loadFromFile(StringRef FilePath,
     return nullptr;
   }
   std::unique_ptr<JSONCompilationDatabase> Database(
-      new JSONCompilationDatabase(std::move(*DatabaseBuffer)));
+      new JSONCompilationDatabase(std::move(*DatabaseBuffer), Syntax));
   if (!Database->parse(ErrorMessage))
     return nullptr;
   return Database;
@@ -159,11 +187,12 @@ JSONCompilationDatabase::loadFromFile(StringRef FilePath,
 
 std::unique_ptr<JSONCompilationDatabase>
 JSONCompilationDatabase::loadFromBuffer(StringRef DatabaseString,
-                                        std::string &ErrorMessage) {
+                                        std::string &ErrorMessage,
+                                        JSONCommandLineSyntax Syntax) {
   std::unique_ptr<llvm::MemoryBuffer> DatabaseBuffer(
       llvm::MemoryBuffer::getMemBuffer(DatabaseString));
   std::unique_ptr<JSONCompilationDatabase> Database(
-      new JSONCompilationDatabase(std::move(DatabaseBuffer)));
+      new JSONCompilationDatabase(std::move(DatabaseBuffer), Syntax));
   if (!Database->parse(ErrorMessage))
     return nullptr;
   return Database;
@@ -211,10 +240,11 @@ JSONCompilationDatabase::getAllCompileCommands() const {
 }
 
 static std::vector<std::string>
-nodeToCommandLine(const std::vector<llvm::yaml::ScalarNode *> &Nodes) {
+nodeToCommandLine(JSONCommandLineSyntax Syntax,
+                  const std::vector<llvm::yaml::ScalarNode *> &Nodes) {
   SmallString<1024> Storage;
   if (Nodes.size() == 1) {
-    return unescapeCommandLine(Nodes[0]->getValue(Storage));
+    return unescapeCommandLine(Syntax, Nodes[0]->getValue(Storage));
   }
   std::vector<std::string> Arguments;
   for (auto *Node : Nodes) {
@@ -230,9 +260,9 @@ void JSONCompilationDatabase::getCommands(
     SmallString<8> DirectoryStorage;
     SmallString<32> FilenameStorage;
     Commands.emplace_back(
-      std::get<0>(CommandsRef[I])->getValue(DirectoryStorage),
-      std::get<1>(CommandsRef[I])->getValue(FilenameStorage),
-      nodeToCommandLine(std::get<2>(CommandsRef[I])));
+        std::get<0>(CommandsRef[I])->getValue(DirectoryStorage),
+        std::get<1>(CommandsRef[I])->getValue(FilenameStorage),
+        nodeToCommandLine(Syntax, std::get<2>(CommandsRef[I])));
   }
 }
 

diff --git a/unittests/Tooling/CompilationDatabaseTest.cpp b/unittests/Tooling/CompilationDatabaseTest.cpp
index 13d60230496a807e823064535ef65d9a2fa1c92f..48539836e5f1c1d0662b34bcac94b83bd5ded37e 100644 (file)
--- a/unittests/Tooling/CompilationDatabaseTest.cpp
+++ b/unittests/Tooling/CompilationDatabaseTest.cpp
@@ -21,9 +21,10 @@ namespace tooling {
 
 static void expectFailure(StringRef JSONDatabase, StringRef Explanation) {
   std::string ErrorMessage;
-  EXPECT_EQ(nullptr, JSONCompilationDatabase::loadFromBuffer(JSONDatabase,
-                                                             ErrorMessage))
-    << "Expected an error because of: " << Explanation.str();
+  EXPECT_EQ(nullptr,
+            JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage,
+                                                    JSONCommandLineSyntax::Gnu))
+      << "Expected an error because of: " << Explanation.str();
 }
 
 TEST(JSONCompilationDatabase, ErrsOnInvalidFormat) {
@@ -46,9 +47,11 @@ TEST(JSONCompilationDatabase, ErrsOnInvalidFormat) {
 }
 
 static std::vector<std::string> getAllFiles(StringRef JSONDatabase,
-                                            std::string &ErrorMessage) {
+                                            std::string &ErrorMessage,
+                                            JSONCommandLineSyntax Syntax) {
   std::unique_ptr<CompilationDatabase> Database(
-      JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage));
+      JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage,
+                                              Syntax));
   if (!Database) {
     ADD_FAILURE() << ErrorMessage;
     return std::vector<std::string>();
@@ -56,10 +59,12 @@ static std::vector<std::string> getAllFiles(StringRef JSONDatabase,
   return Database->getAllFiles();
 }
 
-static std::vector<CompileCommand> getAllCompileCommands(StringRef JSONDatabase,
-                                                    std::string &ErrorMessage) {
+static std::vector<CompileCommand>
+getAllCompileCommands(JSONCommandLineSyntax Syntax, StringRef JSONDatabase,
+                      std::string &ErrorMessage) {
   std::unique_ptr<CompilationDatabase> Database(
-      JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage));
+      JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage,
+                                              Syntax));
   if (!Database) {
     ADD_FAILURE() << ErrorMessage;
     return std::vector<CompileCommand>();
@@ -70,7 +75,8 @@ static std::vector<CompileCommand> getAllCompileCommands(StringRef JSONDatabase,
 TEST(JSONCompilationDatabase, GetAllFiles) {
   std::string ErrorMessage;
   EXPECT_EQ(std::vector<std::string>(),
-            getAllFiles("[]", ErrorMessage)) << ErrorMessage;
+            getAllFiles("[]", ErrorMessage, JSONCommandLineSyntax::Gnu))
+      << ErrorMessage;
 
   std::vector<std::string> expected_files;
   SmallString<16> PathStorage;
@@ -78,20 +84,23 @@ TEST(JSONCompilationDatabase, GetAllFiles) {
   expected_files.push_back(PathStorage.str());
   llvm::sys::path::native("//net/dir/file2", PathStorage);
   expected_files.push_back(PathStorage.str());
-  EXPECT_EQ(expected_files, getAllFiles(
-    "[{\"directory\":\"//net/dir\","
-      "\"command\":\"command\","
-      "\"file\":\"file1\"},"
-    " {\"directory\":\"//net/dir\","
-      "\"command\":\"command\","
-      "\"file\":\"file2\"}]",
-    ErrorMessage)) << ErrorMessage;
+  EXPECT_EQ(expected_files,
+            getAllFiles("[{\"directory\":\"//net/dir\","
+                        "\"command\":\"command\","
+                        "\"file\":\"file1\"},"
+                        " {\"directory\":\"//net/dir\","
+                        "\"command\":\"command\","
+                        "\"file\":\"file2\"}]",
+                        ErrorMessage, JSONCommandLineSyntax::Gnu))
+      << ErrorMessage;
 }
 
 TEST(JSONCompilationDatabase, GetAllCompileCommands) {
   std::string ErrorMessage;
-  EXPECT_EQ(0u,
-            getAllCompileCommands("[]", ErrorMessage).size()) << ErrorMessage;
+  EXPECT_EQ(
+      0u, getAllCompileCommands(JSONCommandLineSyntax::Gnu, "[]", ErrorMessage)
+              .size())
+      << ErrorMessage;
 
   StringRef Directory1("//net/dir1");
   StringRef FileName1("file1");
@@ -101,12 +110,16 @@ TEST(JSONCompilationDatabase, GetAllCompileCommands) {
   StringRef Command2("command2");
 
   std::vector<CompileCommand> Commands = getAllCompileCommands(
-      ("[{\"directory\":\"" + Directory1 + "\"," +
-             "\"command\":\"" + Command1 + "\","
-             "\"file\":\"" + FileName1 + "\"},"
-       " {\"directory\":\"" + Directory2 + "\"," +
-             "\"command\":\"" + Command2 + "\","
-             "\"file\":\"" + FileName2 + "\"}]").str(),
+      JSONCommandLineSyntax::Gnu,
+      ("[{\"directory\":\"" + Directory1 + "\"," + "\"command\":\"" + Command1 +
+       "\","
+       "\"file\":\"" +
+       FileName1 + "\"},"
+                   " {\"directory\":\"" +
+       Directory2 + "\"," + "\"command\":\"" + Command2 + "\","
+                                                          "\"file\":\"" +
+       FileName2 + "\"}]")
+          .str(),
       ErrorMessage);
   EXPECT_EQ(2U, Commands.size()) << ErrorMessage;
   EXPECT_EQ(Directory1, Commands[0].Directory) << ErrorMessage;
@@ -120,12 +133,16 @@ TEST(JSONCompilationDatabase, GetAllCompileCommands) {
 
   // Check that order is preserved.
   Commands = getAllCompileCommands(
-      ("[{\"directory\":\"" + Directory2 + "\"," +
-             "\"command\":\"" + Command2 + "\","
-             "\"file\":\"" + FileName2 + "\"},"
-       " {\"directory\":\"" + Directory1 + "\"," +
-             "\"command\":\"" + Command1 + "\","
-             "\"file\":\"" + FileName1 + "\"}]").str(),
+      JSONCommandLineSyntax::Gnu,
+      ("[{\"directory\":\"" + Directory2 + "\"," + "\"command\":\"" + Command2 +
+       "\","
+       "\"file\":\"" +
+       FileName2 + "\"},"
+                   " {\"directory\":\"" +
+       Directory1 + "\"," + "\"command\":\"" + Command1 + "\","
+                                                          "\"file\":\"" +
+       FileName1 + "\"}]")
+          .str(),
       ErrorMessage);
   EXPECT_EQ(2U, Commands.size()) << ErrorMessage;
   EXPECT_EQ(Directory2, Commands[0].Directory) << ErrorMessage;
@@ -142,7 +159,8 @@ static CompileCommand findCompileArgsInJsonDatabase(StringRef FileName,
                                                     StringRef JSONDatabase,
                                                     std::string &ErrorMessage) {
   std::unique_ptr<CompilationDatabase> Database(
-      JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage));
+      JSONCompilationDatabase::loadFromBuffer(JSONDatabase, ErrorMessage,
+                                              JSONCommandLineSyntax::Gnu));
   if (!Database)
     return CompileCommand();
   std::vector<CompileCommand> Commands = Database->getCompileCommands(FileName);