From ec358cd538687a1e13f766c976a67e93443543e3 Mon Sep 17 00:00:00 2001 From: Daniel Sanders Date: Thu, 3 Oct 2019 19:13:39 +0000 Subject: [PATCH] [gicombiner] Add a CodeExpander to handle C++ fragments with variable expansion Summary: This will handle expansion of C++ fragments in the declarative combiner including custom predicates, and escapes into C++ to aid the migration effort. Fixed the -DLLVM_LINK_LLVM_DYLIB=ON using DISABLE_LLVM_LINK_LLVM_DYLIB when creating the library. Apparently it automatically links to libLLVM.dylib and we don't want that from tablegen. Reviewers: bogner, volkan Subscribers: mgorny, llvm-commits Tags: #llvm Differential Revision: https://reviews.llvm.org/D68288 llvm-svn: 373551 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@373651 91177308-0d34-0410-b5e6-96231b3b80d8 --- unittests/CMakeLists.txt | 3 +- unittests/TableGen/CMakeLists.txt | 10 + unittests/TableGen/CodeExpanderTest.cpp | 203 +++++++++++++++++++++ utils/TableGen/CMakeLists.txt | 3 + utils/TableGen/GICombinerEmitter.cpp | 5 + utils/TableGen/GlobalISel/CMakeLists.txt | 7 + utils/TableGen/GlobalISel/CodeExpander.cpp | 93 ++++++++++ utils/TableGen/GlobalISel/CodeExpander.h | 55 ++++++ utils/TableGen/GlobalISel/CodeExpansions.h | 43 +++++ 9 files changed, 421 insertions(+), 1 deletion(-) create mode 100644 unittests/TableGen/CMakeLists.txt create mode 100644 unittests/TableGen/CodeExpanderTest.cpp create mode 100644 utils/TableGen/GlobalISel/CMakeLists.txt create mode 100644 utils/TableGen/GlobalISel/CodeExpander.cpp create mode 100644 utils/TableGen/GlobalISel/CodeExpander.h create mode 100644 utils/TableGen/GlobalISel/CodeExpansions.h diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 6bb2fb8eb92..9384bdad043 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -31,8 +31,9 @@ add_subdirectory(Remarks) add_subdirectory(Passes) add_subdirectory(ProfileData) add_subdirectory(Support) -add_subdirectory(TextAPI) +add_subdirectory(TableGen) add_subdirectory(Target) +add_subdirectory(TextAPI) add_subdirectory(Transforms) add_subdirectory(XRay) add_subdirectory(tools) diff --git a/unittests/TableGen/CMakeLists.txt b/unittests/TableGen/CMakeLists.txt new file mode 100644 index 00000000000..c725de7a066 --- /dev/null +++ b/unittests/TableGen/CMakeLists.txt @@ -0,0 +1,10 @@ +set(LLVM_LINK_COMPONENTS + TableGen + Support + ) + +add_llvm_unittest(TableGenTests + CodeExpanderTest.cpp + ) +include_directories(${CMAKE_SOURCE_DIR}/utils/TableGen) +target_link_libraries(TableGenTests PRIVATE LLVMTableGenGlobalISel LLVMTableGen) diff --git a/unittests/TableGen/CodeExpanderTest.cpp b/unittests/TableGen/CodeExpanderTest.cpp new file mode 100644 index 00000000000..75b9b737370 --- /dev/null +++ b/unittests/TableGen/CodeExpanderTest.cpp @@ -0,0 +1,203 @@ +//===- llvm/unittest/TableGen/CodeExpanderTest.cpp - Tests ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "GlobalISel/CodeExpander.h" +#include "GlobalISel/CodeExpansions.h" + +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Error.h" +#include "gtest/gtest.h" + +using namespace llvm; + +static StringRef bufferize(StringRef Str) { + std::unique_ptr Buffer = + MemoryBuffer::getMemBufferCopy(Str, "TestBuffer"); + StringRef StrBufferRef = Buffer->getBuffer(); + SrcMgr.AddNewSourceBuffer(std::move(Buffer), SMLoc()); + return StrBufferRef; +} + +class RAIIDiagnosticChecker { + std::string EmittedDiags; + raw_string_ostream OS; + std::vector Expected; + std::vector Received; + +public: + RAIIDiagnosticChecker() : OS(EmittedDiags) { + SrcMgr.setDiagHandler(handler, this); + } + ~RAIIDiagnosticChecker() { + SrcMgr.setDiagHandler(nullptr); + EXPECT_EQ(Received.size(), Expected.size()); + for (unsigned i = 0; i < Received.size() && i < Expected.size(); ++i) { + EXPECT_EQ(Received[i].getLoc(), Expected[i].getLoc()); + EXPECT_EQ(Received[i].getFilename(), Expected[i].getFilename()); + EXPECT_EQ(Received[i].getKind(), Expected[i].getKind()); + EXPECT_EQ(Received[i].getLineNo(), Expected[i].getLineNo()); + EXPECT_EQ(Received[i].getColumnNo(), Expected[i].getColumnNo()); + EXPECT_EQ(Received[i].getMessage(), Expected[i].getMessage()); + EXPECT_EQ(Received[i].getLineContents(), Expected[i].getLineContents()); + EXPECT_EQ(Received[i].getRanges(), Expected[i].getRanges()); + } + + if (testing::Test::HasFailure()) + errs() << "Emitted diagnostic:\n" << OS.str(); + } + + void expect(SMDiagnostic D) { Expected.push_back(D); } + + void diag(const SMDiagnostic &D) { + Received.push_back(D); + } + + static void handler(const SMDiagnostic &D, void *Context) { + RAIIDiagnosticChecker *Self = static_cast(Context); + Self->diag(D); + SrcMgr.setDiagHandler(nullptr); + SrcMgr.PrintMessage(Self->OS, D); + SrcMgr.setDiagHandler(handler, Context); + }; +}; + +TEST(CodeExpander, NoExpansions) { + std::string Result; + raw_string_ostream OS(Result); + CodeExpansions Expansions; + + RAIIDiagnosticChecker DiagChecker; + CodeExpander("No expansions", Expansions, SMLoc(), false).emit(OS); + EXPECT_EQ(OS.str(), "No expansions"); +} + +// Indentation is applied to all lines except the first +TEST(CodeExpander, Indentation) { + std::string Result; + raw_string_ostream OS(Result); + CodeExpansions Expansions; + + RAIIDiagnosticChecker DiagChecker; + CodeExpander("No expansions\nsecond line\nthird line", Expansions, SMLoc(), + false, " ") + .emit(OS); + EXPECT_EQ(OS.str(), "No expansions\n second line\n third line"); +} + +// \ is an escape character that removes special meanings from the next +// character. +TEST(CodeExpander, Escape) { + std::string Result; + raw_string_ostream OS(Result); + CodeExpansions Expansions; + + RAIIDiagnosticChecker DiagChecker; + CodeExpander("\\\\\\a\\$", Expansions, SMLoc(), false).emit(OS); + EXPECT_EQ(OS.str(), "\\a$"); +} + +// $foo is not an expansion. It should warn though. +TEST(CodeExpander, NotAnExpansion) { + std::string Result; + raw_string_ostream OS(Result); + CodeExpansions Expansions; + + RAIIDiagnosticChecker DiagChecker; + StringRef In = bufferize(" $foo"); + CodeExpander(" $foo", Expansions, SMLoc::getFromPointer(In.data()), false) + .emit(OS); + EXPECT_EQ(OS.str(), " $foo"); + DiagChecker.expect(SMDiagnostic( + SrcMgr, SMLoc::getFromPointer(In.data() + 1), "TestBuffer", 1, 1, + SourceMgr::DK_Warning, "Assuming missing escape character", " $foo", {})); +} + +// \$foo is not an expansion but shouldn't warn as it's using the escape. +TEST(CodeExpander, EscapedNotAnExpansion) { + std::string Result; + raw_string_ostream OS(Result); + CodeExpansions Expansions; + + RAIIDiagnosticChecker DiagChecker; + CodeExpander("\\$foo", Expansions, SMLoc(), false).emit(OS); + EXPECT_EQ(OS.str(), "$foo"); +} + +// \${foo is not an expansion but shouldn't warn as it's using the escape. +TEST(CodeExpander, EscapedUnterminatedExpansion) { + std::string Result; + raw_string_ostream OS(Result); + CodeExpansions Expansions; + + RAIIDiagnosticChecker DiagChecker; + CodeExpander("\\${foo", Expansions, SMLoc(), false).emit(OS); + EXPECT_EQ(OS.str(), "${foo"); +} + +// \${foo is not an expansion but shouldn't warn as it's using the escape. +TEST(CodeExpander, EscapedExpansion) { + std::string Result; + raw_string_ostream OS(Result); + CodeExpansions Expansions; + + RAIIDiagnosticChecker DiagChecker; + CodeExpander("\\${foo}", Expansions, SMLoc(), false).emit(OS); + EXPECT_EQ(OS.str(), "${foo}"); +} + +// ${foo} is an undefined expansion and should error. +TEST(CodeExpander, UndefinedExpansion) { + std::string Result; + raw_string_ostream OS(Result); + CodeExpansions Expansions; + Expansions.declare("bar", "expansion"); + + RAIIDiagnosticChecker DiagChecker; + CodeExpander("${foo}${bar}", Expansions, SMLoc(), false).emit(OS); + EXPECT_EQ(OS.str(), "expansion"); + DiagChecker.expect( + SMDiagnostic(SrcMgr, SMLoc(), "", 0, -1, SourceMgr::DK_Error, + "Attempting to expand an undeclared variable foo", "", {})); +} + +// ${foo} is an undefined expansion and should error. When given a valid +// location for the start of the buffer it should correctly point at the +// expansion being performed. +TEST(CodeExpander, UndefinedExpansionWithLoc) { + std::string Result; + raw_string_ostream OS(Result); + CodeExpansions Expansions; + Expansions.declare("bar", "expansion"); + + RAIIDiagnosticChecker DiagChecker; + StringRef In = bufferize("Padding ${foo}${bar}"); + CodeExpander(In, Expansions, SMLoc::getFromPointer(In.data()), false) + .emit(OS); + EXPECT_EQ(OS.str(), "Padding expansion"); + DiagChecker.expect(SMDiagnostic( + SrcMgr, SMLoc::getFromPointer(In.data() + 8), "TestBuffer", 1, 8, + SourceMgr::DK_Error, "Attempting to expand an undeclared variable foo", + "Padding ${foo}${bar}", {})); +} + +// ${bar is an unterminated expansion. Warn and implicitly terminate it. +TEST(CodeExpander, UnterminatedExpansion) { + std::string Result; + raw_string_ostream OS(Result); + CodeExpansions Expansions; + Expansions.declare("bar", "expansion"); + + RAIIDiagnosticChecker DiagChecker; + StringRef In = bufferize(" ${bar"); + CodeExpander(In, Expansions, SMLoc::getFromPointer(In.data()), false) + .emit(OS); + EXPECT_EQ(OS.str(), " expansion"); + DiagChecker.expect(SMDiagnostic(SrcMgr, SMLoc::getFromPointer(In.data() + 1), + "TestBuffer", 1, 1, SourceMgr::DK_Warning, + "Unterminated expansion", " ${bar", {})); +} diff --git a/utils/TableGen/CMakeLists.txt b/utils/TableGen/CMakeLists.txt index d97f9359f54..77ef764d2f2 100644 --- a/utils/TableGen/CMakeLists.txt +++ b/utils/TableGen/CMakeLists.txt @@ -1,3 +1,5 @@ +add_subdirectory(GlobalISel) + set(LLVM_LINK_COMPONENTS Support) add_tablegen(llvm-tblgen LLVM @@ -50,4 +52,5 @@ add_tablegen(llvm-tblgen LLVM WebAssemblyDisassemblerEmitter.cpp CTagsEmitter.cpp ) +target_link_libraries(llvm-tblgen PRIVATE LLVMTableGenGlobalISel) set_target_properties(llvm-tblgen PROPERTIES FOLDER "Tablegenning") diff --git a/utils/TableGen/GICombinerEmitter.cpp b/utils/TableGen/GICombinerEmitter.cpp index a85462b5aa8..c2b64bcfb7c 100644 --- a/utils/TableGen/GICombinerEmitter.cpp +++ b/utils/TableGen/GICombinerEmitter.cpp @@ -26,6 +26,11 @@ cl::OptionCategory static cl::list SelectedCombiners("combiners", cl::desc("Emit the specified combiners"), cl::cat(GICombinerEmitterCat), cl::CommaSeparated); +static cl::opt ShowExpansions( + "gicombiner-show-expansions", + cl::desc("Use C++ comments to indicate occurence of code expansion"), + cl::cat(GICombinerEmitterCat)); + namespace { class GICombinerEmitter { StringRef Name; diff --git a/utils/TableGen/GlobalISel/CMakeLists.txt b/utils/TableGen/GlobalISel/CMakeLists.txt new file mode 100644 index 00000000000..2f74d1087bc --- /dev/null +++ b/utils/TableGen/GlobalISel/CMakeLists.txt @@ -0,0 +1,7 @@ +set(LLVM_LINK_COMPONENTS + Support + ) + +llvm_add_library(LLVMTableGenGlobalISel STATIC DISABLE_LLVM_LINK_LLVM_DYLIB + CodeExpander.cpp + ) diff --git a/utils/TableGen/GlobalISel/CodeExpander.cpp b/utils/TableGen/GlobalISel/CodeExpander.cpp new file mode 100644 index 00000000000..d59a9b8e3b6 --- /dev/null +++ b/utils/TableGen/GlobalISel/CodeExpander.cpp @@ -0,0 +1,93 @@ +//===- CodeExpander.cpp - Expand variables in a string --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file Expand the variables in a string. +// +//===----------------------------------------------------------------------===// + +#include "CodeExpander.h" +#include "CodeExpansions.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Error.h" + +using namespace llvm; + +void CodeExpander::emit(raw_ostream &OS) const { + StringRef Current = Code; + + while (!Current.empty()) { + size_t Pos = Current.find_first_of("$\n\\"); + if (Pos == StringRef::npos) { + OS << Current; + Current = ""; + continue; + } + + OS << Current.substr(0, Pos); + Current = Current.substr(Pos); + + if (Current.startswith("\n")) { + OS << "\n" << Indent; + Current = Current.drop_front(1); + continue; + } + + if (Current.startswith("\\$") || Current.startswith("\\\\")) { + OS << Current[1]; + Current = Current.drop_front(2); + continue; + } + + if (Current.startswith("\\")) { + Current = Current.drop_front(1); + continue; + } + + if (Current.startswith("${")) { + StringRef StartVar = Current; + Current = Current.drop_front(2); + StringRef Var; + std::tie(Var, Current) = Current.split("}"); + + // Warn if we split because no terminator was found. + StringRef EndVar = StartVar.drop_front(2 /* ${ */ + Var.size()); + if (EndVar.empty()) { + size_t LocOffset = StartVar.data() - Code.data(); + PrintWarning( + Loc.size() > 0 && Loc[0].isValid() + ? SMLoc::getFromPointer(Loc[0].getPointer() + LocOffset) + : SMLoc(), + "Unterminated expansion"); + } + + auto ValueI = Expansions.find(Var); + if (ValueI == Expansions.end()) { + size_t LocOffset = StartVar.data() - Code.data(); + PrintError(Loc.size() > 0 && Loc[0].isValid() + ? SMLoc::getFromPointer(Loc[0].getPointer() + LocOffset) + : SMLoc(), + "Attempting to expand an undeclared variable " + Var); + } + if (ShowExpansions) + OS << "/*$" << Var << "{*/"; + OS << Expansions.lookup(Var); + if (ShowExpansions) + OS << "/*}*/"; + continue; + } + + size_t LocOffset = Current.data() - Code.data(); + PrintWarning(Loc.size() > 0 && Loc[0].isValid() + ? SMLoc::getFromPointer(Loc[0].getPointer() + LocOffset) + : SMLoc(), + "Assuming missing escape character"); + OS << "$"; + Current = Current.drop_front(1); + } +} diff --git a/utils/TableGen/GlobalISel/CodeExpander.h b/utils/TableGen/GlobalISel/CodeExpander.h new file mode 100644 index 00000000000..bd6946de592 --- /dev/null +++ b/utils/TableGen/GlobalISel/CodeExpander.h @@ -0,0 +1,55 @@ +//===- CodeExpander.h - Expand variables in a string ----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file Expand the variables in a string. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_CODEEXPANDER_H +#define LLVM_UTILS_TABLEGEN_CODEEXPANDER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/SMLoc.h" + +namespace llvm { +class CodeExpansions; +class raw_ostream; + +/// Emit the given code with all '${foo}' placeholders expanded to their +/// replacements. +/// +/// It's an error to use an undefined expansion and expansion-like output that +/// needs to be emitted verbatim can be escaped as '\${foo}' +/// +/// The emitted code can be given a custom indent to enable both indentation by +/// an arbitrary amount of whitespace and emission of the code as a comment. +class CodeExpander { + StringRef Code; + const CodeExpansions &Expansions; + const ArrayRef &Loc; + bool ShowExpansions; + StringRef Indent; + +public: + CodeExpander(StringRef Code, const CodeExpansions &Expansions, + const ArrayRef &Loc, bool ShowExpansions, + StringRef Indent = " ") + : Code(Code), Expansions(Expansions), Loc(Loc), + ShowExpansions(ShowExpansions), Indent(Indent) {} + + void emit(raw_ostream &OS) const; +}; + +inline raw_ostream &operator<<(raw_ostream &OS, const CodeExpander &Expander) { + Expander.emit(OS); + return OS; +} +} // end namespace llvm + +#endif // ifndef LLVM_UTILS_TABLEGEN_CODEEXPANDER_H diff --git a/utils/TableGen/GlobalISel/CodeExpansions.h b/utils/TableGen/GlobalISel/CodeExpansions.h new file mode 100644 index 00000000000..bb890ec8f57 --- /dev/null +++ b/utils/TableGen/GlobalISel/CodeExpansions.h @@ -0,0 +1,43 @@ +//===- CodeExpansions.h - Record expansions for CodeExpander --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file Record the expansions to use in a CodeExpander. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/StringMap.h" + +#ifndef LLVM_UTILS_TABLEGEN_CODEEXPANSIONS_H +#define LLVM_UTILS_TABLEGEN_CODEEXPANSIONS_H +namespace llvm { +class CodeExpansions { +public: + using const_iterator = StringMap::const_iterator; + +protected: + StringMap Expansions; + +public: + void declare(StringRef Name, StringRef Expansion) { + bool Inserted = Expansions.try_emplace(Name, Expansion).second; + assert(Inserted && "Declared variable twice"); + (void)Inserted; + } + + std::string lookup(StringRef Variable) const { + return Expansions.lookup(Variable); + } + + const_iterator begin() const { return Expansions.begin(); } + const_iterator end() const { return Expansions.end(); } + const_iterator find(StringRef Variable) const { + return Expansions.find(Variable); + } +}; +} // end namespace llvm +#endif // ifndef LLVM_UTILS_TABLEGEN_CODEEXPANSIONS_H -- 2.40.0