From: Manuel Klimek <klimek@google.com>
Date: Tue, 28 May 2013 13:42:28 +0000 (+0000)
Subject: Make UnwrappedLines and AnnotatedToken contain pointers to FormatToken.
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=dcb3f2aab329c80e97a44312d15299df4dc059be;p=clang

Make UnwrappedLines and AnnotatedToken contain pointers to FormatToken.

The FormatToken is now not copyable any more.

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@182772 91177308-0d34-0410-b5e6-96231b3b80d8
---

diff --git a/lib/Format/Format.cpp b/lib/Format/Format.cpp
index c126249ef8..5e74ed188d 100644
--- a/lib/Format/Format.cpp
+++ b/lib/Format/Format.cpp
@@ -266,7 +266,7 @@ public:
     // If everything fits on a single line, just put it there.
     unsigned ColumnLimit = Style.ColumnLimit;
     if (NextLine && NextLine->InPPDirective &&
-        !NextLine->First.FormatTok.HasUnescapedNewline)
+        !NextLine->First.FormatTok->HasUnescapedNewline)
       ColumnLimit = getColumnLimit();
     if (Line.Last->TotalLength <= ColumnLimit - FirstIndent) {
       while (State.NextToken != NULL) {
@@ -285,7 +285,7 @@ public:
 
 private:
   void DebugTokenState(const AnnotatedToken &AnnotatedTok) {
-    const Token &Tok = AnnotatedTok.FormatTok.Tok;
+    const Token &Tok = AnnotatedTok.FormatTok->Tok;
     llvm::dbgs() << StringRef(SourceMgr.getCharacterData(Tok.getLocation()),
                               Tok.getLength());
     llvm::dbgs();
@@ -478,10 +478,11 @@ private:
       // FIXME: Is this correct?
       int WhitespaceLength =
           SourceMgr.getSpellingColumnNumber(
-              State.NextToken->FormatTok.WhitespaceRange.getEnd()) -
+              State.NextToken->FormatTok->WhitespaceRange.getEnd()) -
           SourceMgr.getSpellingColumnNumber(
-              State.NextToken->FormatTok.WhitespaceRange.getBegin());
-      State.Column += WhitespaceLength + State.NextToken->FormatTok.TokenLength;
+              State.NextToken->FormatTok->WhitespaceRange.getBegin());
+      State.Column +=
+          WhitespaceLength + State.NextToken->FormatTok->TokenLength;
       if (State.NextToken->Children.empty())
         State.NextToken = NULL;
       else
@@ -520,13 +521,13 @@ private:
                   Line.StartsDefinition)) {
         State.Column = State.Stack.back().Indent;
       } else if (Current.Type == TT_ObjCSelectorName) {
-        if (State.Stack.back().ColonPos > Current.FormatTok.TokenLength) {
+        if (State.Stack.back().ColonPos > Current.FormatTok->TokenLength) {
           State.Column =
-              State.Stack.back().ColonPos - Current.FormatTok.TokenLength;
+              State.Stack.back().ColonPos - Current.FormatTok->TokenLength;
         } else {
           State.Column = State.Stack.back().Indent;
           State.Stack.back().ColonPos =
-              State.Column + Current.FormatTok.TokenLength;
+              State.Column + Current.FormatTok->TokenLength;
         }
       } else if (Current.Type == TT_StartOfName ||
                  Previous.isOneOf(tok::coloncolon, tok::equal) ||
@@ -553,7 +554,7 @@ private:
         unsigned NewLines = 1;
         if (Current.Type == TT_LineComment)
           NewLines =
-              std::max(NewLines, std::min(Current.FormatTok.NewlinesBefore,
+              std::max(NewLines, std::min(Current.FormatTok->NewlinesBefore,
                                           Style.MaxEmptyLinesToKeep + 1));
         Whitespaces.replaceWhitespace(Current, NewLines, State.Column,
                                       State.Column, Line.InPPDirective);
@@ -562,7 +563,7 @@ private:
       State.Stack.back().LastSpace = State.Column;
       if (Current.isOneOf(tok::arrow, tok::period) &&
           Current.Type != TT_DesignatedInitializerPeriod)
-        State.Stack.back().LastSpace += Current.FormatTok.TokenLength;
+        State.Stack.back().LastSpace += Current.FormatTok->TokenLength;
       State.StartOfLineLevel = State.ParenLevel;
       State.LowestLevelOnLine = State.ParenLevel;
 
@@ -598,8 +599,8 @@ private:
         // Move over * and & if they are bound to the variable name.
         const AnnotatedToken *Tok = &Previous;
         while (Tok &&
-               State.Stack.back().VariablePos >= Tok->FormatTok.TokenLength) {
-          State.Stack.back().VariablePos -= Tok->FormatTok.TokenLength;
+               State.Stack.back().VariablePos >= Tok->FormatTok->TokenLength) {
+          State.Stack.back().VariablePos -= Tok->FormatTok->TokenLength;
           if (Tok->SpacesRequiredBefore != 0)
             break;
           Tok = Tok->Parent;
@@ -617,12 +618,12 @@ private:
       if (Current.Type == TT_ObjCSelectorName &&
           State.Stack.back().ColonPos == 0) {
         if (State.Stack.back().Indent + Current.LongestObjCSelectorName >
-            State.Column + Spaces + Current.FormatTok.TokenLength)
+            State.Column + Spaces + Current.FormatTok->TokenLength)
           State.Stack.back().ColonPos =
               State.Stack.back().Indent + Current.LongestObjCSelectorName;
         else
           State.Stack.back().ColonPos =
-              State.Column + Spaces + Current.FormatTok.TokenLength;
+              State.Column + Spaces + Current.FormatTok->TokenLength;
       }
 
       if (Previous.opensScope() && Previous.Type != TT_ObjCMethodExpr &&
@@ -674,7 +675,7 @@ private:
         Line.Type == LT_BuilderTypeCall && State.ParenLevel == 0)
       State.Stack.back().StartOfFunctionCall =
           Current.LastInChainOfCalls ? 0 : State.Column +
-                                               Current.FormatTok.TokenLength;
+                                               Current.FormatTok->TokenLength;
     if (Current.Type == TT_CtorInitializerColon) {
       // Indent 2 from the column, so:
       // SomeClass::SomeClass()
@@ -783,7 +784,7 @@ private:
       State.StartOfStringLiteral = 0;
     }
 
-    State.Column += Current.FormatTok.TokenLength;
+    State.Column += Current.FormatTok->TokenLength;
 
     if (State.NextToken->Children.empty())
       State.NextToken = NULL;
@@ -805,28 +806,30 @@ private:
                                 bool DryRun) {
     unsigned UnbreakableTailLength = Current.UnbreakableTailLength;
     llvm::OwningPtr<BreakableToken> Token;
-    unsigned StartColumn = State.Column - Current.FormatTok.TokenLength;
-    unsigned OriginalStartColumn = SourceMgr.getSpellingColumnNumber(
-        Current.FormatTok.getStartOfNonWhitespace()) - 1;
+    unsigned StartColumn = State.Column - Current.FormatTok->TokenLength;
+    unsigned OriginalStartColumn =
+        SourceMgr.getSpellingColumnNumber(
+            Current.FormatTok->getStartOfNonWhitespace()) -
+        1;
 
     if (Current.is(tok::string_literal) &&
         Current.Type != TT_ImplicitStringLiteral) {
       // Only break up default narrow strings.
       const char *LiteralData = SourceMgr.getCharacterData(
-          Current.FormatTok.getStartOfNonWhitespace());
+          Current.FormatTok->getStartOfNonWhitespace());
       if (!LiteralData || *LiteralData != '"')
         return 0;
 
-      Token.reset(new BreakableStringLiteral(Current.FormatTok, StartColumn));
+      Token.reset(new BreakableStringLiteral(*Current.FormatTok, StartColumn));
     } else if (Current.Type == TT_BlockComment) {
       BreakableBlockComment *BBC =
-          new BreakableBlockComment(Style, Current.FormatTok, StartColumn,
+          new BreakableBlockComment(Style, *Current.FormatTok, StartColumn,
                                     OriginalStartColumn, !Current.Parent);
       Token.reset(BBC);
     } else if (Current.Type == TT_LineComment &&
                (Current.Parent == NULL ||
                 Current.Parent->Type != TT_ImplicitStringLiteral)) {
-      Token.reset(new BreakableLineComment(Current.FormatTok, StartColumn));
+      Token.reset(new BreakableLineComment(*Current.FormatTok, StartColumn));
     } else {
       return 0;
     }
@@ -967,7 +970,7 @@ private:
       if (Current->NewLine) {
         llvm::dbgs()
             << "Penalty for splitting before "
-            << Current->Previous->State.NextToken->FormatTok.Tok.getName()
+            << Current->Previous->State.NextToken->FormatTok->Tok.getName()
             << ": " << Current->Previous->State.NextToken->SplitPenalty << "\n";
       }
     });
@@ -1108,12 +1111,15 @@ public:
 private:
   FormatToken *getNextToken() {
     if (GreaterStashed) {
-      FormatTok = new (Allocator.Allocate()) FormatToken(*FormatTok);
-      FormatTok->NewlinesBefore = 0;
+      // Create a synthesized second '>' token.
+      Token Greater = FormatTok->Tok;
+      FormatTok = new (Allocator.Allocate()) FormatToken;
+      FormatTok->Tok = Greater;
       SourceLocation GreaterLocation =
           FormatTok->Tok.getLocation().getLocWithOffset(1);
       FormatTok->WhitespaceRange =
           SourceRange(GreaterLocation, GreaterLocation);
+      FormatTok->TokenLength = 1;
       GreaterStashed = false;
       return FormatTok;
     }
@@ -1251,11 +1257,11 @@ public:
                                               E = AnnotatedLines.end();
          I != E; ++I) {
       const AnnotatedLine &TheLine = *I;
-      const FormatToken &FirstTok = TheLine.First.FormatTok;
+      const FormatToken *FirstTok = TheLine.First.FormatTok;
       int Offset = getIndentOffset(TheLine.First);
 
       // Check whether this line is part of a formatted preprocessor directive.
-      if (FirstTok.HasUnescapedNewline)
+      if (FirstTok->HasUnescapedNewline)
         FormatPPDirective = false;
       if (!FormatPPDirective && TheLine.InPPDirective &&
           (touchesLine(TheLine) || touchesPPDirective(I + 1, E)))
@@ -1270,25 +1276,26 @@ public:
         Indent += Offset;
       tryFitMultipleLinesInOne(Indent, I, E);
 
-      bool WasMoved = PreviousLineWasTouched && FirstTok.NewlinesBefore == 0;
+      bool WasMoved = PreviousLineWasTouched && FirstTok->NewlinesBefore == 0;
       if (TheLine.First.is(tok::eof)) {
         if (PreviousLineWasTouched) {
-          unsigned NewLines = std::min(FirstTok.NewlinesBefore, 1u);
+          unsigned NewLines = std::min(FirstTok->NewlinesBefore, 1u);
           Whitespaces.replaceWhitespace(TheLine.First, NewLines, /*Indent*/ 0,
                                         /*TargetColumn*/ 0);
         }
       } else if (TheLine.Type != LT_Invalid &&
                  (WasMoved || FormatPPDirective || touchesLine(TheLine))) {
         unsigned LevelIndent = getIndent(IndentForLevel, TheLine.Level);
-        if (FirstTok.WhitespaceRange.isValid() &&
+        if (FirstTok->WhitespaceRange.isValid() &&
             // Insert a break even if there is a structural error in case where
             // we break apart a line consisting of multiple unwrapped lines.
-            (FirstTok.NewlinesBefore == 0 || !StructuralError)) {
+            (FirstTok->NewlinesBefore == 0 || !StructuralError)) {
           formatFirstToken(TheLine.First, PreviousLineLastToken, Indent,
                            TheLine.InPPDirective);
         } else {
           Indent = LevelIndent =
-              SourceMgr.getSpellingColumnNumber(FirstTok.Tok.getLocation()) - 1;
+              SourceMgr.getSpellingColumnNumber(FirstTok->Tok.getLocation()) -
+              1;
         }
         UnwrappedLineFormatter Formatter(Style, SourceMgr, TheLine, Indent,
                                          TheLine.First, Whitespaces);
@@ -1301,9 +1308,9 @@ public:
         for (const AnnotatedToken *Tok = &TheLine.First; Tok != NULL;
              Tok = Tok->Children.empty() ? NULL : &Tok->Children[0]) {
           if (Tok == &TheLine.First &&
-              (Tok->FormatTok.NewlinesBefore > 0 || Tok->FormatTok.IsFirst)) {
+              (Tok->FormatTok->NewlinesBefore > 0 || Tok->FormatTok->IsFirst)) {
             unsigned LevelIndent = SourceMgr.getSpellingColumnNumber(
-                Tok->FormatTok.Tok.getLocation()) -
+                                       Tok->FormatTok->Tok.getLocation()) -
                                    1;
             // Remove trailing whitespace of the previous line if it was
             // touched.
@@ -1311,7 +1318,7 @@ public:
               formatFirstToken(*Tok, PreviousLineLastToken, LevelIndent,
                                TheLine.InPPDirective);
             } else {
-              Whitespaces.addUntouchableToken(Tok->FormatTok,
+              Whitespaces.addUntouchableToken(*Tok->FormatTok,
                                               TheLine.InPPDirective);
             }
 
@@ -1320,7 +1327,7 @@ public:
             if (Tok->isNot(tok::comment))
               IndentForLevel[TheLine.Level] = LevelIndent;
           } else {
-            Whitespaces.addUntouchableToken(Tok->FormatTok,
+            Whitespaces.addUntouchableToken(*Tok->FormatTok,
                                             TheLine.InPPDirective);
           }
         }
@@ -1345,11 +1352,11 @@ private:
       AnnotatedToken *Tok = &AnnotatedLines[i].First.Children[0];
       while (!Tok->Children.empty()) {
         if (Tok->Type == TT_PointerOrReference) {
-          bool SpacesBefore = Tok->FormatTok.WhitespaceRange.getBegin() !=
-                              Tok->FormatTok.WhitespaceRange.getEnd();
+          bool SpacesBefore = Tok->FormatTok->WhitespaceRange.getBegin() !=
+                              Tok->FormatTok->WhitespaceRange.getEnd();
           bool SpacesAfter =
-              Tok->Children[0].FormatTok.WhitespaceRange.getBegin() !=
-              Tok->Children[0].FormatTok.WhitespaceRange.getEnd();
+              Tok->Children[0].FormatTok->WhitespaceRange.getBegin() !=
+              Tok->Children[0].FormatTok->WhitespaceRange.getEnd();
           if (SpacesBefore && !SpacesAfter)
             ++CountBoundToVariable;
           else if (!SpacesBefore && SpacesAfter)
@@ -1358,8 +1365,8 @@ private:
 
         if (Tok->Type == TT_TemplateCloser &&
             Tok->Parent->Type == TT_TemplateCloser &&
-            Tok->FormatTok.WhitespaceRange.getBegin() ==
-                Tok->FormatTok.WhitespaceRange.getEnd())
+            Tok->FormatTok->WhitespaceRange.getBegin() ==
+                Tok->FormatTok->WhitespaceRange.getEnd())
           HasCpp03IncompatibleFormat = true;
         Tok = &Tok->Children[0];
       }
@@ -1426,8 +1433,8 @@ private:
     } else if (Style.AllowShortLoopsOnASingleLine &&
                I->First.isOneOf(tok::kw_for, tok::kw_while)) {
       tryMergeSimpleControlStatement(I, E, Limit);
-    } else if (I->InPPDirective && (I->First.FormatTok.HasUnescapedNewline ||
-                                    I->First.FormatTok.IsFirst)) {
+    } else if (I->InPPDirective && (I->First.FormatTok->HasUnescapedNewline ||
+                                    I->First.FormatTok->IsFirst)) {
       tryMergeSimplePPDirective(I, E, Limit);
     }
   }
@@ -1438,10 +1445,11 @@ private:
     if (Limit == 0)
       return;
     AnnotatedLine &Line = *I;
-    if (!(I + 1)->InPPDirective || (I + 1)->First.FormatTok.HasUnescapedNewline)
+    if (!(I + 1)->InPPDirective ||
+        (I + 1)->First.FormatTok->HasUnescapedNewline)
       return;
     if (I + 2 != E && (I + 2)->InPPDirective &&
-        !(I + 2)->First.FormatTok.HasUnescapedNewline)
+        !(I + 2)->First.FormatTok->HasUnescapedNewline)
       return;
     if (1 + (I + 1)->Last->TotalLength > Limit)
       return;
@@ -1455,7 +1463,7 @@ private:
       return;
     if ((I + 1)->InPPDirective != I->InPPDirective ||
         ((I + 1)->InPPDirective &&
-         (I + 1)->First.FormatTok.HasUnescapedNewline))
+         (I + 1)->First.FormatTok->HasUnescapedNewline))
       return;
     AnnotatedLine &Line = *I;
     if (Line.Last->isNot(tok::r_paren))
@@ -1555,8 +1563,8 @@ private:
   }
 
   bool touchesLine(const AnnotatedLine &TheLine) {
-    const FormatToken *First = &TheLine.First.FormatTok;
-    const FormatToken *Last = &TheLine.Last->FormatTok;
+    const FormatToken *First = TheLine.First.FormatTok;
+    const FormatToken *Last = TheLine.Last->FormatTok;
     CharSourceRange LineRange = CharSourceRange::getCharRange(
         First->WhitespaceRange.getBegin().getLocWithOffset(
             First->LastNewlineOffset),
@@ -1567,7 +1575,7 @@ private:
   bool touchesPPDirective(std::vector<AnnotatedLine>::iterator I,
                           std::vector<AnnotatedLine>::iterator E) {
     for (; I != E; ++I) {
-      if (I->First.FormatTok.HasUnescapedNewline)
+      if (I->First.FormatTok->HasUnescapedNewline)
         return false;
       if (touchesLine(*I))
         return true;
@@ -1576,7 +1584,7 @@ private:
   }
 
   bool touchesEmptyLineBefore(const AnnotatedLine &TheLine) {
-    const FormatToken *First = &TheLine.First.FormatTok;
+    const FormatToken *First = TheLine.First.FormatTok;
     CharSourceRange LineRange = CharSourceRange::getCharRange(
         First->WhitespaceRange.getBegin(),
         First->WhitespaceRange.getBegin().getLocWithOffset(
@@ -1594,20 +1602,20 @@ private:
   void formatFirstToken(const AnnotatedToken &RootToken,
                         const AnnotatedToken *PreviousToken, unsigned Indent,
                         bool InPPDirective) {
-    const FormatToken &Tok = RootToken.FormatTok;
+    const FormatToken *Tok = RootToken.FormatTok;
 
     unsigned Newlines =
-        std::min(Tok.NewlinesBefore, Style.MaxEmptyLinesToKeep + 1);
-    if (Newlines == 0 && !Tok.IsFirst)
+        std::min(Tok->NewlinesBefore, Style.MaxEmptyLinesToKeep + 1);
+    if (Newlines == 0 && !Tok->IsFirst)
       Newlines = 1;
 
     // Insert extra new line before access specifiers.
     if (PreviousToken && PreviousToken->isOneOf(tok::semi, tok::r_brace) &&
-        RootToken.isAccessSpecifier() && Tok.NewlinesBefore == 1)
+        RootToken.isAccessSpecifier() && Tok->NewlinesBefore == 1)
       ++Newlines;
 
     Whitespaces.replaceWhitespace(RootToken, Newlines, Indent, Indent,
-                                  InPPDirective && !Tok.HasUnescapedNewline);
+                                  InPPDirective && !Tok->HasUnescapedNewline);
   }
 
   FormatStyle Style;
diff --git a/lib/Format/TokenAnnotator.cpp b/lib/Format/TokenAnnotator.cpp
index ed39a573f6..67726234a9 100644
--- a/lib/Format/TokenAnnotator.cpp
+++ b/lib/Format/TokenAnnotator.cpp
@@ -22,7 +22,7 @@ namespace clang {
 namespace format {
 
 bool AnnotatedToken::isUnaryOperator() const {
-  switch (FormatTok.Tok.getKind()) {
+  switch (FormatTok->Tok.getKind()) {
   case tok::plus:
   case tok::plusplus:
   case tok::minus:
@@ -44,7 +44,7 @@ bool AnnotatedToken::isBinaryOperator() const {
 
 bool AnnotatedToken::isTrailingComment() const {
   return is(tok::comment) &&
-         (Children.empty() || Children[0].FormatTok.NewlinesBefore > 0);
+         (Children.empty() || Children[0].FormatTok->NewlinesBefore > 0);
 }
 
 AnnotatedToken *AnnotatedToken::getPreviousNoneComment() const {
@@ -203,7 +203,7 @@ private:
                                     tok::kw_return, tok::kw_throw) ||
          Parent->isUnaryOperator() || Parent->Type == TT_ObjCForIn ||
          Parent->Type == TT_CastRParen ||
-         getBinOpPrecedence(Parent->FormatTok.Tok.getKind(), true, true) >
+         getBinOpPrecedence(Parent->FormatTok->Tok.getKind(), true, true) >
              prec::Unknown);
     ScopedContextCreator ContextCreator(*this, tok::l_square, 10);
     Contexts.back().IsExpression = true;
@@ -321,7 +321,7 @@ private:
   bool consumeToken() {
     AnnotatedToken *Tok = CurrentToken;
     next();
-    switch (Tok->FormatTok.Tok.getKind()) {
+    switch (Tok->FormatTok->Tok.getKind()) {
     case tok::plus:
     case tok::minus:
       if (Tok->Parent == NULL && Line.MustBeDeclaration)
@@ -339,10 +339,10 @@ private:
                  Line.First.Type == TT_ObjCMethodSpecifier) {
         Tok->Type = TT_ObjCMethodExpr;
         Tok->Parent->Type = TT_ObjCSelectorName;
-        if (Tok->Parent->FormatTok.TokenLength >
+        if (Tok->Parent->FormatTok->TokenLength >
             Contexts.back().LongestObjCSelectorName)
           Contexts.back().LongestObjCSelectorName =
-              Tok->Parent->FormatTok.TokenLength;
+              Tok->Parent->FormatTok->TokenLength;
         if (Contexts.back().FirstObjCSelectorName == NULL)
           Contexts.back().FirstObjCSelectorName = Tok->Parent;
       } else if (Contexts.back().ColonIsForRangeExpr) {
@@ -421,7 +421,7 @@ private:
       break;
     case tok::identifier:
       if (Line.First.is(tok::kw_for) &&
-          Tok->FormatTok.Tok.getIdentifierInfo() == &Ident_in)
+          Tok->FormatTok->Tok.getIdentifierInfo() == &Ident_in)
         Tok->Type = TT_ObjCForIn;
       break;
     case tok::comma:
@@ -472,9 +472,10 @@ private:
       return;
     // Hashes in the middle of a line can lead to any strange token
     // sequence.
-    if (CurrentToken->FormatTok.Tok.getIdentifierInfo() == NULL)
+    if (CurrentToken->FormatTok->Tok.getIdentifierInfo() == NULL)
       return;
-    switch (CurrentToken->FormatTok.Tok.getIdentifierInfo()->getPPKeywordID()) {
+    switch (
+        CurrentToken->FormatTok->Tok.getIdentifierInfo()->getPPKeywordID()) {
     case tok::pp_include:
     case tok::pp_import:
       parseIncludeDirective();
@@ -630,7 +631,7 @@ private:
     if (Current.Type == TT_Unknown) {
       if (Current.Parent && Current.is(tok::identifier) &&
           ((Current.Parent->is(tok::identifier) &&
-            Current.Parent->FormatTok.Tok.getIdentifierInfo()
+            Current.Parent->FormatTok->Tok.getIdentifierInfo()
                     ->getPPKeywordID() ==
                 tok::pp_not_keyword) ||
            isSimpleTypeSpecifier(*Current.Parent) ||
@@ -651,7 +652,7 @@ private:
       } else if (Current.isBinaryOperator()) {
         Current.Type = TT_BinaryOperator;
       } else if (Current.is(tok::comment)) {
-        std::string Data(Lexer::getSpelling(Current.FormatTok.Tok, SourceMgr,
+        std::string Data(Lexer::getSpelling(Current.FormatTok->Tok, SourceMgr,
                                             Lex.getLangOpts()));
         if (StringRef(Data).startswith("//"))
           Current.Type = TT_LineComment;
@@ -673,7 +674,7 @@ private:
           // FIXME: We need to get smarter and understand more cases of casts.
           Current.Type = TT_CastRParen;
       } else if (Current.is(tok::at) && Current.Children.size()) {
-        switch (Current.Children[0].FormatTok.Tok.getObjCKeywordID()) {
+        switch (Current.Children[0].FormatTok->Tok.getObjCKeywordID()) {
         case tok::objc_interface:
         case tok::objc_implementation:
         case tok::objc_protocol:
@@ -719,9 +720,9 @@ private:
     if (NextToken->is(tok::l_square))
       return TT_PointerOrReference;
 
-    if (PrevToken->FormatTok.Tok.isLiteral() ||
+    if (PrevToken->FormatTok->Tok.isLiteral() ||
         PrevToken->isOneOf(tok::r_paren, tok::r_square) ||
-        NextToken->FormatTok.Tok.isLiteral() || NextToken->isUnaryOperator())
+        NextToken->FormatTok->Tok.isLiteral() || NextToken->isUnaryOperator())
       return TT_BinaryOperator;
 
     // It is very unlikely that we are going to find a pointer or reference type
@@ -766,7 +767,7 @@ private:
   // duplication.
   /// \brief Determine whether the token kind starts a simple-type-specifier.
   bool isSimpleTypeSpecifier(const AnnotatedToken &Tok) const {
-    switch (Tok.FormatTok.Tok.getKind()) {
+    switch (Tok.FormatTok->Tok.getKind()) {
     case tok::kw_short:
     case tok::kw_long:
     case tok::kw___int64:
@@ -894,7 +895,7 @@ void TokenAnnotator::annotate(AnnotatedLine &Line) {
     Line.Type = LT_ObjCProperty;
 
   Line.First.SpacesRequiredBefore = 1;
-  Line.First.MustBreakBefore = Line.First.FormatTok.MustBreakBefore;
+  Line.First.MustBreakBefore = Line.First.FormatTok->MustBreakBefore;
   Line.First.CanBreakBefore = Line.First.MustBreakBefore;
 }
 
@@ -909,10 +910,10 @@ void TokenAnnotator::calculateFormattingInformation(AnnotatedLine &Line) {
       Current->SpacesRequiredBefore =
           spaceRequiredBefore(Line, *Current) ? 1 : 0;
 
-    if (Current->FormatTok.MustBreakBefore) {
+    if (Current->FormatTok->MustBreakBefore) {
       Current->MustBreakBefore = true;
     } else if (Current->Type == TT_LineComment) {
-      Current->MustBreakBefore = Current->FormatTok.NewlinesBefore > 0;
+      Current->MustBreakBefore = Current->FormatTok->NewlinesBefore > 0;
     } else if (Current->Parent->isTrailingComment() ||
                (Current->is(tok::string_literal) &&
                 Current->Parent->is(tok::string_literal))) {
@@ -930,7 +931,7 @@ void TokenAnnotator::calculateFormattingInformation(AnnotatedLine &Line) {
       Current->TotalLength = Current->Parent->TotalLength + Style.ColumnLimit;
     else
       Current->TotalLength =
-          Current->Parent->TotalLength + Current->FormatTok.TokenLength +
+          Current->Parent->TotalLength + Current->FormatTok->TokenLength +
           Current->SpacesRequiredBefore;
     // FIXME: Only calculate this if CanBreakBefore is true once static
     // initializers etc. are sorted out.
@@ -957,7 +958,7 @@ void TokenAnnotator::calculateUnbreakableTailLengths(AnnotatedLine &Line) {
       UnbreakableTailLength = 0;
     } else {
       UnbreakableTailLength +=
-          Current->FormatTok.TokenLength + Current->SpacesRequiredBefore;
+          Current->FormatTok->TokenLength + Current->SpacesRequiredBefore;
     }
     Current = Current->Parent;
   }
@@ -1025,8 +1026,8 @@ unsigned TokenAnnotator::splitPenalty(const AnnotatedLine &Line,
 
   if (Right.is(tok::lessless)) {
     if (Left.is(tok::string_literal)) {
-      StringRef Content = StringRef(Left.FormatTok.Tok.getLiteralData(),
-                                    Left.FormatTok.TokenLength);
+      StringRef Content = StringRef(Left.FormatTok->Tok.getLiteralData(),
+                                    Left.FormatTok->TokenLength);
       Content = Content.drop_back(1).drop_front(1).trim();
       if (Content.size() > 1 &&
           (Content.back() == ':' || Content.back() == '='))
@@ -1073,14 +1074,14 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
   if (Left.is(tok::less) || Right.isOneOf(tok::greater, tok::less))
     return false;
   if (Right.Type == TT_PointerOrReference)
-    return Left.FormatTok.Tok.isLiteral() ||
+    return Left.FormatTok->Tok.isLiteral() ||
            ((Left.Type != TT_PointerOrReference) && Left.isNot(tok::l_paren) &&
             !Style.PointerBindsToType);
   if (Right.Type == TT_FunctionTypeLParen &&
       (Left.Type != TT_PointerOrReference || Style.PointerBindsToType))
     return true;
   if (Left.Type == TT_PointerOrReference)
-    return Right.FormatTok.Tok.isLiteral() ||
+    return Right.FormatTok->Tok.isLiteral() ||
            ((Right.Type != TT_PointerOrReference) &&
             Right.isNot(tok::l_paren) && Style.PointerBindsToType &&
             Left.Parent &&
@@ -1106,7 +1107,7 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
                         tok::kw_delete, tok::semi);
   }
   if (Left.is(tok::at) &&
-      Right.FormatTok.Tok.getObjCKeywordID() != tok::objc_not_keyword)
+      Right.FormatTok->Tok.getObjCKeywordID() != tok::objc_not_keyword)
     return false;
   if (Left.is(tok::l_brace) && Right.is(tok::r_brace))
     return false; // No spaces in "{}".
@@ -1127,8 +1128,8 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
 
 bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line,
                                          const AnnotatedToken &Tok) {
-  if (Tok.FormatTok.Tok.getIdentifierInfo() &&
-      Tok.Parent->FormatTok.Tok.getIdentifierInfo())
+  if (Tok.FormatTok->Tok.getIdentifierInfo() &&
+      Tok.Parent->FormatTok->Tok.getIdentifierInfo())
     return true; // Never ever merge two identifiers.
   if (Line.Type == LT_ObjCMethodDecl) {
     if (Tok.Parent->Type == TT_ObjCMethodSpecifier)
@@ -1147,7 +1148,7 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line,
     return false;
   if (Tok.Type == TT_CtorInitializerColon || Tok.Type == TT_ObjCBlockLParen)
     return true;
-  if (Tok.Parent->FormatTok.Tok.is(tok::kw_operator))
+  if (Tok.Parent->FormatTok->Tok.is(tok::kw_operator))
     return false;
   if (Tok.Type == TT_OverloadedOperatorLParen)
     return false;
@@ -1244,11 +1245,11 @@ void TokenAnnotator::printDebugInfo(const AnnotatedLine &Line) {
   llvm::errs() << "AnnotatedTokens:\n";
   const AnnotatedToken *Tok = &Line.First;
   while (Tok) {
-    llvm::errs() << " M=" << Tok->MustBreakBefore
-                 << " C=" << Tok->CanBreakBefore << " T=" << Tok->Type
-                 << " S=" << Tok->SpacesRequiredBefore
-                 << " P=" << Tok->SplitPenalty
-                 << " Name=" << Tok->FormatTok.Tok.getName() << " FakeLParens=";
+    llvm::errs()
+        << " M=" << Tok->MustBreakBefore << " C=" << Tok->CanBreakBefore
+        << " T=" << Tok->Type << " S=" << Tok->SpacesRequiredBefore
+        << " P=" << Tok->SplitPenalty
+        << " Name=" << Tok->FormatTok->Tok.getName() << " FakeLParens=";
     for (unsigned i = 0, e = Tok->FakeLParens.size(); i != e; ++i)
       llvm::errs() << Tok->FakeLParens[i] << "/";
     llvm::errs() << " FakeRParens=" << Tok->FakeRParens << "\n";
diff --git a/lib/Format/TokenAnnotator.h b/lib/Format/TokenAnnotator.h
index 37704ad6a7..dba5e4d44c 100644
--- a/lib/Format/TokenAnnotator.h
+++ b/lib/Format/TokenAnnotator.h
@@ -74,16 +74,16 @@ enum LineType {
 
 class AnnotatedToken {
 public:
-  explicit AnnotatedToken(const FormatToken &FormatTok)
+  explicit AnnotatedToken(FormatToken *FormatTok)
       : FormatTok(FormatTok), Type(TT_Unknown), SpacesRequiredBefore(0),
         CanBreakBefore(false), MustBreakBefore(false),
         ClosesTemplateDeclaration(false), MatchingParen(NULL),
-        ParameterCount(0), TotalLength(FormatTok.TokenLength),
+        ParameterCount(0), TotalLength(FormatTok->TokenLength),
         UnbreakableTailLength(0), BindingStrength(0), SplitPenalty(0),
         LongestObjCSelectorName(0), Parent(NULL), FakeRParens(0),
         LastInChainOfCalls(false), PartOfMultiVariableDeclStmt(false) {}
 
-  bool is(tok::TokenKind Kind) const { return FormatTok.Tok.is(Kind); }
+  bool is(tok::TokenKind Kind) const { return FormatTok->Tok.is(Kind); }
 
   bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const {
     return is(K1) || is(K2);
@@ -105,10 +105,10 @@ public:
            is(K8) || is(K9) || is(K10) || is(K11) || is(K12);
   }
 
-  bool isNot(tok::TokenKind Kind) const { return FormatTok.Tok.isNot(Kind); }
+  bool isNot(tok::TokenKind Kind) const { return FormatTok->Tok.isNot(Kind); }
 
   bool isObjCAtKeyword(tok::ObjCKeywordKind Kind) const {
-    return FormatTok.Tok.isObjCAtKeyword(Kind);
+    return FormatTok->Tok.isObjCAtKeyword(Kind);
   }
 
   bool isAccessSpecifier(bool ColonRequired = true) const {
@@ -134,7 +134,7 @@ public:
   bool isBinaryOperator() const;
   bool isTrailingComment() const;
 
-  FormatToken FormatTok;
+  FormatToken *FormatTok;
 
   TokenType Type;
 
@@ -208,8 +208,8 @@ public:
         StartsDefinition(false) {
     assert(!Line.Tokens.empty());
     AnnotatedToken *Current = &First;
-    for (std::list<FormatToken>::const_iterator I = ++Line.Tokens.begin(),
-                                                E = Line.Tokens.end();
+    for (std::list<FormatToken *>::const_iterator I = ++Line.Tokens.begin(),
+                                                  E = Line.Tokens.end();
          I != E; ++I) {
       Current->Children.push_back(AnnotatedToken(*I));
       Current->Children[0].Parent = Current;
@@ -242,7 +242,7 @@ public:
 };
 
 inline prec::Level getPrecedence(const AnnotatedToken &Tok) {
-  return getBinOpPrecedence(Tok.FormatTok.Tok.getKind(), true, true);
+  return getBinOpPrecedence(Tok.FormatTok->Tok.getKind(), true, true);
 }
 
 /// \brief Determines extra information about the tokens comprising an
diff --git a/lib/Format/UnwrappedLineParser.cpp b/lib/Format/UnwrappedLineParser.cpp
index 2d44cdb12f..f74fbc10ec 100644
--- a/lib/Format/UnwrappedLineParser.cpp
+++ b/lib/Format/UnwrappedLineParser.cpp
@@ -1022,10 +1022,10 @@ void UnwrappedLineParser::addUnwrappedLine() {
   DEBUG({
     llvm::dbgs() << "Line(" << Line->Level << ")"
                  << (Line->InPPDirective ? " MACRO" : "") << ": ";
-    for (std::list<FormatToken>::iterator I = Line->Tokens.begin(),
-                                          E = Line->Tokens.end();
+    for (std::list<FormatToken *>::iterator I = Line->Tokens.begin(),
+                                            E = Line->Tokens.end();
          I != E; ++I) {
-      llvm::dbgs() << I->Tok.getName() << " ";
+      llvm::dbgs() << (*I)->Tok.getName() << " ";
 
     }
     llvm::dbgs() << "\n";
@@ -1107,9 +1107,9 @@ void UnwrappedLineParser::readToken() {
 }
 
 void UnwrappedLineParser::pushToken(FormatToken *Tok) {
-  Line->Tokens.push_back(*Tok);
+  Line->Tokens.push_back(Tok);
   if (MustBreakBeforeNextToken) {
-    Line->Tokens.back().MustBreakBefore = true;
+    Line->Tokens.back()->MustBreakBefore = true;
     MustBreakBeforeNextToken = false;
   }
 }
diff --git a/lib/Format/UnwrappedLineParser.h b/lib/Format/UnwrappedLineParser.h
index 03505ff69a..8e92559742 100644
--- a/lib/Format/UnwrappedLineParser.h
+++ b/lib/Format/UnwrappedLineParser.h
@@ -79,6 +79,11 @@ struct FormatToken {
   /// Contains the raw token text without leading whitespace and without leading
   /// escaped newlines.
   StringRef TokenText;
+
+private:
+  // Disallow copying.
+  FormatToken(const FormatToken &);
+  void operator=(const FormatToken &);
 };
 
 /// \brief An unwrapped line is a sequence of \c Token, that we would like to
@@ -93,7 +98,7 @@ struct UnwrappedLine {
 
   // FIXME: Don't use std::list here.
   /// \brief The \c Tokens comprising this \c UnwrappedLine.
-  std::list<FormatToken> Tokens;
+  std::list<FormatToken *> Tokens;
 
   /// \brief The indent level of the \c UnwrappedLine.
   unsigned Level;
diff --git a/lib/Format/WhitespaceManager.cpp b/lib/Format/WhitespaceManager.cpp
index daf63697b0..069b0c497d 100644
--- a/lib/Format/WhitespaceManager.cpp
+++ b/lib/Format/WhitespaceManager.cpp
@@ -42,10 +42,10 @@ void WhitespaceManager::replaceWhitespace(const AnnotatedToken &Tok,
                                           unsigned Newlines, unsigned Spaces,
                                           unsigned StartOfTokenColumn,
                                           bool InPPDirective) {
-  Changes.push_back(Change(
-      true, Tok.FormatTok.WhitespaceRange,
-      Spaces, StartOfTokenColumn, Newlines, "", "", Tok.FormatTok.Tok.getKind(),
-      InPPDirective && !Tok.FormatTok.IsFirst));
+  Changes.push_back(
+      Change(true, Tok.FormatTok->WhitespaceRange, Spaces, StartOfTokenColumn,
+             Newlines, "", "", Tok.FormatTok->Tok.getKind(),
+             InPPDirective && !Tok.FormatTok->IsFirst));
 }
 
 void WhitespaceManager::addUntouchableToken(const FormatToken &Tok,