Clang Project

clang_source_code/lib/AST/CommentParser.cpp
1//===--- CommentParser.cpp - Doxygen comment parser -----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/AST/CommentParser.h"
10#include "clang/AST/CommentCommandTraits.h"
11#include "clang/AST/CommentDiagnostic.h"
12#include "clang/AST/CommentSema.h"
13#include "clang/Basic/CharInfo.h"
14#include "clang/Basic/SourceManager.h"
15#include "llvm/Support/ErrorHandling.h"
16
17namespace clang {
18
19static inline bool isWhitespace(llvm::StringRef S) {
20  for (StringRef::const_iterator I = S.begin(), E = S.end(); I != E; ++I) {
21    if (!isWhitespace(*I))
22      return false;
23  }
24  return true;
25}
26
27namespace comments {
28
29/// Re-lexes a sequence of tok::text tokens.
30class TextTokenRetokenizer {
31  llvm::BumpPtrAllocator &Allocator;
32  Parser &P;
33
34  /// This flag is set when there are no more tokens we can fetch from lexer.
35  bool NoMoreInterestingTokens;
36
37  /// Token buffer: tokens we have processed and lookahead.
38  SmallVector<Token, 16> Toks;
39
40  /// A position in \c Toks.
41  struct Position {
42    const char *BufferStart;
43    const char *BufferEnd;
44    const char *BufferPtr;
45    SourceLocation BufferStartLoc;
46    unsigned CurToken;
47  };
48
49  /// Current position in Toks.
50  Position Pos;
51
52  bool isEnd() const {
53    return Pos.CurToken >= Toks.size();
54  }
55
56  /// Sets up the buffer pointers to point to current token.
57  void setupBuffer() {
58    assert(!isEnd());
59    const Token &Tok = Toks[Pos.CurToken];
60
61    Pos.BufferStart = Tok.getText().begin();
62    Pos.BufferEnd = Tok.getText().end();
63    Pos.BufferPtr = Pos.BufferStart;
64    Pos.BufferStartLoc = Tok.getLocation();
65  }
66
67  SourceLocation getSourceLocation() const {
68    const unsigned CharNo = Pos.BufferPtr - Pos.BufferStart;
69    return Pos.BufferStartLoc.getLocWithOffset(CharNo);
70  }
71
72  char peek() const {
73    assert(!isEnd());
74    assert(Pos.BufferPtr != Pos.BufferEnd);
75    return *Pos.BufferPtr;
76  }
77
78  void consumeChar() {
79    assert(!isEnd());
80    assert(Pos.BufferPtr != Pos.BufferEnd);
81    Pos.BufferPtr++;
82    if (Pos.BufferPtr == Pos.BufferEnd) {
83      Pos.CurToken++;
84      if (isEnd() && !addToken())
85        return;
86
87      assert(!isEnd());
88      setupBuffer();
89    }
90  }
91
92  /// Add a token.
93  /// Returns true on success, false if there are no interesting tokens to
94  /// fetch from lexer.
95  bool addToken() {
96    if (NoMoreInterestingTokens)
97      return false;
98
99    if (P.Tok.is(tok::newline)) {
100      // If we see a single newline token between text tokens, skip it.
101      Token Newline = P.Tok;
102      P.consumeToken();
103      if (P.Tok.isNot(tok::text)) {
104        P.putBack(Newline);
105        NoMoreInterestingTokens = true;
106        return false;
107      }
108    }
109    if (P.Tok.isNot(tok::text)) {
110      NoMoreInterestingTokens = true;
111      return false;
112    }
113
114    Toks.push_back(P.Tok);
115    P.consumeToken();
116    if (Toks.size() == 1)
117      setupBuffer();
118    return true;
119  }
120
121  void consumeWhitespace() {
122    while (!isEnd()) {
123      if (isWhitespace(peek()))
124        consumeChar();
125      else
126        break;
127    }
128  }
129
130  void formTokenWithChars(Token &Result,
131                          SourceLocation Loc,
132                          const char *TokBegin,
133                          unsigned TokLength,
134                          StringRef Text) {
135    Result.setLocation(Loc);
136    Result.setKind(tok::text);
137    Result.setLength(TokLength);
138#ifndef NDEBUG
139    Result.TextPtr = "<UNSET>";
140    Result.IntVal = 7;
141#endif
142    Result.setText(Text);
143  }
144
145public:
146  TextTokenRetokenizer(llvm::BumpPtrAllocator &Allocator, Parser &P):
147      Allocator(Allocator), P(P), NoMoreInterestingTokens(false) {
148    Pos.CurToken = 0;
149    addToken();
150  }
151
152  /// Extract a word -- sequence of non-whitespace characters.
153  bool lexWord(Token &Tok) {
154    if (isEnd())
155      return false;
156
157    Position SavedPos = Pos;
158
159    consumeWhitespace();
160    SmallString<32> WordText;
161    const char *WordBegin = Pos.BufferPtr;
162    SourceLocation Loc = getSourceLocation();
163    while (!isEnd()) {
164      const char C = peek();
165      if (!isWhitespace(C)) {
166        WordText.push_back(C);
167        consumeChar();
168      } else
169        break;
170    }
171    const unsigned Length = WordText.size();
172    if (Length == 0) {
173      Pos = SavedPos;
174      return false;
175    }
176
177    char *TextPtr = Allocator.Allocate<char>(Length + 1);
178
179    memcpy(TextPtr, WordText.c_str(), Length + 1);
180    StringRef Text = StringRef(TextPtr, Length);
181
182    formTokenWithChars(Tok, Loc, WordBegin, Length, Text);
183    return true;
184  }
185
186  bool lexDelimitedSeq(Token &Tok, char OpenDelim, char CloseDelim) {
187    if (isEnd())
188      return false;
189
190    Position SavedPos = Pos;
191
192    consumeWhitespace();
193    SmallString<32> WordText;
194    const char *WordBegin = Pos.BufferPtr;
195    SourceLocation Loc = getSourceLocation();
196    bool Error = false;
197    if (!isEnd()) {
198      const char C = peek();
199      if (C == OpenDelim) {
200        WordText.push_back(C);
201        consumeChar();
202      } else
203        Error = true;
204    }
205    char C = '\0';
206    while (!Error && !isEnd()) {
207      C = peek();
208      WordText.push_back(C);
209      consumeChar();
210      if (C == CloseDelim)
211        break;
212    }
213    if (!Error && C != CloseDelim)
214      Error = true;
215
216    if (Error) {
217      Pos = SavedPos;
218      return false;
219    }
220
221    const unsigned Length = WordText.size();
222    char *TextPtr = Allocator.Allocate<char>(Length + 1);
223
224    memcpy(TextPtr, WordText.c_str(), Length + 1);
225    StringRef Text = StringRef(TextPtr, Length);
226
227    formTokenWithChars(Tok, Loc, WordBegin,
228                       Pos.BufferPtr - WordBegin, Text);
229    return true;
230  }
231
232  /// Put back tokens that we didn't consume.
233  void putBackLeftoverTokens() {
234    if (isEnd())
235      return;
236
237    bool HavePartialTok = false;
238    Token PartialTok;
239    if (Pos.BufferPtr != Pos.BufferStart) {
240      formTokenWithChars(PartialTok, getSourceLocation(),
241                         Pos.BufferPtr, Pos.BufferEnd - Pos.BufferPtr,
242                         StringRef(Pos.BufferPtr,
243                                   Pos.BufferEnd - Pos.BufferPtr));
244      HavePartialTok = true;
245      Pos.CurToken++;
246    }
247
248    P.putBack(llvm::makeArrayRef(Toks.begin() + Pos.CurToken, Toks.end()));
249    Pos.CurToken = Toks.size();
250
251    if (HavePartialTok)
252      P.putBack(PartialTok);
253  }
254};
255
256Parser::Parser(Lexer &L, Sema &S, llvm::BumpPtrAllocator &Allocator,
257               const SourceManager &SourceMgr, DiagnosticsEngine &Diags,
258               const CommandTraits &Traits):
259    L(L), S(S), Allocator(Allocator), SourceMgr(SourceMgr), Diags(Diags),
260    Traits(Traits) {
261  consumeToken();
262}
263
264void Parser::parseParamCommandArgs(ParamCommandComment *PC,
265                                   TextTokenRetokenizer &Retokenizer) {
266  Token Arg;
267  // Check if argument looks like direction specification: [dir]
268  // e.g., [in], [out], [in,out]
269  if (Retokenizer.lexDelimitedSeq(Arg, '[', ']'))
270    S.actOnParamCommandDirectionArg(PC,
271                                    Arg.getLocation(),
272                                    Arg.getEndLocation(),
273                                    Arg.getText());
274
275  if (Retokenizer.lexWord(Arg))
276    S.actOnParamCommandParamNameArg(PC,
277                                    Arg.getLocation(),
278                                    Arg.getEndLocation(),
279                                    Arg.getText());
280}
281
282void Parser::parseTParamCommandArgs(TParamCommandComment *TPC,
283                                    TextTokenRetokenizer &Retokenizer) {
284  Token Arg;
285  if (Retokenizer.lexWord(Arg))
286    S.actOnTParamCommandParamNameArg(TPC,
287                                     Arg.getLocation(),
288                                     Arg.getEndLocation(),
289                                     Arg.getText());
290}
291
292void Parser::parseBlockCommandArgs(BlockCommandComment *BC,
293                                   TextTokenRetokenizer &Retokenizer,
294                                   unsigned NumArgs) {
295  typedef BlockCommandComment::Argument Argument;
296  Argument *Args =
297      new (Allocator.Allocate<Argument>(NumArgs)) Argument[NumArgs];
298  unsigned ParsedArgs = 0;
299  Token Arg;
300  while (ParsedArgs < NumArgs && Retokenizer.lexWord(Arg)) {
301    Args[ParsedArgs] = Argument(SourceRange(Arg.getLocation(),
302                                            Arg.getEndLocation()),
303                                Arg.getText());
304    ParsedArgs++;
305  }
306
307  S.actOnBlockCommandArgs(BC, llvm::makeArrayRef(Args, ParsedArgs));
308}
309
310BlockCommandComment *Parser::parseBlockCommand() {
311  assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command));
312
313  ParamCommandComment *PC = nullptr;
314  TParamCommandComment *TPC = nullptr;
315  BlockCommandComment *BC = nullptr;
316  const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
317  CommandMarkerKind CommandMarker =
318      Tok.is(tok::backslash_command) ? CMK_Backslash : CMK_At;
319  if (Info->IsParamCommand) {
320    PC = S.actOnParamCommandStart(Tok.getLocation(),
321                                  Tok.getEndLocation(),
322                                  Tok.getCommandID(),
323                                  CommandMarker);
324  } else if (Info->IsTParamCommand) {
325    TPC = S.actOnTParamCommandStart(Tok.getLocation(),
326                                    Tok.getEndLocation(),
327                                    Tok.getCommandID(),
328                                    CommandMarker);
329  } else {
330    BC = S.actOnBlockCommandStart(Tok.getLocation(),
331                                  Tok.getEndLocation(),
332                                  Tok.getCommandID(),
333                                  CommandMarker);
334  }
335  consumeToken();
336
337  if (isTokBlockCommand()) {
338    // Block command ahead.  We can't nest block commands, so pretend that this
339    // command has an empty argument.
340    ParagraphComment *Paragraph = S.actOnParagraphComment(None);
341    if (PC) {
342      S.actOnParamCommandFinish(PC, Paragraph);
343      return PC;
344    } else if (TPC) {
345      S.actOnTParamCommandFinish(TPC, Paragraph);
346      return TPC;
347    } else {
348      S.actOnBlockCommandFinish(BC, Paragraph);
349      return BC;
350    }
351  }
352
353  if (PC || TPC || Info->NumArgs > 0) {
354    // In order to parse command arguments we need to retokenize a few
355    // following text tokens.
356    TextTokenRetokenizer Retokenizer(Allocator, *this);
357
358    if (PC)
359      parseParamCommandArgs(PC, Retokenizer);
360    else if (TPC)
361      parseTParamCommandArgs(TPC, Retokenizer);
362    else
363      parseBlockCommandArgs(BC, Retokenizer, Info->NumArgs);
364
365    Retokenizer.putBackLeftoverTokens();
366  }
367
368  // If there's a block command ahead, we will attach an empty paragraph to
369  // this command.
370  bool EmptyParagraph = false;
371  if (isTokBlockCommand())
372    EmptyParagraph = true;
373  else if (Tok.is(tok::newline)) {
374    Token PrevTok = Tok;
375    consumeToken();
376    EmptyParagraph = isTokBlockCommand();
377    putBack(PrevTok);
378  }
379
380  ParagraphComment *Paragraph;
381  if (EmptyParagraph)
382    Paragraph = S.actOnParagraphComment(None);
383  else {
384    BlockContentComment *Block = parseParagraphOrBlockCommand();
385    // Since we have checked for a block command, we should have parsed a
386    // paragraph.
387    Paragraph = cast<ParagraphComment>(Block);
388  }
389
390  if (PC) {
391    S.actOnParamCommandFinish(PC, Paragraph);
392    return PC;
393  } else if (TPC) {
394    S.actOnTParamCommandFinish(TPC, Paragraph);
395    return TPC;
396  } else {
397    S.actOnBlockCommandFinish(BC, Paragraph);
398    return BC;
399  }
400}
401
402InlineCommandComment *Parser::parseInlineCommand() {
403  assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command));
404
405  const Token CommandTok = Tok;
406  consumeToken();
407
408  TextTokenRetokenizer Retokenizer(Allocator, *this);
409
410  Token ArgTok;
411  bool ArgTokValid = Retokenizer.lexWord(ArgTok);
412
413  InlineCommandComment *IC;
414  if (ArgTokValid) {
415    IC = S.actOnInlineCommand(CommandTok.getLocation(),
416                              CommandTok.getEndLocation(),
417                              CommandTok.getCommandID(),
418                              ArgTok.getLocation(),
419                              ArgTok.getEndLocation(),
420                              ArgTok.getText());
421  } else {
422    IC = S.actOnInlineCommand(CommandTok.getLocation(),
423                              CommandTok.getEndLocation(),
424                              CommandTok.getCommandID());
425  }
426
427  Retokenizer.putBackLeftoverTokens();
428
429  return IC;
430}
431
432HTMLStartTagComment *Parser::parseHTMLStartTag() {
433  assert(Tok.is(tok::html_start_tag));
434  HTMLStartTagComment *HST =
435      S.actOnHTMLStartTagStart(Tok.getLocation(),
436                               Tok.getHTMLTagStartName());
437  consumeToken();
438
439  SmallVector<HTMLStartTagComment::Attribute, 2> Attrs;
440  while (true) {
441    switch (Tok.getKind()) {
442    case tok::html_ident: {
443      Token Ident = Tok;
444      consumeToken();
445      if (Tok.isNot(tok::html_equals)) {
446        Attrs.push_back(HTMLStartTagComment::Attribute(Ident.getLocation(),
447                                                       Ident.getHTMLIdent()));
448        continue;
449      }
450      Token Equals = Tok;
451      consumeToken();
452      if (Tok.isNot(tok::html_quoted_string)) {
453        Diag(Tok.getLocation(),
454             diag::warn_doc_html_start_tag_expected_quoted_string)
455          << SourceRange(Equals.getLocation());
456        Attrs.push_back(HTMLStartTagComment::Attribute(Ident.getLocation(),
457                                                       Ident.getHTMLIdent()));
458        while (Tok.is(tok::html_equals) ||
459               Tok.is(tok::html_quoted_string))
460          consumeToken();
461        continue;
462      }
463      Attrs.push_back(HTMLStartTagComment::Attribute(
464                              Ident.getLocation(),
465                              Ident.getHTMLIdent(),
466                              Equals.getLocation(),
467                              SourceRange(Tok.getLocation(),
468                                          Tok.getEndLocation()),
469                              Tok.getHTMLQuotedString()));
470      consumeToken();
471      continue;
472    }
473
474    case tok::html_greater:
475      S.actOnHTMLStartTagFinish(HST,
476                                S.copyArray(llvm::makeArrayRef(Attrs)),
477                                Tok.getLocation(),
478                                /* IsSelfClosing = */ false);
479      consumeToken();
480      return HST;
481
482    case tok::html_slash_greater:
483      S.actOnHTMLStartTagFinish(HST,
484                                S.copyArray(llvm::makeArrayRef(Attrs)),
485                                Tok.getLocation(),
486                                /* IsSelfClosing = */ true);
487      consumeToken();
488      return HST;
489
490    case tok::html_equals:
491    case tok::html_quoted_string:
492      Diag(Tok.getLocation(),
493           diag::warn_doc_html_start_tag_expected_ident_or_greater);
494      while (Tok.is(tok::html_equals) ||
495             Tok.is(tok::html_quoted_string))
496        consumeToken();
497      if (Tok.is(tok::html_ident) ||
498          Tok.is(tok::html_greater) ||
499          Tok.is(tok::html_slash_greater))
500        continue;
501
502      S.actOnHTMLStartTagFinish(HST,
503                                S.copyArray(llvm::makeArrayRef(Attrs)),
504                                SourceLocation(),
505                                /* IsSelfClosing = */ false);
506      return HST;
507
508    default:
509      // Not a token from an HTML start tag.  Thus HTML tag prematurely ended.
510      S.actOnHTMLStartTagFinish(HST,
511                                S.copyArray(llvm::makeArrayRef(Attrs)),
512                                SourceLocation(),
513                                /* IsSelfClosing = */ false);
514      bool StartLineInvalid;
515      const unsigned StartLine = SourceMgr.getPresumedLineNumber(
516                                                  HST->getLocation(),
517                                                  &StartLineInvalid);
518      bool EndLineInvalid;
519      const unsigned EndLine = SourceMgr.getPresumedLineNumber(
520                                                  Tok.getLocation(),
521                                                  &EndLineInvalid);
522      if (StartLineInvalid || EndLineInvalid || StartLine == EndLine)
523        Diag(Tok.getLocation(),
524             diag::warn_doc_html_start_tag_expected_ident_or_greater)
525          << HST->getSourceRange();
526      else {
527        Diag(Tok.getLocation(),
528             diag::warn_doc_html_start_tag_expected_ident_or_greater);
529        Diag(HST->getLocation(), diag::note_doc_html_tag_started_here)
530          << HST->getSourceRange();
531      }
532      return HST;
533    }
534  }
535}
536
537HTMLEndTagComment *Parser::parseHTMLEndTag() {
538  assert(Tok.is(tok::html_end_tag));
539  Token TokEndTag = Tok;
540  consumeToken();
541  SourceLocation Loc;
542  if (Tok.is(tok::html_greater)) {
543    Loc = Tok.getLocation();
544    consumeToken();
545  }
546
547  return S.actOnHTMLEndTag(TokEndTag.getLocation(),
548                           Loc,
549                           TokEndTag.getHTMLTagEndName());
550}
551
552BlockContentComment *Parser::parseParagraphOrBlockCommand() {
553  SmallVector<InlineContentComment *, 8> Content;
554
555  while (true) {
556    switch (Tok.getKind()) {
557    case tok::verbatim_block_begin:
558    case tok::verbatim_line_name:
559    case tok::eof:
560      break; // Block content or EOF ahead, finish this parapgaph.
561
562    case tok::unknown_command:
563      Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
564                                              Tok.getEndLocation(),
565                                              Tok.getUnknownCommandName()));
566      consumeToken();
567      continue;
568
569    case tok::backslash_command:
570    case tok::at_command: {
571      const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
572      if (Info->IsBlockCommand) {
573        if (Content.size() == 0)
574          return parseBlockCommand();
575        break; // Block command ahead, finish this parapgaph.
576      }
577      if (Info->IsVerbatimBlockEndCommand) {
578        Diag(Tok.getLocation(),
579             diag::warn_verbatim_block_end_without_start)
580          << Tok.is(tok::at_command)
581          << Info->Name
582          << SourceRange(Tok.getLocation(), Tok.getEndLocation());
583        consumeToken();
584        continue;
585      }
586      if (Info->IsUnknownCommand) {
587        Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
588                                                Tok.getEndLocation(),
589                                                Info->getID()));
590        consumeToken();
591        continue;
592      }
593      IsInlineCommand", "/home/seafit/code_projects/clang_source/clang/lib/AST/CommentParser.cpp", 593, __PRETTY_FUNCTION__))" file_link="../../../include/assert.h.html#88" macro="true">assert(Info->IsInlineCommand);
594      Content.push_back(parseInlineCommand());
595      continue;
596    }
597
598    case tok::newline: {
599      consumeToken();
600      if (Tok.is(tok::newline) || Tok.is(tok::eof)) {
601        consumeToken();
602        break; // Two newlines -- end of paragraph.
603      }
604      // Also allow [tok::newline, tok::text, tok::newline] if the middle
605      // tok::text is just whitespace.
606      if (Tok.is(tok::text) && isWhitespace(Tok.getText())) {
607        Token WhitespaceTok = Tok;
608        consumeToken();
609        if (Tok.is(tok::newline) || Tok.is(tok::eof)) {
610          consumeToken();
611          break;
612        }
613        // We have [tok::newline, tok::text, non-newline].  Put back tok::text.
614        putBack(WhitespaceTok);
615      }
616      if (Content.size() > 0)
617        Content.back()->addTrailingNewline();
618      continue;
619    }
620
621    // Don't deal with HTML tag soup now.
622    case tok::html_start_tag:
623      Content.push_back(parseHTMLStartTag());
624      continue;
625
626    case tok::html_end_tag:
627      Content.push_back(parseHTMLEndTag());
628      continue;
629
630    case tok::text:
631      Content.push_back(S.actOnText(Tok.getLocation(),
632                                    Tok.getEndLocation(),
633                                    Tok.getText()));
634      consumeToken();
635      continue;
636
637    case tok::verbatim_block_line:
638    case tok::verbatim_block_end:
639    case tok::verbatim_line_text:
640    case tok::html_ident:
641    case tok::html_equals:
642    case tok::html_quoted_string:
643    case tok::html_greater:
644    case tok::html_slash_greater:
645      llvm_unreachable("should not see this token");
646    }
647    break;
648  }
649
650  return S.actOnParagraphComment(S.copyArray(llvm::makeArrayRef(Content)));
651}
652
653VerbatimBlockComment *Parser::parseVerbatimBlock() {
654  assert(Tok.is(tok::verbatim_block_begin));
655
656  VerbatimBlockComment *VB =
657      S.actOnVerbatimBlockStart(Tok.getLocation(),
658                                Tok.getVerbatimBlockID());
659  consumeToken();
660
661  // Don't create an empty line if verbatim opening command is followed
662  // by a newline.
663  if (Tok.is(tok::newline))
664    consumeToken();
665
666  SmallVector<VerbatimBlockLineComment *, 8> Lines;
667  while (Tok.is(tok::verbatim_block_line) ||
668         Tok.is(tok::newline)) {
669    VerbatimBlockLineComment *Line;
670    if (Tok.is(tok::verbatim_block_line)) {
671      Line = S.actOnVerbatimBlockLine(Tok.getLocation(),
672                                      Tok.getVerbatimBlockText());
673      consumeToken();
674      if (Tok.is(tok::newline)) {
675        consumeToken();
676      }
677    } else {
678      // Empty line, just a tok::newline.
679      Line = S.actOnVerbatimBlockLine(Tok.getLocation(), "");
680      consumeToken();
681    }
682    Lines.push_back(Line);
683  }
684
685  if (Tok.is(tok::verbatim_block_end)) {
686    const CommandInfo *Info = Traits.getCommandInfo(Tok.getVerbatimBlockID());
687    S.actOnVerbatimBlockFinish(VB, Tok.getLocation(),
688                               Info->Name,
689                               S.copyArray(llvm::makeArrayRef(Lines)));
690    consumeToken();
691  } else {
692    // Unterminated \\verbatim block
693    S.actOnVerbatimBlockFinish(VB, SourceLocation(), "",
694                               S.copyArray(llvm::makeArrayRef(Lines)));
695  }
696
697  return VB;
698}
699
700VerbatimLineComment *Parser::parseVerbatimLine() {
701  assert(Tok.is(tok::verbatim_line_name));
702
703  Token NameTok = Tok;
704  consumeToken();
705
706  SourceLocation TextBegin;
707  StringRef Text;
708  // Next token might not be a tok::verbatim_line_text if verbatim line
709  // starting command comes just before a newline or comment end.
710  if (Tok.is(tok::verbatim_line_text)) {
711    TextBegin = Tok.getLocation();
712    Text = Tok.getVerbatimLineText();
713  } else {
714    TextBegin = NameTok.getEndLocation();
715    Text = "";
716  }
717
718  VerbatimLineComment *VL = S.actOnVerbatimLine(NameTok.getLocation(),
719                                                NameTok.getVerbatimLineID(),
720                                                TextBegin,
721                                                Text);
722  consumeToken();
723  return VL;
724}
725
726BlockContentComment *Parser::parseBlockContent() {
727  switch (Tok.getKind()) {
728  case tok::text:
729  case tok::unknown_command:
730  case tok::backslash_command:
731  case tok::at_command:
732  case tok::html_start_tag:
733  case tok::html_end_tag:
734    return parseParagraphOrBlockCommand();
735
736  case tok::verbatim_block_begin:
737    return parseVerbatimBlock();
738
739  case tok::verbatim_line_name:
740    return parseVerbatimLine();
741
742  case tok::eof:
743  case tok::newline:
744  case tok::verbatim_block_line:
745  case tok::verbatim_block_end:
746  case tok::verbatim_line_text:
747  case tok::html_ident:
748  case tok::html_equals:
749  case tok::html_quoted_string:
750  case tok::html_greater:
751  case tok::html_slash_greater:
752    llvm_unreachable("should not see this token");
753  }
754  llvm_unreachable("bogus token kind");
755}
756
757FullComment *Parser::parseFullComment() {
758  // Skip newlines at the beginning of the comment.
759  while (Tok.is(tok::newline))
760    consumeToken();
761
762  SmallVector<BlockContentComment *, 8> Blocks;
763  while (Tok.isNot(tok::eof)) {
764    Blocks.push_back(parseBlockContent());
765
766    // Skip extra newlines after paragraph end.
767    while (Tok.is(tok::newline))
768      consumeToken();
769  }
770  return S.actOnFullComment(S.copyArray(llvm::makeArrayRef(Blocks)));
771}
772
773} // end namespace comments
774} // end namespace clang
775
clang::comments::TextTokenRetokenizer::Allocator
clang::comments::TextTokenRetokenizer::P
clang::comments::TextTokenRetokenizer::NoMoreInterestingTokens
clang::comments::TextTokenRetokenizer::Toks
clang::comments::TextTokenRetokenizer::Position
clang::comments::TextTokenRetokenizer::Pos
clang::comments::TextTokenRetokenizer::isEnd
clang::comments::TextTokenRetokenizer::setupBuffer
clang::comments::TextTokenRetokenizer::getSourceLocation
clang::comments::TextTokenRetokenizer::peek
clang::comments::TextTokenRetokenizer::consumeChar
clang::comments::TextTokenRetokenizer::addToken
clang::comments::TextTokenRetokenizer::consumeWhitespace
clang::comments::TextTokenRetokenizer::formTokenWithChars
clang::comments::TextTokenRetokenizer::lexWord
clang::comments::TextTokenRetokenizer::lexDelimitedSeq
clang::comments::TextTokenRetokenizer::putBackLeftoverTokens
clang::comments::Parser::parseParamCommandArgs
clang::comments::Parser::parseTParamCommandArgs
clang::comments::Parser::parseBlockCommandArgs
clang::comments::Parser::parseBlockCommand
clang::comments::Parser::parseInlineCommand
clang::comments::Parser::parseHTMLStartTag
clang::comments::Parser::parseHTMLEndTag
clang::comments::Parser::parseParagraphOrBlockCommand
clang::comments::Parser::parseVerbatimBlock
clang::comments::Parser::parseVerbatimLine
clang::comments::Parser::parseBlockContent
clang::comments::Parser::parseFullComment
CodeThinker, Enjoy your reading code