Vendor import of clang trunk r321017:

https://llvm.org/svn/llvm-project/cfe/trunk@321017

tools/CMakeLists.txt

@@ -2,6 +2,7 @@ create_subdirectory_options(CLANG TOOL)
 
 add_clang_subdirectory(diagtool)
 add_clang_subdirectory(driver)
+add_clang_subdirectory(clang-diff)
 add_clang_subdirectory(clang-format)
 add_clang_subdirectory(clang-format-vs)
 add_clang_subdirectory(clang-fuzzer)
@@ -11,6 +12,7 @@ add_clang_subdirectory(clang-offload-bundler)
 add_clang_subdirectory(c-index-test)
 
 add_clang_subdirectory(clang-rename)
+add_clang_subdirectory(clang-refactor)
 
 if(CLANG_ENABLE_ARCMT)
   add_clang_subdirectory(arcmt-test)
@@ -19,6 +21,7 @@ endif()
 
 if(CLANG_ENABLE_STATIC_ANALYZER)
   add_clang_subdirectory(clang-check)
+  add_clang_subdirectory(clang-func-mapping)
   add_clang_subdirectory(scan-build)
   add_clang_subdirectory(scan-view)
 endif()

tools/arcmt-test/CMakeLists.txt

@@ -7,6 +7,7 @@ add_clang_executable(arcmt-test
   )
 
 target_link_libraries(arcmt-test
+  PRIVATE
   clangARCMigrate
   clangBasic
   clangFrontend

tools/c-arcmt-test/CMakeLists.txt

@@ -4,10 +4,12 @@ add_clang_executable(c-arcmt-test
 
 if (LLVM_BUILD_STATIC)
   target_link_libraries(c-arcmt-test
+    PRIVATE
     libclang_static
     )
 else()
   target_link_libraries(c-arcmt-test
+    PRIVATE
     libclang
     )
 endif()

tools/c-index-test/CMakeLists.txt

@@ -16,12 +16,14 @@ endif()
 
 if (LLVM_BUILD_STATIC)
   target_link_libraries(c-index-test
+    PRIVATE
     libclang_static
     clangCodeGen
     clangIndex
   )
 else()
   target_link_libraries(c-index-test
+    PRIVATE
     libclang
     clangAST
     clangBasic
@@ -39,7 +41,7 @@ set_target_properties(c-index-test
 # If libxml2 is available, make it available for c-index-test.
 if (CLANG_HAVE_LIBXML)
   include_directories(SYSTEM ${LIBXML2_INCLUDE_DIR})
-  target_link_libraries(c-index-test ${LIBXML2_LIBRARIES})
+  target_link_libraries(c-index-test PRIVATE ${LIBXML2_LIBRARIES})
 endif()
 
 if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY)
@@ -56,10 +58,8 @@ if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY)
     COMPONENT c-index-test)
 
   if (NOT CMAKE_CONFIGURATION_TYPES) # don't add this for IDE's.
-    add_custom_target(install-c-index-test
-      DEPENDS c-index-test
-      COMMAND "${CMAKE_COMMAND}"
-              -DCMAKE_INSTALL_COMPONENT=c-index-test
-              -P "${CMAKE_BINARY_DIR}/cmake_install.cmake")
+    add_llvm_install_targets(install-c-index-test
+                             DEPENDS c-index-test
+                             COMPONENT c-index-test)
   endif()
 endif()

tools/c-index-test/c-index-test.c

@@ -804,6 +804,8 @@ static void PrintCursor(CXCursor Cursor, const char *CommentSchemaFile) {
       printf(" (const)");
     if (clang_CXXMethod_isPureVirtual(Cursor))
       printf(" (pure)");
+    if (clang_CXXRecord_isAbstract(Cursor))
+      printf(" (abstract)");
     if (clang_EnumDecl_isScoped(Cursor))
       printf(" (scoped)");
     if (clang_Cursor_isVariadic(Cursor))
@@ -1563,10 +1565,19 @@ static enum CXChildVisitResult PrintManglings(CXCursor cursor, CXCursor p,
     return CXChildVisit_Continue;
   PrintCursor(cursor, NULL);
   Manglings = clang_Cursor_getCXXManglings(cursor);
-  for (I = 0, E = Manglings->Count; I < E; ++I)
-    printf(" [mangled=%s]", clang_getCString(Manglings->Strings[I]));
-  clang_disposeStringSet(Manglings);
-  printf("\n");
+  if (Manglings) {
+    for (I = 0, E = Manglings->Count; I < E; ++I)
+      printf(" [mangled=%s]", clang_getCString(Manglings->Strings[I]));
+    clang_disposeStringSet(Manglings);
+    printf("\n");
+  }
+  Manglings = clang_Cursor_getObjCManglings(cursor);
+  if (Manglings) {
+    for (I = 0, E = Manglings->Count; I < E; ++I)
+      printf(" [mangled=%s]", clang_getCString(Manglings->Strings[I]));
+    clang_disposeStringSet(Manglings);
+    printf("\n");
+  }
   return CXChildVisit_Recurse;
 }
 
@@ -1738,11 +1749,15 @@ int perform_test_load_source(int argc, const char **argv,
   int result;
   unsigned Repeats = 0;
   unsigned I;
+  const char *InvocationPath;
 
   Idx = clang_createIndex(/* excludeDeclsFromPCH */
                           (!strcmp(filter, "local") || 
                            !strcmp(filter, "local-display"))? 1 : 0,
                           /* displayDiagnostics=*/1);
+  InvocationPath = getenv("CINDEXTEST_INVOCATION_EMISSION_PATH");
+  if (InvocationPath)
+    clang_CXIndex_setInvocationEmissionPathOption(Idx, InvocationPath);
 
   if ((CommentSchemaFile = parse_comments_schema(argc, argv))) {
     argc--;
@@ -2303,7 +2318,8 @@ int perform_code_completion(int argc, const char **argv, int timing_only) {
   CXTranslationUnit TU;
   unsigned I, Repeats = 1;
   unsigned completionOptions = clang_defaultCodeCompleteOptions();
-  
+  const char *InvocationPath;
+
   if (getenv("CINDEXTEST_CODE_COMPLETE_PATTERNS"))
     completionOptions |= CXCodeComplete_IncludeCodePatterns;
   if (getenv("CINDEXTEST_COMPLETION_BRIEF_COMMENTS"))
@@ -2322,7 +2338,10 @@ int perform_code_completion(int argc, const char **argv, int timing_only) {
     return -1;
 
   CIdx = clang_createIndex(0, 0);
-  
+  InvocationPath = getenv("CINDEXTEST_INVOCATION_EMISSION_PATH");
+  if (InvocationPath)
+    clang_CXIndex_setInvocationEmissionPathOption(CIdx, InvocationPath);
+
   if (getenv("CINDEXTEST_EDITING"))
     Repeats = 5;
 

tools/clang-check/CMakeLists.txt

@@ -9,6 +9,7 @@ add_clang_executable(clang-check
   )
 
 target_link_libraries(clang-check
+  PRIVATE
   clangAST
   clangBasic
   clangDriver

tools/clang-diff/CMakeLists.txt

@@ -0,0 +1,15 @@
+set(LLVM_LINK_COMPONENTS
+  Support
+  )
+
+add_clang_executable(clang-diff
+  ClangDiff.cpp
+  )
+
+target_link_libraries(clang-diff
+  PRIVATE
+  clangBasic
+  clangFrontend
+  clangTooling
+  clangToolingASTDiff
+  )

tools/clang-diff/ClangDiff.cpp

@@ -0,0 +1,537 @@
+//===- ClangDiff.cpp - compare source files by AST nodes ------*- C++ -*- -===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a tool for syntax tree based comparison using
+// Tooling/ASTDiff.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/ASTDiff/ASTDiff.h"
+#include "clang/Tooling/CommonOptionsParser.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/Support/CommandLine.h"
+
+using namespace llvm;
+using namespace clang;
+using namespace clang::tooling;
+
+static cl::OptionCategory ClangDiffCategory("clang-diff options");
+
+static cl::opt<bool>
+    ASTDump("ast-dump",
+            cl::desc("Print the internal representation of the AST."),
+            cl::init(false), cl::cat(ClangDiffCategory));
+
+static cl::opt<bool> ASTDumpJson(
+    "ast-dump-json",
+    cl::desc("Print the internal representation of the AST as JSON."),
+    cl::init(false), cl::cat(ClangDiffCategory));
+
+static cl::opt<bool> PrintMatches("dump-matches",
+                                  cl::desc("Print the matched nodes."),
+                                  cl::init(false), cl::cat(ClangDiffCategory));
+
+static cl::opt<bool> HtmlDiff("html",
+                              cl::desc("Output a side-by-side diff in HTML."),
+                              cl::init(false), cl::cat(ClangDiffCategory));
+
+static cl::opt<std::string> SourcePath(cl::Positional, cl::desc("<source>"),
+                                       cl::Required,
+                                       cl::cat(ClangDiffCategory));
+
+static cl::opt<std::string> DestinationPath(cl::Positional,
+                                            cl::desc("<destination>"),
+                                            cl::Optional,
+                                            cl::cat(ClangDiffCategory));
+
+static cl::opt<std::string> StopAfter("stop-diff-after",
+                                      cl::desc("<topdown|bottomup>"),
+                                      cl::Optional, cl::init(""),
+                                      cl::cat(ClangDiffCategory));
+
+static cl::opt<int> MaxSize("s", cl::desc("<maxsize>"), cl::Optional,
+                            cl::init(-1), cl::cat(ClangDiffCategory));
+
+static cl::opt<std::string> BuildPath("p", cl::desc("Build path"), cl::init(""),
+                                      cl::Optional, cl::cat(ClangDiffCategory));
+
+static cl::list<std::string> ArgsAfter(
+    "extra-arg",
+    cl::desc("Additional argument to append to the compiler command line"),
+    cl::cat(ClangDiffCategory));
+
+static cl::list<std::string> ArgsBefore(
+    "extra-arg-before",
+    cl::desc("Additional argument to prepend to the compiler command line"),
+    cl::cat(ClangDiffCategory));
+
+static void addExtraArgs(std::unique_ptr<CompilationDatabase> &Compilations) {
+  if (!Compilations)
+    return;
+  auto AdjustingCompilations =
+      llvm::make_unique<ArgumentsAdjustingCompilations>(
+          std::move(Compilations));
+  AdjustingCompilations->appendArgumentsAdjuster(
+      getInsertArgumentAdjuster(ArgsBefore, ArgumentInsertPosition::BEGIN));
+  AdjustingCompilations->appendArgumentsAdjuster(
+      getInsertArgumentAdjuster(ArgsAfter, ArgumentInsertPosition::END));
+  Compilations = std::move(AdjustingCompilations);
+}
+
+static std::unique_ptr<ASTUnit>
+getAST(const std::unique_ptr<CompilationDatabase> &CommonCompilations,
+       const StringRef Filename) {
+  std::string ErrorMessage;
+  std::unique_ptr<CompilationDatabase> Compilations;
+  if (!CommonCompilations) {
+    Compilations = CompilationDatabase::autoDetectFromSource(
+        BuildPath.empty() ? Filename : BuildPath, ErrorMessage);
+    if (!Compilations) {
+      llvm::errs()
+          << "Error while trying to load a compilation database, running "
+             "without flags.\n"
+          << ErrorMessage;
+      Compilations =
+          llvm::make_unique<clang::tooling::FixedCompilationDatabase>(
+              ".", std::vector<std::string>());
+    }
+  }
+  addExtraArgs(Compilations);
+  std::array<std::string, 1> Files = {{Filename}};
+  ClangTool Tool(Compilations ? *Compilations : *CommonCompilations, Files);
+  std::vector<std::unique_ptr<ASTUnit>> ASTs;
+  Tool.buildASTs(ASTs);
+  if (ASTs.size() != Files.size())
+    return nullptr;
+  return std::move(ASTs[0]);
+}
+
+static char hexdigit(int N) { return N &= 0xf, N + (N < 10 ? '0' : 'a' - 10); }
+
+static const char HtmlDiffHeader[] = R"(
+<html>
+<head>
+<meta charset='utf-8'/>
+<style>
+span.d { color: red; }
+span.u { color: #cc00cc; }
+span.i { color: green; }
+span.m { font-weight: bold; }
+span   { font-weight: normal; color: black; }
+div.code {
+  width: 48%;
+  height: 98%;
+  overflow: scroll;
+  float: left;
+  padding: 0 0 0.5% 0.5%;
+  border: solid 2px LightGrey;
+  border-radius: 5px;
+}
+</style>
+</head>
+<script type='text/javascript'>
+highlightStack = []
+function clearHighlight() {
+  while (highlightStack.length) {
+    var [l, r] = highlightStack.pop()
+    document.getElementById(l).style.backgroundColor = 'inherit'
+    if (r[1] != '-')
+      document.getElementById(r).style.backgroundColor = 'inherit'
+  }
+}
+function highlight(event) {
+  var id = event.target['id']
+  doHighlight(id)
+}
+function doHighlight(id) {
+  clearHighlight()
+  source = document.getElementById(id)
+  if (!source.attributes['tid'])
+    return
+  var mapped = source
+  while (mapped && mapped.parentElement && mapped.attributes['tid'].value.substr(1) === '-1')
+    mapped = mapped.parentElement
+  var tid = null, target = null
+  if (mapped) {
+    tid = mapped.attributes['tid'].value
+    target = document.getElementById(tid)
+  }
+  if (source.parentElement && source.parentElement.classList.contains('code'))
+    return
+  source.style.backgroundColor = 'lightgrey'
+  source.scrollIntoView()
+  if (target) {
+    if (mapped === source)
+      target.style.backgroundColor = 'lightgrey'
+    target.scrollIntoView()
+  }
+  highlightStack.push([id, tid])
+  location.hash = '#' + id
+}
+function scrollToBoth() {
+  doHighlight(location.hash.substr(1))
+}
+function changed(elem) {
+  return elem.classList.length == 0
+}
+function nextChangedNode(prefix, increment, number) {
+  do {
+    number += increment
+    var elem = document.getElementById(prefix + number)
+  } while(elem && !changed(elem))
+  return elem ? number : null
+}
+function handleKey(e) {
+  var down = e.code === "KeyJ"
+  var up = e.code === "KeyK"
+  if (!down && !up)
+    return
+  var id = highlightStack[0] ? highlightStack[0][0] : 'R0'
+  var oldelem = document.getElementById(id)
+  var number = parseInt(id.substr(1))
+  var increment = down ? 1 : -1
+  var lastnumber = number
+  var prefix = id[0]
+  do {
+    number = nextChangedNode(prefix, increment, number)
+    var elem = document.getElementById(prefix + number)
+    if (up && elem) {
+      while (elem.parentElement && changed(elem.parentElement))
+        elem = elem.parentElement
+      number = elem.id.substr(1)
+    }
+  } while ((down && id !== 'R0' && oldelem.contains(elem)))
+  if (!number)
+    number = lastnumber
+  elem = document.getElementById(prefix + number)
+  doHighlight(prefix + number)
+}
+window.onload = scrollToBoth
+window.onkeydown = handleKey
+</script>
+<body>
+<div onclick='highlight(event)'>
+)";
+
+static void printHtml(raw_ostream &OS, char C) {
+  switch (C) {
+  case '&':
+    OS << "&amp;";
+    break;
+  case '<':
+    OS << "&lt;";
+    break;
+  case '>':
+    OS << "&gt;";
+    break;
+  case '\'':
+    OS << "&#x27;";
+    break;
+  case '"':
+    OS << "&quot;";
+    break;
+  default:
+    OS << C;
+  }
+}
+
+static void printHtml(raw_ostream &OS, const StringRef Str) {
+  for (char C : Str)
+    printHtml(OS, C);
+}
+
+static std::string getChangeKindAbbr(diff::ChangeKind Kind) {
+  switch (Kind) {
+  case diff::None:
+    return "";
+  case diff::Delete:
+    return "d";
+  case diff::Update:
+    return "u";
+  case diff::Insert:
+    return "i";
+  case diff::Move:
+    return "m";
+  case diff::UpdateMove:
+    return "u m";
+  }
+  llvm_unreachable("Invalid enumeration value.");
+}
+
+static unsigned printHtmlForNode(raw_ostream &OS, const diff::ASTDiff &Diff,
+                                 diff::SyntaxTree &Tree, bool IsLeft,
+                                 diff::NodeId Id, unsigned Offset) {
+  const diff::Node &Node = Tree.getNode(Id);
+  char MyTag, OtherTag;
+  diff::NodeId LeftId, RightId;
+  diff::NodeId TargetId = Diff.getMapped(Tree, Id);
+  if (IsLeft) {
+    MyTag = 'L';
+    OtherTag = 'R';
+    LeftId = Id;
+    RightId = TargetId;
+  } else {
+    MyTag = 'R';
+    OtherTag = 'L';
+    LeftId = TargetId;
+    RightId = Id;
+  }
+  unsigned Begin, End;
+  std::tie(Begin, End) = Tree.getSourceRangeOffsets(Node);
+  const SourceManager &SrcMgr = Tree.getASTContext().getSourceManager();
+  auto Code = SrcMgr.getBuffer(SrcMgr.getMainFileID())->getBuffer();
+  for (; Offset < Begin; ++Offset)
+    printHtml(OS, Code[Offset]);
+  OS << "<span id='" << MyTag << Id << "' "
+     << "tid='" << OtherTag << TargetId << "' ";
+  OS << "title='";
+  printHtml(OS, Node.getTypeLabel());
+  OS << "\n" << LeftId << " -> " << RightId;
+  std::string Value = Tree.getNodeValue(Node);
+  if (!Value.empty()) {
+    OS << "\n";
+    printHtml(OS, Value);
+  }
+  OS << "'";
+  if (Node.Change != diff::None)
+    OS << " class='" << getChangeKindAbbr(Node.Change) << "'";
+  OS << ">";
+
+  for (diff::NodeId Child : Node.Children)
+    Offset = printHtmlForNode(OS, Diff, Tree, IsLeft, Child, Offset);
+
+  for (; Offset < End; ++Offset)
+    printHtml(OS, Code[Offset]);
+  if (Id == Tree.getRootId()) {
+    End = Code.size();
+    for (; Offset < End; ++Offset)
+      printHtml(OS, Code[Offset]);
+  }
+  OS << "</span>";
+  return Offset;
+}
+
+static void printJsonString(raw_ostream &OS, const StringRef Str) {
+  for (signed char C : Str) {
+    switch (C) {
+    case '"':
+      OS << R"(\")";
+      break;
+    case '\\':
+      OS << R"(\\)";
+      break;
+    case '\n':
+      OS << R"(\n)";
+      break;
+    case '\t':
+      OS << R"(\t)";
+      break;
+    default:
+      if ('\x00' <= C && C <= '\x1f') {
+        OS << R"(\u00)" << hexdigit(C >> 4) << hexdigit(C);
+      } else {
+        OS << C;
+      }
+    }
+  }
+}
+
+static void printNodeAttributes(raw_ostream &OS, diff::SyntaxTree &Tree,
+                                diff::NodeId Id) {
+  const diff::Node &N = Tree.getNode(Id);
+  OS << R"("id":)" << int(Id);
+  OS << R"(,"type":")" << N.getTypeLabel() << '"';
+  auto Offsets = Tree.getSourceRangeOffsets(N);
+  OS << R"(,"begin":)" << Offsets.first;
+  OS << R"(,"end":)" << Offsets.second;
+  std::string Value = Tree.getNodeValue(N);
+  if (!Value.empty()) {
+    OS << R"(,"value":")";
+    printJsonString(OS, Value);
+    OS << '"';
+  }
+}
+
+static void printNodeAsJson(raw_ostream &OS, diff::SyntaxTree &Tree,
+                            diff::NodeId Id) {
+  const diff::Node &N = Tree.getNode(Id);
+  OS << "{";
+  printNodeAttributes(OS, Tree, Id);
+  auto Identifier = N.getIdentifier();
+  auto QualifiedIdentifier = N.getQualifiedIdentifier();
+  if (Identifier) {
+    OS << R"(,"identifier":")";
+    printJsonString(OS, *Identifier);
+    OS << R"(")";
+    if (QualifiedIdentifier && *Identifier != *QualifiedIdentifier) {
+      OS << R"(,"qualified_identifier":")";
+      printJsonString(OS, *QualifiedIdentifier);
+      OS << R"(")";
+    }
+  }
+  OS << R"(,"children":[)";
+  if (N.Children.size() > 0) {
+    printNodeAsJson(OS, Tree, N.Children[0]);
+    for (size_t I = 1, E = N.Children.size(); I < E; ++I) {
+      OS << ",";
+      printNodeAsJson(OS, Tree, N.Children[I]);
+    }
+  }
+  OS << "]}";
+}
+
+static void printNode(raw_ostream &OS, diff::SyntaxTree &Tree,
+                      diff::NodeId Id) {
+  if (Id.isInvalid()) {
+    OS << "None";
+    return;
+  }
+  OS << Tree.getNode(Id).getTypeLabel();
+  std::string Value = Tree.getNodeValue(Id);
+  if (!Value.empty())
+    OS << ": " << Value;
+  OS << "(" << Id << ")";
+}
+
+static void printTree(raw_ostream &OS, diff::SyntaxTree &Tree) {
+  for (diff::NodeId Id : Tree) {
+    for (int I = 0; I < Tree.getNode(Id).Depth; ++I)
+      OS << " ";
+    printNode(OS, Tree, Id);
+    OS << "\n";
+  }
+}
+
+static void printDstChange(raw_ostream &OS, diff::ASTDiff &Diff,
+                           diff::SyntaxTree &SrcTree, diff::SyntaxTree &DstTree,
+                           diff::NodeId Dst) {
+  const diff::Node &DstNode = DstTree.getNode(Dst);
+  diff::NodeId Src = Diff.getMapped(DstTree, Dst);
+  switch (DstNode.Change) {
+  case diff::None:
+    break;
+  case diff::Delete:
+    llvm_unreachable("The destination tree can't have deletions.");
+  case diff::Update:
+    OS << "Update ";
+    printNode(OS, SrcTree, Src);
+    OS << " to " << DstTree.getNodeValue(Dst) << "\n";
+    break;
+  case diff::Insert:
+  case diff::Move:
+  case diff::UpdateMove:
+    if (DstNode.Change == diff::Insert)
+      OS << "Insert";
+    else if (DstNode.Change == diff::Move)
+      OS << "Move";
+    else if (DstNode.Change == diff::UpdateMove)
+      OS << "Update and Move";
+    OS << " ";
+    printNode(OS, DstTree, Dst);
+    OS << " into ";
+    printNode(OS, DstTree, DstNode.Parent);
+    OS << " at " << DstTree.findPositionInParent(Dst) << "\n";
+    break;
+  }
+}
+
+int main(int argc, const char **argv) {
+  std::string ErrorMessage;
+  std::unique_ptr<CompilationDatabase> CommonCompilations =
+      FixedCompilationDatabase::loadFromCommandLine(argc, argv, ErrorMessage);
+  if (!CommonCompilations && !ErrorMessage.empty())
+    llvm::errs() << ErrorMessage;
+  cl::HideUnrelatedOptions(ClangDiffCategory);
+  if (!cl::ParseCommandLineOptions(argc, argv)) {
+    cl::PrintOptionValues();
+    return 1;
+  }
+
+  addExtraArgs(CommonCompilations);
+
+  if (ASTDump || ASTDumpJson) {
+    if (!DestinationPath.empty()) {
+      llvm::errs() << "Error: Please specify exactly one filename.\n";
+      return 1;
+    }
+    std::unique_ptr<ASTUnit> AST = getAST(CommonCompilations, SourcePath);
+    if (!AST)
+      return 1;
+    diff::SyntaxTree Tree(AST->getASTContext());
+    if (ASTDump) {
+      printTree(llvm::outs(), Tree);
+      return 0;
+    }
+    llvm::outs() << R"({"filename":")";
+    printJsonString(llvm::outs(), SourcePath);
+    llvm::outs() << R"(","root":)";
+    printNodeAsJson(llvm::outs(), Tree, Tree.getRootId());
+    llvm::outs() << "}\n";
+    return 0;
+  }
+
+  if (DestinationPath.empty()) {
+    llvm::errs() << "Error: Exactly two paths are required.\n";
+    return 1;
+  }
+
+  std::unique_ptr<ASTUnit> Src = getAST(CommonCompilations, SourcePath);
+  std::unique_ptr<ASTUnit> Dst = getAST(CommonCompilations, DestinationPath);
+  if (!Src || !Dst)
+    return 1;
+
+  diff::ComparisonOptions Options;
+  if (MaxSize != -1)
+    Options.MaxSize = MaxSize;
+  if (!StopAfter.empty()) {
+    if (StopAfter == "topdown")
+      Options.StopAfterTopDown = true;
+    else if (StopAfter != "bottomup") {
+      llvm::errs() << "Error: Invalid argument for -stop-after\n";
+      return 1;
+    }
+  }
+  diff::SyntaxTree SrcTree(Src->getASTContext());
+  diff::SyntaxTree DstTree(Dst->getASTContext());
+  diff::ASTDiff Diff(SrcTree, DstTree, Options);
+
+  if (HtmlDiff) {
+    llvm::outs() << HtmlDiffHeader << "<pre>";
+    llvm::outs() << "<div id='L' class='code'>";
+    printHtmlForNode(llvm::outs(), Diff, SrcTree, true, SrcTree.getRootId(), 0);
+    llvm::outs() << "</div>";
+    llvm::outs() << "<div id='R' class='code'>";
+    printHtmlForNode(llvm::outs(), Diff, DstTree, false, DstTree.getRootId(),
+                     0);
+    llvm::outs() << "</div>";
+    llvm::outs() << "</pre></div></body></html>\n";
+    return 0;
+  }
+
+  for (diff::NodeId Dst : DstTree) {
+    diff::NodeId Src = Diff.getMapped(DstTree, Dst);
+    if (PrintMatches && Src.isValid()) {
+      llvm::outs() << "Match ";
+      printNode(llvm::outs(), SrcTree, Src);
+      llvm::outs() << " to ";
+      printNode(llvm::outs(), DstTree, Dst);
+      llvm::outs() << "\n";
+    }
+    printDstChange(llvm::outs(), Diff, SrcTree, DstTree, Dst);
+  }
+  for (diff::NodeId Src : SrcTree) {
+    if (Diff.getMapped(SrcTree, Src).isInvalid()) {
+      llvm::outs() << "Delete ";
+      printNode(llvm::outs(), SrcTree, Src);
+      llvm::outs() << "\n";
+    }
+  }
+
+  return 0;
+}

tools/clang-format-vs/ClangFormat/ClangFormatPackage.cs

@@ -326,7 +326,13 @@ namespace LLVM.ClangFormat
 
             string filePath = Vsix.GetDocumentPath(view);
             var path = Path.GetDirectoryName(filePath);
+
             string text = view.TextBuffer.CurrentSnapshot.GetText();
+            if (!text.EndsWith(Environment.NewLine))
+            {
+                view.TextBuffer.Insert(view.TextBuffer.CurrentSnapshot.Length, Environment.NewLine);
+                text += Environment.NewLine;
+            }
 
             RunClangFormatAndApplyReplacements(text, 0, text.Length, path, filePath, options, view);
         }

tools/clang-format-vs/source.extension.vsixmanifest.in

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <PackageManifest Version="2.0.0" xmlns="http://schemas.microsoft.com/developer/vsx-schema/2011" xmlns:d="http://schemas.microsoft.com/developer/vsx-schema-design/2011">
   <Metadata>
-    <Identity Id="20dbc914-1c7a-4992-b236-ef58b37850eb" Version="@CLANG_FORMAT_VS_VERSION@" Language="en-US" Publisher="LLVM"/>
+    <Identity Id="3cb18a5e-97e9-11e7-abc4-cec278b6b50a" Version="@CLANG_FORMAT_VS_VERSION@" Language="en-US" Publisher="LLVM"/>
     <DisplayName>ClangFormat</DisplayName>
     <Description xml:space="preserve">A tool to format C/C++/Obj-C code.</Description>
     <MoreInfo>http://clang.llvm.org/docs/ClangFormat.html</MoreInfo>

tools/clang-format/CMakeLists.txt

@@ -12,10 +12,11 @@ set(CLANG_FORMAT_LIB_DEPS
   )
 
 target_link_libraries(clang-format
+  PRIVATE
   ${CLANG_FORMAT_LIB_DEPS}
   )
 
-if( LLVM_USE_SANITIZE_COVERAGE )
+if( LLVM_LIB_FUZZING_ENGINE OR LLVM_USE_SANITIZE_COVERAGE )
   add_subdirectory(fuzzer)
 endif()
 

tools/clang-format/ClangFormat.cpp

@@ -102,6 +102,10 @@ static cl::opt<bool> SortIncludes(
              "SortIncludes style flag"),
     cl::cat(ClangFormatCategory));
 
+static cl::opt<bool>
+    Verbose("verbose", cl::desc("If set, shows the list of processed files"),
+            cl::cat(ClangFormatCategory));
+
 static cl::list<std::string> FileNames(cl::Positional, cl::desc("[<file> ...]"),
                                        cl::cat(ClangFormatCategory));
 
@@ -285,7 +289,7 @@ static bool format(StringRef FileName) {
               "xml:space='preserve' incomplete_format='"
            << (Status.FormatComplete ? "false" : "true") << "'";
     if (!Status.FormatComplete)
-      outs() << " line=" << Status.Line;
+      outs() << " line='" << Status.Line << "'";
     outs() << ">\n";
     if (Cursor.getNumOccurrences() != 0)
       outs() << "<cursor>"
@@ -328,8 +332,7 @@ static bool format(StringRef FileName) {
 }  // namespace format
 }  // namespace clang
 
-static void PrintVersion() {
-  raw_ostream &OS = outs();
+static void PrintVersion(raw_ostream &OS) {
   OS << clang::getClangToolFullVersion("clang-format") << '\n';
 }
 
@@ -348,8 +351,10 @@ int main(int argc, const char **argv) {
       "together with <file>s, the files are edited in-place. Otherwise, the\n"
       "result is written to the standard output.\n");
 
-  if (Help)
+  if (Help) {
     cl::PrintHelpMessage();
+    return 0;
+  }
 
   if (DumpConfig) {
     llvm::Expected<clang::format::FormatStyle> FormatStyle =
@@ -366,23 +371,19 @@ int main(int argc, const char **argv) {
   }
 
   bool Error = false;
-  switch (FileNames.size()) {
-  case 0:
+  if (FileNames.empty()) {
     Error = clang::format::format("-");
-    break;
-  case 1:
-    Error = clang::format::format(FileNames[0]);
-    break;
-  default:
-    if (!Offsets.empty() || !Lengths.empty() || !LineRanges.empty()) {
-      errs() << "error: -offset, -length and -lines can only be used for "
-                "single file.\n";
-      return 1;
-    }
-    for (unsigned i = 0; i < FileNames.size(); ++i)
-      Error |= clang::format::format(FileNames[i]);
-    break;
+    return Error ? 1 : 0;
+  }
+  if (FileNames.size() != 1 && (!Offsets.empty() || !Lengths.empty() || !LineRanges.empty())) {
+    errs() << "error: -offset, -length and -lines can only be used for "
+              "single file.\n";
+    return 1;
+  }
+  for (const auto &FileName : FileNames) {
+    if (Verbose)
+      errs() << "Formatting " << FileName << "\n";
+    Error |= clang::format::format(FileName);
   }
   return Error ? 1 : 0;
 }
-

tools/clang-format/clang-format.el

@@ -119,10 +119,12 @@ is a zero-based file offset, assuming ‘utf-8-unix’ coding."
       (byte-to-position (1+ byte)))))
 
 ;;;###autoload
-(defun clang-format-region (start end &optional style)
+(defun clang-format-region (start end &optional style assume-file-name)
   "Use clang-format to format the code between START and END according to STYLE.
-If called interactively uses the region or the current statement if there
-is no active region.  If no style is given uses `clang-format-style'."
+If called interactively uses the region or the current statement if there is no
+no active region. If no STYLE is given uses `clang-format-style'. Use
+ASSUME-FILE-NAME to locate a style config file, if no ASSUME-FILE-NAME is given
+uses the function `buffer-file-name'."
   (interactive
    (if (use-region-p)
        (list (region-beginning) (region-end))
@@ -131,6 +133,9 @@ is no active region.  If no style is given uses `clang-format-style'."
   (unless style
     (setq style clang-format-style))
 
+  (unless assume-file-name
+    (setq assume-file-name buffer-file-name))
+
   (let ((file-start (clang-format--bufferpos-to-filepos start 'approximate
                                                         'utf-8-unix))
         (file-end (clang-format--bufferpos-to-filepos end 'approximate
@@ -144,16 +149,21 @@ is no active region.  If no style is given uses `clang-format-style'."
         ;; always use ‘utf-8-unix’ and ignore the buffer coding system.
         (default-process-coding-system '(utf-8-unix . utf-8-unix)))
     (unwind-protect
-        (let ((status (call-process-region
-                       nil nil clang-format-executable
-                       nil `(,temp-buffer ,temp-file) nil
-
-                       "-output-replacements-xml"
-                       "-assume-filename" (or (buffer-file-name) "")
-                       "-style" style
-                       "-offset" (number-to-string file-start)
-                       "-length" (number-to-string (- file-end file-start))
-                       "-cursor" (number-to-string cursor)))
+        (let ((status (apply #'call-process-region
+                             nil nil clang-format-executable
+                             nil `(,temp-buffer ,temp-file) nil
+                             `("-output-replacements-xml"
+                               ;; Gaurd against a nil assume-file-name.
+                               ;; If the clang-format option -assume-filename
+                               ;; is given a blank string it will crash as per
+                               ;; the following bug report
+                               ;; https://bugs.llvm.org/show_bug.cgi?id=34667
+                               ,@(and assume-file-name
+                                      (list "-assume-filename" assume-file-name))
+                               "-style" ,style
+                               "-offset" ,(number-to-string file-start)
+                               "-length" ,(number-to-string (- file-end file-start))
+                               "-cursor" ,(number-to-string cursor))))
               (stderr (with-temp-buffer
                         (unless (zerop (cadr (insert-file-contents temp-file)))
                           (insert ": "))
@@ -181,10 +191,13 @@ is no active region.  If no style is given uses `clang-format-style'."
       (when (buffer-name temp-buffer) (kill-buffer temp-buffer)))))
 
 ;;;###autoload
-(defun clang-format-buffer (&optional style)
-  "Use clang-format to format the current buffer according to STYLE."
+(defun clang-format-buffer (&optional style assume-file-name)
+  "Use clang-format to format the current buffer according to STYLE.
+If no STYLE is given uses `clang-format-style'. Use ASSUME-FILE-NAME
+to locate a style config file. If no ASSUME-FILE-NAME is given uses
+the function `buffer-file-name'."
   (interactive)
-  (clang-format-region (point-min) (point-max) style))
+  (clang-format-region (point-min) (point-max) style assume-file-name))
 
 ;;;###autoload
 (defalias 'clang-format 'clang-format-region)

tools/clang-format/clang-format.py

@@ -92,7 +92,7 @@ def main():
 
   # Call formatter.
   command = [binary, '-style', style, '-cursor', str(cursor)]
-  if lines != 'all':
+  if lines != ['-lines', 'all']:
     command += lines
   if fallback_style:
     command.extend(['-fallback-style', fallback_style])

tools/clang-format/fuzzer/CMakeLists.txt

@@ -1,11 +1,16 @@
 set(LLVM_LINK_COMPONENTS support)
 
+if(LLVM_USE_SANITIZE_COVERAGE)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=fuzzer")
+endif()
+
 add_clang_executable(clang-format-fuzzer
   EXCLUDE_FROM_ALL
   ClangFormatFuzzer.cpp
   )
 
 target_link_libraries(clang-format-fuzzer
+  PRIVATE
   ${CLANG_FORMAT_LIB_DEPS}
-  LLVMFuzzer
+  ${LLVM_LIB_FUZZING_ENGINE}
   )

tools/clang-format/fuzzer/ClangFormatFuzzer.cpp

@@ -20,7 +20,10 @@ extern "C" int LLVMFuzzerTestOneInput(uint8_t *data, size_t size) {
   std::string s((const char *)data, size);
   auto Style = getGoogleStyle(clang::format::FormatStyle::LK_Cpp);
   Style.ColumnLimit = 60;
-  applyAllReplacements(s, clang::format::reformat(
-                              Style, s, {clang::tooling::Range(0, s.size())}));
+  auto Replaces = reformat(Style, s, clang::tooling::Range(0, s.size()));
+  auto Result = applyAllReplacements(s, Replaces);
+
+  // Output must be checked, as otherwise we crash.
+  if (!Result) {}
   return 0;
 }

tools/clang-func-mapping/CMakeLists.txt

@@ -0,0 +1,23 @@
+set(LLVM_LINK_COMPONENTS
+  ${LLVM_TARGETS_TO_BUILD}
+  asmparser
+  support
+  mc
+  )
+
+add_clang_executable(clang-func-mapping
+  ClangFnMapGen.cpp
+  )
+
+target_link_libraries(clang-func-mapping
+  PRIVATE
+  clangAST
+  clangBasic
+  clangCrossTU
+  clangFrontend
+  clangIndex
+  clangTooling
+  )
+
+install(TARGETS clang-func-mapping
+  RUNTIME DESTINATION bin)

tools/clang-func-mapping/ClangFnMapGen.cpp

@@ -0,0 +1,124 @@
+//===- ClangFnMapGen.cpp -----------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--------------------------------------------------------------------===//
+//
+// Clang tool which creates a list of defined functions and the files in which
+// they are defined.
+//
+//===--------------------------------------------------------------------===//
+
+#include "clang/AST/ASTConsumer.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/GlobalDecl.h"
+#include "clang/AST/Mangle.h"
+#include "clang/AST/StmtVisitor.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Basic/TargetInfo.h"
+#include "clang/CrossTU/CrossTranslationUnit.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/FrontendActions.h"
+#include "clang/Index/USRGeneration.h"
+#include "clang/Tooling/CommonOptionsParser.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Signals.h"
+#include <sstream>
+#include <string>
+#include <vector>
+
+using namespace llvm;
+using namespace clang;
+using namespace clang::cross_tu;
+using namespace clang::tooling;
+
+static cl::OptionCategory ClangFnMapGenCategory("clang-fnmapgen options");
+
+class MapFunctionNamesConsumer : public ASTConsumer {
+public:
+  MapFunctionNamesConsumer(ASTContext &Context) : Ctx(Context) {}
+
+  ~MapFunctionNamesConsumer() {
+    // Flush results to standard output.
+    llvm::outs() << createCrossTUIndexString(Index);
+  }
+
+  virtual void HandleTranslationUnit(ASTContext &Ctx) {
+    handleDecl(Ctx.getTranslationUnitDecl());
+  }
+
+private:
+  void handleDecl(const Decl *D);
+
+  ASTContext &Ctx;
+  llvm::StringMap<std::string> Index;
+  std::string CurrentFileName;
+};
+
+void MapFunctionNamesConsumer::handleDecl(const Decl *D) {
+  if (!D)
+    return;
+
+  if (const auto *FD = dyn_cast<FunctionDecl>(D)) {
+    if (FD->isThisDeclarationADefinition()) {
+      if (const Stmt *Body = FD->getBody()) {
+        std::string LookupName = CrossTranslationUnitContext::getLookupName(FD);
+        const SourceManager &SM = Ctx.getSourceManager();
+        if (CurrentFileName.empty()) {
+          CurrentFileName =
+              SM.getFileEntryForID(SM.getMainFileID())->tryGetRealPathName();
+          if (CurrentFileName.empty())
+            CurrentFileName = "invalid_file";
+        }
+
+        switch (FD->getLinkageInternal()) {
+        case ExternalLinkage:
+        case VisibleNoLinkage:
+        case UniqueExternalLinkage:
+          if (SM.isInMainFile(Body->getLocStart()))
+            Index[LookupName] = CurrentFileName;
+        default:
+          break;
+        }
+      }
+    }
+  }
+
+  if (const auto *DC = dyn_cast<DeclContext>(D))
+    for (const Decl *D : DC->decls())
+      handleDecl(D);
+}
+
+class MapFunctionNamesAction : public ASTFrontendAction {
+protected:
+  std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
+                                                 llvm::StringRef) {
+    std::unique_ptr<ASTConsumer> PFC(
+        new MapFunctionNamesConsumer(CI.getASTContext()));
+    return PFC;
+  }
+};
+
+static cl::extrahelp CommonHelp(CommonOptionsParser::HelpMessage);
+
+int main(int argc, const char **argv) {
+  // Print a stack trace if we signal out.
+  sys::PrintStackTraceOnErrorSignal(argv[0], false);
+  PrettyStackTraceProgram X(argc, argv);
+
+  const char *Overview = "\nThis tool collects the USR name and location "
+                         "of all functions definitions in the source files "
+                         "(excluding headers).\n";
+  CommonOptionsParser OptionsParser(argc, argv, ClangFnMapGenCategory,
+                                    cl::ZeroOrMore, Overview);
+
+  ClangTool Tool(OptionsParser.getCompilations(),
+                 OptionsParser.getSourcePathList());
+  Tool.run(newFrontendActionFactory<MapFunctionNamesAction>().get());
+  return 0;
+}

tools/clang-fuzzer/CMakeLists.txt

@@ -1,21 +1,73 @@
-if( LLVM_USE_SANITIZE_COVERAGE )
-  set(LLVM_LINK_COMPONENTS ${LLVM_TARGETS_TO_BUILD})
+set(LLVM_LINK_COMPONENTS ${LLVM_TARGETS_TO_BUILD} FuzzMutate)
+set(CXX_FLAGS_NOFUZZ ${CMAKE_CXX_FLAGS})
+set(DUMMY_MAIN DummyClangFuzzer.cpp)
+if(LLVM_LIB_FUZZING_ENGINE)
+  unset(DUMMY_MAIN)
+elseif(LLVM_USE_SANITIZE_COVERAGE)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=fuzzer")
+  set(CXX_FLAGS_NOFUZZ "${CXX_FLAGS_NOFUZZ} -fsanitize=fuzzer-no-link")
+  unset(DUMMY_MAIN)
+endif()
 
-  add_clang_executable(clang-fuzzer
-    EXCLUDE_FROM_ALL
-    ClangFuzzer.cpp
+# Hack to bypass LLVM's cmake sources check and allow multiple libraries and
+# executables from this directory.
+set(LLVM_OPTIONAL_SOURCES
+  ClangFuzzer.cpp
+  DummyClangFuzzer.cpp
+  ExampleClangProtoFuzzer.cpp
+  )
+
+if(CLANG_ENABLE_PROTO_FUZZER)
+  # Create protobuf .h and .cc files, and put them in a library for use by
+  # clang-proto-fuzzer components.
+  find_package(Protobuf REQUIRED)
+  add_definitions(-DGOOGLE_PROTOBUF_NO_RTTI)
+  include_directories(${PROTOBUF_INCLUDE_DIRS})
+  include_directories(${CMAKE_CURRENT_BINARY_DIR})
+  protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS cxx_proto.proto)
+  set(LLVM_OPTIONAL_SOURCES ${LLVM_OPTIONAL_SOURCES} ${PROTO_SRCS})
+  add_clang_library(clangCXXProto
+    ${PROTO_SRCS}
+    ${PROTO_HDRS}
+
+    LINK_LIBS
+    ${PROTOBUF_LIBRARIES}
     )
 
-  target_link_libraries(clang-fuzzer
-    ${CLANG_FORMAT_LIB_DEPS}
-    clangAST
-    clangBasic
-    clangCodeGen
-    clangDriver
-    clangFrontend
-    clangRewriteFrontend
-    clangStaticAnalyzerFrontend
-    clangTooling
-    LLVMFuzzer
+  # Build and include libprotobuf-mutator
+  include(ProtobufMutator)
+  include_directories(${ProtobufMutator_INCLUDE_DIRS})
+
+  # Build the protobuf->C++ translation library and driver.
+  add_clang_subdirectory(proto-to-cxx)
+
+  # Build the protobuf fuzzer
+  add_clang_executable(clang-proto-fuzzer
+    ${DUMMY_MAIN}
+    ExampleClangProtoFuzzer.cpp
+    )
+
+  target_link_libraries(clang-proto-fuzzer
+    PRIVATE
+    ${ProtobufMutator_LIBRARIES}
+    ${PROTOBUF_LIBRARIES}
+    ${LLVM_LIB_FUZZING_ENGINE}
+    clangCXXProto
+    clangHandleCXX
+    clangProtoToCXX
     )
 endif()
+
+add_clang_subdirectory(handle-cxx)
+
+add_clang_executable(clang-fuzzer
+  EXCLUDE_FROM_ALL
+  ${DUMMY_MAIN}
+  ClangFuzzer.cpp
+  )
+
+target_link_libraries(clang-fuzzer
+  PRIVATE
+  ${LLVM_LIB_FUZZING_ENGINE}
+  clangHandleCXX
+  )

tools/clang-fuzzer/ClangFuzzer.cpp

@@ -13,43 +13,14 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#include "clang/Tooling/Tooling.h"
-#include "clang/CodeGen/CodeGenAction.h"
-#include "clang/Frontend/CompilerInstance.h"
-#include "clang/Lex/PreprocessorOptions.h"
-#include "llvm/Option/Option.h"
-#include "llvm/Support/TargetSelect.h"
+#include "handle-cxx/handle_cxx.h"
 
-using namespace clang;
+using namespace clang_fuzzer;
+
+extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv) { return 0; }
 
 extern "C" int LLVMFuzzerTestOneInput(uint8_t *data, size_t size) {
   std::string s((const char *)data, size);
-  llvm::InitializeAllTargets();
-  llvm::InitializeAllTargetMCs();
-  llvm::InitializeAllAsmPrinters();
-  llvm::InitializeAllAsmParsers();
-
-  llvm::opt::ArgStringList CC1Args;
-  CC1Args.push_back("-cc1");
-  CC1Args.push_back("./test.cc");
-  CC1Args.push_back("-O2");
-  llvm::IntrusiveRefCntPtr<FileManager> Files(
-      new FileManager(FileSystemOptions()));
-  IgnoringDiagConsumer Diags;
-  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
-  DiagnosticsEngine Diagnostics(
-      IntrusiveRefCntPtr<clang::DiagnosticIDs>(new DiagnosticIDs()), &*DiagOpts,
-      &Diags, false);
-  std::unique_ptr<clang::CompilerInvocation> Invocation(
-      tooling::newInvocation(&Diagnostics, CC1Args));
-  std::unique_ptr<llvm::MemoryBuffer> Input =
-      llvm::MemoryBuffer::getMemBuffer(s);
-  Invocation->getPreprocessorOpts().addRemappedFile("./test.cc", Input.release());
-  std::unique_ptr<tooling::ToolAction> action(
-      tooling::newFrontendActionFactory<clang::EmitObjAction>());
-  std::shared_ptr<PCHContainerOperations> PCHContainerOps =
-      std::make_shared<PCHContainerOperations>();
-  action->runInvocation(std::move(Invocation), Files.get(), PCHContainerOps,
-                        &Diags);
+  HandleCXX(s, {"-O2"});
   return 0;
 }

tools/clang-fuzzer/Dockerfile

@@ -0,0 +1,37 @@
+#===- llvm/tools/clang/tools/clang-fuzzer ---------------------------------===//
+#
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+#
+#===----------------------------------------------------------------------===//
+# Produces an image that builds clang-proto-fuzzer
+FROM ubuntu:16.04
+RUN apt-get update -y
+RUN apt-get install -y autoconf automake libtool curl make g++ unzip wget git \
+    binutils liblzma-dev libz-dev python-all cmake ninja-build subversion \
+    pkg-config docbook2x
+
+WORKDIR /root
+
+# Get protobuf
+RUN wget -qO- https://github.com/google/protobuf/releases/download/v3.3.0/protobuf-cpp-3.3.0.tar.gz | tar zxf -
+RUN cd protobuf-3.3.0 && ./autogen.sh && ./configure && make -j $(nproc) && make check -j $(nproc) && make install && ldconfig
+# Get LLVM
+RUN svn co http://llvm.org/svn/llvm-project/llvm/trunk llvm
+RUN cd llvm/tools && svn co http://llvm.org/svn/llvm-project/cfe/trunk clang -r $(cd ../ && svn info | grep Revision | awk '{print $2}')
+RUN cd llvm/projects && svn co http://llvm.org/svn/llvm-project/compiler-rt/trunk compiler-rt -r $(cd ../ && svn info | grep Revision | awk '{print $2}')
+# Build plain LLVM (stage 0)
+RUN mkdir build0 && cd build0 && cmake -GNinja -DCMAKE_BUILD_TYPE=Release ../llvm && ninja
+# Configure instrumented LLVM (stage 1)
+RUN mkdir build1 && cd build1 && cmake -GNinja -DCMAKE_BUILD_TYPE=Release ../llvm \
+    -DLLVM_ENABLE_ASSERTIONS=ON \
+    -DCMAKE_C_COMPILER=`pwd`/../build0/bin/clang \
+    -DCMAKE_CXX_COMPILER=`pwd`/../build0/bin/clang++ \
+    -DLLVM_USE_SANITIZE_COVERAGE=YES \
+    -DLLVM_USE_SANITIZER=Address -DCLANG_ENABLE_PROTO_FUZZER=ON
+# Build the fuzzers
+RUN cd build1 && ninja clang-fuzzer
+RUN cd build1 && ninja clang-proto-fuzzer
+RUN cd build1 && ninja clang-proto-to-cxx

tools/clang-fuzzer/DummyClangFuzzer.cpp

@@ -0,0 +1,21 @@
+//===-- DummyClangFuzzer.cpp - Entry point to sanity check fuzzers --------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Provides a main() to build without linking libFuzzer.
+//
+//===----------------------------------------------------------------------===//
+#include "llvm/FuzzMutate/FuzzerCLI.h"
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size);
+extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv);
+
+int main(int argc, char *argv[]) {
+  return llvm::runFuzzerOnInputs(argc, argv, LLVMFuzzerTestOneInput,
+                                 LLVMFuzzerInitialize);
+}

tools/clang-fuzzer/ExampleClangProtoFuzzer.cpp

@@ -0,0 +1,44 @@
+//===-- ExampleClangProtoFuzzer.cpp - Fuzz Clang --------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file implements a function that runs Clang on a single
+///  input and uses libprotobuf-mutator to find new inputs. This function is
+///  then linked into the Fuzzer library.
+///
+//===----------------------------------------------------------------------===//
+
+#include "cxx_proto.pb.h"
+#include "handle-cxx/handle_cxx.h"
+#include "proto-to-cxx/proto_to_cxx.h"
+
+#include "src/libfuzzer/libfuzzer_macro.h"
+
+#include <cstring>
+
+using namespace clang_fuzzer;
+
+static std::vector<const char *> CLArgs;
+
+extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv) {
+  CLArgs.push_back("-O2");
+  for (int I = 1; I < *argc; I++) {
+    if (strcmp((*argv)[I], "-ignore_remaining_args=1") == 0) {
+      for (I++; I < *argc; I++)
+        CLArgs.push_back((*argv)[I]);
+      break;
+    }
+  }
+  return 0;
+}
+
+DEFINE_BINARY_PROTO_FUZZER(const Function& input) {
+  auto S = FunctionToString(input);
+  HandleCXX(S, CLArgs);
+}

tools/clang-fuzzer/README.txt

@@ -0,0 +1,82 @@
+This directory contains two utilities for fuzzing Clang: clang-fuzzer and
+clang-proto-fuzzer.  Both use libFuzzer to generate inputs to clang via
+coverage-guided mutation.
+
+The two utilities differ, however, in how they structure inputs to Clang.
+clang-fuzzer makes no attempt to generate valid C++ programs and is therefore
+primarily useful for stressing the surface layers of Clang (i.e. lexer, parser).
+clang-proto-fuzzer uses a protobuf class to describe a subset of the C++
+language and then uses libprotobuf-mutator to mutate instantiations of that
+class, producing valid C++ programs in the process.  As a result,
+clang-proto-fuzzer is better at stressing deeper layers of Clang and LLVM.
+
+===================================
+ Building clang-fuzzer
+===================================
+Within your LLVM build directory, run CMake with the following variable
+definitions:
+- CMAKE_C_COMPILER=clang
+- CMAKE_CXX_COMPILER=clang++
+- LLVM_USE_SANITIZE_COVERAGE=YES
+- LLVM_USE_SANITIZER=Address
+
+Then build the clang-fuzzer target.
+
+Example:
+  cd $LLVM_SOURCE_DIR
+  mkdir build && cd build
+  cmake .. -GNinja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
+    -DLLVM_USE_SANITIZE_COVERAGE=YES -DLLVM_USE_SANITIZER=Address
+  ninja clang-fuzzer
+
+======================
+ Running clang-fuzzer
+======================
+  bin/clang-fuzzer CORPUS_DIR
+
+
+=======================================================
+ Building clang-proto-fuzzer (Linux-only instructions)
+=======================================================
+Install the necessary dependencies:
+- binutils  // needed for libprotobuf-mutator
+- liblzma-dev  // needed for libprotobuf-mutator
+- libz-dev  // needed for libprotobuf-mutator
+- docbook2x  // needed for libprotobuf-mutator
+- Recent version of protobuf [3.3.0 is known to work]
+
+Within your LLVM build directory, run CMake with the following variable
+definitions:
+- CMAKE_C_COMPILER=clang
+- CMAKE_CXX_COMPILER=clang++
+- LLVM_USE_SANITIZE_COVERAGE=YES
+- LLVM_USE_SANITIZER=Address
+- CLANG_ENABLE_PROTO_FUZZER=ON
+
+Then build the clang-proto-fuzzer and clang-proto-to-cxx targets.  Optionally,
+you may also build clang-fuzzer with this setup.
+
+Example:
+  cd $LLVM_SOURCE_DIR
+  mkdir build && cd build
+  cmake .. -GNinja -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
+    -DLLVM_USE_SANITIZE_COVERAGE=YES -DLLVM_USE_SANITIZER=Address \
+    -DCLANG_ENABLE_PROTO_FUZZER=ON
+  ninja clang-proto-fuzzer clang-proto-to-cxx
+
+This directory also contains a Dockerfile which sets up all required
+dependencies and builds the fuzzers.
+
+============================
+ Running clang-proto-fuzzer
+============================
+  bin/clang-proto-fuzzer CORPUS_DIR
+
+Arguments can be specified after -ignore_remaining_args=1 to modify the compiler
+invocation.  For example, the following command line will fuzz LLVM with a
+custom optimization level and target triple:
+  bin/clang-proto-fuzzer CORPUS_DIR -ignore_remaining_args=1 -O3 -triple \
+      arm64apple-ios9
+
+To translate a clang-proto-fuzzer corpus output to C++:
+  bin/clang-proto-to-cxx CORPUS_OUTPUT_FILE

tools/clang-fuzzer/cxx_proto.proto

@@ -0,0 +1,93 @@
+//===-- cxx_proto.proto - Protobuf description of C++ ---------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file describes a subset of C++ as a protobuf.  It is used to
+///  more easily find interesting inputs for fuzzing Clang.
+///
+//===----------------------------------------------------------------------===//
+
+syntax = "proto2";
+
+message VarRef {
+  required int32 varnum = 1;
+}
+
+message Lvalue {
+  required VarRef varref = 1;
+}
+
+message Const {
+  required int32 val = 1;
+}
+
+message BinaryOp {
+  enum Op {
+    PLUS = 0;
+    MINUS = 1;
+    MUL = 2;
+    DIV = 3;
+    MOD = 4;
+    XOR = 5;
+    AND = 6;
+    OR = 7;
+    EQ = 8;
+    NE = 9;
+    LE = 10;
+    GE = 11;
+    LT = 12;
+    GT = 13;
+  };
+  required Op op = 1;
+  required Rvalue left = 2;
+  required Rvalue right = 3;
+}
+
+message Rvalue {
+  oneof rvalue_oneof {
+    VarRef varref = 1;
+    Const cons = 2;
+    BinaryOp binop = 3;
+  }
+}
+
+message AssignmentStatement {
+  required Lvalue lvalue = 1;
+  required Rvalue rvalue = 2;
+}
+
+
+message IfElse {
+  required Rvalue cond = 1;
+  required StatementSeq if_body = 2;
+  required StatementSeq else_body = 3;
+}
+
+message While {
+  required Rvalue cond = 1;
+  required StatementSeq body = 2;
+}
+
+message Statement {
+  oneof stmt_oneof {
+    AssignmentStatement assignment = 1;
+    IfElse              ifelse     = 2;
+    While               while_loop = 3;
+  }
+}
+
+message StatementSeq {
+  repeated Statement statements = 1;
+}
+
+message Function {
+  required StatementSeq statements = 1;
+}
+
+package clang_fuzzer;

tools/clang-fuzzer/handle-cxx/CMakeLists.txt

@@ -0,0 +1,12 @@
+set(LLVM_LINK_COMPONENTS ${LLVM_TARGETS_TO_BUILD} Support)
+
+add_clang_library(clangHandleCXX
+  handle_cxx.cpp
+
+  LINK_LIBS
+  clangBasic
+  clangCodeGen
+  clangFrontend
+  clangLex
+  clangTooling
+  )

tools/clang-fuzzer/handle-cxx/handle_cxx.cpp

@@ -0,0 +1,58 @@
+//==-- handle_cxx.cpp - Helper function for Clang fuzzers ------------------==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Implements HandleCXX for use by the Clang fuzzers.
+//
+//===----------------------------------------------------------------------===//
+
+#include "handle_cxx.h"
+
+#include "clang/CodeGen/CodeGenAction.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Lex/PreprocessorOptions.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/Option/Option.h"
+#include "llvm/Support/TargetSelect.h"
+
+using namespace clang;
+
+void clang_fuzzer::HandleCXX(const std::string &S,
+                             const std::vector<const char *> &ExtraArgs) {
+  llvm::InitializeAllTargets();
+  llvm::InitializeAllTargetMCs();
+  llvm::InitializeAllAsmPrinters();
+  llvm::InitializeAllAsmParsers();
+
+  llvm::opt::ArgStringList CC1Args;
+  CC1Args.push_back("-cc1");
+  for (auto &A : ExtraArgs)
+    CC1Args.push_back(A);
+  CC1Args.push_back("./test.cc");
+
+  llvm::IntrusiveRefCntPtr<FileManager> Files(
+      new FileManager(FileSystemOptions()));
+  IgnoringDiagConsumer Diags;
+  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
+  DiagnosticsEngine Diagnostics(
+      IntrusiveRefCntPtr<clang::DiagnosticIDs>(new DiagnosticIDs()), &*DiagOpts,
+      &Diags, false);
+  std::unique_ptr<clang::CompilerInvocation> Invocation(
+      tooling::newInvocation(&Diagnostics, CC1Args));
+  std::unique_ptr<llvm::MemoryBuffer> Input =
+      llvm::MemoryBuffer::getMemBuffer(S);
+  Invocation->getPreprocessorOpts().addRemappedFile("./test.cc",
+                                                    Input.release());
+  std::unique_ptr<tooling::ToolAction> action(
+      tooling::newFrontendActionFactory<clang::EmitObjAction>());
+  std::shared_ptr<PCHContainerOperations> PCHContainerOps =
+      std::make_shared<PCHContainerOperations>();
+  action->runInvocation(std::move(Invocation), Files.get(), PCHContainerOps,
+                        &Diags);
+}
+

tools/clang-fuzzer/handle-cxx/handle_cxx.h

@@ -0,0 +1,25 @@
+//==-- handle_cxx.h - Helper function for Clang fuzzers --------------------==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines HandleCXX for use by the Clang fuzzers.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_CLANG_FUZZER_HANDLE_CXX_HANDLECXX_H
+#define LLVM_CLANG_TOOLS_CLANG_FUZZER_HANDLE_CXX_HANDLECXX_H
+
+#include <string>
+#include <vector>
+
+namespace clang_fuzzer {
+void HandleCXX(const std::string &S,
+               const std::vector<const char *> &ExtraArgs);
+} // namespace clang_fuzzer
+
+#endif

tools/clang-fuzzer/proto-to-cxx/CMakeLists.txt

@@ -0,0 +1,14 @@
+set(LLVM_LINK_COMPONENTS ${LLVM_TARGETS_TO_BUILD})
+set(CMAKE_CXX_FLAGS ${CXX_FLAGS_NOFUZZ})
+
+# Hack to bypass LLVM's CMake source checks so we can have both a library and
+# an executable built from this directory.
+set(LLVM_OPTIONAL_SOURCES proto_to_cxx.cpp proto_to_cxx_main.cpp)
+
+add_clang_library(clangProtoToCXX proto_to_cxx.cpp
+                  DEPENDS clangCXXProto
+                  LINK_LIBS clangCXXProto ${PROTOBUF_LIBRARIES}
+                  )
+
+add_clang_executable(clang-proto-to-cxx proto_to_cxx_main.cpp)
+target_link_libraries(clang-proto-to-cxx PRIVATE clangProtoToCXX)

tools/clang-fuzzer/proto-to-cxx/proto_to_cxx.cpp

@@ -0,0 +1,102 @@
+//==-- proto_to_cxx.cpp - Protobuf-C++ conversion --------------------------==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Implements functions for converting between protobufs and C++.
+//
+//===----------------------------------------------------------------------===//
+
+#include "proto_to_cxx.h"
+#include "cxx_proto.pb.h"
+
+#include <ostream>
+#include <sstream>
+
+namespace clang_fuzzer {
+
+// Forward decls.
+std::ostream &operator<<(std::ostream &os, const BinaryOp &x);
+std::ostream &operator<<(std::ostream &os, const StatementSeq &x);
+
+// Proto to C++.
+std::ostream &operator<<(std::ostream &os, const Const &x) {
+  return os << "(" << x.val() << ")";
+}
+std::ostream &operator<<(std::ostream &os, const VarRef &x) {
+  return os << "a[" << (static_cast<uint32_t>(x.varnum()) % 100) << "]";
+}
+std::ostream &operator<<(std::ostream &os, const Lvalue &x) {
+  return os << x.varref();
+}
+std::ostream &operator<<(std::ostream &os, const Rvalue &x) {
+    if (x.has_varref()) return os << x.varref();
+    if (x.has_cons())   return os << x.cons();
+    if (x.has_binop())  return os << x.binop();
+    return os << "1";
+}
+std::ostream &operator<<(std::ostream &os, const BinaryOp &x) {
+  os << "(" << x.left();
+  switch (x.op()) {
+    case BinaryOp::PLUS: os << "+"; break;
+    case BinaryOp::MINUS: os << "-"; break;
+    case BinaryOp::MUL: os << "*"; break;
+    case BinaryOp::DIV: os << "/"; break;
+    case BinaryOp::MOD: os << "%"; break;
+    case BinaryOp::XOR: os << "^"; break;
+    case BinaryOp::AND: os << "&"; break;
+    case BinaryOp::OR: os << "|"; break;
+    case BinaryOp::EQ: os << "=="; break;
+    case BinaryOp::NE: os << "!="; break;
+    case BinaryOp::LE: os << "<="; break;
+    case BinaryOp::GE: os << ">="; break;
+    case BinaryOp::LT: os << "<"; break;
+    case BinaryOp::GT: os << ">"; break;
+  }
+  return os << x.right() << ")";
+}
+std::ostream &operator<<(std::ostream &os, const AssignmentStatement &x) {
+  return os << x.lvalue() << "=" << x.rvalue() << ";\n";
+}
+std::ostream &operator<<(std::ostream &os, const IfElse &x) {
+  return os << "if (" << x.cond() << "){\n"
+            << x.if_body() << "} else { \n"
+            << x.else_body() << "}\n";
+}
+std::ostream &operator<<(std::ostream &os, const While &x) {
+  return os << "while (" << x.cond() << "){\n" << x.body() << "}\n";
+}
+std::ostream &operator<<(std::ostream &os, const Statement &x) {
+  if (x.has_assignment()) return os << x.assignment();
+  if (x.has_ifelse())     return os << x.ifelse();
+  if (x.has_while_loop()) return os << x.while_loop();
+  return os << "(void)0;\n";
+}
+std::ostream &operator<<(std::ostream &os, const StatementSeq &x) {
+  for (auto &st : x.statements()) os << st;
+  return os;
+}
+std::ostream &operator<<(std::ostream &os, const Function &x) {
+  return os << "void foo(int *a) {\n" << x.statements() << "}\n";
+}
+
+// ---------------------------------
+
+std::string FunctionToString(const Function &input) {
+  std::ostringstream os;
+  os << input;
+  return os.str();
+
+}
+std::string ProtoToCxx(const uint8_t *data, size_t size) {
+  Function message;
+  if (!message.ParseFromArray(data, size))
+    return "#error invalid proto\n";
+  return FunctionToString(message);
+}
+
+} // namespace clang_fuzzer

tools/clang-fuzzer/proto-to-cxx/proto_to_cxx.h

@@ -0,0 +1,22 @@
+//==-- proto_to_cxx.h - Protobuf-C++ conversion ----------------------------==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines functions for converting between protobufs and C++.
+//
+//===----------------------------------------------------------------------===//
+
+#include <cstdint>
+#include <cstddef>
+#include <string>
+
+namespace clang_fuzzer {
+class Function;
+std::string FunctionToString(const Function &input);
+std::string ProtoToCxx(const uint8_t *data, size_t size);
+}

tools/clang-fuzzer/proto-to-cxx/proto_to_cxx_main.cpp

@@ -0,0 +1,30 @@
+//==-- proto_to_cxx_main.cpp - Driver for protobuf-C++ conversion ----------==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Implements a simple driver to print a C++ program from a protobuf.
+//
+//===----------------------------------------------------------------------===//
+#include <fstream>
+#include <iostream>
+#include <streambuf>
+#include <string>
+
+#include "proto_to_cxx.h"
+
+int main(int argc, char **argv) {
+  for (int i = 1; i < argc; i++) {
+    std::fstream in(argv[i]);
+    std::string str((std::istreambuf_iterator<char>(in)),
+                    std::istreambuf_iterator<char>());
+    std::cout << "// " << argv[i] << std::endl;
+    std::cout << clang_fuzzer::ProtoToCxx(
+        reinterpret_cast<const uint8_t *>(str.data()), str.size());
+  }
+}
+

tools/clang-import-test/CMakeLists.txt

@@ -17,11 +17,13 @@ set(CLANG_IMPORT_TEST_LIB_DEPS
   clangAST
   clangBasic
   clangCodeGen
+  clangDriver
   clangFrontend
   clangLex
   clangParse
   )
 
 target_link_libraries(clang-import-test
+  PRIVATE
   ${CLANG_IMPORT_TEST_LIB_DEPS}
   )

tools/clang-import-test/clang-import-test.cpp

@@ -17,6 +17,7 @@
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Basic/TargetOptions.h"
 #include "clang/CodeGen/ModuleBuilder.h"
+#include "clang/Driver/Types.h"
 #include "clang/Frontend/ASTConsumers.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/MultiplexConsumer.h"
@@ -26,6 +27,7 @@
 #include "clang/Parse/ParseAST.h"
 
 #include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/Host.h"
@@ -46,16 +48,27 @@ static llvm::cl::list<std::string>
 
 static llvm::cl::opt<bool>
     Direct("direct", llvm::cl::Optional,
-             llvm::cl::desc("Use the parsed declarations without indirection"));
+           llvm::cl::desc("Use the parsed declarations without indirection"));
+
+static llvm::cl::opt<bool>
+    UseOrigins("use-origins", llvm::cl::Optional,
+           llvm::cl::desc("Use DeclContext origin information for more accurate lookups"));  
 
 static llvm::cl::list<std::string>
     ClangArgs("Xcc", llvm::cl::ZeroOrMore,
               llvm::cl::desc("Argument to pass to the CompilerInvocation"),
               llvm::cl::CommaSeparated);
 
-static llvm::cl::opt<bool>
-DumpAST("dump-ast", llvm::cl::init(false),
-        llvm::cl::desc("Dump combined AST"));
+static llvm::cl::opt<std::string>
+    Input("x", llvm::cl::Optional,
+          llvm::cl::desc("The language to parse (default: c++)"),
+          llvm::cl::init("c++"));
+
+static llvm::cl::opt<bool> DumpAST("dump-ast", llvm::cl::init(false),
+                                   llvm::cl::desc("Dump combined AST"));
+
+static llvm::cl::opt<bool> DumpIR("dump-ir", llvm::cl::init(false),
+                                  llvm::cl::desc("Dump IR from final parse"));
 
 namespace init_convenience {
 class TestDiagnosticConsumer : public DiagnosticConsumer {
@@ -110,6 +123,7 @@ class TestDiagnosticConsumer : public DiagnosticConsumer {
     llvm::errs() << LineString << '\n';
     llvm::errs().indent(LocColumn);
     llvm::errs() << '^';
+    llvm::errs() << '\n';
   }
 
   virtual void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
@@ -142,8 +156,7 @@ class TestDiagnosticConsumer : public DiagnosticConsumer {
   }
 };
 
-std::unique_ptr<CompilerInstance>
-BuildCompilerInstance(ArrayRef<const char *> ClangArgv) {
+std::unique_ptr<CompilerInstance> BuildCompilerInstance() {
   auto Ins = llvm::make_unique<CompilerInstance>();
   auto DC = llvm::make_unique<TestDiagnosticConsumer>();
   const bool ShouldOwnClient = true;
@@ -151,13 +164,27 @@ BuildCompilerInstance(ArrayRef<const char *> ClangArgv) {
 
   auto Inv = llvm::make_unique<CompilerInvocation>();
 
+  std::vector<const char *> ClangArgv(ClangArgs.size());
+  std::transform(ClangArgs.begin(), ClangArgs.end(), ClangArgv.begin(),
+                 [](const std::string &s) -> const char * { return s.data(); });
   CompilerInvocation::CreateFromArgs(*Inv, ClangArgv.data(),
                                      &ClangArgv.data()[ClangArgv.size()],
                                      Ins->getDiagnostics());
 
-  Inv->getLangOpts()->CPlusPlus = true;
-  Inv->getLangOpts()->CPlusPlus11 = true;
-  Inv->getHeaderSearchOpts().UseLibcxx = true;
+  {
+    using namespace driver::types;
+    ID Id = lookupTypeForTypeSpecifier(Input.c_str());
+    assert(Id != TY_INVALID);
+    if (isCXX(Id)) {
+      Inv->getLangOpts()->CPlusPlus = true;
+      Inv->getLangOpts()->CPlusPlus11 = true;
+      Inv->getHeaderSearchOpts().UseLibcxx = true;
+    }
+    if (isObjC(Id)) {
+      Inv->getLangOpts()->ObjC1 = 1;
+      Inv->getLangOpts()->ObjC2 = 1;
+    }
+  }
   Inv->getLangOpts()->Bool = true;
   Inv->getLangOpts()->WChar = true;
   Inv->getLangOpts()->Blocks = true;
@@ -201,32 +228,54 @@ std::unique_ptr<CodeGenerator> BuildCodeGen(CompilerInstance &CI,
 } // end namespace
 
 namespace {
- 
-void AddExternalSource(
-    CompilerInstance &CI,
-    llvm::ArrayRef<std::unique_ptr<CompilerInstance>> Imports) {
-  ExternalASTMerger::ImporterEndpoint Target({CI.getASTContext(), CI.getFileManager()});
-  llvm::SmallVector<ExternalASTMerger::ImporterEndpoint, 3> Sources;
-  for (const std::unique_ptr<CompilerInstance> &CI : Imports) {
-    Sources.push_back({CI->getASTContext(), CI->getFileManager()});
+
+/// A container for a CompilerInstance (possibly with an ExternalASTMerger
+/// attached to its ASTContext).
+///
+/// Provides an accessor for the DeclContext origins associated with the
+/// ExternalASTMerger (or an empty list of origins if no ExternalASTMerger is
+/// attached).
+///
+/// This is the main unit of parsed source code maintained by clang-import-test.
+struct CIAndOrigins {
+  using OriginMap = clang::ExternalASTMerger::OriginMap;
+  std::unique_ptr<CompilerInstance> CI;
+
+  ASTContext &getASTContext() { return CI->getASTContext(); }
+  FileManager &getFileManager() { return CI->getFileManager(); }
+  const OriginMap &getOriginMap() {
+    static const OriginMap EmptyOriginMap;
+    if (ExternalASTSource *Source = CI->getASTContext().getExternalSource())
+      return static_cast<ExternalASTMerger *>(Source)->GetOrigins();
+    return EmptyOriginMap;
   }
+  DiagnosticConsumer &getDiagnosticClient() {
+    return CI->getDiagnosticClient();
+  }
+  CompilerInstance &getCompilerInstance() { return *CI; }
+};
+
+void AddExternalSource(CIAndOrigins &CI,
+                       llvm::MutableArrayRef<CIAndOrigins> Imports) {
+  ExternalASTMerger::ImporterTarget Target(
+      {CI.getASTContext(), CI.getFileManager()});
+  llvm::SmallVector<ExternalASTMerger::ImporterSource, 3> Sources;
+  for (CIAndOrigins &Import : Imports)
+    Sources.push_back(
+        {Import.getASTContext(), Import.getFileManager(), Import.getOriginMap()});
   auto ES = llvm::make_unique<ExternalASTMerger>(Target, Sources);
   CI.getASTContext().setExternalSource(ES.release());
   CI.getASTContext().getTranslationUnitDecl()->setHasExternalVisibleStorage();
 }
 
-std::unique_ptr<CompilerInstance> BuildIndirect(std::unique_ptr<CompilerInstance> &CI) {
-  std::vector<const char *> ClangArgv(ClangArgs.size());
-  std::transform(ClangArgs.begin(), ClangArgs.end(), ClangArgv.begin(),
-                 [](const std::string &s) -> const char * { return s.data(); });
-  std::unique_ptr<CompilerInstance> IndirectCI =
-      init_convenience::BuildCompilerInstance(ClangArgv);
+CIAndOrigins BuildIndirect(CIAndOrigins &CI) {
+  CIAndOrigins IndirectCI{init_convenience::BuildCompilerInstance()};
   auto ST = llvm::make_unique<SelectorTable>();
   auto BC = llvm::make_unique<Builtin::Context>();
-  std::unique_ptr<ASTContext> AST =
-      init_convenience::BuildASTContext(*IndirectCI, *ST, *BC);
-  IndirectCI->setASTContext(AST.release());
-  AddExternalSource(*IndirectCI, CI);
+  std::unique_ptr<ASTContext> AST = init_convenience::BuildASTContext(
+      IndirectCI.getCompilerInstance(), *ST, *BC);
+  IndirectCI.getCompilerInstance().setASTContext(AST.release());
+  AddExternalSource(IndirectCI, CI);
   return IndirectCI;
 }
 
@@ -243,46 +292,53 @@ llvm::Error ParseSource(const std::string &Path, CompilerInstance &CI,
   return llvm::Error::success();
 }
 
-llvm::Expected<std::unique_ptr<CompilerInstance>>
-Parse(const std::string &Path,
-      llvm::ArrayRef<std::unique_ptr<CompilerInstance>> Imports,
-      bool ShouldDumpAST) {
-  std::vector<const char *> ClangArgv(ClangArgs.size());
-  std::transform(ClangArgs.begin(), ClangArgs.end(), ClangArgv.begin(),
-                 [](const std::string &s) -> const char * { return s.data(); });
-  std::unique_ptr<CompilerInstance> CI =
-      init_convenience::BuildCompilerInstance(ClangArgv);
+llvm::Expected<CIAndOrigins> Parse(const std::string &Path,
+                                   llvm::MutableArrayRef<CIAndOrigins> Imports,
+                                   bool ShouldDumpAST, bool ShouldDumpIR) {
+  CIAndOrigins CI{init_convenience::BuildCompilerInstance()};
   auto ST = llvm::make_unique<SelectorTable>();
   auto BC = llvm::make_unique<Builtin::Context>();
   std::unique_ptr<ASTContext> AST =
-      init_convenience::BuildASTContext(*CI, *ST, *BC);
-  CI->setASTContext(AST.release());
+      init_convenience::BuildASTContext(CI.getCompilerInstance(), *ST, *BC);
+  CI.getCompilerInstance().setASTContext(AST.release());
   if (Imports.size())
-    AddExternalSource(*CI, Imports);
+    AddExternalSource(CI, Imports);
 
   std::vector<std::unique_ptr<ASTConsumer>> ASTConsumers;
 
   auto LLVMCtx = llvm::make_unique<llvm::LLVMContext>();
-  ASTConsumers.push_back(init_convenience::BuildCodeGen(*CI, *LLVMCtx));
+  ASTConsumers.push_back(
+      init_convenience::BuildCodeGen(CI.getCompilerInstance(), *LLVMCtx));
+  auto &CG = *static_cast<CodeGenerator *>(ASTConsumers.back().get());
 
   if (ShouldDumpAST)
     ASTConsumers.push_back(CreateASTDumper("", true, false, false));
 
-  CI->getDiagnosticClient().BeginSourceFile(CI->getLangOpts(),
-                                            &CI->getPreprocessor());
+  CI.getDiagnosticClient().BeginSourceFile(
+      CI.getCompilerInstance().getLangOpts(),
+      &CI.getCompilerInstance().getPreprocessor());
   MultiplexConsumer Consumers(std::move(ASTConsumers));
-  Consumers.Initialize(CI->getASTContext());
+  Consumers.Initialize(CI.getASTContext());
 
-  if (llvm::Error PE = ParseSource(Path, *CI, Consumers)) {
+  if (llvm::Error PE = ParseSource(Path, CI.getCompilerInstance(), Consumers))
     return std::move(PE);
-  }
-  CI->getDiagnosticClient().EndSourceFile();
-  if (CI->getDiagnosticClient().getNumErrors()) {
+  CI.getDiagnosticClient().EndSourceFile();
+  if (ShouldDumpIR)
+    CG.GetModule()->print(llvm::outs(), nullptr);
+  if (CI.getDiagnosticClient().getNumErrors())
     return llvm::make_error<llvm::StringError>(
         "Errors occured while parsing the expression.", std::error_code());
-  } else {
-    return std::move(CI);
-  }
+  return std::move(CI);
+}
+
+void Forget(CIAndOrigins &CI, llvm::MutableArrayRef<CIAndOrigins> Imports) {
+  llvm::SmallVector<ExternalASTMerger::ImporterSource, 3> Sources;
+  for (CIAndOrigins &Import : Imports)
+    Sources.push_back(
+        {Import.getASTContext(), Import.getFileManager(), Import.getOriginMap()});
+  ExternalASTSource *Source = CI.CI->getASTContext().getExternalSource();
+  auto *Merger = static_cast<ExternalASTMerger *>(Source);
+  Merger->RemoveSources(Sources);
 }
 
 } // end namespace
@@ -291,31 +347,32 @@ int main(int argc, const char **argv) {
   const bool DisableCrashReporting = true;
   llvm::sys::PrintStackTraceOnErrorSignal(argv[0], DisableCrashReporting);
   llvm::cl::ParseCommandLineOptions(argc, argv);
-  std::vector<std::unique_ptr<CompilerInstance>> ImportCIs;
+  std::vector<CIAndOrigins> ImportCIs;
   for (auto I : Imports) {
-    llvm::Expected<std::unique_ptr<CompilerInstance>> ImportCI =
-      Parse(I, {}, false);
+    llvm::Expected<CIAndOrigins> ImportCI = Parse(I, {}, false, false);
     if (auto E = ImportCI.takeError()) {
       llvm::errs() << llvm::toString(std::move(E));
       exit(-1);
-    } else {
-      ImportCIs.push_back(std::move(*ImportCI));
     }
+    ImportCIs.push_back(std::move(*ImportCI));
   }
-  std::vector<std::unique_ptr<CompilerInstance>> IndirectCIs;
-  if (!Direct) {
+  std::vector<CIAndOrigins> IndirectCIs;
+  if (!Direct || UseOrigins) {
     for (auto &ImportCI : ImportCIs) {
-      std::unique_ptr<CompilerInstance> IndirectCI = BuildIndirect(ImportCI);
+      CIAndOrigins IndirectCI = BuildIndirect(ImportCI);
       IndirectCIs.push_back(std::move(IndirectCI));
     }
   }
-  llvm::Expected<std::unique_ptr<CompilerInstance>> ExpressionCI =
-      Parse(Expression, Direct ? ImportCIs : IndirectCIs, DumpAST);
+  if (UseOrigins)
+    for (auto &ImportCI : ImportCIs)
+      IndirectCIs.push_back(std::move(ImportCI));
+  llvm::Expected<CIAndOrigins> ExpressionCI =
+      Parse(Expression, (Direct && !UseOrigins) ? ImportCIs : IndirectCIs,
+            DumpAST, DumpIR);
   if (auto E = ExpressionCI.takeError()) {
     llvm::errs() << llvm::toString(std::move(E));
     exit(-1);
-  } else {
-    return 0;
   }
+  Forget(*ExpressionCI, (Direct && !UseOrigins) ? ImportCIs : IndirectCIs);
+  return 0;
 }
-

tools/clang-offload-bundler/CMakeLists.txt

@@ -18,6 +18,7 @@ set(CLANG_OFFLOAD_BUNDLER_LIB_DEPS
 add_dependencies(clang clang-offload-bundler)
 
 target_link_libraries(clang-offload-bundler
+  PRIVATE
   ${CLANG_OFFLOAD_BUNDLER_LIB_DEPS}
   )
 

tools/clang-offload-bundler/ClangOffloadBundler.cpp

@@ -915,8 +915,7 @@ static bool UnbundleFiles() {
   return false;
 }
 
-static void PrintVersion() {
-  raw_ostream &OS = outs();
+static void PrintVersion(raw_ostream &OS) {
   OS << clang::getClangToolFullVersion("clang-offload-bundler") << '\n';
 }
 
@@ -932,8 +931,10 @@ int main(int argc, const char **argv) {
       "one. The resulting file can also be unbundled into different files by \n"
       "this tool if -unbundle is provided.\n");
 
-  if (Help)
+  if (Help) {
     cl::PrintHelpMessage();
+    return 0;
+  }
 
   bool Error = false;
   if (Unbundle) {

tools/clang-refactor/CMakeLists.txt

@@ -0,0 +1,24 @@
+set(LLVM_LINK_COMPONENTS
+  Option
+  Support
+  )
+
+add_clang_tool(clang-refactor
+  ClangRefactor.cpp
+  TestSupport.cpp
+  )
+
+target_link_libraries(clang-refactor
+  PRIVATE
+  clangAST
+  clangBasic
+  clangFormat
+  clangFrontend
+  clangLex
+  clangRewrite
+  clangTooling
+  clangToolingCore
+  clangToolingRefactor
+  )
+
+install(TARGETS clang-refactor RUNTIME DESTINATION bin)

tools/clang-refactor/ClangRefactor.cpp

@@ -0,0 +1,638 @@
+//===--- ClangRefactor.cpp - Clang-based refactoring tool -----------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file implements a clang-refactor tool that performs various
+/// source transformations.
+///
+//===----------------------------------------------------------------------===//
+
+#include "TestSupport.h"
+#include "clang/Frontend/CommandLineSourceLoc.h"
+#include "clang/Frontend/TextDiagnosticPrinter.h"
+#include "clang/Rewrite/Core/Rewriter.h"
+#include "clang/Tooling/CommonOptionsParser.h"
+#include "clang/Tooling/Refactoring.h"
+#include "clang/Tooling/Refactoring/RefactoringAction.h"
+#include "clang/Tooling/Refactoring/RefactoringOptions.h"
+#include "clang/Tooling/Refactoring/Rename/RenamingAction.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Signals.h"
+#include "llvm/Support/raw_ostream.h"
+#include <string>
+
+using namespace clang;
+using namespace tooling;
+using namespace refactor;
+namespace cl = llvm::cl;
+
+namespace opts {
+
+static cl::OptionCategory CommonRefactorOptions("Refactoring options");
+
+static cl::opt<bool> Verbose("v", cl::desc("Use verbose output"),
+                             cl::cat(cl::GeneralCategory),
+                             cl::sub(*cl::AllSubCommands));
+
+static cl::opt<bool> Inplace("i", cl::desc("Inplace edit <file>s"),
+                             cl::cat(cl::GeneralCategory),
+                             cl::sub(*cl::AllSubCommands));
+
+} // end namespace opts
+
+namespace {
+
+/// Stores the parsed `-selection` argument.
+class SourceSelectionArgument {
+public:
+  virtual ~SourceSelectionArgument() {}
+
+  /// Parse the `-selection` argument.
+  ///
+  /// \returns A valid argument when the parse succedeed, null otherwise.
+  static std::unique_ptr<SourceSelectionArgument> fromString(StringRef Value);
+
+  /// Prints any additional state associated with the selection argument to
+  /// the given output stream.
+  virtual void print(raw_ostream &OS) {}
+
+  /// Returns a replacement refactoring result consumer (if any) that should
+  /// consume the results of a refactoring operation.
+  ///
+  /// The replacement refactoring result consumer is used by \c
+  /// TestSourceSelectionArgument to inject a test-specific result handling
+  /// logic into the refactoring operation. The test-specific consumer
+  /// ensures that the individual results in a particular test group are
+  /// identical.
+  virtual std::unique_ptr<ClangRefactorToolConsumerInterface>
+  createCustomConsumer() {
+    return nullptr;
+  }
+
+  /// Runs the give refactoring function for each specified selection.
+  ///
+  /// \returns true if an error occurred, false otherwise.
+  virtual bool
+  forAllRanges(const SourceManager &SM,
+               llvm::function_ref<void(SourceRange R)> Callback) = 0;
+};
+
+/// Stores the parsed -selection=test:<filename> option.
+class TestSourceSelectionArgument final : public SourceSelectionArgument {
+public:
+  TestSourceSelectionArgument(TestSelectionRangesInFile TestSelections)
+      : TestSelections(std::move(TestSelections)) {}
+
+  void print(raw_ostream &OS) override { TestSelections.dump(OS); }
+
+  std::unique_ptr<ClangRefactorToolConsumerInterface>
+  createCustomConsumer() override {
+    return TestSelections.createConsumer();
+  }
+
+  /// Testing support: invokes the selection action for each selection range in
+  /// the test file.
+  bool forAllRanges(const SourceManager &SM,
+                    llvm::function_ref<void(SourceRange R)> Callback) override {
+    return TestSelections.foreachRange(SM, Callback);
+  }
+
+private:
+  TestSelectionRangesInFile TestSelections;
+};
+
+/// Stores the parsed -selection=filename:line:column[-line:column] option.
+class SourceRangeSelectionArgument final : public SourceSelectionArgument {
+public:
+  SourceRangeSelectionArgument(ParsedSourceRange Range)
+      : Range(std::move(Range)) {}
+
+  bool forAllRanges(const SourceManager &SM,
+                    llvm::function_ref<void(SourceRange R)> Callback) override {
+    const FileEntry *FE = SM.getFileManager().getFile(Range.FileName);
+    FileID FID = FE ? SM.translateFile(FE) : FileID();
+    if (!FE || FID.isInvalid()) {
+      llvm::errs() << "error: -selection=" << Range.FileName
+                   << ":... : given file is not in the target TU\n";
+      return true;
+    }
+
+    SourceLocation Start = SM.getMacroArgExpandedLocation(
+        SM.translateLineCol(FID, Range.Begin.first, Range.Begin.second));
+    SourceLocation End = SM.getMacroArgExpandedLocation(
+        SM.translateLineCol(FID, Range.End.first, Range.End.second));
+    if (Start.isInvalid() || End.isInvalid()) {
+      llvm::errs() << "error: -selection=" << Range.FileName << ':'
+                   << Range.Begin.first << ':' << Range.Begin.second << '-'
+                   << Range.End.first << ':' << Range.End.second
+                   << " : invalid source location\n";
+      return true;
+    }
+    Callback(SourceRange(Start, End));
+    return false;
+  }
+
+private:
+  ParsedSourceRange Range;
+};
+
+std::unique_ptr<SourceSelectionArgument>
+SourceSelectionArgument::fromString(StringRef Value) {
+  if (Value.startswith("test:")) {
+    StringRef Filename = Value.drop_front(strlen("test:"));
+    Optional<TestSelectionRangesInFile> ParsedTestSelection =
+        findTestSelectionRanges(Filename);
+    if (!ParsedTestSelection)
+      return nullptr; // A parsing error was already reported.
+    return llvm::make_unique<TestSourceSelectionArgument>(
+        std::move(*ParsedTestSelection));
+  }
+  Optional<ParsedSourceRange> Range = ParsedSourceRange::fromString(Value);
+  if (Range)
+    return llvm::make_unique<SourceRangeSelectionArgument>(std::move(*Range));
+  llvm::errs() << "error: '-selection' option must be specified using "
+                  "<file>:<line>:<column> or "
+                  "<file>:<line>:<column>-<line>:<column> format\n";
+  return nullptr;
+}
+
+/// A container that stores the command-line options used by a single
+/// refactoring option.
+class RefactoringActionCommandLineOptions {
+public:
+  void addStringOption(const RefactoringOption &Option,
+                       std::unique_ptr<cl::opt<std::string>> CLOption) {
+    StringOptions[&Option] = std::move(CLOption);
+  }
+
+  const cl::opt<std::string> &
+  getStringOption(const RefactoringOption &Opt) const {
+    auto It = StringOptions.find(&Opt);
+    return *It->second;
+  }
+
+private:
+  llvm::DenseMap<const RefactoringOption *,
+                 std::unique_ptr<cl::opt<std::string>>>
+      StringOptions;
+};
+
+/// Passes the command-line option values to the options used by a single
+/// refactoring action rule.
+class CommandLineRefactoringOptionVisitor final
+    : public RefactoringOptionVisitor {
+public:
+  CommandLineRefactoringOptionVisitor(
+      const RefactoringActionCommandLineOptions &Options)
+      : Options(Options) {}
+
+  void visit(const RefactoringOption &Opt,
+             Optional<std::string> &Value) override {
+    const cl::opt<std::string> &CLOpt = Options.getStringOption(Opt);
+    if (!CLOpt.getValue().empty()) {
+      Value = CLOpt.getValue();
+      return;
+    }
+    Value = None;
+    if (Opt.isRequired())
+      MissingRequiredOptions.push_back(&Opt);
+  }
+
+  ArrayRef<const RefactoringOption *> getMissingRequiredOptions() const {
+    return MissingRequiredOptions;
+  }
+
+private:
+  llvm::SmallVector<const RefactoringOption *, 4> MissingRequiredOptions;
+  const RefactoringActionCommandLineOptions &Options;
+};
+
+/// Creates the refactoring options used by all the rules in a single
+/// refactoring action.
+class CommandLineRefactoringOptionCreator final
+    : public RefactoringOptionVisitor {
+public:
+  CommandLineRefactoringOptionCreator(
+      cl::OptionCategory &Category, cl::SubCommand &Subcommand,
+      RefactoringActionCommandLineOptions &Options)
+      : Category(Category), Subcommand(Subcommand), Options(Options) {}
+
+  void visit(const RefactoringOption &Opt, Optional<std::string> &) override {
+    if (Visited.insert(&Opt).second)
+      Options.addStringOption(Opt, create<std::string>(Opt));
+  }
+
+private:
+  template <typename T>
+  std::unique_ptr<cl::opt<T>> create(const RefactoringOption &Opt) {
+    if (!OptionNames.insert(Opt.getName()).second)
+      llvm::report_fatal_error("Multiple identical refactoring options "
+                               "specified for one refactoring action");
+    // FIXME: cl::Required can be specified when this option is present
+    // in all rules in an action.
+    return llvm::make_unique<cl::opt<T>>(
+        Opt.getName(), cl::desc(Opt.getDescription()), cl::Optional,
+        cl::cat(Category), cl::sub(Subcommand));
+  }
+
+  llvm::SmallPtrSet<const RefactoringOption *, 8> Visited;
+  llvm::StringSet<> OptionNames;
+  cl::OptionCategory &Category;
+  cl::SubCommand &Subcommand;
+  RefactoringActionCommandLineOptions &Options;
+};
+
+/// A subcommand that corresponds to individual refactoring action.
+class RefactoringActionSubcommand : public cl::SubCommand {
+public:
+  RefactoringActionSubcommand(std::unique_ptr<RefactoringAction> Action,
+                              RefactoringActionRules ActionRules,
+                              cl::OptionCategory &Category)
+      : SubCommand(Action->getCommand(), Action->getDescription()),
+        Action(std::move(Action)), ActionRules(std::move(ActionRules)) {
+    // Check if the selection option is supported.
+    for (const auto &Rule : this->ActionRules) {
+      if (Rule->hasSelectionRequirement()) {
+        Selection = llvm::make_unique<cl::opt<std::string>>(
+            "selection",
+            cl::desc(
+                "The selected source range in which the refactoring should "
+                "be initiated (<file>:<line>:<column>-<line>:<column> or "
+                "<file>:<line>:<column>)"),
+            cl::cat(Category), cl::sub(*this));
+        break;
+      }
+    }
+    // Create the refactoring options.
+    for (const auto &Rule : this->ActionRules) {
+      CommandLineRefactoringOptionCreator OptionCreator(Category, *this,
+                                                        Options);
+      Rule->visitRefactoringOptions(OptionCreator);
+    }
+  }
+
+  ~RefactoringActionSubcommand() { unregisterSubCommand(); }
+
+  const RefactoringActionRules &getActionRules() const { return ActionRules; }
+
+  /// Parses the "-selection" command-line argument.
+  ///
+  /// \returns true on error, false otherwise.
+  bool parseSelectionArgument() {
+    if (Selection) {
+      ParsedSelection = SourceSelectionArgument::fromString(*Selection);
+      if (!ParsedSelection)
+        return true;
+    }
+    return false;
+  }
+
+  SourceSelectionArgument *getSelection() const {
+    assert(Selection && "selection not supported!");
+    return ParsedSelection.get();
+  }
+
+  const RefactoringActionCommandLineOptions &getOptions() const {
+    return Options;
+  }
+
+private:
+  std::unique_ptr<RefactoringAction> Action;
+  RefactoringActionRules ActionRules;
+  std::unique_ptr<cl::opt<std::string>> Selection;
+  std::unique_ptr<SourceSelectionArgument> ParsedSelection;
+  RefactoringActionCommandLineOptions Options;
+};
+
+class ClangRefactorConsumer final : public ClangRefactorToolConsumerInterface {
+public:
+  ClangRefactorConsumer(AtomicChanges &Changes) : SourceChanges(&Changes) {}
+
+  void handleError(llvm::Error Err) override {
+    Optional<PartialDiagnosticAt> Diag = DiagnosticError::take(Err);
+    if (!Diag) {
+      llvm::errs() << llvm::toString(std::move(Err)) << "\n";
+      return;
+    }
+    llvm::cantFail(std::move(Err)); // This is a success.
+    DiagnosticBuilder DB(
+        getDiags().Report(Diag->first, Diag->second.getDiagID()));
+    Diag->second.Emit(DB);
+  }
+
+  void handle(AtomicChanges Changes) override {
+    SourceChanges->insert(SourceChanges->begin(), Changes.begin(),
+                          Changes.end());
+  }
+
+  void handle(SymbolOccurrences Occurrences) override {
+    llvm_unreachable("symbol occurrence results are not handled yet");
+  }
+
+private:
+  AtomicChanges *SourceChanges;
+};
+
+class ClangRefactorTool {
+public:
+  ClangRefactorTool()
+      : SelectedSubcommand(nullptr), MatchingRule(nullptr),
+        Consumer(new ClangRefactorConsumer(Changes)), HasFailed(false) {
+    std::vector<std::unique_ptr<RefactoringAction>> Actions =
+        createRefactoringActions();
+
+    // Actions must have unique command names so that we can map them to one
+    // subcommand.
+    llvm::StringSet<> CommandNames;
+    for (const auto &Action : Actions) {
+      if (!CommandNames.insert(Action->getCommand()).second) {
+        llvm::errs() << "duplicate refactoring action command '"
+                     << Action->getCommand() << "'!";
+        exit(1);
+      }
+    }
+
+    // Create subcommands and command-line options.
+    for (auto &Action : Actions) {
+      SubCommands.push_back(llvm::make_unique<RefactoringActionSubcommand>(
+          std::move(Action), Action->createActiveActionRules(),
+          opts::CommonRefactorOptions));
+    }
+  }
+
+  // Initializes the selected subcommand and refactoring rule based on the
+  // command line options.
+  llvm::Error Init() {
+    auto Subcommand = getSelectedSubcommand();
+    if (!Subcommand)
+      return Subcommand.takeError();
+    auto Rule = getMatchingRule(**Subcommand);
+    if (!Rule)
+      return Rule.takeError();
+
+    SelectedSubcommand = *Subcommand;
+    MatchingRule = *Rule;
+
+    return llvm::Error::success();
+  }
+
+  bool hasFailed() const { return HasFailed; }
+
+  using TUCallbackType = std::function<void(ASTContext &)>;
+
+  // Callback of an AST action. This invokes the matching rule on the given AST.
+  void callback(ASTContext &AST) {
+    assert(SelectedSubcommand && MatchingRule && Consumer);
+    RefactoringRuleContext Context(AST.getSourceManager());
+    Context.setASTContext(AST);
+
+    // If the selection option is test specific, we use a test-specific
+    // consumer.
+    std::unique_ptr<ClangRefactorToolConsumerInterface> TestConsumer;
+    bool HasSelection = MatchingRule->hasSelectionRequirement();
+    if (HasSelection)
+      TestConsumer = SelectedSubcommand->getSelection()->createCustomConsumer();
+    ClangRefactorToolConsumerInterface *ActiveConsumer =
+        TestConsumer ? TestConsumer.get() : Consumer.get();
+    ActiveConsumer->beginTU(AST);
+
+    auto InvokeRule = [&](RefactoringResultConsumer &Consumer) {
+      if (opts::Verbose)
+        logInvocation(*SelectedSubcommand, Context);
+      MatchingRule->invoke(*ActiveConsumer, Context);
+    };
+    if (HasSelection) {
+      assert(SelectedSubcommand->getSelection() &&
+             "Missing selection argument?");
+      if (opts::Verbose)
+        SelectedSubcommand->getSelection()->print(llvm::outs());
+      if (SelectedSubcommand->getSelection()->forAllRanges(
+              Context.getSources(), [&](SourceRange R) {
+                Context.setSelectionRange(R);
+                InvokeRule(*ActiveConsumer);
+              }))
+        HasFailed = true;
+      ActiveConsumer->endTU();
+      return;
+    }
+    InvokeRule(*ActiveConsumer);
+    ActiveConsumer->endTU();
+  }
+
+  llvm::Expected<std::unique_ptr<FrontendActionFactory>>
+  getFrontendActionFactory() {
+    class ToolASTConsumer : public ASTConsumer {
+    public:
+      TUCallbackType Callback;
+      ToolASTConsumer(TUCallbackType Callback)
+          : Callback(std::move(Callback)) {}
+
+      void HandleTranslationUnit(ASTContext &Context) override {
+        Callback(Context);
+      }
+    };
+    class ToolASTAction : public ASTFrontendAction {
+    public:
+      explicit ToolASTAction(TUCallbackType Callback)
+          : Callback(std::move(Callback)) {}
+
+    protected:
+      std::unique_ptr<clang::ASTConsumer>
+      CreateASTConsumer(clang::CompilerInstance &compiler,
+                        StringRef /* dummy */) override {
+        std::unique_ptr<clang::ASTConsumer> Consumer{
+            new ToolASTConsumer(Callback)};
+        return Consumer;
+      }
+
+    private:
+      TUCallbackType Callback;
+    };
+
+    class ToolActionFactory : public FrontendActionFactory {
+    public:
+      ToolActionFactory(TUCallbackType Callback)
+          : Callback(std::move(Callback)) {}
+
+      FrontendAction *create() override { return new ToolASTAction(Callback); }
+
+    private:
+      TUCallbackType Callback;
+    };
+
+    return llvm::make_unique<ToolActionFactory>(
+        [this](ASTContext &AST) { return callback(AST); });
+  }
+
+  // FIXME(ioeric): this seems to only works for changes in a single file at
+  // this point.
+  bool applySourceChanges() {
+    std::set<std::string> Files;
+    for (const auto &Change : Changes)
+      Files.insert(Change.getFilePath());
+    // FIXME: Add automatic formatting support as well.
+    tooling::ApplyChangesSpec Spec;
+    // FIXME: We should probably cleanup the result by default as well.
+    Spec.Cleanup = false;
+    for (const auto &File : Files) {
+      llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferErr =
+          llvm::MemoryBuffer::getFile(File);
+      if (!BufferErr) {
+        llvm::errs() << "error: failed to open " << File << " for rewriting\n";
+        return true;
+      }
+      auto Result = tooling::applyAtomicChanges(File, (*BufferErr)->getBuffer(),
+                                                Changes, Spec);
+      if (!Result) {
+        llvm::errs() << toString(Result.takeError());
+        return true;
+      }
+
+      if (opts::Inplace) {
+        std::error_code EC;
+        llvm::raw_fd_ostream OS(File, EC, llvm::sys::fs::F_Text);
+        if (EC) {
+          llvm::errs() << EC.message() << "\n";
+          return true;
+        }
+        OS << *Result;
+        continue;
+      }
+
+      llvm::outs() << *Result;
+    }
+    return false;
+  }
+
+private:
+  /// Logs an individual refactoring action invocation to STDOUT.
+  void logInvocation(RefactoringActionSubcommand &Subcommand,
+                     const RefactoringRuleContext &Context) {
+    llvm::outs() << "invoking action '" << Subcommand.getName() << "':\n";
+    if (Context.getSelectionRange().isValid()) {
+      SourceRange R = Context.getSelectionRange();
+      llvm::outs() << "  -selection=";
+      R.getBegin().print(llvm::outs(), Context.getSources());
+      llvm::outs() << " -> ";
+      R.getEnd().print(llvm::outs(), Context.getSources());
+      llvm::outs() << "\n";
+    }
+  }
+
+  llvm::Expected<RefactoringActionRule *>
+  getMatchingRule(RefactoringActionSubcommand &Subcommand) {
+    SmallVector<RefactoringActionRule *, 4> MatchingRules;
+    llvm::StringSet<> MissingOptions;
+
+    for (const auto &Rule : Subcommand.getActionRules()) {
+      CommandLineRefactoringOptionVisitor Visitor(Subcommand.getOptions());
+      Rule->visitRefactoringOptions(Visitor);
+      if (Visitor.getMissingRequiredOptions().empty()) {
+        if (!Rule->hasSelectionRequirement()) {
+          MatchingRules.push_back(Rule.get());
+        } else {
+          Subcommand.parseSelectionArgument();
+          if (Subcommand.getSelection()) {
+            MatchingRules.push_back(Rule.get());
+          } else {
+            MissingOptions.insert("selection");
+          }
+        }
+      }
+      for (const RefactoringOption *Opt : Visitor.getMissingRequiredOptions())
+        MissingOptions.insert(Opt->getName());
+    }
+    if (MatchingRules.empty()) {
+      std::string Error;
+      llvm::raw_string_ostream OS(Error);
+      OS << "ERROR: '" << Subcommand.getName()
+         << "' can't be invoked with the given arguments:\n";
+      for (const auto &Opt : MissingOptions)
+        OS << "  missing '-" << Opt.getKey() << "' option\n";
+      OS.flush();
+      return llvm::make_error<llvm::StringError>(
+          Error, llvm::inconvertibleErrorCode());
+    }
+    if (MatchingRules.size() != 1) {
+      return llvm::make_error<llvm::StringError>(
+          llvm::Twine("ERROR: more than one matching rule of action") +
+              Subcommand.getName() + "was found with given options.",
+          llvm::inconvertibleErrorCode());
+    }
+    return MatchingRules.front();
+  }
+  // Figure out which action is specified by the user. The user must specify the
+  // action using a command-line subcommand, e.g. the invocation `clang-refactor
+  // local-rename` corresponds to the `LocalRename` refactoring action. All
+  // subcommands must have a unique names. This allows us to figure out which
+  // refactoring action should be invoked by looking at the first subcommand
+  // that's enabled by LLVM's command-line parser.
+  llvm::Expected<RefactoringActionSubcommand *> getSelectedSubcommand() {
+    auto It = llvm::find_if(
+        SubCommands,
+        [](const std::unique_ptr<RefactoringActionSubcommand> &SubCommand) {
+          return !!(*SubCommand);
+        });
+    if (It == SubCommands.end()) {
+      std::string Error;
+      llvm::raw_string_ostream OS(Error);
+      OS << "error: no refactoring action given\n";
+      OS << "note: the following actions are supported:\n";
+      for (const auto &Subcommand : SubCommands)
+        OS.indent(2) << Subcommand->getName() << "\n";
+      OS.flush();
+      return llvm::make_error<llvm::StringError>(
+          Error, llvm::inconvertibleErrorCode());
+    }
+    RefactoringActionSubcommand *Subcommand = &(**It);
+    return Subcommand;
+  }
+
+  std::vector<std::unique_ptr<RefactoringActionSubcommand>> SubCommands;
+  RefactoringActionSubcommand *SelectedSubcommand;
+  RefactoringActionRule *MatchingRule;
+  std::unique_ptr<ClangRefactorToolConsumerInterface> Consumer;
+  AtomicChanges Changes;
+  bool HasFailed;
+};
+
+} // end anonymous namespace
+
+int main(int argc, const char **argv) {
+  llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
+
+  ClangRefactorTool RefactorTool;
+
+  CommonOptionsParser Options(
+      argc, argv, cl::GeneralCategory, cl::ZeroOrMore,
+      "Clang-based refactoring tool for C, C++ and Objective-C");
+
+  if (auto Err = RefactorTool.Init()) {
+    llvm::errs() << llvm::toString(std::move(Err)) << "\n";
+    return 1;
+  }
+
+  auto ActionFactory = RefactorTool.getFrontendActionFactory();
+  if (!ActionFactory) {
+    llvm::errs() << llvm::toString(ActionFactory.takeError()) << "\n";
+    return 1;
+  }
+  ClangTool Tool(Options.getCompilations(), Options.getSourcePathList());
+  bool Failed = false;
+  if (Tool.run(ActionFactory->get()) != 0) {
+    llvm::errs() << "Failed to run refactoring action on files\n";
+    // It is possible that TUs are broken while changes are generated correctly,
+    // so we still try applying changes.
+    Failed = true;
+  }
+  return RefactorTool.applySourceChanges() || Failed ||
+         RefactorTool.hasFailed();
+}

tools/clang-refactor/TestSupport.cpp

@@ -0,0 +1,392 @@
+//===--- TestSupport.cpp - Clang-based refactoring tool -------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file implements routines that provide refactoring testing
+/// utilities.
+///
+//===----------------------------------------------------------------------===//
+
+#include "TestSupport.h"
+#include "clang/Basic/DiagnosticError.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Lex/Lexer.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/ErrorOr.h"
+#include "llvm/Support/LineIterator.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Regex.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+namespace clang {
+namespace refactor {
+
+void TestSelectionRangesInFile::dump(raw_ostream &OS) const {
+  for (const auto &Group : GroupedRanges) {
+    OS << "Test selection group '" << Group.Name << "':\n";
+    for (const auto &Range : Group.Ranges) {
+      OS << "  " << Range.Begin << "-" << Range.End << "\n";
+    }
+  }
+}
+
+bool TestSelectionRangesInFile::foreachRange(
+    const SourceManager &SM,
+    llvm::function_ref<void(SourceRange)> Callback) const {
+  const FileEntry *FE = SM.getFileManager().getFile(Filename);
+  FileID FID = FE ? SM.translateFile(FE) : FileID();
+  if (!FE || FID.isInvalid()) {
+    llvm::errs() << "error: -selection=test:" << Filename
+                 << " : given file is not in the target TU";
+    return true;
+  }
+  SourceLocation FileLoc = SM.getLocForStartOfFile(FID);
+  for (const auto &Group : GroupedRanges) {
+    for (const TestSelectionRange &Range : Group.Ranges) {
+      // Translate the offset pair to a true source range.
+      SourceLocation Start =
+          SM.getMacroArgExpandedLocation(FileLoc.getLocWithOffset(Range.Begin));
+      SourceLocation End =
+          SM.getMacroArgExpandedLocation(FileLoc.getLocWithOffset(Range.End));
+      assert(Start.isValid() && End.isValid() && "unexpected invalid range");
+      Callback(SourceRange(Start, End));
+    }
+  }
+  return false;
+}
+
+namespace {
+
+void dumpChanges(const tooling::AtomicChanges &Changes, raw_ostream &OS) {
+  for (const auto &Change : Changes)
+    OS << const_cast<tooling::AtomicChange &>(Change).toYAMLString() << "\n";
+}
+
+bool areChangesSame(const tooling::AtomicChanges &LHS,
+                    const tooling::AtomicChanges &RHS) {
+  if (LHS.size() != RHS.size())
+    return false;
+  for (const auto &I : llvm::zip(LHS, RHS)) {
+    if (!(std::get<0>(I) == std::get<1>(I)))
+      return false;
+  }
+  return true;
+}
+
+bool printRewrittenSources(const tooling::AtomicChanges &Changes,
+                           raw_ostream &OS) {
+  std::set<std::string> Files;
+  for (const auto &Change : Changes)
+    Files.insert(Change.getFilePath());
+  tooling::ApplyChangesSpec Spec;
+  Spec.Cleanup = false;
+  for (const auto &File : Files) {
+    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferErr =
+        llvm::MemoryBuffer::getFile(File);
+    if (!BufferErr) {
+      llvm::errs() << "failed to open" << File << "\n";
+      return true;
+    }
+    auto Result = tooling::applyAtomicChanges(File, (*BufferErr)->getBuffer(),
+                                              Changes, Spec);
+    if (!Result) {
+      llvm::errs() << toString(Result.takeError());
+      return true;
+    }
+    OS << *Result;
+  }
+  return false;
+}
+
+class TestRefactoringResultConsumer final
+    : public ClangRefactorToolConsumerInterface {
+public:
+  TestRefactoringResultConsumer(const TestSelectionRangesInFile &TestRanges)
+      : TestRanges(TestRanges) {
+    Results.push_back({});
+  }
+
+  ~TestRefactoringResultConsumer() {
+    // Ensure all results are checked.
+    for (auto &Group : Results) {
+      for (auto &Result : Group) {
+        if (!Result) {
+          (void)llvm::toString(Result.takeError());
+        }
+      }
+    }
+  }
+
+  void handleError(llvm::Error Err) override { handleResult(std::move(Err)); }
+
+  void handle(tooling::AtomicChanges Changes) override {
+    handleResult(std::move(Changes));
+  }
+
+  void handle(tooling::SymbolOccurrences Occurrences) override {
+    tooling::RefactoringResultConsumer::handle(std::move(Occurrences));
+  }
+
+private:
+  bool handleAllResults();
+
+  void handleResult(Expected<tooling::AtomicChanges> Result) {
+    Results.back().push_back(std::move(Result));
+    size_t GroupIndex = Results.size() - 1;
+    if (Results.back().size() >=
+        TestRanges.GroupedRanges[GroupIndex].Ranges.size()) {
+      ++GroupIndex;
+      if (GroupIndex >= TestRanges.GroupedRanges.size()) {
+        if (handleAllResults())
+          exit(1); // error has occurred.
+        return;
+      }
+      Results.push_back({});
+    }
+  }
+
+  const TestSelectionRangesInFile &TestRanges;
+  std::vector<std::vector<Expected<tooling::AtomicChanges>>> Results;
+};
+
+std::pair<unsigned, unsigned> getLineColumn(StringRef Filename,
+                                            unsigned Offset) {
+  ErrorOr<std::unique_ptr<MemoryBuffer>> ErrOrFile =
+      MemoryBuffer::getFile(Filename);
+  if (!ErrOrFile)
+    return {0, 0};
+  StringRef Source = ErrOrFile.get()->getBuffer();
+  Source = Source.take_front(Offset);
+  size_t LastLine = Source.find_last_of("\r\n");
+  return {Source.count('\n') + 1,
+          (LastLine == StringRef::npos ? Offset : Offset - LastLine) + 1};
+}
+
+} // end anonymous namespace
+
+bool TestRefactoringResultConsumer::handleAllResults() {
+  bool Failed = false;
+  for (auto &Group : llvm::enumerate(Results)) {
+    // All ranges in the group must produce the same result.
+    Optional<tooling::AtomicChanges> CanonicalResult;
+    Optional<std::string> CanonicalErrorMessage;
+    for (auto &I : llvm::enumerate(Group.value())) {
+      Expected<tooling::AtomicChanges> &Result = I.value();
+      std::string ErrorMessage;
+      bool HasResult = !!Result;
+      if (!HasResult) {
+        handleAllErrors(
+            Result.takeError(),
+            [&](StringError &Err) { ErrorMessage = Err.getMessage(); },
+            [&](DiagnosticError &Err) {
+              const PartialDiagnosticAt &Diag = Err.getDiagnostic();
+              llvm::SmallString<100> DiagText;
+              Diag.second.EmitToString(getDiags(), DiagText);
+              ErrorMessage = DiagText.str().str();
+            });
+      }
+      if (!CanonicalResult && !CanonicalErrorMessage) {
+        if (HasResult)
+          CanonicalResult = std::move(*Result);
+        else
+          CanonicalErrorMessage = std::move(ErrorMessage);
+        continue;
+      }
+
+      // Verify that this result corresponds to the canonical result.
+      if (CanonicalErrorMessage) {
+        // The error messages must match.
+        if (!HasResult && ErrorMessage == *CanonicalErrorMessage)
+          continue;
+      } else {
+        assert(CanonicalResult && "missing canonical result");
+        // The results must match.
+        if (HasResult && areChangesSame(*Result, *CanonicalResult))
+          continue;
+      }
+      Failed = true;
+      // Report the mismatch.
+      std::pair<unsigned, unsigned> LineColumn = getLineColumn(
+          TestRanges.Filename,
+          TestRanges.GroupedRanges[Group.index()].Ranges[I.index()].Begin);
+      llvm::errs()
+          << "error: unexpected refactoring result for range starting at "
+          << LineColumn.first << ':' << LineColumn.second << " in group '"
+          << TestRanges.GroupedRanges[Group.index()].Name << "':\n  ";
+      if (HasResult)
+        llvm::errs() << "valid result";
+      else
+        llvm::errs() << "error '" << ErrorMessage << "'";
+      llvm::errs() << " does not match initial ";
+      if (CanonicalErrorMessage)
+        llvm::errs() << "error '" << *CanonicalErrorMessage << "'\n";
+      else
+        llvm::errs() << "valid result\n";
+      if (HasResult && !CanonicalErrorMessage) {
+        llvm::errs() << "  Expected to Produce:\n";
+        dumpChanges(*CanonicalResult, llvm::errs());
+        llvm::errs() << "  Produced:\n";
+        dumpChanges(*Result, llvm::errs());
+      }
+    }
+
+    // Dump the results:
+    const auto &TestGroup = TestRanges.GroupedRanges[Group.index()];
+    if (!CanonicalResult) {
+      llvm::outs() << TestGroup.Ranges.size() << " '" << TestGroup.Name
+                   << "' results:\n";
+      llvm::outs() << *CanonicalErrorMessage << "\n";
+    } else {
+      llvm::outs() << TestGroup.Ranges.size() << " '" << TestGroup.Name
+                   << "' results:\n";
+      if (printRewrittenSources(*CanonicalResult, llvm::outs()))
+        return true;
+    }
+  }
+  return Failed;
+}
+
+std::unique_ptr<ClangRefactorToolConsumerInterface>
+TestSelectionRangesInFile::createConsumer() const {
+  return llvm::make_unique<TestRefactoringResultConsumer>(*this);
+}
+
+/// Adds the \p ColumnOffset to file offset \p Offset, without going past a
+/// newline.
+static unsigned addColumnOffset(StringRef Source, unsigned Offset,
+                                unsigned ColumnOffset) {
+  if (!ColumnOffset)
+    return Offset;
+  StringRef Substr = Source.drop_front(Offset).take_front(ColumnOffset);
+  size_t NewlinePos = Substr.find_first_of("\r\n");
+  return Offset +
+         (NewlinePos == StringRef::npos ? ColumnOffset : (unsigned)NewlinePos);
+}
+
+static unsigned addEndLineOffsetAndEndColumn(StringRef Source, unsigned Offset,
+                                             unsigned LineNumberOffset,
+                                             unsigned Column) {
+  StringRef Line = Source.drop_front(Offset);
+  unsigned LineOffset = 0;
+  for (; LineNumberOffset != 0; --LineNumberOffset) {
+    size_t NewlinePos = Line.find_first_of("\r\n");
+    // Line offset goes out of bounds.
+    if (NewlinePos == StringRef::npos)
+      break;
+    LineOffset += NewlinePos + 1;
+    Line = Line.drop_front(NewlinePos + 1);
+  }
+  // Source now points to the line at +lineOffset;
+  size_t LineStart = Source.find_last_of("\r\n", /*From=*/Offset + LineOffset);
+  return addColumnOffset(
+      Source, LineStart == StringRef::npos ? 0 : LineStart + 1, Column - 1);
+}
+
+Optional<TestSelectionRangesInFile>
+findTestSelectionRanges(StringRef Filename) {
+  ErrorOr<std::unique_ptr<MemoryBuffer>> ErrOrFile =
+      MemoryBuffer::getFile(Filename);
+  if (!ErrOrFile) {
+    llvm::errs() << "error: -selection=test:" << Filename
+                 << " : could not open the given file";
+    return None;
+  }
+  StringRef Source = ErrOrFile.get()->getBuffer();
+
+  // See the doc comment for this function for the explanation of this
+  // syntax.
+  static Regex RangeRegex("range[[:blank:]]*([[:alpha:]_]*)?[[:blank:]]*=[[:"
+                          "blank:]]*(\\+[[:digit:]]+)?[[:blank:]]*(->[[:blank:]"
+                          "]*[\\+\\:[:digit:]]+)?");
+
+  std::map<std::string, SmallVector<TestSelectionRange, 8>> GroupedRanges;
+
+  LangOptions LangOpts;
+  LangOpts.CPlusPlus = 1;
+  LangOpts.CPlusPlus11 = 1;
+  Lexer Lex(SourceLocation::getFromRawEncoding(0), LangOpts, Source.begin(),
+            Source.begin(), Source.end());
+  Lex.SetCommentRetentionState(true);
+  Token Tok;
+  for (Lex.LexFromRawLexer(Tok); Tok.isNot(tok::eof);
+       Lex.LexFromRawLexer(Tok)) {
+    if (Tok.isNot(tok::comment))
+      continue;
+    StringRef Comment =
+        Source.substr(Tok.getLocation().getRawEncoding(), Tok.getLength());
+    SmallVector<StringRef, 4> Matches;
+    // Try to detect mistyped 'range:' comments to ensure tests don't miss
+    // anything.
+    auto DetectMistypedCommand = [&]() -> bool {
+      if (Comment.contains_lower("range") && Comment.contains("=") &&
+          !Comment.contains_lower("run") && !Comment.contains("CHECK")) {
+        llvm::errs() << "error: suspicious comment '" << Comment
+                     << "' that "
+                        "resembles the range command found\n";
+        llvm::errs() << "note: please reword if this isn't a range command\n";
+      }
+      return false;
+    };
+    // Allow CHECK: comments to contain range= commands.
+    if (!RangeRegex.match(Comment, &Matches) || Comment.contains("CHECK")) {
+      if (DetectMistypedCommand())
+        return None;
+      continue;
+    }
+    unsigned Offset = Tok.getEndLoc().getRawEncoding();
+    unsigned ColumnOffset = 0;
+    if (!Matches[2].empty()) {
+      // Don't forget to drop the '+'!
+      if (Matches[2].drop_front().getAsInteger(10, ColumnOffset))
+        assert(false && "regex should have produced a number");
+    }
+    Offset = addColumnOffset(Source, Offset, ColumnOffset);
+    unsigned EndOffset;
+
+    if (!Matches[3].empty()) {
+      static Regex EndLocRegex(
+          "->[[:blank:]]*(\\+[[:digit:]]+):([[:digit:]]+)");
+      SmallVector<StringRef, 4> EndLocMatches;
+      if (!EndLocRegex.match(Matches[3], &EndLocMatches)) {
+        if (DetectMistypedCommand())
+          return None;
+        continue;
+      }
+      unsigned EndLineOffset = 0, EndColumn = 0;
+      if (EndLocMatches[1].drop_front().getAsInteger(10, EndLineOffset) ||
+          EndLocMatches[2].getAsInteger(10, EndColumn))
+        assert(false && "regex should have produced a number");
+      EndOffset = addEndLineOffsetAndEndColumn(Source, Offset, EndLineOffset,
+                                               EndColumn);
+    } else {
+      EndOffset = Offset;
+    }
+    TestSelectionRange Range = {Offset, EndOffset};
+    auto It = GroupedRanges.insert(std::make_pair(
+        Matches[1].str(), SmallVector<TestSelectionRange, 8>{Range}));
+    if (!It.second)
+      It.first->second.push_back(Range);
+  }
+  if (GroupedRanges.empty()) {
+    llvm::errs() << "error: -selection=test:" << Filename
+                 << ": no 'range' commands";
+    return None;
+  }
+
+  TestSelectionRangesInFile TestRanges = {Filename.str(), {}};
+  for (auto &Group : GroupedRanges)
+    TestRanges.GroupedRanges.push_back({Group.first, std::move(Group.second)});
+  return std::move(TestRanges);
+}
+
+} // end namespace refactor
+} // end namespace clang

tools/clang-refactor/TestSupport.h

@@ -0,0 +1,107 @@
+//===--- TestSupport.h - Clang-based refactoring tool -----------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief Declares datatypes and routines that are used by test-specific code
+/// in clang-refactor.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_CLANG_REFACTOR_TEST_SUPPORT_H
+#define LLVM_CLANG_TOOLS_CLANG_REFACTOR_TEST_SUPPORT_H
+
+#include "ToolRefactoringResultConsumer.h"
+#include "clang/Basic/LLVM.h"
+#include "clang/Basic/SourceLocation.h"
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Error.h"
+#include <map>
+#include <string>
+
+namespace clang {
+
+class SourceManager;
+
+namespace refactor {
+
+/// A source selection range that's specified in a test file using an inline
+/// command in the comment. These commands can take the following forms:
+///
+/// - /*range=*/ will create an empty selection range in the default group
+///   right after the comment.
+/// - /*range a=*/ will create an empty selection range in the 'a' group right
+///   after the comment.
+/// - /*range = +1*/ will create an empty selection range at a location that's
+///   right after the comment with one offset to the column.
+/// - /*range= -> +2:3*/ will create a selection range that starts at the
+///   location right after the comment, and ends at column 3 of the 2nd line
+///   after the line of the starting location.
+///
+/// Clang-refactor will expected all ranges in one test group to produce
+/// identical results.
+struct TestSelectionRange {
+  unsigned Begin, End;
+};
+
+/// A set of test selection ranges specified in one file.
+struct TestSelectionRangesInFile {
+  std::string Filename;
+  struct RangeGroup {
+    std::string Name;
+    SmallVector<TestSelectionRange, 8> Ranges;
+  };
+  std::vector<RangeGroup> GroupedRanges;
+
+  TestSelectionRangesInFile(TestSelectionRangesInFile &&) = default;
+  TestSelectionRangesInFile &operator=(TestSelectionRangesInFile &&) = default;
+
+  bool foreachRange(const SourceManager &SM,
+                    llvm::function_ref<void(SourceRange)> Callback) const;
+
+  std::unique_ptr<ClangRefactorToolConsumerInterface> createConsumer() const;
+
+  void dump(llvm::raw_ostream &OS) const;
+};
+
+/// Extracts the grouped selection ranges from the file that's specified in
+/// the -selection=test:<filename> option.
+///
+/// The grouped ranges are specified in comments using the following syntax:
+/// "range" [ group-name ] "=" [ "+" starting-column-offset ] [ "->"
+///                              "+" ending-line-offset ":"
+///                                  ending-column-position ]
+///
+/// The selection range is then computed from this command by taking the ending
+/// location of the comment, and adding 'starting-column-offset' to the column
+/// for that location. That location in turns becomes the whole selection range,
+/// unless 'ending-line-offset' and 'ending-column-position' are specified. If
+/// they are specified, then the ending location of the selection range is
+/// the starting location's line + 'ending-line-offset' and the
+/// 'ending-column-position' column.
+///
+/// All selection ranges in one group are expected to produce the same
+/// refactoring result.
+///
+/// When testing, zero is returned from clang-refactor even when a group
+/// produces an initiation error, which is different from normal invocation
+/// that returns a non-zero value. This is done on purpose, to ensure that group
+/// consistency checks can return non-zero, but still print the output of
+/// the group. So even if a test matches the output of group, it will still fail
+/// because clang-refactor should return zero on exit when the group results are
+/// consistent.
+///
+/// \returns None on failure (errors are emitted to stderr), or a set of
+/// grouped source ranges in the given file otherwise.
+Optional<TestSelectionRangesInFile> findTestSelectionRanges(StringRef Filename);
+
+} // end namespace refactor
+} // end namespace clang
+
+#endif // LLVM_CLANG_TOOLS_CLANG_REFACTOR_TEST_SUPPORT_H

tools/clang-refactor/ToolRefactoringResultConsumer.h

@@ -0,0 +1,48 @@
+//===--- ToolRefactoringResultConsumer.h - ----------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_CLANG_REFACTOR_TOOL_REFACTORING_RESULT_CONSUMER_H
+#define LLVM_CLANG_TOOLS_CLANG_REFACTOR_TOOL_REFACTORING_RESULT_CONSUMER_H
+
+#include "clang/AST/ASTContext.h"
+#include "clang/Tooling/Refactoring/RefactoringResultConsumer.h"
+
+namespace clang {
+namespace refactor {
+
+/// An interface that subclasses the \c RefactoringResultConsumer interface
+/// that stores the reference to the TU-specific diagnostics engine.
+class ClangRefactorToolConsumerInterface
+    : public tooling::RefactoringResultConsumer {
+public:
+  /// Called when a TU is entered.
+  void beginTU(ASTContext &Context) {
+    assert(!Diags && "Diags has been set");
+    Diags = &Context.getDiagnostics();
+  }
+
+  /// Called when the tool is done with a TU.
+  void endTU() {
+    assert(Diags && "Diags unset");
+    Diags = nullptr;
+  }
+
+  DiagnosticsEngine &getDiags() const {
+    assert(Diags && "no diags");
+    return *Diags;
+  }
+
+private:
+  DiagnosticsEngine *Diags = nullptr;
+};
+
+} // end namespace refactor
+} // end namespace clang
+
+#endif // LLVM_CLANG_TOOLS_CLANG_REFACTOR_TOOL_REFACTORING_RESULT_CONSUMER_H

tools/clang-rename/CMakeLists.txt

@@ -3,9 +3,10 @@ set(LLVM_LINK_COMPONENTS
   Support
   )
 
-add_clang_executable(clang-rename ClangRename.cpp)
+add_clang_tool(clang-rename ClangRename.cpp)
 
 target_link_libraries(clang-rename
+  PRIVATE
   clangBasic
   clangFrontend
   clangRewrite
@@ -14,8 +15,6 @@ target_link_libraries(clang-rename
   clangToolingRefactor
   )
 
-install(TARGETS clang-rename RUNTIME DESTINATION bin)
-
 install(PROGRAMS clang-rename.py
   DESTINATION share/clang
   COMPONENT clang-rename)

tools/clang-rename/ClangRename.cpp

@@ -138,7 +138,7 @@ int main(int argc, const char **argv) {
   // Check if NewNames is a valid identifier in C++17.
   LangOptions Options;
   Options.CPlusPlus = true;
-  Options.CPlusPlus1z = true;
+  Options.CPlusPlus17 = true;
   IdentifierTable Table(Options);
   for (const auto &NewName : NewNames) {
     auto NewNameTokKind = Table.get(NewName).getTokenID();
@@ -175,12 +175,6 @@ int main(int argc, const char **argv) {
     return 1;
   }
 
-  if (Force && PrevNames.size() < NewNames.size()) {
-    // No matching PrevName for all NewNames. Without Force this is an error
-    // above already.
-    return 0;
-  }
-
   // Perform the renaming.
   tooling::RenamingAction RenameAction(NewNames, PrevNames, USRList,
                                        Tool.getReplacements(), PrintLocations);

tools/diagtool/CMakeLists.txt

@@ -13,6 +13,7 @@ add_clang_executable(diagtool
 )
 
 target_link_libraries(diagtool
+  PRIVATE
   clangBasic
   clangFrontend
   )

tools/diagtool/DiagnosticNames.cpp

@@ -32,6 +32,7 @@ static const DiagnosticRecord BuiltinDiagnosticsByID[] = {
              SFINAE,NOWERROR,SHOWINSYSHEADER,CATEGORY)            \
   { #ENUM, diag::ENUM, STR_SIZE(#ENUM, uint8_t) },
 #include "clang/Basic/DiagnosticCommonKinds.inc"
+#include "clang/Basic/DiagnosticCrossTUKinds.inc"
 #include "clang/Basic/DiagnosticDriverKinds.inc"
 #include "clang/Basic/DiagnosticFrontendKinds.inc"
 #include "clang/Basic/DiagnosticSerializationKinds.inc"
@@ -41,6 +42,7 @@ static const DiagnosticRecord BuiltinDiagnosticsByID[] = {
 #include "clang/Basic/DiagnosticCommentKinds.inc"
 #include "clang/Basic/DiagnosticSemaKinds.inc"
 #include "clang/Basic/DiagnosticAnalysisKinds.inc"
+#include "clang/Basic/DiagnosticRefactoringKinds.inc"
 #undef DIAG
 };
 
@@ -84,6 +86,11 @@ GroupRecord::subgroup_iterator GroupRecord::subgroup_end() const {
   return nullptr;
 }
 
+llvm::iterator_range<diagtool::GroupRecord::subgroup_iterator>
+GroupRecord::subgroups() const {
+  return llvm::make_range(subgroup_begin(), subgroup_end());
+}
+
 GroupRecord::diagnostics_iterator GroupRecord::diagnostics_begin() const {
   return DiagArrays + Members;
 }
@@ -92,6 +99,11 @@ GroupRecord::diagnostics_iterator GroupRecord::diagnostics_end() const {
   return nullptr;
 }
 
+llvm::iterator_range<diagtool::GroupRecord::diagnostics_iterator>
+GroupRecord::diagnostics() const {
+  return llvm::make_range(diagnostics_begin(), diagnostics_end());
+}
+
 llvm::ArrayRef<GroupRecord> diagtool::getDiagnosticGroups() {
   return llvm::makeArrayRef(OptionTable);
 }

tools/diagtool/DiagnosticNames.h

@@ -20,7 +20,7 @@ namespace diagtool {
     const char *NameStr;
     short DiagID;
     uint8_t NameLen;
-    
+
     llvm::StringRef getName() const {
       return llvm::StringRef(NameStr, NameLen);
     }
@@ -80,7 +80,7 @@ namespace diagtool {
       bool operator==(group_iterator &Other) const {
         return CurrentID == Other.CurrentID;
       }
-      
+
       bool operator!=(group_iterator &Other) const {
         return CurrentID != Other.CurrentID;
       }
@@ -89,10 +89,12 @@ namespace diagtool {
     typedef group_iterator<GroupRecord> subgroup_iterator;
     subgroup_iterator subgroup_begin() const;
     subgroup_iterator subgroup_end() const;
+    llvm::iterator_range<subgroup_iterator> subgroups() const;
 
     typedef group_iterator<DiagnosticRecord> diagnostics_iterator;
     diagnostics_iterator diagnostics_begin() const;
     diagnostics_iterator diagnostics_end() const;
+    llvm::iterator_range<diagnostics_iterator> diagnostics() const;
 
     bool operator<(llvm::StringRef Other) const {
       return getName() < Other;

tools/diagtool/FindDiagnosticID.cpp

@@ -18,6 +18,15 @@ DEF_DIAGTOOL("find-diagnostic-id", "Print the id of the given diagnostic",
 using namespace clang;
 using namespace diagtool;
 
+static StringRef getNameFromID(StringRef Name) {
+  int DiagID;
+  if(!Name.getAsInteger(0, DiagID)) {
+    const DiagnosticRecord &Diag = getDiagnosticForID(DiagID);
+    return Diag.getName();
+  }
+  return StringRef();
+}
+
 static Optional<DiagnosticRecord>
 findDiagnostic(ArrayRef<DiagnosticRecord> Diagnostics, StringRef Name) {
   for (const auto &Diag : Diagnostics) {
@@ -38,7 +47,7 @@ int FindDiagnosticID::run(unsigned int argc, char **argv,
       llvm::cl::Required, llvm::cl::cat(FindDiagnosticIDOptions));
 
   std::vector<const char *> Args;
-  Args.push_back("find-diagnostic-id");
+  Args.push_back("diagtool find-diagnostic-id");
   for (const char *A : llvm::makeArrayRef(argv, argc))
     Args.push_back(A);
 
@@ -50,6 +59,13 @@ int FindDiagnosticID::run(unsigned int argc, char **argv,
   Optional<DiagnosticRecord> Diag =
       findDiagnostic(AllDiagnostics, DiagnosticName);
   if (!Diag) {
+    // Name to id failed, so try id to name.
+    auto Name = getNameFromID(DiagnosticName);
+    if (!Name.empty()) {
+      OS << Name << '\n';
+      return 0;
+    }
+
     llvm::errs() << "error: invalid diagnostic '" << DiagnosticName << "'\n";
     return 1;
   }

tools/diagtool/ListWarnings.cpp

@@ -23,7 +23,7 @@
 DEF_DIAGTOOL("list-warnings",
              "List warnings and their corresponding flags",
              ListWarnings)
-  
+
 using namespace clang;
 using namespace diagtool;
 
@@ -31,20 +31,19 @@ namespace {
 struct Entry {
   llvm::StringRef DiagName;
   llvm::StringRef Flag;
-  
+
   Entry(llvm::StringRef diagN, llvm::StringRef flag)
     : DiagName(diagN), Flag(flag) {}
-  
+
   bool operator<(const Entry &x) const { return DiagName < x.DiagName; }
 };
 }
 
 static void printEntries(std::vector<Entry> &entries, llvm::raw_ostream &out) {
-  for (std::vector<Entry>::iterator it = entries.begin(), ei = entries.end();
-       it != ei; ++it) {
-    out << "  " << it->DiagName;
-    if (!it->Flag.empty())
-      out << " [-W" << it->Flag << "]";
+  for (const Entry &E : entries) {
+    out << "  " << E.DiagName;
+    if (!E.Flag.empty())
+      out << " [-W" << E.Flag << "]";
     out << '\n';
   }
 }
@@ -52,23 +51,18 @@ static void printEntries(std::vector<Entry> &entries, llvm::raw_ostream &out) {
 int ListWarnings::run(unsigned int argc, char **argv, llvm::raw_ostream &out) {
   std::vector<Entry> Flagged, Unflagged;
   llvm::StringMap<std::vector<unsigned> > flagHistogram;
-  
-  ArrayRef<DiagnosticRecord> AllDiagnostics = getBuiltinDiagnosticsByName();
 
-  for (ArrayRef<DiagnosticRecord>::iterator di = AllDiagnostics.begin(),
-                                            de = AllDiagnostics.end();
-       di != de; ++di) {
-    unsigned diagID = di->DiagID;
-    
+  for (const DiagnosticRecord &DR : getBuiltinDiagnosticsByName()) {
+    const unsigned diagID = DR.DiagID;
+
     if (DiagnosticIDs::isBuiltinNote(diagID))
       continue;
-        
+
     if (!DiagnosticIDs::isBuiltinWarningOrExtension(diagID))
       continue;
-  
-    Entry entry(di->getName(),
-                DiagnosticIDs::getWarningOptionForDiag(diagID));
-    
+
+    Entry entry(DR.getName(), DiagnosticIDs::getWarningOptionForDiag(diagID));
+
     if (entry.Flag.empty())
       Unflagged.push_back(entry);
     else {
@@ -76,24 +70,24 @@ int ListWarnings::run(unsigned int argc, char **argv, llvm::raw_ostream &out) {
       flagHistogram[entry.Flag].push_back(diagID);
     }
   }
-  
+
   out << "Warnings with flags (" << Flagged.size() << "):\n";
   printEntries(Flagged, out);
-  
+
   out << "Warnings without flags (" << Unflagged.size() << "):\n";
   printEntries(Unflagged, out);
 
   out << "\nSTATISTICS:\n\n";
 
-  double percentFlagged = ((double) Flagged.size()) 
-    / (Flagged.size() + Unflagged.size()) * 100.0;
-  
-  out << "  Percentage of warnings with flags: " 
-      << llvm::format("%.4g",percentFlagged) << "%\n";
-  
+  double percentFlagged =
+      ((double)Flagged.size()) / (Flagged.size() + Unflagged.size()) * 100.0;
+
+  out << "  Percentage of warnings with flags: "
+      << llvm::format("%.4g", percentFlagged) << "%\n";
+
   out << "  Number of unique flags: "
       << flagHistogram.size() << '\n';
-  
+
   double avgDiagsPerFlag = (double) Flagged.size() / flagHistogram.size();
   out << "  Average number of diagnostics per flag: "
       << llvm::format("%.4g", avgDiagsPerFlag) << '\n';
@@ -102,7 +96,7 @@ int ListWarnings::run(unsigned int argc, char **argv, llvm::raw_ostream &out) {
       << flagHistogram["pedantic"].size() << '\n';
 
   out << '\n';
-  
+
   return 0;
 }
 

tools/diagtool/ShowEnabledWarnings.cpp

@@ -112,17 +112,14 @@ int ShowEnabledWarnings::run(unsigned int argc, char **argv, raw_ostream &Out) {
   // which ones are turned on.
   // FIXME: It would be very nice to print which flags are turning on which
   // diagnostics, but this can be done with a diff.
-  ArrayRef<DiagnosticRecord> AllDiagnostics = getBuiltinDiagnosticsByName();
   std::vector<PrettyDiag> Active;
 
-  for (ArrayRef<DiagnosticRecord>::iterator I = AllDiagnostics.begin(),
-                                            E = AllDiagnostics.end();
-       I != E; ++I) {
-    unsigned DiagID = I->DiagID;
-    
+  for (const DiagnosticRecord &DR : getBuiltinDiagnosticsByName()) {
+    unsigned DiagID = DR.DiagID;
+
     if (DiagnosticIDs::isBuiltinNote(DiagID))
       continue;
-    
+
     if (!DiagnosticIDs::isBuiltinWarningOrExtension(DiagID))
       continue;
 
@@ -132,17 +129,16 @@ int ShowEnabledWarnings::run(unsigned int argc, char **argv, raw_ostream &Out) {
       continue;
 
     StringRef WarningOpt = DiagnosticIDs::getWarningOptionForDiag(DiagID);
-    Active.push_back(PrettyDiag(I->getName(), WarningOpt, DiagLevel));
+    Active.push_back(PrettyDiag(DR.getName(), WarningOpt, DiagLevel));
   }
 
   // Print them all out.
-  for (std::vector<PrettyDiag>::const_iterator I = Active.begin(),
-       E = Active.end(); I != E; ++I) {
+  for (const PrettyDiag &PD : Active) {
     if (ShouldShowLevels)
-      Out << getCharForLevel(I->Level) << "  ";
-    Out << I->Name;
-    if (!I->Flag.empty())
-      Out << " [-W" << I->Flag << "]";
+      Out << getCharForLevel(PD.Level) << "  ";
+    Out << PD.Name;
+    if (!PD.Flag.empty())
+      Out << " [-W" << PD.Flag << "]";
     Out << '\n';
   }
 

tools/diagtool/TreeView.cpp

@@ -32,10 +32,10 @@ class TreePrinter {
 public:
   llvm::raw_ostream &out;
   const bool ShowColors;
-  bool FlagsOnly;
+  bool Internal;
 
   TreePrinter(llvm::raw_ostream &out)
-      : out(out), ShowColors(hasColors(out)), FlagsOnly(false) {}
+      : out(out), ShowColors(hasColors(out)), Internal(false) {}
 
   void setColor(llvm::raw_ostream::Colors Color) {
     if (ShowColors)
@@ -54,28 +54,42 @@ class TreePrinter {
     return Diags.isIgnored(DiagID, SourceLocation());
   }
 
+  static bool enabledByDefault(const GroupRecord &Group) {
+    for (const DiagnosticRecord &DR : Group.diagnostics()) {
+      if (isIgnored(DR.DiagID))
+        return false;
+    }
+
+    for (const GroupRecord &GR : Group.subgroups()) {
+      if (!enabledByDefault(GR))
+        return false;
+    }
+
+    return true;
+  }
+
   void printGroup(const GroupRecord &Group, unsigned Indent = 0) {
     out.indent(Indent * 2);
 
-    setColor(llvm::raw_ostream::YELLOW);
+    if (enabledByDefault(Group))
+      setColor(llvm::raw_ostream::GREEN);
+    else
+      setColor(llvm::raw_ostream::YELLOW);
+
     out << "-W" << Group.getName() << "\n";
     resetColor();
 
     ++Indent;
-    for (GroupRecord::subgroup_iterator I = Group.subgroup_begin(),
-                                        E = Group.subgroup_end();
-         I != E; ++I) {
-      printGroup(*I, Indent);
+    for (const GroupRecord &GR : Group.subgroups()) {
+      printGroup(GR, Indent);
     }
 
-    if (!FlagsOnly) {
-      for (GroupRecord::diagnostics_iterator I = Group.diagnostics_begin(),
-                                             E = Group.diagnostics_end();
-           I != E; ++I) {
-        if (ShowColors && !isIgnored(I->DiagID))
+    if (Internal) {
+      for (const DiagnosticRecord &DR : Group.diagnostics()) {
+        if (ShowColors && !isIgnored(DR.DiagID))
           setColor(llvm::raw_ostream::GREEN);
         out.indent(Indent * 2);
-        out << I->getName();
+        out << DR.getName();
         resetColor();
         out << "\n";
       }
@@ -107,12 +121,9 @@ class TreePrinter {
     ArrayRef<GroupRecord> AllGroups = getDiagnosticGroups();
     llvm::DenseSet<unsigned> NonRootGroupIDs;
 
-    for (ArrayRef<GroupRecord>::iterator I = AllGroups.begin(),
-                                         E = AllGroups.end();
-         I != E; ++I) {
-      for (GroupRecord::subgroup_iterator SI = I->subgroup_begin(),
-                                          SE = I->subgroup_end();
-           SI != SE; ++SI) {
+    for (const GroupRecord &GR : AllGroups) {
+      for (auto SI = GR.subgroup_begin(), SE = GR.subgroup_end(); SI != SE;
+           ++SI) {
         NonRootGroupIDs.insert((unsigned)SI.getID());
       }
     }
@@ -139,16 +150,16 @@ class TreePrinter {
 };
 
 static void printUsage() {
-  llvm::errs() << "Usage: diagtool tree [--flags-only] [<diagnostic-group>]\n";
+  llvm::errs() << "Usage: diagtool tree [--internal] [<diagnostic-group>]\n";
 }
 
 int TreeView::run(unsigned int argc, char **argv, llvm::raw_ostream &out) {
   // First check our one flag (--flags-only).
-  bool FlagsOnly = false;
+  bool Internal = false;
   if (argc > 0) {
     StringRef FirstArg(*argv);
-    if (FirstArg.equals("--flags-only")) {
-      FlagsOnly = true;
+    if (FirstArg.equals("--internal")) {
+      Internal = true;
       --argc;
       ++argv;
     }
@@ -175,7 +186,7 @@ int TreeView::run(unsigned int argc, char **argv, llvm::raw_ostream &out) {
   }
 
   TreePrinter TP(out);
-  TP.FlagsOnly = FlagsOnly;
+  TP.Internal = Internal;
   TP.showKey();
   return ShowAll ? TP.showAll() : TP.showGroup(RootGroup);
 }

tools/driver/CMakeLists.txt

@@ -38,6 +38,7 @@ add_clang_tool(clang
   )
 
 target_link_libraries(clang
+  PRIVATE
   clangBasic
   clangCodeGen
   clangDriver
@@ -85,6 +86,7 @@ if (APPLE)
 
   set(TOOL_INFO_PLIST_OUT "${CMAKE_CURRENT_BINARY_DIR}/${TOOL_INFO_PLIST}")
   target_link_libraries(clang
+    PRIVATE
     "-Wl,-sectcreate,__TEXT,__info_plist,${TOOL_INFO_PLIST_OUT}")
   configure_file("${TOOL_INFO_PLIST}.in" "${TOOL_INFO_PLIST_OUT}" @ONLY)
 
@@ -121,11 +123,11 @@ if(CLANG_ORDER_FILE AND (LD64_EXECUTABLE OR GOLD_EXECUTABLE))
   if("${ORDER_FILE}" STREQUAL "\n")
     set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CLANG_ORDER_FILE})
   elseif(LINKER_ORDER_FILE_WORKS)
-    target_link_libraries(clang ${LINKER_ORDER_FILE_OPTION})
+    target_link_libraries(clang PRIVATE ${LINKER_ORDER_FILE_OPTION})
     set_target_properties(clang PROPERTIES LINK_DEPENDS ${CLANG_ORDER_FILE})
   endif()
 endif()
 
 if(WITH_POLLY AND LINK_POLLY_INTO_TOOLS)
-  target_link_libraries(clang Polly)
+  target_link_libraries(clang PRIVATE Polly)
 endif(WITH_POLLY AND LINK_POLLY_INTO_TOOLS)

tools/driver/cc1as_main.cpp

@@ -356,7 +356,7 @@ static bool ExecuteAssembler(AssemblerInvocation &Opts,
     PIC = false;
   }
 
-  MOFI->InitMCObjectFileInfo(Triple(Opts.Triple), PIC, CodeModel::Default, Ctx);
+  MOFI->InitMCObjectFileInfo(Triple(Opts.Triple), PIC, Ctx);
   if (Opts.SaveTemporaryLabels)
     Ctx.setAllowTemporaryLabels(false);
   if (Opts.GenDwarfForAssembly)
@@ -419,8 +419,8 @@ static bool ExecuteAssembler(AssemblerInvocation &Opts,
                                                       Opts.CPU, Options);
     Triple T(Opts.Triple);
     Str.reset(TheTarget->createMCObjectStreamer(
-        T, Ctx, *MAB, *Out, CE, *STI, Opts.RelaxAll,
-        Opts.IncrementalLinkerCompatible,
+                T, Ctx, std::unique_ptr<MCAsmBackend>(MAB), *Out, std::unique_ptr<MCCodeEmitter>(CE), *STI,
+        Opts.RelaxAll, Opts.IncrementalLinkerCompatible,
         /*DWARFMustBeAtTheEnd*/ true));
     Str.get()->InitSections(Opts.NoExecStack);
   }
@@ -504,7 +504,8 @@ int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
   if (Asm.ShowHelp) {
     std::unique_ptr<OptTable> Opts(driver::createDriverOptTable());
     Opts->PrintHelp(llvm::outs(), "clang -cc1as", "Clang Integrated Assembler",
-                    /*Include=*/driver::options::CC1AsOption, /*Exclude=*/0);
+                    /*Include=*/driver::options::CC1AsOption, /*Exclude=*/0,
+                    /*ShowAllAliases=*/false);
     return 0;
   }
 

tools/driver/driver.cpp

@@ -206,23 +206,26 @@ extern int cc1_main(ArrayRef<const char *> Argv, const char *Argv0,
 extern int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0,
                       void *MainAddr);
 
-static void insertTargetAndModeArgs(StringRef Target, StringRef Mode,
+static void insertTargetAndModeArgs(const ParsedClangName &NameParts,
                                     SmallVectorImpl<const char *> &ArgVector,
                                     std::set<std::string> &SavedStrings) {
-  if (!Mode.empty()) {
+  // Put target and mode arguments at the start of argument list so that
+  // arguments specified in command line could override them. Avoid putting
+  // them at index 0, as an option like '-cc1' must remain the first.
+  auto InsertionPoint = ArgVector.begin();
+  if (InsertionPoint != ArgVector.end())
+    ++InsertionPoint;
+
+  if (NameParts.DriverMode) {
     // Add the mode flag to the arguments.
-    auto it = ArgVector.begin();
-    if (it != ArgVector.end())
-      ++it;
-    ArgVector.insert(it, GetStableCStr(SavedStrings, Mode));
+    ArgVector.insert(InsertionPoint,
+                     GetStableCStr(SavedStrings, NameParts.DriverMode));
   }
 
-  if (!Target.empty()) {
-    auto it = ArgVector.begin();
-    if (it != ArgVector.end())
-      ++it;
-    const char *arr[] = {"-target", GetStableCStr(SavedStrings, Target)};
-    ArgVector.insert(it, std::begin(arr), std::end(arr));
+  if (NameParts.TargetIsValid) {
+    const char *arr[] = {"-target", GetStableCStr(SavedStrings,
+                                                  NameParts.TargetPrefix)};
+    ArgVector.insert(InsertionPoint, std::begin(arr), std::end(arr));
   }
 }
 
@@ -330,9 +333,7 @@ int main(int argc_, const char **argv_) {
   }
 
   llvm::InitializeAllTargets();
-  std::string ProgName = argv[0];
-  std::pair<std::string, std::string> TargetAndMode =
-      ToolChain::getTargetAndModeFromProgramName(ProgName);
+  auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(argv[0]);
 
   llvm::BumpPtrAllocator A;
   llvm::StringSaver Saver(A);
@@ -345,7 +346,7 @@ int main(int argc_, const char **argv_) {
   // Finally, our -cc1 tools don't care which tokenization mode we use because
   // response files written by clang will tokenize the same way in either mode.
   bool ClangCLMode = false;
-  if (TargetAndMode.second == "--driver-mode=cl" ||
+  if (StringRef(TargetAndMode.DriverMode).equals("--driver-mode=cl") ||
       std::find_if(argv.begin(), argv.end(), [](const char *F) {
         return F && strcmp(F, "--driver-mode=cl") == 0;
       }) != argv.end()) {
@@ -454,9 +455,9 @@ int main(int argc_, const char **argv_) {
 
   Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags);
   SetInstallDir(argv, TheDriver, CanonicalPrefixes);
+  TheDriver.setTargetAndMode(TargetAndMode);
 
-  insertTargetAndModeArgs(TargetAndMode.first, TargetAndMode.second, argv,
-                          SavedStrings);
+  insertTargetAndModeArgs(TargetAndMode, argv, SavedStrings);
 
   SetBackdoorDriverOutputsFromEnvVars(TheDriver);
 

tools/libclang/ARCMigrate.cpp

@@ -34,7 +34,7 @@ struct Remap {
 //===----------------------------------------------------------------------===//
 
 CXRemapping clang_getRemappings(const char *migrate_dir_path) {
-#ifndef CLANG_ENABLE_ARCMT
+#if !CLANG_ENABLE_ARCMT
   llvm::errs() << "error: feature not enabled in this build\n";
   return nullptr;
 #else
@@ -77,7 +77,7 @@ CXRemapping clang_getRemappings(const char *migrate_dir_path) {
 
 CXRemapping clang_getRemappingsFromFileList(const char **filePaths,
                                             unsigned numFiles) {
-#ifndef CLANG_ENABLE_ARCMT
+#if !CLANG_ENABLE_ARCMT
   llvm::errs() << "error: feature not enabled in this build\n";
   return nullptr;
 #else

tools/libclang/CIndex.cpp

@@ -81,6 +81,8 @@ CXTranslationUnit cxtu::MakeCXTranslationUnit(CIndexer *CIdx,
   D->Diagnostics = nullptr;
   D->OverridenCursorsPool = createOverridenCXCursorsPool();
   D->CommentToXML = nullptr;
+  D->ParsingOptions = 0;
+  D->Arguments = {};
   return D;
 }
 
@@ -877,6 +879,9 @@ bool CursorVisitor::VisitFieldDecl(FieldDecl *D) {
   if (Expr *BitWidth = D->getBitWidth())
     return Visit(MakeCXCursor(BitWidth, StmtParent, TU, RegionOfInterest));
 
+  if (Expr *Init = D->getInClassInitializer())
+    return Visit(MakeCXCursor(Init, StmtParent, TU, RegionOfInterest));
+
   return false;
 }
 
@@ -907,7 +912,8 @@ bool CursorVisitor::VisitFunctionTemplateDecl(FunctionTemplateDecl *D) {
   if (VisitTemplateParameters(D->getTemplateParameters()))
     return true;
   
-  return VisitFunctionDecl(D->getTemplatedDecl());
+  auto* FD = D->getTemplatedDecl();
+  return VisitAttributes(FD) || VisitFunctionDecl(FD);
 }
 
 bool CursorVisitor::VisitClassTemplateDecl(ClassTemplateDecl *D) {
@@ -916,7 +922,8 @@ bool CursorVisitor::VisitClassTemplateDecl(ClassTemplateDecl *D) {
   if (VisitTemplateParameters(D->getTemplateParameters()))
     return true;
   
-  return VisitCXXRecordDecl(D->getTemplatedDecl());
+  auto* CD = D->getTemplatedDecl();
+  return VisitAttributes(CD) || VisitCXXRecordDecl(CD);
 }
 
 bool CursorVisitor::VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D) {
@@ -1020,8 +1027,9 @@ bool CursorVisitor::VisitObjCContainerDecl(ObjCContainerDecl *D) {
             [&SM](Decl *A, Decl *B) {
     SourceLocation L_A = A->getLocStart();
     SourceLocation L_B = B->getLocStart();
-    assert(L_A.isValid() && L_B.isValid());
-    return SM.isBeforeInTranslationUnit(L_A, L_B);
+    return L_A != L_B ?
+           SM.isBeforeInTranslationUnit(L_A, L_B) :
+           SM.isBeforeInTranslationUnit(A->getLocEnd(), B->getLocEnd());
   });
 
   // Now visit the decls.
@@ -1742,6 +1750,7 @@ DEFAULT_TYPELOC_IMPL(ConstantArray, ArrayType)
 DEFAULT_TYPELOC_IMPL(IncompleteArray, ArrayType)
 DEFAULT_TYPELOC_IMPL(VariableArray, ArrayType)
 DEFAULT_TYPELOC_IMPL(DependentSizedArray, ArrayType)
+DEFAULT_TYPELOC_IMPL(DependentAddressSpace, Type)
 DEFAULT_TYPELOC_IMPL(DependentSizedExtVector, Type)
 DEFAULT_TYPELOC_IMPL(Vector, Type)
 DEFAULT_TYPELOC_IMPL(ExtVector, VectorType)
@@ -2281,6 +2290,25 @@ void OMPClauseEnqueue::VisitOMPTaskReductionClause(
     Visitor->AddStmt(E);
   }
 }
+void OMPClauseEnqueue::VisitOMPInReductionClause(
+    const OMPInReductionClause *C) {
+  VisitOMPClauseList(C);
+  VisitOMPClauseWithPostUpdate(C);
+  for (auto *E : C->privates()) {
+    Visitor->AddStmt(E);
+  }
+  for (auto *E : C->lhs_exprs()) {
+    Visitor->AddStmt(E);
+  }
+  for (auto *E : C->rhs_exprs()) {
+    Visitor->AddStmt(E);
+  }
+  for (auto *E : C->reduction_ops()) {
+    Visitor->AddStmt(E);
+  }
+  for (auto *E : C->taskgroup_descriptors())
+    Visitor->AddStmt(E);
+}
 void OMPClauseEnqueue::VisitOMPLinearClause(const OMPLinearClause *C) {
   VisitOMPClauseList(C);
   VisitOMPClauseWithPostUpdate(C);
@@ -2738,6 +2766,8 @@ void EnqueueVisitor::VisitOMPTaskwaitDirective(const OMPTaskwaitDirective *D) {
 void EnqueueVisitor::VisitOMPTaskgroupDirective(
     const OMPTaskgroupDirective *D) {
   VisitOMPExecutableDirective(D);
+  if (const Expr *E = D->getReductionRef())
+    VisitStmt(E);
 }
 
 void EnqueueVisitor::VisitOMPFlushDirective(const OMPFlushDirective *D) {
@@ -3227,6 +3257,12 @@ unsigned clang_CXIndex_getGlobalOptions(CXIndex CIdx) {
   return 0;
 }
 
+void clang_CXIndex_setInvocationEmissionPathOption(CXIndex CIdx,
+                                                   const char *Path) {
+  if (CIdx)
+    static_cast<CIndexer *>(CIdx)->setInvocationEmissionPath(Path ? Path : "");
+}
+
 void clang_toggleCrashRecovery(unsigned isEnabled) {
   if (isEnabled)
     llvm::CrashRecoveryContext::Enable();
@@ -3403,6 +3439,11 @@ clang_parseTranslationUnit_Impl(CXIndex CIdx, const char *source_filename,
   // faster, trading for a slower (first) reparse.
   unsigned PrecompilePreambleAfterNParses =
       !PrecompilePreamble ? 0 : 2 - CreatePreambleOnFirstParse;
+
+  LibclangInvocationReporter InvocationReporter(
+      *CXXIdx, LibclangInvocationReporter::OperationKind::ParseOperation,
+      options, llvm::makeArrayRef(*Args), /*InvocationArgs=*/None,
+      unsaved_files);
   std::unique_ptr<ASTUnit> Unit(ASTUnit::LoadFromCommandLine(
       Args->data(), Args->data() + Args->size(),
       CXXIdx->getPCHContainerOperations(), Diags,
@@ -3429,7 +3470,14 @@ clang_parseTranslationUnit_Impl(CXIndex CIdx, const char *source_filename,
     return CXError_ASTReadError;
 
   *out_TU = MakeCXTranslationUnit(CXXIdx, std::move(Unit));
-  return *out_TU ? CXError_Success : CXError_Failure;
+  if (CXTranslationUnitImpl *TU = *out_TU) {
+    TU->ParsingOptions = options;
+    TU->Arguments.reserve(Args->size());
+    for (const char *Arg : *Args)
+      TU->Arguments.push_back(Arg);
+    return CXError_Success;
+  }
+  return CXError_Failure;
 }
 
 CXTranslationUnit
@@ -3483,6 +3531,7 @@ enum CXErrorCode clang_parseTranslationUnit2FullArgv(
         CIdx, source_filename, command_line_args, num_command_line_args,
         llvm::makeArrayRef(unsaved_files, num_unsaved_files), options, out_TU);
   };
+
   llvm::CrashRecoveryContext CRC;
 
   if (!RunSafely(CRC, ParseTranslationUnitImpl)) {
@@ -3891,8 +3940,7 @@ int clang_saveTranslationUnit(CXTranslationUnit TU, const char *FileName,
     result = clang_saveTranslationUnit_Impl(TU, FileName, options);
   };
 
-  if (!CXXUnit->getDiagnostics().hasUnrecoverableErrorOccurred() ||
-      getenv("LIBCLANG_NOTHREADS")) {
+  if (!CXXUnit->getDiagnostics().hasUnrecoverableErrorOccurred()) {
     SaveTranslationUnitImpl();
 
     if (getenv("LIBCLANG_RESOURCE_USAGE"))
@@ -4015,11 +4063,6 @@ int clang_reparseTranslationUnit(CXTranslationUnit TU,
         TU, llvm::makeArrayRef(unsaved_files, num_unsaved_files), options);
   };
 
-  if (getenv("LIBCLANG_NOTHREADS")) {
-    ReparseTranslationUnitImpl();
-    return result;
-  }
-
   llvm::CrashRecoveryContext CRC;
 
   if (!RunSafely(CRC, ReparseTranslationUnitImpl)) {
@@ -4129,6 +4172,27 @@ CXFile clang_getFile(CXTranslationUnit TU, const char *file_name) {
   return const_cast<FileEntry *>(FMgr.getFile(file_name));
 }
 
+const char *clang_getFileContents(CXTranslationUnit TU, CXFile file,
+                                  size_t *size) {
+  if (isNotUsableTU(TU)) {
+    LOG_BAD_TU(TU);
+    return nullptr;
+  }
+
+  const SourceManager &SM = cxtu::getASTUnit(TU)->getSourceManager();
+  FileID fid = SM.translateFile(static_cast<FileEntry *>(file));
+  bool Invalid = true;
+  llvm::MemoryBuffer *buf = SM.getBuffer(fid, &Invalid);
+  if (Invalid) {
+    if (size)
+      *size = 0;
+    return nullptr;
+  }
+  if (size)
+    *size = buf->getBufferSize();
+  return buf->getBufferStart();
+}
+
 unsigned clang_isFileMultipleIncludeGuarded(CXTranslationUnit TU,
                                             CXFile file) {
   if (isNotUsableTU(TU)) {
@@ -4612,6 +4676,20 @@ CXStringSet *clang_Cursor_getCXXManglings(CXCursor C) {
   return cxstring::createSet(Manglings);
 }
 
+CXStringSet *clang_Cursor_getObjCManglings(CXCursor C) {
+  if (clang_isInvalid(C.kind) || !clang_isDeclaration(C.kind))
+    return nullptr;
+
+  const Decl *D = getCursorDecl(C);
+  if (!(isa<ObjCInterfaceDecl>(D) || isa<ObjCImplementationDecl>(D)))
+    return nullptr;
+
+  ASTContext &Ctx = D->getASTContext();
+  index::CodegenNameGenerator CGNameGen(Ctx);
+  std::vector<std::string> Manglings = CGNameGen.getAllManglings(D);
+  return cxstring::createSet(Manglings);
+}
+
 CXString clang_getCursorDisplayName(CXCursor C) {
   if (!clang_isDeclaration(C.kind))
     return clang_getCursorSpelling(C);
@@ -4682,12 +4760,12 @@ CXString clang_getCursorDisplayName(CXCursor C) {
     // If the type was explicitly written, use that.
     if (TypeSourceInfo *TSInfo = ClassSpec->getTypeAsWritten())
       return cxstring::createDup(TSInfo->getType().getAsString(Policy));
-    
+
     SmallString<128> Str;
     llvm::raw_svector_ostream OS(Str);
     OS << *ClassSpec;
-    TemplateSpecializationType::PrintTemplateArgumentList(
-        OS, ClassSpec->getTemplateArgs().asArray(), Policy);
+    printTemplateArgumentList(OS, ClassSpec->getTemplateArgs().asArray(),
+                              Policy);
     return cxstring::createDup(OS.str());
   }
   
@@ -7282,7 +7360,8 @@ static void getCursorPlatformAvailabilityForDecl(
 
   std::sort(AvailabilityAttrs.begin(), AvailabilityAttrs.end(),
             [](AvailabilityAttr *LHS, AvailabilityAttr *RHS) {
-              return LHS->getPlatform() > RHS->getPlatform();
+              return LHS->getPlatform()->getName() <
+                     RHS->getPlatform()->getName();
             });
   ASTContext &Ctx = D->getASTContext();
   auto It = std::unique(
@@ -7384,6 +7463,22 @@ CXLanguageKind clang_getCursorLanguage(CXCursor cursor) {
   return CXLanguage_Invalid;
 }
 
+CXTLSKind clang_getCursorTLSKind(CXCursor cursor) {
+  const Decl *D = cxcursor::getCursorDecl(cursor);
+  if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
+    switch (VD->getTLSKind()) {
+    case VarDecl::TLS_None:
+      return CXTLS_None;
+    case VarDecl::TLS_Dynamic:
+      return CXTLS_Dynamic;
+    case VarDecl::TLS_Static:
+      return CXTLS_Static;
+    }
+  }
+
+  return CXTLS_None;
+}
+
  /// \brief If the given cursor is the "templated" declaration
  /// descibing a class or function template, return the class or
  /// function template.
@@ -7824,6 +7919,17 @@ unsigned clang_CXXMethod_isVirtual(CXCursor C) {
   return (Method && Method->isVirtual()) ? 1 : 0;
 }
 
+unsigned clang_CXXRecord_isAbstract(CXCursor C) {
+  if (!clang_isDeclaration(C.kind))
+    return 0;
+
+  const auto *D = cxcursor::getCursorDecl(C);
+  const auto *RD = dyn_cast_or_null<CXXRecordDecl>(D);
+  if (RD)
+    RD = RD->getDefinition();
+  return (RD && RD->isAbstract()) ? 1 : 0;
+}
+
 unsigned clang_EnumDecl_isScoped(CXCursor C) {
   if (!clang_isDeclaration(C.kind))
     return 0;
@@ -8103,7 +8209,7 @@ bool RunSafely(llvm::CrashRecoveryContext &CRC, llvm::function_ref<void()> Fn,
                unsigned Size) {
   if (!Size)
     Size = GetSafetyThreadStackSize();
-  if (Size)
+  if (Size && !getenv("LIBCLANG_NOTHREADS"))
     return CRC.RunSafelyOnThread(Fn, Size);
   return CRC.RunSafely(Fn);
 }

tools/libclang/CIndexCodeCompletion.cpp

@@ -32,6 +32,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/CrashRecoveryContext.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/Timer.h"
@@ -691,6 +692,16 @@ clang_codeCompleteAt_Impl(CXTranslationUnit TU, const char *complete_filename,
   CaptureCompletionResults Capture(Opts, *Results, &TU);
 
   // Perform completion.
+  std::vector<const char *> CArgs;
+  for (const auto &Arg : TU->Arguments)
+    CArgs.push_back(Arg.c_str());
+  std::string CompletionInvocation =
+      llvm::formatv("-code-completion-at={0}:{1}:{2}", complete_filename,
+                    complete_line, complete_column)
+          .str();
+  LibclangInvocationReporter InvocationReporter(
+      *CXXIdx, LibclangInvocationReporter::OperationKind::CompletionOperation,
+      TU->ParsingOptions, CArgs, CompletionInvocation, unsaved_files);
   AST->CodeComplete(complete_filename, complete_line, complete_column,
                     RemappedFiles, (options & CXCodeComplete_IncludeMacros),
                     (options & CXCodeComplete_IncludeCodePatterns),
@@ -806,11 +817,6 @@ CXCodeCompleteResults *clang_codeCompleteAt(CXTranslationUnit TU,
         llvm::makeArrayRef(unsaved_files, num_unsaved_files), options);
   };
 
-  if (getenv("LIBCLANG_NOTHREADS")) {
-    CodeCompleteAtImpl();
-    return result;
-  }
-
   llvm::CrashRecoveryContext CRC;
 
   if (!RunSafely(CRC, CodeCompleteAtImpl)) {

tools/libclang/CIndexer.cpp

@@ -12,10 +12,14 @@
 //===----------------------------------------------------------------------===//
 
 #include "CIndexer.h"
+#include "CXString.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/Version.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Config/llvm-config.h"
+#include "llvm/Support/MD5.h"
+#include "llvm/Support/MutexGuard.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
 #include <cstdio>
@@ -76,3 +80,83 @@ const std::string &CIndexer::getClangResourcesPath() {
   ResourcesPath = LibClangPath.str();
   return ResourcesPath;
 }
+
+StringRef CIndexer::getClangToolchainPath() {
+  if (!ToolchainPath.empty())
+    return ToolchainPath;
+  StringRef ResourcePath = getClangResourcesPath();
+  ToolchainPath = llvm::sys::path::parent_path(
+      llvm::sys::path::parent_path(llvm::sys::path::parent_path(ResourcePath)));
+  return ToolchainPath;
+}
+
+LibclangInvocationReporter::LibclangInvocationReporter(
+    CIndexer &Idx, OperationKind Op, unsigned ParseOptions,
+    llvm::ArrayRef<const char *> Args,
+    llvm::ArrayRef<std::string> InvocationArgs,
+    llvm::ArrayRef<CXUnsavedFile> UnsavedFiles) {
+  StringRef Path = Idx.getInvocationEmissionPath();
+  if (Path.empty())
+    return;
+
+  // Create a temporary file for the invocation log.
+  SmallString<256> TempPath;
+  TempPath = Path;
+  llvm::sys::path::append(TempPath, "libclang-%%%%%%%%%%%%");
+  int FD;
+  if (llvm::sys::fs::createUniqueFile(TempPath, FD, TempPath))
+    return;
+  File = std::string(TempPath.begin(), TempPath.end());
+  llvm::raw_fd_ostream OS(FD, /*ShouldClose=*/true);
+
+  // Write out the information about the invocation to it.
+  auto WriteStringKey = [&OS](StringRef Key, StringRef Value) {
+    OS << R"(")" << Key << R"(":")";
+    OS << Value << '"';
+  };
+  OS << '{';
+  WriteStringKey("toolchain", Idx.getClangToolchainPath());
+  OS << ',';
+  WriteStringKey("libclang.operation",
+                 Op == OperationKind::ParseOperation ? "parse" : "complete");
+  OS << ',';
+  OS << R"("libclang.opts":)" << ParseOptions;
+  OS << ',';
+  OS << R"("args":[)";
+  for (const auto &I : llvm::enumerate(Args)) {
+    if (I.index())
+      OS << ',';
+    OS << '"' << I.value() << '"';
+  }
+  if (!InvocationArgs.empty()) {
+    OS << R"(],"invocation-args":[)";
+    for (const auto &I : llvm::enumerate(InvocationArgs)) {
+      if (I.index())
+        OS << ',';
+      OS << '"' << I.value() << '"';
+    }
+  }
+  if (!UnsavedFiles.empty()) {
+    OS << R"(],"unsaved_file_hashes":[)";
+    for (const auto &UF : llvm::enumerate(UnsavedFiles)) {
+      if (UF.index())
+        OS << ',';
+      OS << '{';
+      WriteStringKey("name", UF.value().Filename);
+      OS << ',';
+      llvm::MD5 Hash;
+      Hash.update(getContents(UF.value()));
+      llvm::MD5::MD5Result Result;
+      Hash.final(Result);
+      SmallString<32> Digest = Result.digest();
+      WriteStringKey("md5", Digest);
+      OS << '}';
+    }
+  }
+  OS << "]}";
+}
+
+LibclangInvocationReporter::~LibclangInvocationReporter() {
+  if (!File.empty())
+    llvm::sys::fs::remove(File);
+}

tools/libclang/CIndexer.h

@@ -18,6 +18,7 @@
 #include "clang-c/Index.h"
 #include "clang/Frontend/PCHContainerOperations.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/Mutex.h"
 #include <utility>
 
 namespace llvm {
@@ -40,6 +41,10 @@ class CIndexer {
   std::string ResourcesPath;
   std::shared_ptr<PCHContainerOperations> PCHContainerOps;
 
+  std::string ToolchainPath;
+
+  std::string InvocationEmissionPath;
+
 public:
   CIndexer(std::shared_ptr<PCHContainerOperations> PCHContainerOps =
                std::make_shared<PCHContainerOperations>())
@@ -71,6 +76,31 @@ class CIndexer {
 
   /// \brief Get the path of the clang resource files.
   const std::string &getClangResourcesPath();
+
+  StringRef getClangToolchainPath();
+
+  void setInvocationEmissionPath(StringRef Str) {
+    InvocationEmissionPath = Str;
+  }
+
+  StringRef getInvocationEmissionPath() const { return InvocationEmissionPath; }
+};
+
+/// Logs information about a particular libclang operation like parsing to
+/// a new file in the invocation emission path.
+class LibclangInvocationReporter {
+public:
+  enum class OperationKind { ParseOperation, CompletionOperation };
+
+  LibclangInvocationReporter(CIndexer &Idx, OperationKind Op,
+                             unsigned ParseOptions,
+                             llvm::ArrayRef<const char *> Args,
+                             llvm::ArrayRef<std::string> InvocationArgs,
+                             llvm::ArrayRef<CXUnsavedFile> UnsavedFiles);
+  ~LibclangInvocationReporter();
+
+private:
+  std::string File;
 };
 
   /// \brief Return the current size to request for "safety".

tools/libclang/CMakeLists.txt

@@ -51,6 +51,9 @@ if (TARGET clangTidyPlugin)
   add_definitions(-DCLANG_TOOL_EXTRA_BUILD)
   list(APPEND LIBS clangTidyPlugin)
   list(APPEND LIBS clangIncludeFixerPlugin)
+  if(LLVM_ENABLE_MODULES)
+    list(APPEND LLVM_COMPILE_FLAGS "-fmodules-ignore-macro=CLANG_TOOL_EXTRA_BUILD")
+  endif()
 endif ()
 
 find_library(DL_LIBRARY_PATH dl)
@@ -115,6 +118,12 @@ if(ENABLE_SHARED)
       PROPERTIES
       VERSION ${LIBCLANG_LIBRARY_VERSION}
       DEFINE_SYMBOL _CINDEX_LIB_)
+    # FIXME: _CINDEX_LIB_ affects dllexport/dllimport on Win32.
+    if(LLVM_ENABLE_MODULES AND NOT WIN32)
+      target_compile_options(libclang PRIVATE
+        "-fmodules-ignore-macro=_CINDEX_LIB_"
+        )
+    endif()
   endif()
 endif()
 
@@ -132,10 +141,13 @@ install(DIRECTORY ../../include/clang-c
   PATTERN ".svn" EXCLUDE
   )
 
+# LLVM_DISTRIBUTION_COMPONENTS requires that each component have both a
+# component and an install-component target, so add a dummy libclang-headers
+# target to allow using it in LLVM_DISTRIBUTION_COMPONENTS.
+add_custom_target(libclang-headers)
+set_target_properties(libclang-headers PROPERTIES FOLDER "Misc")
+
 if (NOT CMAKE_CONFIGURATION_TYPES) # don't add this for IDE's.
-  add_custom_target(install-libclang-headers
-    DEPENDS
-    COMMAND "${CMAKE_COMMAND}"
-            -DCMAKE_INSTALL_COMPONENT=libclang-headers
-            -P "${CMAKE_BINARY_DIR}/cmake_install.cmake")
+  add_llvm_install_targets(install-libclang-headers
+                           COMPONENT libclang-headers)
 endif()

tools/libclang/CXIndexDataConsumer.cpp

@@ -1258,6 +1258,7 @@ static CXIdxEntityKind getEntityKindFromSymbolKind(SymbolKind K, SymbolLanguage
   case SymbolKind::Module:
   case SymbolKind::Macro:
   case SymbolKind::ClassProperty:
+  case SymbolKind::Using:
     return CXIdxEntity_Unexposed;
 
   case SymbolKind::Enum: return CXIdxEntity_Enum;

tools/libclang/CXIndexDataConsumer.h

@@ -342,7 +342,7 @@ class CXIndexDataConsumer : public index::IndexDataConsumer {
   CXTranslationUnit getCXTU() const { return CXTU; }
 
   void setASTContext(ASTContext &ctx);
-  void setPreprocessor(std::shared_ptr<Preprocessor> PP);
+  void setPreprocessor(std::shared_ptr<Preprocessor> PP) override;
 
   bool shouldSuppressRefs() const {
     return IndexOptions & CXIndexOpt_SuppressRedundantRefs;

tools/libclang/CXTranslationUnit.h

@@ -33,6 +33,8 @@ struct CXTranslationUnitImpl {
   void *Diagnostics;
   void *OverridenCursorsPool;
   clang::index::CommentToXMLConverter *CommentToXML;
+  unsigned ParsingOptions;
+  std::vector<std::string> Arguments;
 };
 
 struct CXTargetInfoImpl {

tools/libclang/CXType.cpp

@@ -53,6 +53,7 @@ static CXTypeKind GetBuiltinTypeKind(const BuiltinType *BT) {
     BTCASE(Float);
     BTCASE(Double);
     BTCASE(LongDouble);
+    BTCASE(Float16);
     BTCASE(Float128);
     BTCASE(NullPtr);
     BTCASE(Overload);
@@ -402,7 +403,10 @@ unsigned clang_getAddressSpace(CXType CT) {
   if (T.getAddressSpace() >= LangAS::FirstTargetAddressSpace) {
     return T.getQualifiers().getAddressSpaceAttributePrintValue();
   }
-  return T.getAddressSpace();
+  // FIXME: this function returns either a LangAS or a target AS
+  // Those values can overlap which makes this function rather unpredictable
+  // for any caller
+  return (unsigned)T.getAddressSpace();
 }
 
 CXString clang_getTypedefName(CXType CT) {
@@ -520,7 +524,7 @@ CXString clang_getTypeKindSpelling(enum CXTypeKind K) {
     TKIND(Char_U);
     TKIND(UChar);
     TKIND(Char16);
-    TKIND(Char32);  
+    TKIND(Char32);
     TKIND(UShort);
     TKIND(UInt);
     TKIND(ULong);
@@ -538,6 +542,7 @@ CXString clang_getTypeKindSpelling(enum CXTypeKind K) {
     TKIND(Float);
     TKIND(Double);
     TKIND(LongDouble);
+    TKIND(Float16);
     TKIND(Float128);
     TKIND(NullPtr);
     TKIND(Overload);

tools/libclang/Indexing.cpp

@@ -272,7 +272,8 @@ class IndexPPCallbacks : public PPCallbacks {
   /// SourceRangeSkipped - This hook is called when a source range is skipped.
   /// \param Range The SourceRange that was skipped. The range begins at the
   /// #if/#else directive and ends after the #endif/#else directive.
-  void SourceRangeSkipped(SourceRange Range) override {}
+  void SourceRangeSkipped(SourceRange Range, SourceLocation EndifLoc) override {
+  }
 };
 
 //===----------------------------------------------------------------------===//
@@ -879,11 +880,6 @@ int clang_indexSourceFileFullArgv(
         TU_options);
   };
 
-  if (getenv("LIBCLANG_NOTHREADS")) {
-    IndexSourceFileImpl();
-    return result;
-  }
-
   llvm::CrashRecoveryContext CRC;
 
   if (!RunSafely(CRC, IndexSourceFileImpl)) {
@@ -933,11 +929,6 @@ int clang_indexTranslationUnit(CXIndexAction idxAction,
         index_options, TU);
   };
 
-  if (getenv("LIBCLANG_NOTHREADS")) {
-    IndexTranslationUnitImpl();
-    return result;
-  }
-
   llvm::CrashRecoveryContext CRC;
 
   if (!RunSafely(CRC, IndexTranslationUnitImpl)) {

tools/libclang/libclang.exports

@@ -2,6 +2,7 @@ clang_CXCursorSet_contains
 clang_CXCursorSet_insert
 clang_CXIndex_getGlobalOptions
 clang_CXIndex_setGlobalOptions
+clang_CXIndex_setInvocationEmissionPathOption
 clang_CXXConstructor_isConvertingConstructor
 clang_CXXConstructor_isCopyConstructor
 clang_CXXConstructor_isDefaultConstructor
@@ -12,6 +13,7 @@ clang_CXXMethod_isConst
 clang_CXXMethod_isPureVirtual
 clang_CXXMethod_isStatic
 clang_CXXMethod_isVirtual
+clang_CXXRecord_isAbstract
 clang_EnumDecl_isScoped
 clang_Cursor_getArgument
 clang_Cursor_getNumTemplateArguments
@@ -23,6 +25,7 @@ clang_Cursor_getBriefCommentText
 clang_Cursor_getCommentRange
 clang_Cursor_getCXXManglings
 clang_Cursor_getMangling
+clang_Cursor_getObjCManglings
 clang_Cursor_getParsedComment
 clang_Cursor_getRawCommentText
 clang_Cursor_getNumArguments
@@ -189,6 +192,7 @@ clang_getCursorReferenced
 clang_getCursorResultType
 clang_getCursorSemanticParent
 clang_getCursorSpelling
+clang_getCursorTLSKind
 clang_getCursorType
 clang_getCursorUSR
 clang_getCursorVisibility