[demangler] Initial support for the new Rust mangling scheme

Add a demangling support for a small subset of a new Rust mangling
scheme, with complete support planned as a follow up work.

Intergate Rust demangling into llvm-cxxfilt and use llvm-cxxfilt for
end-to-end testing. The new Rust mangling scheme uses "_R" as a prefix,
which makes it easy to disambiguate it from other mangling schemes.

The public API is modeled after __cxa_demangle / llvm::itaniumDemangle,
since potential candidates for further integration use those.

Reviewed By: dblaikie

Differential Revision: https://reviews.llvm.org/D101444

Author: tmiasko

llvm/include/llvm/Demangle/Demangle.h

@@ -57,6 +57,9 @@ char *microsoftDemangle(const char *mangled_name, size_t *n_read,
                         char *buf, size_t *n_buf,
                         int *status, MSDemangleFlags Flags = MSDF_None);
 
+// Demangles a Rust v0 mangled symbol. The API follows that of __cxa_demangle.
+char *rustDemangle(const char *MangledName, char *Buf, size_t *N, int *Status);
+
 /// Attempt to demangle a string using different demangling schemes.
 /// The function uses heuristics to determine which demangling scheme to use.
 /// \param MangledName - reference to string to demangle.

llvm/include/llvm/Demangle/RustDemangle.h

@@ -0,0 +1,118 @@
+//===--- RustDemangle.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEMANGLE_RUSTDEMANGLE_H
+#define LLVM_DEMANGLE_RUSTDEMANGLE_H
+
+#include "llvm/Demangle/DemangleConfig.h"
+#include "llvm/Demangle/StringView.h"
+#include "llvm/Demangle/Utility.h"
+
+namespace llvm {
+namespace rust_demangle {
+
+using llvm::itanium_demangle::OutputStream;
+using llvm::itanium_demangle::StringView;
+
+struct Identifier {
+  StringView Name;
+  bool Punycode;
+
+  bool empty() const { return Name.empty(); }
+};
+
+class Demangler {
+  // Maximum recursion level. Used to avoid stack overflow.
+  size_t MaxRecursionLevel;
+  // Current recursion level.
+  size_t RecursionLevel;
+
+  // Input string that is being demangled with "_R" prefix removed.
+  StringView Input;
+  // Position in the input string.
+  size_t Position;
+
+  // True if an error occurred.
+  bool Error;
+
+public:
+  // Demangled output.
+  OutputStream Output;
+
+  Demangler(size_t MaxRecursionLevel = 500);
+
+  bool demangle(StringView MangledName);
+
+private:
+  void demanglePath();
+
+  Identifier parseIdentifier();
+  void parseOptionalBase62Number(char Tag);
+  uint64_t parseBase62Number();
+  uint64_t parseDecimalNumber();
+
+  void print(StringView S) {
+    if (Error)
+      return;
+
+    Output += S;
+  }
+
+  char look() const {
+    if (Error || Position >= Input.size())
+      return 0;
+
+    return Input[Position];
+  }
+
+  char consume() {
+    if (Error || Position >= Input.size()) {
+      Error = true;
+      return 0;
+    }
+
+    return Input[Position++];
+  }
+
+  bool consumeIf(char Prefix) {
+    if (Error || Position >= Input.size() || Input[Position] != Prefix)
+      return false;
+
+    Position += 1;
+    return true;
+  }
+
+  /// Computes A + B. When computation wraps around sets the error and returns
+  /// false. Otherwise assigns the result to A and returns true.
+  bool addAssign(uint64_t &A, const uint64_t B) {
+    if (A > std::numeric_limits<uint64_t>::max() - B) {
+      Error = true;
+      return false;
+    }
+
+    A += B;
+    return true;
+  }
+
+  /// Computes A * B. When computation wraps around sets the error and returns
+  /// false. Otherwise assigns the result to A and returns true.
+  bool mulAssign(uint64_t &A, const uint64_t B) {
+    if (B != 0 && A > std::numeric_limits<uint64_t>::max() / B) {
+      Error = true;
+      return false;
+    }
+
+    A *= B;
+    return true;
+  }
+};
+
+} // namespace rust_demangle
+} // namespace llvm
+
+#endif

llvm/lib/Demangle/CMakeLists.txt

@@ -3,6 +3,7 @@ add_llvm_component_library(LLVMDemangle
   ItaniumDemangle.cpp
   MicrosoftDemangle.cpp
   MicrosoftDemangleNodes.cpp
+  RustDemangle.cpp
 
   ADDITIONAL_HEADER_DIRS
   "${LLVM_MAIN_INCLUDE_DIR}/llvm/Demangle"