Revert "Revert "[lld-macho] Implement -dependency_info (partially - more opcodes needed)""

This reverts commit 2554b95.

Relanding [lld-macho] Implement -dependency_info (D98559) with changes:
 - inline functions removed from cpp file.
 - updated tests to not check libSystem.tbd with other input files (because of possible indeterministic ordering)

Author: oontvoo

lld/MachO/Driver.cpp

@@ -54,7 +54,8 @@ using namespace llvm::sys;
 using namespace lld;
 using namespace lld::macho;
 
-Configuration *lld::macho::config;
+Configuration *macho::config;
+DependencyTracker *macho::depTracker;
 
 static HeaderFileType getOutputType(const InputArgList &args) {
   // TODO: -r, -dylinker, -preload...
@@ -84,6 +85,8 @@ findAlongPathsWithExtensions(StringRef name, ArrayRef<StringRef> extensions) {
       Twine location = base + ext;
       if (fs::exists(location))
         return location.str();
+      else
+        depTracker->logFileNotFound(location);
     }
   }
   return {};
@@ -815,6 +818,9 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly,
   symtab = make<SymbolTable>();
   target = createTargetInfo(args);
 
+  depTracker =
+      make<DependencyTracker>(args.getLastArgValue(OPT_dependency_info, ""));
+
   config->entry = symtab->addUndefined(args.getLastArgValue(OPT_e, "_main"),
                                        /*file=*/nullptr,
                                        /*isWeakRef=*/false);
@@ -1066,6 +1072,8 @@ bool macho::link(ArrayRef<const char *> argsArr, bool canExitEarly,
 
     // Write to an output file.
     writeResult();
+
+    depTracker->write(getLLDVersion(), inputFiles, config->outputFile);
   }
 
   if (config->timeTraceEnabled) {

lld/MachO/Driver.h

@@ -11,10 +11,14 @@
 
 #include "lld/Common/LLVM.h"
 #include "llvm/ADT/Optional.h"
+#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Option/OptTable.h"
 #include "llvm/Support/MemoryBuffer.h"
 
+#include <set>
+#include <type_traits>
+
 namespace llvm {
 namespace MachO {
 class InterfaceFile;
@@ -61,6 +65,52 @@ uint32_t getModTime(llvm::StringRef path);
 
 void printArchiveMemberLoad(StringRef reason, const InputFile *);
 
+// Helper class to export dependency info.
+class DependencyTracker {
+public:
+  explicit DependencyTracker(llvm::StringRef path);
+
+  // Adds the given path to the set of not-found files.
+  inline void logFileNotFound(std::string path) {
+    if (active)
+      notFounds.insert(std::move(path));
+  }
+
+  inline void logFileNotFound(const Twine &path) {
+    if (active)
+      notFounds.insert(path.str());
+  }
+
+  // Writes the dependencies to specified path.
+  // The content is sorted by its Op Code, then within each section,
+  // alphabetical order.
+  void write(llvm::StringRef version,
+             const llvm::SetVector<InputFile *> &inputs,
+             llvm::StringRef output);
+
+private:
+  enum DepOpCode : char {
+    // Denotes the linker version.
+    Version = 0x00,
+    // Denotes the input files.
+    Input = 0x10,
+    // Denotes the files that do not exist(?)
+    NotFound = 0x11,
+    // Denotes the output files.
+    Output = 0x40,
+  };
+
+  const llvm::StringRef path;
+  bool active;
+
+  // The paths need to be alphabetically ordered.
+  // We need to own the paths because some of them are temporarily
+  // constructed.
+  std::set<std::string> notFounds;
+};
+
+extern DependencyTracker *depTracker;
+
 } // namespace macho
 } // namespace lld
 

lld/MachO/DriverUtils.cpp

@@ -23,6 +23,7 @@
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/Option.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/TextAPI/MachO/InterfaceFile.h"
 #include "llvm/TextAPI/MachO/TextAPIReader.h"
@@ -164,12 +165,15 @@ Optional<std::string> macho::resolveDylibPath(StringRef path) {
   // they are consistent.
   if (fs::exists(path))
     return std::string(path);
+  else
+    depTracker->logFileNotFound(path);
 
   SmallString<261> location = path;
   path::replace_extension(location, ".tbd");
   if (fs::exists(location))
     return std::string(location);
-
+  else
+    depTracker->logFileNotFound(location);
   return {};
 }
 
@@ -240,3 +244,49 @@ void macho::printArchiveMemberLoad(StringRef reason, const InputFile *f) {
   if (config->printWhyLoad)
     message(reason + " forced load of " + toString(f));
 }
+
+macho::DependencyTracker::DependencyTracker(StringRef path)
+    : path(path), active(!path.empty()) {
+  if (active && fs::exists(path) && !fs::can_write(path)) {
+    warn("Ignoring dependency_info option since specified path is not "
+         "writeable.");
+    active = false;
+  }
+}
+
+void macho::DependencyTracker::write(llvm::StringRef version,
+                                     const llvm::SetVector<InputFile *> &inputs,
+                                     llvm::StringRef output) {
+  if (!active)
+    return;
+
+  std::error_code ec;
+  llvm::raw_fd_ostream os(path, ec, llvm::sys::fs::OF_None);
+  if (ec) {
+    warn("Error writing dependency info to file");
+    return;
+  }
+
+  auto addDep = [&os](DepOpCode opcode, const StringRef &path) {
+    os << opcode;
+    os << path;
+    os << '\0';
+  };
+
+  addDep(DepOpCode::Version, version);
+
+  // Sort the input by its names.
+  std::vector<StringRef> inputNames;
+  inputNames.reserve(inputs.size());
+  for (InputFile *f : inputs)
+    inputNames.push_back(f->getName());
+  llvm::sort(inputNames,
+             [](const StringRef &a, const StringRef &b) { return a < b; });
+  for (const StringRef &in : inputNames)
+    addDep(DepOpCode::Input, in);
+
+  for (const std::string &f : notFounds)
+    addDep(DepOpCode::NotFound, f);
+
+  addDep(DepOpCode::Output, output);
+}

lld/MachO/Options.td

@@ -504,7 +504,6 @@ def map : Separate<["-"], "map">,
 def dependency_info : Separate<["-"], "dependency_info">,
     MetaVarName<"<path>">,
     HelpText<"Dump dependency info">,
-    Flags<[HelpHidden]>,
     Group<grp_introspect>;
 def save_temps : Flag<["-"], "save-temps">,
     HelpText<"Save intermediate LTO compilation results">,

lld/test/MachO/Inputs/DependencyDump.py

@@ -0,0 +1,26 @@
+#
+# Dump the dependency file (produced with -dependency_info) to text
+# format for testing purposes.
+#
+
+import sys
+
+f = open(sys.argv[1], "rb")
+byte = f.read(1)
+while byte != b'':
+    if byte == b'\x00':
+        sys.stdout.write("lld-version: ")
+    elif byte == b'\x10':
+        sys.stdout.write("input-file: ")
+    elif byte == b'\x11':
+        sys.stdout.write("not-found: ")
+    elif byte == b'\x40':
+        sys.stdout.write("output-file: ")
+    byte = f.read(1)
+    while byte != b'\x00':
+        sys.stdout.write(byte.decode("ascii"))
+        byte = f.read(1)
+    sys.stdout.write("\n")
+    byte = f.read(1)
+
+f.close()

lld/test/MachO/dependency-info.s

@@ -0,0 +1,55 @@
+# REQUIRES: x86
+## FIXME: Paths on windows have both `\` and '/', as a result, they are in a different
+## order when sorted. Maybe create a separate test for that?
+# UNSUPPORTED: system-windows
+#
+# RUN: rm -rf %t
+# RUN: split-file %s %t
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/foo.o %t/foo.s
+# RUN: %lld -dylib -o %t/libfoo.dylib %t/foo.o
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/bar.o %t/bar.s
+# RUN: llvm-ar csr  %t/bar.a %t/bar.o
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-apple-macos -o %t/main.o %t/main.s
+
+# RUN: %lld %t/main.o %t/bar.a %t/libfoo.dylib -lSystem -dependency_info %t/deps_info.out
+
+## Smoke check to make sure that `libSystem.tbd` exists in the set of input.
+# RUN: %python %S/Inputs/DependencyDump.py %t/deps_info.out | FileCheck %s --check-prefix "SYS-CHECK"
+# SYS-CHECK: input-file: {{.*}}/libSystem.tbd
+
+## Filter out libSystem.tbd because it is in the source tree, which is different from the rest,
+## therefore ordering might be unpredictable.
+# RUN: %python %S/Inputs/DependencyDump.py %t/deps_info.out | grep -v 'libSystem.tbd' | FileCheck %s
+
+# CHECK: lld-version: LLD {{.*}}
+
+# CHECK-NEXT: input-file: {{.*}}/bar.a
+# CHECK-NEXT: input-file: {{.*}}/libfoo.dylib
+# CHECK-NEXT: input-file: {{.*}}/main.o
+# CHECK-NEXT: bar.o
+
+# CHECK-NEXT: not-found: {{.*}}/libdyld.dylib
+## There could be more not-found here but we are not checking those because it's brittle.
+
+# CHECK: output-file: a.out
+
+#--- foo.s
+.globl __Z3foo
+__Z3foo:
+  ret
+
+#--- bar.s
+.globl _bar
+_bar:
+  callq __Z3foo
+  ret
+
+#--- main.s
+.globl _main
+_main:
+  callq _bar
+  callq __Z3foo
+  ret