Leverage package graph, add include paths. (#8750)

Take advantage of the fact we're in swiftpm and calculate things out of
the package graph instead of hard coding. In particular, we compute the
list of targets for the requested list of products, as well as the list
of C modules and their include paths.

Swift build is better at ensuring the include path is added to all
targets that need it. Unfortunately when mixing libraries that use
swift-syntax and macros, we end up adding both the C modules from the
checkouts and the prebuilts causing duplicate definition of the C
modules. By adding the above include paths and leveraging that we always
checkout the source anyway, switch to using the checked out include
paths, thus avoiding the duplicate.

Also some prep for adding a config.json file so the list of prebuilts
can be changed without a PR to SwiftPM. Removes the magic we were using
to add the prebuilt library to the package manifest and just add them at
the end.

Author: dschaefer2

Sources/PackageLoading/PackageBuilder.swift

@@ -290,17 +290,33 @@ public struct PrebuiltLibrary {
     /// The path to the extracted prebuilt artifacts
     public let path: AbsolutePath
 
+    /// The path to the checked out source
+    public let checkoutPath: AbsolutePath?
+
     /// The products in the library
     public let products: [String]
 
+    /// The include path relative to the checkouts dir
+    public let includePath: [RelativePath]?
+
     /// The C modules that need their includes directory added to the include path
     public let cModules: [String]
 
-    public init(identity: PackageIdentity, libraryName: String, path: AbsolutePath, products: [String], cModules: [String]) {
+    public init(
+        identity: PackageIdentity,
+        libraryName: String,
+        path: AbsolutePath,
+        checkoutPath: AbsolutePath?,
+        products: [String],
+        includePath: [RelativePath]? = nil,
+        cModules: [String] = []
+    ) {
         self.identity = identity
         self.libraryName = libraryName
         self.path = path
+        self.checkoutPath = checkoutPath
         self.products = products
+        self.includePath = includePath
         self.cModules = cModules
     }
 }
@@ -1318,8 +1334,14 @@ public final class PackageBuilder {
                 table.add(ldFlagsAssignment, for: .OTHER_LDFLAGS)
 
                 var includeDirs: [AbsolutePath] = [prebuilt.path.appending(component: "Modules")]
-                for cModule in prebuilt.cModules {
-                    includeDirs.append(prebuilt.path.appending(components: "include", cModule))
+                if let checkoutPath = prebuilt.checkoutPath, let includePath = prebuilt.includePath {
+                    for includeDir in includePath {
+                        includeDirs.append(checkoutPath.appending(includeDir))
+                    }
+                } else {
+                    for cModule in prebuilt.cModules {
+                        includeDirs.append(prebuilt.path.appending(components: "include", cModule))
+                    }
                 }
                 var includeAssignment = BuildSettings.Assignment()
                 includeAssignment.values = includeDirs.map({ "-I\($0.pathString)" })

Sources/Workspace/ManagedPrebuilt.swift

@@ -31,9 +31,15 @@ extension Workspace {
         /// The path to the extracted prebuilt artifacts
         public let path: AbsolutePath
 
+        /// The path to the checked out source
+        public let checkoutPath: AbsolutePath?
+
         /// The products in the library
         public let products: [String]
 
+        /// The include path for the C modules
+        public let includePath: [RelativePath]?
+
         /// The C modules that need their includes directory added to the include path
         public let cModules: [String]
     }

Sources/Workspace/Workspace+Prebuilts.swift

@@ -53,8 +53,9 @@ extension Workspace {
         public struct Library: Identifiable, Codable {
             public let name: String
             public var products: [String]
-            public var cModules: [String]
-            public var artifacts: [Artifact]
+            public var cModules: [String]?
+            public var includePath: [RelativePath]?
+            public var artifacts: [Artifact]?
 
             public var id: String { name }
 
@@ -73,12 +74,14 @@ extension Workspace {
             public init(
                 name: String,
                 products: [String] = [],
-                cModules: [String] = [],
-                artifacts: [Artifact] = []
+                cModules: [String]? = nil,
+                includePath: [RelativePath]? = nil,
+                artifacts: [Artifact]? = nil
             ) {
                 self.name = name
                 self.products = products
                 self.cModules = cModules
+                self.includePath = includePath
                 self.artifacts = artifacts
             }
         }
@@ -600,7 +603,7 @@ extension Workspace {
             let hostPlatform = prebuiltsManager.hostPlatform
 
             for library in prebuiltManifest.libraries {
-                for artifact in library.artifacts where artifact.platform == hostPlatform {
+                for artifact in library.artifacts ?? [] where artifact.platform == hostPlatform {
                     if let path = try await prebuiltsManager
                         .downloadPrebuilt(
                             package: prebuilt,
@@ -611,13 +614,17 @@ extension Workspace {
                         )
                     {
                         // Add to workspace state
+                        let checkoutPath = self.location.repositoriesCheckoutsDirectory
+                            .appending(component: prebuilt.identity.description)
                         let managedPrebuilt = ManagedPrebuilt(
                             identity: prebuilt.identity,
                             version: packageVersion,
                             libraryName: library.name,
                             path: path,
+                            checkoutPath: checkoutPath,
                             products: library.products,
-                            cModules: library.cModules
+                            includePath: library.includePath,
+                            cModules: library.cModules ?? []
                         )
                         addedPrebuilts.add(managedPrebuilt)
                         await self.state.prebuilts.add(managedPrebuilt)

Sources/Workspace/Workspace+State.swift

@@ -464,15 +464,19 @@ extension WorkspaceStateStorage {
             let version: TSCUtility.Version
             let libraryName: String
             let path: Basics.AbsolutePath
+            let checkoutPath: Basics.AbsolutePath?
             let products: [String]
+            let includePath: [Basics.RelativePath]?
             let cModules: [String]
 
             init(_ managedPrebuilt: Workspace.ManagedPrebuilt) {
                 self.identity = managedPrebuilt.identity
                 self.version = managedPrebuilt.version
                 self.libraryName = managedPrebuilt.libraryName
                 self.path = managedPrebuilt.path
+                self.checkoutPath = managedPrebuilt.checkoutPath
                 self.products = managedPrebuilt.products
+                self.includePath = managedPrebuilt.includePath
                 self.cModules = managedPrebuilt.cModules
             }
         }
@@ -546,7 +550,9 @@ extension Workspace.ManagedPrebuilt {
             version: prebuilt.version,
             libraryName: prebuilt.libraryName,
             path: prebuilt.path,
+            checkoutPath: prebuilt.checkoutPath,
             products: prebuilt.products,
+            includePath: prebuilt.includePath,
             cModules: prebuilt.cModules
         )
     }

Sources/Workspace/Workspace.swift

@@ -969,7 +969,14 @@ extension Workspace {
             }
 
         let prebuilts: [PackageIdentity: [String: PrebuiltLibrary]] = await self.state.prebuilts.reduce(into: .init()) {
-            let prebuilt = PrebuiltLibrary(identity: $1.identity, libraryName: $1.libraryName, path: $1.path, products: $1.products, cModules: $1.cModules)
+            let prebuilt = PrebuiltLibrary(
+                identity: $1.identity,
+                libraryName: $1.libraryName,
+                path: $1.path,
+                checkoutPath: $1.checkoutPath,
+                products: $1.products,
+                includePath: $1.includePath,
+                cModules: $1.cModules)
             for product in $1.products {
                 $0[$1.identity, default: [:]][product] = prebuilt
             }