Fix Swift SDK handling URLs without filename components (#6708)

Currently when invoking `swift experimental-sdk install
https://server.org/abc?xy=z` installation will fail as we assume URLs
always have a filename component similar to
`https://server.org/swift-sdk.tar.gz`. Let's make the installation logic
more flexible so that bundle name is inferred from the downloaded
extract archive contents and not the URL.

Also improved logging so that final bundle name is printed for such URLs
without a filename component. Additionally we now print a message when
starting a download, which is helpful if the download is long-running.

Author: MaxDesiatov

Fixtures/SwiftSDKs/test-sdk.artifactbundle.tar.gz

@@ -496,7 +496,7 @@ extension FileSystem {
         }
     }
 
-    public func getSharedSwiftSDKsDirectory(explicitDirectory: AbsolutePath?) throws -> AbsolutePath? {
+    public func getSharedSwiftSDKsDirectory(explicitDirectory: AbsolutePath?) throws -> AbsolutePath {
         if let explicitDirectory {
             // Create the explicit SDKs path if necessary
             if !exists(explicitDirectory) {

Sources/Basics/FileSystem/FileSystem+Extensions.swift

@@ -163,6 +163,6 @@ public struct SwiftBuildTool: SwiftCommand {
 
 public extension _SwiftCommand {
     func buildSystemProvider(_ swiftTool: SwiftTool) throws -> BuildSystemProvider {
-        return try swiftTool.defaultBuildSystemProvider
+        swiftTool.defaultBuildSystemProvider
     }
 }

Sources/Commands/SwiftBuildTool.swift

@@ -199,16 +199,16 @@ public final class SwiftTool {
     public let scratchDirectory: AbsolutePath
 
     /// Path to the shared security directory
-    public let sharedSecurityDirectory: AbsolutePath?
+    public let sharedSecurityDirectory: AbsolutePath
 
     /// Path to the shared cache directory
-    public let sharedCacheDirectory: AbsolutePath?
+    public let sharedCacheDirectory: AbsolutePath
 
     /// Path to the shared configuration directory
-    public let sharedConfigurationDirectory: AbsolutePath?
+    public let sharedConfigurationDirectory: AbsolutePath
 
     /// Path to the cross-compilation Swift SDKs directory.
-    public let sharedSwiftSDKsDirectory: AbsolutePath?
+    public let sharedSwiftSDKsDirectory: AbsolutePath
 
     /// Cancellator to handle cancellation of outstanding work when handling SIGINT
     public let cancellator: Cancellator
@@ -747,11 +747,11 @@ public final class SwiftTool {
             let hostTriple = hostToolchain.targetTriple
 
             // Create custom toolchain if present.
-            if let customDestination = options.locations.customCompileDestination {
+            if let customDestination = self.options.locations.customCompileDestination {
                 let swiftSDKs = try SwiftSDK.decode(
                     fromFile: customDestination,
-                    fileSystem: fileSystem,
-                    observabilityScope: observabilityScope
+                    fileSystem: self.fileSystem,
+                    observabilityScope: self.observabilityScope
                 )
                 if swiftSDKs.count == 1 {
                     swiftSDK = swiftSDKs[0]
@@ -768,13 +768,13 @@ public final class SwiftTool {
             {
                 swiftSDK = targetSwiftSDK
             } else if let swiftSDKSelector = options.build.swiftSDKSelector {
-                swiftSDK = try SwiftSDKBundle.selectBundle(
-                    fromBundlesAt: sharedSwiftSDKsDirectory,
-                    fileSystem: fileSystem,
-                    matching: swiftSDKSelector,
-                    hostTriple: hostTriple,
-                    observabilityScope: observabilityScope
+                let store = SwiftSDKBundleStore(
+                    swiftSDKsDirectory: self.sharedSwiftSDKsDirectory,
+                    fileSystem: self.fileSystem,
+                    observabilityScope: self.observabilityScope,
+                    outputHandler: { print($0.description) }
                 )
+                swiftSDK = try store.selectBundle(matching: swiftSDKSelector, hostTriple: hostTriple)
             } else {
                 // Otherwise use the host toolchain.
                 swiftSDK = hostSwiftSDK
@@ -824,7 +824,7 @@ public final class SwiftTool {
             case (false, .local):
                 cachePath = self.scratchDirectory
             case (false, .shared):
-                cachePath = self.sharedCacheDirectory.map{ Workspace.DefaultLocations.manifestsDirectory(at: $0) }
+                cachePath = Workspace.DefaultLocations.manifestsDirectory(at: self.sharedCacheDirectory)
             }
 
             var extraManifestFlags = self.options.build.manifestFlags
@@ -876,7 +876,7 @@ private func findPackageRoot(fileSystem: FileSystem) -> AbsolutePath? {
     return root
 }
 
-private func getSharedSecurityDirectory(options: GlobalOptions, fileSystem: FileSystem) throws -> AbsolutePath? {
+private func getSharedSecurityDirectory(options: GlobalOptions, fileSystem: FileSystem) throws -> AbsolutePath {
     if let explicitSecurityDirectory = options.locations.securityDirectory {
         // Create the explicit security path if necessary
         if !fileSystem.exists(explicitSecurityDirectory) {
@@ -889,7 +889,7 @@ private func getSharedSecurityDirectory(options: GlobalOptions, fileSystem: File
     }
 }
 
-private func getSharedConfigurationDirectory(options: GlobalOptions, fileSystem: FileSystem) throws -> AbsolutePath? {
+private func getSharedConfigurationDirectory(options: GlobalOptions, fileSystem: FileSystem) throws -> AbsolutePath {
     if let explicitConfigurationDirectory = options.locations.configurationDirectory {
         // Create the explicit config path if necessary
         if !fileSystem.exists(explicitConfigurationDirectory) {
@@ -902,7 +902,7 @@ private func getSharedConfigurationDirectory(options: GlobalOptions, fileSystem:
     }
 }
 
-private func getSharedCacheDirectory(options: GlobalOptions, fileSystem: FileSystem) throws -> AbsolutePath? {
+private func getSharedCacheDirectory(options: GlobalOptions, fileSystem: FileSystem) throws -> AbsolutePath {
     if let explicitCacheDirectory = options.locations.cacheDirectory {
         // Create the explicit cache path if necessary
         if !fileSystem.exists(explicitCacheDirectory) {

Sources/CoreCommands/SwiftTool.swift

@@ -15,6 +15,8 @@ import Foundation
 
 import struct TSCUtility.Version
 
+public let artifactBundleExtension = "artifactbundle"
+
 public struct ArtifactsArchiveMetadata: Equatable {
     public let schemaVersion: String
     public let artifacts: [String: Artifact]

Sources/PackageModel/ArtifactsArchiveMetadata.swift

@@ -43,9 +43,10 @@ add_library(PackageModel
   Sources.swift
   SupportedLanguageExtension.swift
   SwiftLanguageVersion.swift
-  SwiftSDK.swift
-  SwiftSDKConfigurationStore.swift
-  SwiftSDKBundle.swift
+  SwiftSDKs/SwiftSDK.swift
+  SwiftSDKs/SwiftSDKConfigurationStore.swift
+  SwiftSDKs/SwiftSDKBundle.swift
+  SwiftSDKs/SwiftSDKBundleStore.swift
   Target/BinaryTarget.swift
   Target/ClangTarget.swift
   Target/PluginTarget.swift