diff --git a/CMakeLists.txt b/CMakeLists.txt index aed3e38110..967c8ddce5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -519,6 +519,7 @@ if(ENABLE_TESTING) TestFoundation/TestTimeZone.swift TestFoundation/TestUnitConverter.swift TestFoundation/TestUnit.swift + TestFoundation/TestURLCache.swift TestFoundation/TestURLCredential.swift TestFoundation/TestURLProtectionSpace.swift TestFoundation/TestURLProtocol.swift diff --git a/Foundation.xcodeproj/project.pbxproj b/Foundation.xcodeproj/project.pbxproj index 6b061c03a8..662756c5ca 100644 --- a/Foundation.xcodeproj/project.pbxproj +++ b/Foundation.xcodeproj/project.pbxproj @@ -12,6 +12,7 @@ 1513A8432044893F00539722 /* FileManager_XDG.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1513A8422044893F00539722 /* FileManager_XDG.swift */; }; 1520469B1D8AEABE00D02E36 /* HTTPServer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1520469A1D8AEABE00D02E36 /* HTTPServer.swift */; }; 1539391422A07007006DFF4F /* TestCachedURLResponse.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1539391322A07007006DFF4F /* TestCachedURLResponse.swift */; }; + 1539391522A07160006DFF4F /* TestNSSortDescriptor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 152EF3932283457B001E1269 /* TestNSSortDescriptor.swift */; }; 153CC8352215E00200BFE8F3 /* ScannerAPI.swift in Sources */ = {isa = PBXBuildFile; fileRef = 153CC8322214C3D100BFE8F3 /* ScannerAPI.swift */; }; 153E951120111DC500F250BE /* CFKnownLocations.h in Headers */ = {isa = PBXBuildFile; fileRef = 153E950F20111DC500F250BE /* CFKnownLocations.h */; settings = {ATTRIBUTES = (Private, ); }; }; 153E951220111DC500F250BE /* CFKnownLocations.c in Sources */ = {isa = PBXBuildFile; fileRef = 153E951020111DC500F250BE /* CFKnownLocations.c */; }; @@ -31,6 +32,7 @@ 1578DA11212B407B003C9516 /* CFString_Internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 1578DA10212B407B003C9516 /* CFString_Internal.h */; }; 1578DA13212B4C35003C9516 /* CFOverflow.h in Headers */ = {isa = PBXBuildFile; fileRef = 1578DA12212B4C35003C9516 /* CFOverflow.h */; }; 1578DA15212B6F33003C9516 /* CFCollections_Internal.h in Headers */ = {isa = PBXBuildFile; fileRef = 1578DA14212B6F33003C9516 /* CFCollections_Internal.h */; }; + 1581706322B1A29100348861 /* TestURLCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1581706222B1A29100348861 /* TestURLCache.swift */; }; 158BCCAA2220A12600750239 /* TestDateIntervalFormatter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 158BCCA92220A12600750239 /* TestDateIntervalFormatter.swift */; }; 158BCCAD2220A18F00750239 /* CFDateIntervalFormatter.h in Headers */ = {isa = PBXBuildFile; fileRef = 158BCCAB2220A18F00750239 /* CFDateIntervalFormatter.h */; settings = {ATTRIBUTES = (Private, ); }; }; 158BCCAE2220A18F00750239 /* CFDateIntervalFormatter.c in Sources */ = {isa = PBXBuildFile; fileRef = 158BCCAC2220A18F00750239 /* CFDateIntervalFormatter.c */; }; @@ -651,6 +653,7 @@ 1578DA10212B407B003C9516 /* CFString_Internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CFString_Internal.h; sourceTree = ""; }; 1578DA12212B4C35003C9516 /* CFOverflow.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CFOverflow.h; sourceTree = ""; }; 1578DA14212B6F33003C9516 /* CFCollections_Internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CFCollections_Internal.h; sourceTree = ""; }; + 1581706222B1A29100348861 /* TestURLCache.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TestURLCache.swift; sourceTree = ""; }; 158BCCA92220A12600750239 /* TestDateIntervalFormatter.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = TestDateIntervalFormatter.swift; sourceTree = ""; }; 158BCCAB2220A18F00750239 /* CFDateIntervalFormatter.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CFDateIntervalFormatter.h; sourceTree = ""; }; 158BCCAC2220A18F00750239 /* CFDateIntervalFormatter.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = CFDateIntervalFormatter.c; sourceTree = ""; }; @@ -1771,6 +1774,7 @@ 84BA558D1C16F90900F48C54 /* TestTimeZone.swift */, EA66F6431BF1619600136161 /* TestURL.swift */, F9E0BB361CA70B8000F7FF3C /* TestURLCredential.swift */, + 1581706222B1A29100348861 /* TestURLCache.swift */, 83712C8D1C1684900049AD49 /* TestNSURLRequest.swift */, DAA79BD820D42C07004AF044 /* TestURLProtectionSpace.swift */, 7A7D6FBA1C16439400957E2E /* TestURLResponse.swift */, @@ -2916,8 +2920,10 @@ 5B13B3271C582D4C00651CE2 /* TestNSArray.swift in Sources */, 5B13B3461C582D4C00651CE2 /* TestProcess.swift in Sources */, 555683BD1C1250E70041D4C6 /* TestUserDefaults.swift in Sources */, + 1539391522A07160006DFF4F /* TestNSSortDescriptor.swift in Sources */, DCA8120B1F046D13000D0C86 /* TestCodable.swift in Sources */, 7900433B1CACD33E00ECCBF1 /* TestNSCompoundPredicate.swift in Sources */, + 1581706322B1A29100348861 /* TestURLCache.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/Foundation/NSKeyedArchiver.swift b/Foundation/NSKeyedArchiver.swift index 7587e49d8c..0b72dfc315 100644 --- a/Foundation/NSKeyedArchiver.swift +++ b/Foundation/NSKeyedArchiver.swift @@ -94,6 +94,13 @@ open class NSKeyedArchiver : NSCoder { /// The archiver’s delegate. open weak var delegate: NSKeyedArchiverDelegate? + /// The latest error. + private var _error: Error? + open override var error: Error? { + get { return _error } + set { _error = newValue } + } + /// The format in which the receiver encodes its data. /// /// The available formats are `xml` and `binary`. diff --git a/Foundation/NSKeyedUnarchiver.swift b/Foundation/NSKeyedUnarchiver.swift index 2ac748a45b..2febebde42 100644 --- a/Foundation/NSKeyedUnarchiver.swift +++ b/Foundation/NSKeyedUnarchiver.swift @@ -79,7 +79,11 @@ open class NSKeyedUnarchiver : NSCoder { unarchiver.requiresSecureCoding = true unarchiver.decodingFailurePolicy = .setErrorAndReturn - return try unarchiver.decodeObject(of: classes, forKey: NSKeyedArchiveRootObjectKey) + let result = unarchiver.decodeObject(of: classes, forKey: NSKeyedArchiveRootObjectKey) + if let error = unarchiver.error { + throw error + } + return result } @available(swift, deprecated: 9999, renamed: "unarchivedObject(ofClass:from:)") diff --git a/Foundation/NSURLRequest.swift b/Foundation/NSURLRequest.swift index b6b82b0f93..481ea3d45f 100644 --- a/Foundation/NSURLRequest.swift +++ b/Foundation/NSURLRequest.swift @@ -564,6 +564,9 @@ open class NSMutableURLRequest : NSURLRequest { get { return super.httpShouldUsePipelining } set { super.httpShouldUsePipelining = newValue } } + + // These properties are settable using URLProtocol's class methods. + var protocolProperties: [String: Any] = [:] } /// Returns an existing key-value pair inside the header fields if it exists. diff --git a/Foundation/URLCache.swift b/Foundation/URLCache.swift index d44e20af1d..e74805e049 100644 --- a/Foundation/URLCache.swift +++ b/Foundation/URLCache.swift @@ -13,6 +13,13 @@ import SwiftFoundation import Foundation #endif +internal extension NSLock { + func performLocked(_ block: () throws -> T) rethrows -> T { + lock(); defer { unlock() } + return try block() + } +} + /*! @enum URLCache.StoragePolicy @@ -40,6 +47,38 @@ extension URLCache { } } +class StoredCachedURLResponse: NSObject, NSSecureCoding { + class var supportsSecureCoding: Bool { return true } + + func encode(with aCoder: NSCoder) { + aCoder.encode(cachedURLResponse.response, forKey: "response") + aCoder.encode(cachedURLResponse.data as NSData, forKey: "data") + aCoder.encode(Int(bitPattern: cachedURLResponse.storagePolicy.rawValue), forKey: "storagePolicy") + aCoder.encode(cachedURLResponse.userInfo as NSDictionary?, forKey: "userInfo") + aCoder.encode(cachedURLResponse.date as NSDate, forKey: "date") + } + + required init?(coder aDecoder: NSCoder) { + guard let response = aDecoder.decodeObject(of: URLResponse.self, forKey: "response"), + let data = aDecoder.decodeObject(of: NSData.self, forKey: "data"), + let storagePolicy = URLCache.StoragePolicy(rawValue: UInt(bitPattern: aDecoder.decodeInteger(forKey: "storagePolicy"))), + let date = aDecoder.decodeObject(of: NSDate.self, forKey: "date") else { + return nil + } + + let userInfo = aDecoder.decodeObject(of: NSDictionary.self, forKey: "userInfo") as? [AnyHashable: Any] + + cachedURLResponse = CachedURLResponse(response: response, data: data as Data, userInfo: userInfo, storagePolicy: storagePolicy) + cachedURLResponse.date = date as Date + } + + let cachedURLResponse: CachedURLResponse + + init(cachedURLResponse: CachedURLResponse) { + self.cachedURLResponse = cachedURLResponse + } +} + /*! @class CachedURLResponse CachedURLResponse is a class whose objects functions as a wrapper for @@ -142,6 +181,8 @@ open class CachedURLResponse : NSObject, NSCopying { self.data == other.data && self.storagePolicy == other.storagePolicy } + + internal fileprivate(set) var date: Date = Date() open override var hash: Int { var hasher = Hasher() @@ -154,6 +195,9 @@ open class CachedURLResponse : NSObject, NSCopying { open class URLCache : NSObject { + private static let sharedLock = NSLock() + private static var _shared: URLCache? + /*! @method sharedURLCache @abstract Returns the shared URLCache instance. @@ -173,13 +217,88 @@ open class URLCache : NSObject { */ open class var shared: URLCache { get { - NSUnimplemented() + return sharedLock.performLocked { + if let shared = _shared { + return shared + } + + let shared = URLCache(memoryCapacity: 4 * 1024 * 1024, diskCapacity: 20 * 1024 * 1024, diskPath: nil) + _shared = shared + return shared + } } set { - NSUnimplemented() + sharedLock.performLocked { + _shared = newValue + } + } + } + + private let cacheDirectory: URL? + + private struct CacheEntry: Hashable { + var identifier: String + var cachedURLResponse: CachedURLResponse + var date: Date + var cost: Int + + init(identifier: String, cachedURLResponse: CachedURLResponse, serializedVersion: Data? = nil) { + self.identifier = identifier + self.cachedURLResponse = cachedURLResponse + self.date = Date() + // Estimate cost if we haven't already had to serialize this. + self.cost = serializedVersion?.count ?? (cachedURLResponse.data.count + 500 * (cachedURLResponse.userInfo?.count ?? 0)) + } + + func hash(into hasher: inout Hasher) { + hasher.combine(identifier) + } + + static func ==(_ lhs: CacheEntry, _ rhs: CacheEntry) -> Bool { + return lhs.identifier == rhs.identifier } } + + private let inMemoryCacheLock = NSLock() + private var inMemoryCacheOrder: [String] = [] + private var inMemoryCacheContents: [String: CacheEntry] = [:] + + func evictFromMemoryCacheAssumingLockHeld(maximumSize: Int) { + var totalSize = inMemoryCacheContents.values.reduce(0) { $0 + $1.cost } + + var countEvicted = 0 + for identifier in inMemoryCacheOrder { + if totalSize > maximumSize { + countEvicted += 1 + let entry = inMemoryCacheContents.removeValue(forKey: identifier)! + totalSize -= entry.cost + } else { + break + } + } + + inMemoryCacheOrder.removeSubrange(0 ..< countEvicted) + } + + func evictFromDiskCache(maximumSize: Int) { + let entries = diskEntries(includingPropertiesForKeys: [.fileSizeKey]).sorted { + $0.date < $1.date + } + + let sizes = entries.map { (entry) in + (try? entry.url.resourceValues(forKeys: [.fileSizeKey]).fileSize) ?? 0 + } + var totalSize = sizes.reduce(0, +) + + for (index, entry) in entries.enumerated() { + if totalSize > maximumSize { + try? FileManager.default.removeItem(at: entry.url) + totalSize -= sizes[index] + } + } + } + /*! @method initWithMemoryCapacity:diskCapacity:diskPath: @abstract Initializes an URLCache with the given capacity and @@ -193,7 +312,140 @@ open class URLCache : NSObject { @result an initialized URLCache, with the given capacity, backed by disk. */ - public init(memoryCapacity: Int, diskCapacity: Int, diskPath path: String?) { NSUnimplemented() } + public init(memoryCapacity: Int, diskCapacity: Int, diskPath path: String?) { + self.memoryCapacity = memoryCapacity + self.diskCapacity = diskCapacity + + let url: URL? + + if let path = path { + url = URL(fileURLWithPath: path) + } else { + do { + let caches = try FileManager.default.url(for: .cachesDirectory, in: .userDomainMask, appropriateFor: nil, create: true) + let directoryName = (Bundle.main.bundleIdentifier ?? ProcessInfo.processInfo.processName) + .replacingOccurrences(of: "/", with: "_") + .replacingOccurrences(of: "\\", with: "_") + .replacingOccurrences(of: ":", with: "_") + + // We append a Swift Foundation identifier to avoid clobbering a Darwin cache that may exist at the same path; + // the two on-disk cache formats aren't compatible. + url = caches + .appendingPathComponent("org.swift.foundation.URLCache", isDirectory: true) + .appendingPathComponent(directoryName, isDirectory: true) + } catch { + url = nil + } + } + + if let url = url { + do { + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) + cacheDirectory = url + } catch { + cacheDirectory = nil + } + } else { + cacheDirectory = nil + } + } + + private func identifier(for request: URLRequest) -> String? { + guard let url = request.url else { return nil } + + if let host = url.host { + var data = Data() + data.append(Data(host.lowercased(with: NSLocale.system).utf8)) + data.append(0) + let port = url.port ?? -1 + data.append(Data("\(port)".utf8)) + data.append(0) + data.append(Data(url.path.utf8)) + data.append(0) + + return data.base64EncodedString() + } else { + return nil + } + } + + private struct DiskEntry { + static let pathExtension = "storedcachedurlresponse" + + var url: URL + var date: Date + var identifier: String + + init?(_ url: URL) { + if url.pathExtension.caseInsensitiveCompare(DiskEntry.pathExtension) != .orderedSame { + return nil + } + + let parts = url.deletingPathExtension().lastPathComponent.components(separatedBy: ".") + guard parts.count == 2 else { return nil } + let (timeString, identifier) = (parts[0], parts[1]) + + guard let time = Int64(timeString) else { return nil } + + self.date = Date(timeIntervalSinceReferenceDate: TimeInterval(time)) + self.identifier = identifier + self.url = url + } + } + + private func enumerateDiskEntries(includingPropertiesForKeys keys: [URLResourceKey] = [], using block: (DiskEntry, inout Bool) -> Void) { + guard let directory = cacheDirectory else { return } + for url in (try? FileManager.default.contentsOfDirectory(at: directory, includingPropertiesForKeys: keys)) ?? [] { + if let entry = DiskEntry(url) { + var stop = false + block(entry, &stop) + if stop { + return + } + } + } + } + + private func diskEntries(includingPropertiesForKeys keys: [URLResourceKey] = []) -> [DiskEntry] { + var entries: [DiskEntry] = [] + enumerateDiskEntries(includingPropertiesForKeys: keys) { (entry, stop) in + entries.append(entry) + } + return entries + } + + private func diskContentLocators(for request: URLRequest, forCreationAt date: Date? = nil) -> (identifier: String, url: URL)? { + guard let directory = cacheDirectory else { return nil } + guard let identifier = self.identifier(for: request) else { return nil } + + if let date = date { + // Create a new URL, which may or may not exist on disk. + let interval = Int64(date.timeIntervalSinceReferenceDate) + return (identifier, directory.appendingPathComponent("\(interval).\(identifier).\(DiskEntry.pathExtension)")) + } else { + var foundURL: URL? + + enumerateDiskEntries { (entry, stop) in + if entry.identifier == identifier { + foundURL = entry.url + stop = true + } + } + + if let foundURL = foundURL { + return (identifier, foundURL) + } + } + + return nil + } + + private func diskContents(for request: URLRequest) throws -> StoredCachedURLResponse? { + guard let url = diskContentLocators(for: request)?.url else { return nil } + + let data = try Data(contentsOf: url) + return try NSKeyedUnarchiver.unarchivedObject(ofClasses: [StoredCachedURLResponse.self], from: data) as? StoredCachedURLResponse + } /*! @method cachedResponseForRequest: @@ -206,7 +458,23 @@ open class URLCache : NSObject { request, or nil if there is no NSCachedURLResponse stored with the given request. */ - open func cachedResponse(for request: URLRequest) -> CachedURLResponse? { NSUnimplemented() } + open func cachedResponse(for request: URLRequest) -> CachedURLResponse? { + let result = inMemoryCacheLock.performLocked { () -> CachedURLResponse? in + if let identifier = identifier(for: request), + let entry = inMemoryCacheContents[identifier] { + return entry.cachedURLResponse + } else { + return nil + } + } + + if let result = result { + return result + } + + guard let contents = try? diskContents(for: request) else { return nil } + return contents.cachedURLResponse + } /*! @method storeCachedResponse:forRequest: @@ -215,7 +483,56 @@ open class URLCache : NSObject { @param cachedResponse The cached response to store. @param request the NSURLRequest to use as a key for the storage. */ - open func storeCachedResponse(_ cachedResponse: CachedURLResponse, for request: URLRequest) { NSUnimplemented() } + open func storeCachedResponse(_ cachedResponse: CachedURLResponse, for request: URLRequest) { + let inMemory = cachedResponse.storagePolicy == .allowed || cachedResponse.storagePolicy == .allowedInMemoryOnly + let onDisk = cachedResponse.storagePolicy == .allowed + guard inMemory || onDisk else { return } + + guard let identifier = identifier(for: request) else { return } + + // Only create a serialized version if we are writing to disk: + let object = StoredCachedURLResponse(cachedURLResponse: cachedResponse) + let serialized = (onDisk && diskCapacity > 0) ? try? NSKeyedArchiver.archivedData(withRootObject: object, requiringSecureCoding: true) : nil + + let entry = CacheEntry(identifier: identifier, cachedURLResponse: cachedResponse, serializedVersion: serialized) + + if inMemory && entry.cost < memoryCapacity { + inMemoryCacheLock.performLocked { + evictFromMemoryCacheAssumingLockHeld(maximumSize: memoryCapacity - entry.cost) + inMemoryCacheOrder.append(identifier) + inMemoryCacheContents[identifier] = entry + } + } + + if onDisk, let serialized = serialized, entry.cost < diskCapacity { + do { + evictFromDiskCache(maximumSize: diskCapacity - entry.cost) + + let locators = diskContentLocators(for: request, forCreationAt: Date()) + if let newURL = locators?.url { + try serialized.write(to: newURL, options: .atomic) + } + + if let identifier = locators?.identifier { + // Multiple threads and/or processes may be writing the same key at the same time. If writing the contents race for the exact same timestamp, we can't do much about that. (One of the two will exist, due to the .atomic; the other will error out.) But if the timestamps differ, we may end up with duplicate keys on disk. + // If so, best-effort clear all entries except the one with the highest date. + + // Refetch a snapshot of the directory contents from disk; do not trust prior state: + let entriesToRemove = diskEntries().filter { + $0.identifier == identifier + }.sorted { + $1.date < $0.date + }.dropFirst() // Keep the one with the latest date. + + for entry in entriesToRemove { + // Do not interrupt cleanup if one fails. + try? FileManager.default.removeItem(at: entry.url) + } + } + + } catch { /* Best effort -- do not store on error. */ } + } + } /*! @method removeCachedResponseForRequest: @@ -225,20 +542,64 @@ open class URLCache : NSObject { stored with the given request. @param request the NSURLRequest to use as a key for the lookup. */ - open func removeCachedResponse(for request: URLRequest) { NSUnimplemented() } + open func removeCachedResponse(for request: URLRequest) { + guard let identifier = identifier(for: request) else { return } + + inMemoryCacheLock.performLocked { + if inMemoryCacheContents[identifier] != nil { + inMemoryCacheOrder.removeAll(where: { $0 == identifier }) + inMemoryCacheContents.removeValue(forKey: identifier) + } + } + + if let oldURL = diskContentLocators(for: request)?.url { + try? FileManager.default.removeItem(at: oldURL) + } + } /*! @method removeAllCachedResponses @abstract Clears the given cache, removing all NSCachedURLResponse objects that it stores. */ - open func removeAllCachedResponses() { NSUnimplemented() } + open func removeAllCachedResponses() { + inMemoryCacheLock.performLocked { + inMemoryCacheContents = [:] + inMemoryCacheOrder = [] + } + + evictFromDiskCache(maximumSize: 0) + } /*! @method removeCachedResponsesSince: @abstract Clears the given cache of any cached responses since the provided date. */ - open func removeCachedResponses(since date: Date) { NSUnimplemented() } + open func removeCachedResponses(since date: Date) { + inMemoryCacheLock.performLocked { // Memory cache: + var identifiersToRemove: Set = [] + for entry in inMemoryCacheContents { + if entry.value.date > date { + identifiersToRemove.insert(entry.key) + } + } + + for toRemove in identifiersToRemove { + inMemoryCacheContents.removeValue(forKey: toRemove) + } + inMemoryCacheOrder.removeAll { identifiersToRemove.contains($0) } + } + + do { // Disk cache: + let entriesToRemove = diskEntries().filter { + $0.date > date + } + + for entry in entriesToRemove { + try? FileManager.default.removeItem(at: entry.url) + } + } + } /*! @method memoryCapacity @@ -246,7 +607,13 @@ open class URLCache : NSObject { @discussion At the time this call is made, the in-memory cache will truncate its contents to the size given, if necessary. @result The in-memory capacity, measured in bytes, for the receiver. */ - open var memoryCapacity: Int + open var memoryCapacity: Int { + didSet { + inMemoryCacheLock.performLocked { + evictFromMemoryCacheAssumingLockHeld(maximumSize: memoryCapacity) + } + } + } /*! @method diskCapacity @@ -254,7 +621,9 @@ open class URLCache : NSObject { @discussion At the time this call is made, the on-disk cache will truncate its contents to the size given, if necessary. @param diskCapacity the new on-disk capacity, measured in bytes, for the receiver. */ - open var diskCapacity: Int + open var diskCapacity: Int { + didSet { evictFromDiskCache(maximumSize: diskCapacity) } + } /*! @method currentMemoryUsage @@ -264,7 +633,13 @@ open class URLCache : NSObject { usage of the in-memory cache. @result the current usage of the in-memory cache of the receiver. */ - open var currentMemoryUsage: Int { NSUnimplemented() } + open var currentMemoryUsage: Int { + return inMemoryCacheLock.performLocked { + return inMemoryCacheContents.values.reduce(0) { (result, entry) in + return result + entry.cost + } + } + } /*! @method currentDiskUsage @@ -274,11 +649,34 @@ open class URLCache : NSObject { usage of the on-disk cache. @result the current usage of the on-disk cache of the receiver. */ - open var currentDiskUsage: Int { NSUnimplemented() } -} + open var currentDiskUsage: Int { + var total = 0 + enumerateDiskEntries(includingPropertiesForKeys: [.fileSizeKey]) { (entry, stop) in + if let size = (try? entry.url.resourceValues(forKeys: [.fileSizeKey]))?.fileSize { + total += size + } + } + + return total + } -extension URLCache { - public func storeCachedResponse(_ cachedResponse: CachedURLResponse, for dataTask: URLSessionDataTask) { NSUnimplemented() } - public func getCachedResponse(for dataTask: URLSessionDataTask, completionHandler: (CachedURLResponse?) -> Void) { NSUnimplemented() } - public func removeCachedResponse(for dataTask: URLSessionDataTask) { NSUnimplemented() } + open func storeCachedResponse(_ cachedResponse: CachedURLResponse, for dataTask: URLSessionDataTask) { + guard let request = dataTask.currentRequest else { return } + storeCachedResponse(cachedResponse, for: request) + } + + open func getCachedResponse(for dataTask: URLSessionDataTask, completionHandler: @escaping (CachedURLResponse?) -> Void) { + guard let request = dataTask.currentRequest else { + completionHandler(nil) + return + } + DispatchQueue.global(qos: .background).async { + completionHandler(self.cachedResponse(for: request)) + } + } + + open func removeCachedResponse(for dataTask: URLSessionDataTask) { + guard let request = dataTask.currentRequest else { return } + removeCachedResponse(for: request) + } } diff --git a/Foundation/URLProtocol.swift b/Foundation/URLProtocol.swift index b5ac41b603..de0c382877 100644 --- a/Foundation/URLProtocol.swift +++ b/Foundation/URLProtocol.swift @@ -148,7 +148,11 @@ public protocol URLProtocolClient : NSObjectProtocol { func urlProtocol(_ protocol: URLProtocol, didCancel challenge: URLAuthenticationChallenge) } -internal class _ProtocolClient : NSObject { } +internal class _ProtocolClient : NSObject { + var cachePolicy: URLCache.StoragePolicy = .notAllowed + var cacheableData: [Data]? + var cacheableResponse: URLResponse? +} /*! @class NSURLProtocol @@ -308,7 +312,9 @@ open class URLProtocol : NSObject { @result The property stored with the given key, or nil if no property had previously been stored with the given key in the given request. */ - open class func property(forKey key: String, in request: URLRequest) -> Any? { NSUnimplemented() } + open class func property(forKey key: String, in request: URLRequest) -> Any? { + return request.protocolProperties[key] + } /*! @method setProperty:forKey:inRequest: @@ -321,7 +327,9 @@ open class URLProtocol : NSObject { @param key The string to use for the property storage. @param request The request in which to store the property. */ - open class func setProperty(_ value: Any, forKey key: String, in request: NSMutableURLRequest) { NSUnimplemented() } + open class func setProperty(_ value: Any, forKey key: String, in request: NSMutableURLRequest) { + request.protocolProperties[key] = value + } /*! @method removePropertyForKey:inRequest: @@ -332,7 +340,9 @@ open class URLProtocol : NSObject { @param key The key whose value should be removed @param request The request to be modified */ - open class func removeProperty(forKey key: String, in request: NSMutableURLRequest) { NSUnimplemented() } + open class func removeProperty(forKey key: String, in request: NSMutableURLRequest) { + request.protocolProperties.removeValue(forKey: key) + } /*! @method registerClass: @@ -408,7 +418,10 @@ open class URLProtocol : NSObject { _classesLock.unlock() } - open class func canInit(with task: URLSessionTask) -> Bool { NSUnimplemented() } + open class func canInit(with task: URLSessionTask) -> Bool { + guard let request = task.currentRequest else { return false } + return canInit(with: request) + } public required convenience init(task: URLSessionTask, cachedResponse: CachedURLResponse?, client: URLProtocolClient?) { let urlRequest = task.originalRequest self.init(request: urlRequest!, cachedResponse: cachedResponse, client: client) diff --git a/Foundation/URLRequest.swift b/Foundation/URLRequest.swift index 27014bee35..67155e92ce 100644 --- a/Foundation/URLRequest.swift +++ b/Foundation/URLRequest.swift @@ -240,6 +240,15 @@ public struct URLRequest : ReferenceConvertible, Equatable, Hashable { public static func ==(lhs: URLRequest, rhs: URLRequest) -> Bool { return lhs._handle._uncopiedReference().isEqual(rhs._handle._uncopiedReference()) } + + var protocolProperties: [String: Any] { + get { + return _handle.map { $0.protocolProperties } + } + set { + _applyMutation { $0.protocolProperties = newValue } + } + } } extension URLRequest : CustomStringConvertible, CustomDebugStringConvertible, CustomReflectable { diff --git a/Foundation/URLResponse.swift b/Foundation/URLResponse.swift index fa77a94b6f..57e2344987 100644 --- a/Foundation/URLResponse.swift +++ b/Foundation/URLResponse.swift @@ -32,21 +32,21 @@ open class URLResponse : NSObject, NSSecureCoding, NSCopying { preconditionFailure("Unkeyed coding is unsupported.") } - if let encodedUrl = aDecoder.decodeObject(forKey: "NS.url") as? NSURL { - self.url = encodedUrl as URL - } + guard let nsurl = aDecoder.decodeObject(of: NSURL.self, forKey: "NS.url") else { return nil } + self.url = nsurl as URL + - if let encodedMimeType = aDecoder.decodeObject(forKey: "NS.mimeType") as? NSString { - self.mimeType = encodedMimeType as String + if let mimetype = aDecoder.decodeObject(of: NSString.self, forKey: "NS.mimeType") { + self.mimeType = mimetype as String } self.expectedContentLength = aDecoder.decodeInt64(forKey: "NS.expectedContentLength") - if let encodedEncodingName = aDecoder.decodeObject(forKey: "NS.textEncodingName") as? NSString { + if let encodedEncodingName = aDecoder.decodeObject(of: NSString.self, forKey: "NS.textEncodingName") { self.textEncodingName = encodedEncodingName as String } - if let encodedFilename = aDecoder.decodeObject(forKey: "NS.suggestedFilename") as? NSString { + if let encodedFilename = aDecoder.decodeObject(of: NSString.self, forKey: "NS.suggestedFilename") { self.suggestedFilename = encodedFilename as String } } @@ -203,8 +203,8 @@ open class HTTPURLResponse : URLResponse { self.statusCode = aDecoder.decodeInteger(forKey: "NS.statusCode") - if let encodedHeaders = aDecoder.decodeObject(forKey: "NS.allHeaderFields") as? NSDictionary { - self.allHeaderFields = encodedHeaders as! [AnyHashable: Any] + if aDecoder.containsValue(forKey: "NS.allHeaderFields") { + self.allHeaderFields = aDecoder.decodeObject(of: NSDictionary.self, forKey: "NS.allHeaderFields") as! [AnyHashable: Any] } else { self.allHeaderFields = [:] } @@ -216,7 +216,7 @@ open class HTTPURLResponse : URLResponse { super.encode(with: aCoder) //Will fail if .allowsKeyedCoding == false aCoder.encode(self.statusCode, forKey: "NS.statusCode") - aCoder.encode(self.allHeaderFields._bridgeToObjectiveC(), forKey: "NS.allHeaderFields") + aCoder.encode(self.allHeaderFields as NSDictionary, forKey: "NS.allHeaderFields") } diff --git a/Foundation/URLSession/NativeProtocol.swift b/Foundation/URLSession/NativeProtocol.swift index 4d0f035266..f1310145f7 100644 --- a/Foundation/URLSession/NativeProtocol.swift +++ b/Foundation/URLSession/NativeProtocol.swift @@ -382,13 +382,44 @@ internal class _NativeProtocol: URLProtocol, _EasyHandleDelegate { guard let r = task?.originalRequest else { fatalError("Task has no original request.") } - startNewTransfer(with: r) + + // Check if the cached response is good to use: + if let cachedResponse = cachedResponse, canRespondFromCache(using: cachedResponse) { + self.internalState = .fulfillingFromCache(cachedResponse) + task?.workQueue.async { + self.client?.urlProtocol(self, cachedResponseIsValid: cachedResponse) + self.client?.urlProtocol(self, didReceive: cachedResponse.response, cacheStoragePolicy: .notAllowed) + if !cachedResponse.data.isEmpty { + self.client?.urlProtocol(self, didLoad: cachedResponse.data) + } + + self.client?.urlProtocolDidFinishLoading(self) + + self.internalState = .taskCompleted + } + + } else { + startNewTransfer(with: r) + } } if case .transferReady(let transferState) = self.internalState { self.internalState = .transferInProgress(transferState) } } + + func canCache(_ response: CachedURLResponse) -> Bool { + return false + } + + /// Allows a native protocol to process a cached response. If `true` is returned, the protocol will replay the cached response instead of starting a new transfer. The default implementation invalidates the response in the cache and returns `false`. + func canRespondFromCache(using response: CachedURLResponse) -> Bool { + // By default, native protocols do not cache. Aggressively remove unexpected cached responses. + if let cache = task?.session.configuration.urlCache, let task = task as? URLSessionDataTask { + cache.removeCachedResponse(for: task) + } + return false + } func suspend() { if case .transferInProgress(let transferState) = self.internalState { @@ -492,6 +523,8 @@ extension _NativeProtocol { enum _InternalState { /// Task has been created, but nothing has been done, yet case initial + /// The task is being fulfilled from the cache rather than the network. + case fulfillingFromCache(CachedURLResponse) /// The easy handle has been fully configured. But it is not added to /// the multi handle. case transferReady(_TransferState) @@ -531,6 +564,7 @@ extension _NativeProtocol._InternalState { var isEasyHandleAddedToMultiHandle: Bool { switch self { case .initial: return false + case .fulfillingFromCache: return false case .transferReady: return false case .transferInProgress: return true case .transferCompleted: return false @@ -544,6 +578,7 @@ extension _NativeProtocol._InternalState { var isEasyHandlePaused: Bool { switch self { case .initial: return false + case .fulfillingFromCache: return false case .transferReady: return false case .transferInProgress: return false case .transferCompleted: return false diff --git a/Foundation/URLSession/URLSession.swift b/Foundation/URLSession/URLSession.swift index 7b14c457db..d3f17b3a7f 100644 --- a/Foundation/URLSession/URLSession.swift +++ b/Foundation/URLSession/URLSession.swift @@ -629,6 +629,8 @@ internal protocol URLSessionProtocol: class { func add(handle: _EasyHandle) func remove(handle: _EasyHandle) func behaviour(for: URLSessionTask) -> URLSession._TaskBehaviour + var configuration: URLSessionConfiguration { get } + var delegate: URLSessionDelegate? { get } } extension URLSession: URLSessionProtocol { func add(handle: _EasyHandle) { @@ -642,6 +644,12 @@ extension URLSession: URLSessionProtocol { /// /// - SeeAlso: URLSessionTask.init() final internal class _MissingURLSession: URLSessionProtocol { + var delegate: URLSessionDelegate? { + fatalError() + } + var configuration: URLSessionConfiguration { + fatalError() + } func add(handle: _EasyHandle) { fatalError() } diff --git a/Foundation/URLSession/URLSessionConfiguration.swift b/Foundation/URLSession/URLSessionConfiguration.swift index 9ce89de4c6..9ba6bae038 100644 --- a/Foundation/URLSession/URLSessionConfiguration.swift +++ b/Foundation/URLSession/URLSessionConfiguration.swift @@ -39,21 +39,21 @@ open class URLSessionConfiguration : NSObject, NSCopying { // -init is silently incorrect in URLSessionCofiguration on the desktop. Ensure code that relied on swift-corelibs-foundation's init() being functional is redirected to the appropriate cross-platform class property. @available(*, deprecated, message: "Use .default instead.", renamed: "URLSessionConfiguration.default") public override init() { - self.requestCachePolicy = .useProtocolCachePolicy - self.timeoutIntervalForRequest = 60 - self.timeoutIntervalForResource = 604800 - self.networkServiceType = .default - self.allowsCellularAccess = true - self.isDiscretionary = false - self.httpShouldUsePipelining = false - self.httpShouldSetCookies = true - self.httpCookieAcceptPolicy = .onlyFromMainDocumentDomain - self.httpMaximumConnectionsPerHost = 6 - self.httpCookieStorage = HTTPCookieStorage.shared - self.urlCredentialStorage = nil - self.urlCache = nil - self.shouldUseExtendedBackgroundIdleMode = false - self.protocolClasses = [_HTTPURLProtocol.self, _FTPURLProtocol.self] + self.requestCachePolicy = URLSessionConfiguration.default.requestCachePolicy + self.timeoutIntervalForRequest = URLSessionConfiguration.default.timeoutIntervalForRequest + self.timeoutIntervalForResource = URLSessionConfiguration.default.timeoutIntervalForResource + self.networkServiceType = URLSessionConfiguration.default.networkServiceType + self.allowsCellularAccess = URLSessionConfiguration.default.allowsCellularAccess + self.isDiscretionary = URLSessionConfiguration.default.isDiscretionary + self.httpShouldUsePipelining = URLSessionConfiguration.default.httpShouldUsePipelining + self.httpShouldSetCookies = URLSessionConfiguration.default.httpShouldSetCookies + self.httpCookieAcceptPolicy = URLSessionConfiguration.default.httpCookieAcceptPolicy + self.httpMaximumConnectionsPerHost = URLSessionConfiguration.default.httpMaximumConnectionsPerHost + self.httpCookieStorage = URLSessionConfiguration.default.httpCookieStorage + self.urlCredentialStorage = URLSessionConfiguration.default.urlCredentialStorage + self.urlCache = URLSessionConfiguration.default.urlCache + self.shouldUseExtendedBackgroundIdleMode = URLSessionConfiguration.default.shouldUseExtendedBackgroundIdleMode + self.protocolClasses = URLSessionConfiguration.default.protocolClasses super.init() } @@ -73,7 +73,7 @@ open class URLSessionConfiguration : NSObject, NSCopying { httpMaximumConnectionsPerHost: 6, httpCookieStorage: .shared, urlCredentialStorage: nil, // Should be .shared once implemented. - urlCache: nil, + urlCache: .shared, shouldUseExtendedBackgroundIdleMode: false, protocolClasses: [_HTTPURLProtocol.self, _FTPURLProtocol.self]) } @@ -149,10 +149,11 @@ open class URLSessionConfiguration : NSObject, NSCopying { open class var ephemeral: URLSessionConfiguration { // Return a new ephemeral URLSessionConfiguration every time this property is invoked - // TODO: urlCache and urlCredentialStorage should also be ephemeral/in-memory - // URLCache and URLCredentialStorage are still unimplemented + // TODO: urlCredentialStorage should also be ephemeral/in-memory + // URLCredentialStorage is still unimplemented let ephemeralConfiguration = URLSessionConfiguration.default.copy() as! URLSessionConfiguration ephemeralConfiguration.httpCookieStorage = .ephemeralStorage() + ephemeralConfiguration.urlCache = URLCache(memoryCapacity: 4 * 1024 * 1024, diskCapacity: 0, diskPath: nil) return ephemeralConfiguration } diff --git a/Foundation/URLSession/URLSessionTask.swift b/Foundation/URLSession/URLSessionTask.swift index 27d788f077..97fb950335 100644 --- a/Foundation/URLSession/URLSessionTask.swift +++ b/Foundation/URLSession/URLSessionTask.swift @@ -22,6 +22,10 @@ import Foundation #endif import CoreFoundation +private class Bag { + var values: [Element] = [] +} + /// A cancelable object that refers to the lifetime /// of processing a given request. open class URLSessionTask : NSObject, NSCopying { @@ -38,7 +42,101 @@ open class URLSessionTask : NSObject, NSCopying { internal var suspendCount = 1 internal var session: URLSessionProtocol! //change to nil when task completes internal let body: _Body - fileprivate var _protocol: URLProtocol? = nil + + fileprivate enum ProtocolState { + case toBeCreated + case awaitingCacheReply(Bag<(URLProtocol?) -> Void>) + case existing(URLProtocol) + case invalidated + } + + fileprivate let _protocolLock = NSLock() + fileprivate var _protocolStorage: ProtocolState = .toBeCreated + + private var _protocolClass: URLProtocol.Type { + guard let request = currentRequest else { fatalError("A protocol class was requested, but we do not have a current request") } + let protocolClasses = session.configuration.protocolClasses ?? [] + if let urlProtocolClass = URLProtocol.getProtocolClass(protocols: protocolClasses, request: request) { + guard let urlProtocol = urlProtocolClass as? URLProtocol.Type else { fatalError("A protocol class specified in the URLSessionConfiguration's .protocolClasses array was not a URLProtocol subclass: \(urlProtocolClass)") } + return urlProtocol + } else { + let protocolClasses = URLProtocol.getProtocols() ?? [] + if let urlProtocolClass = URLProtocol.getProtocolClass(protocols: protocolClasses, request: request) { + guard let urlProtocol = urlProtocolClass as? URLProtocol.Type else { fatalError("A protocol class registered with URLProtocol.register… was not a URLProtocol subclass: \(urlProtocolClass)") } + return urlProtocol + } + } + + fatalError("Couldn't find a protocol appropriate for request: \(request)") + } + + func _getProtocol(_ callback: @escaping (URLProtocol?) -> Void) { + _protocolLock.lock() // Must be balanced below, before we call out ⬇ + + switch _protocolStorage { + case .toBeCreated: + if let cache = session.configuration.urlCache, let me = self as? URLSessionDataTask { + let bag: Bag<(URLProtocol?) -> Void> = Bag() + bag.values.append(callback) + + _protocolStorage = .awaitingCacheReply(bag) + _protocolLock.unlock() // Balances above ⬆ + + cache.getCachedResponse(for: me) { (response) in + let urlProtocol = self._protocolClass.init(task: self, cachedResponse: response, client: nil) + self._satisfyProtocolRequest(with: urlProtocol) + } + } else { + let urlProtocol = _protocolClass.init(task: self, cachedResponse: nil, client: nil) + _protocolStorage = .existing(urlProtocol) + _protocolLock.unlock() // Balances above ⬆ + + callback(urlProtocol) + } + + case .awaitingCacheReply(let bag): + bag.values.append(callback) + _protocolLock.unlock() // Balances above ⬆ + + case .existing(let urlProtocol): + _protocolLock.unlock() // Balances above ⬆ + + callback(urlProtocol) + + case .invalidated: + _protocolLock.unlock() // Balances above ⬆ + + callback(nil) + } + } + + func _satisfyProtocolRequest(with urlProtocol: URLProtocol) { + _protocolLock.lock() // Must be balanced below, before we call out ⬇ + switch _protocolStorage { + case .toBeCreated: + _protocolStorage = .existing(urlProtocol) + _protocolLock.unlock() // Balances above ⬆ + + case .awaitingCacheReply(let bag): + _protocolStorage = .existing(urlProtocol) + _protocolLock.unlock() // Balances above ⬆ + + for callback in bag.values { + callback(urlProtocol) + } + + case .existing(_): fallthrough + case .invalidated: + _protocolLock.unlock() // Balances above ⬆ + } + } + + func _invalidateProtocol() { + _protocolLock.performLocked { + _protocolStorage = .invalidated + } + } + private let syncQ = DispatchQueue(label: "org.swift.URLSessionTask.SyncQ") private var hasTriggeredResume: Bool = false internal var isSuspendedAfterResume: Bool { @@ -80,25 +178,7 @@ open class URLSessionTask : NSObject, NSCopying { self.originalRequest = request self.body = body super.init() - if session.configuration.protocolClasses != nil { - guard let protocolClasses = session.configuration.protocolClasses else { fatalError() } - if let urlProtocolClass = URLProtocol.getProtocolClass(protocols: protocolClasses, request: request) { - guard let urlProtocol = urlProtocolClass as? URLProtocol.Type else { fatalError() } - self._protocol = urlProtocol.init(task: self, cachedResponse: nil, client: nil) - } else { - guard let protocolClasses = URLProtocol.getProtocols() else { fatalError() } - if let urlProtocolClass = URLProtocol.getProtocolClass(protocols: protocolClasses, request: request) { - guard let urlProtocol = urlProtocolClass as? URLProtocol.Type else { fatalError() } - self._protocol = urlProtocol.init(task: self, cachedResponse: nil, client: nil) - } - } - } else { - guard let protocolClasses = URLProtocol.getProtocols() else { fatalError() } - if let urlProtocolClass = URLProtocol.getProtocolClass(protocols: protocolClasses, request: request) { - guard let urlProtocol = urlProtocolClass as? URLProtocol.Type else { fatalError() } - self._protocol = urlProtocol.init(task: self, cachedResponse: nil, client: nil) - } - } + self.currentRequest = request } deinit { //TODO: Do we remove the EasyHandle from the session here? This might run on the wrong thread / queue. @@ -193,11 +273,15 @@ open class URLSessionTask : NSObject, NSCopying { workQueue.sync { guard self.state == .running || self.state == .suspended else { return } self.state = .canceling - self.workQueue.async { - let urlError = URLError(_nsError: NSError(domain: NSURLErrorDomain, code: NSURLErrorCancelled, userInfo: nil)) - self.error = urlError - self._protocol?.stopLoading() - self._protocol?.client?.urlProtocol(self._protocol!, didFailWithError: urlError) + self._getProtocol { (urlProtocol) in + self.workQueue.async { + let urlError = URLError(_nsError: NSError(domain: NSURLErrorDomain, code: NSURLErrorCancelled, userInfo: nil)) + self.error = urlError + if let urlProtocol = urlProtocol { + urlProtocol.stopLoading() + urlProtocol.client?.urlProtocol(urlProtocol, didFailWithError: urlError) + } + } } } } @@ -252,8 +336,10 @@ open class URLSessionTask : NSObject, NSCopying { self.updateTaskState() if self.suspendCount == 1 { - self.workQueue.async { - self._protocol?.stopLoading() + self._getProtocol { (urlProtocol) in + self.workQueue.async { + urlProtocol?.stopLoading() + } } } } @@ -268,21 +354,23 @@ open class URLSessionTask : NSObject, NSCopying { self.updateTaskState() if self.suspendCount == 0 { self.hasTriggeredResume = true - self.workQueue.async { - if let _protocol = self._protocol { - _protocol.startLoading() - } - else if self.error == nil { - var userInfo: [String: Any] = [NSLocalizedDescriptionKey: "unsupported URL"] - if let url = self.originalRequest?.url { - userInfo[NSURLErrorFailingURLErrorKey] = url - userInfo[NSURLErrorFailingURLStringErrorKey] = url.absoluteString + self._getProtocol { (urlProtocol) in + self.workQueue.async { + if let _protocol = urlProtocol { + _protocol.startLoading() + } + else if self.error == nil { + var userInfo: [String: Any] = [NSLocalizedDescriptionKey: "unsupported URL"] + if let url = self.originalRequest?.url { + userInfo[NSURLErrorFailingURLErrorKey] = url + userInfo[NSURLErrorFailingURLStringErrorKey] = url.absoluteString + } + let urlError = URLError(_nsError: NSError(domain: NSURLErrorDomain, + code: NSURLErrorUnsupportedURL, + userInfo: userInfo)) + self.error = urlError + _ProtocolClient().urlProtocol(task: self, didFailWithError: urlError) } - let urlError = URLError(_nsError: NSError(domain: NSURLErrorDomain, - code: NSURLErrorUnsupportedURL, - userInfo: userInfo)) - self.error = urlError - _ProtocolClient().urlProtocol(task: self, didFailWithError: urlError) } } } @@ -546,6 +634,22 @@ extension _ProtocolClient : URLProtocolClient { task.response = response let session = task.session as! URLSession guard let dataTask = task as? URLSessionDataTask else { return } + + // Only cache data tasks: + self.cachePolicy = policy + + if session.configuration.urlCache != nil { + switch policy { + case .allowed: fallthrough + case .allowedInMemoryOnly: + cacheableData = [] + cacheableResponse = response + + case .notAllowed: + break + } + } + switch session.behaviour(for: task) { case .taskDelegate(let delegate as URLSessionDataDelegate): session.delegateQueue.addOperation { @@ -572,8 +676,8 @@ extension _ProtocolClient : URLProtocolClient { } } - func urlProtocolDidFinishLoading(_ protocol: URLProtocol) { - guard let task = `protocol`.task else { fatalError() } + func urlProtocolDidFinishLoading(_ urlProtocol: URLProtocol) { + guard let task = urlProtocol.task else { fatalError() } guard let session = task.session as? URLSession else { fatalError() } guard let urlResponse = task.response else { fatalError("No response") } if let response = urlResponse as? HTTPURLResponse, response.statusCode == 401 { @@ -581,17 +685,38 @@ extension _ProtocolClient : URLProtocolClient { //TODO: Fetch and set proposed credentials if they exist let authenticationChallenge = URLAuthenticationChallenge(protectionSpace: protectionSpace, proposedCredential: nil, previousFailureCount: task.previousFailureCount, failureResponse: response, error: nil, - sender: `protocol` as! _HTTPURLProtocol) + sender: urlProtocol as! _HTTPURLProtocol) task.previousFailureCount += 1 - urlProtocol(`protocol`, didReceive: authenticationChallenge) + self.urlProtocol(urlProtocol, didReceive: authenticationChallenge) return } } + + if let cache = session.configuration.urlCache, + let data = cacheableData, + let response = cacheableResponse, + let task = task as? URLSessionDataTask { + + let cacheable = CachedURLResponse(response: response, data: Data(data.joined()), storagePolicy: cachePolicy) + let protocolAllows = (urlProtocol as? _NativeProtocol)?.canCache(cacheable) ?? false + if protocolAllows { + if let delegate = task.session.delegate as? URLSessionDataDelegate { + delegate.urlSession(task.session as! URLSession, dataTask: task, willCacheResponse: cacheable) { (actualCacheable) in + if let actualCacheable = actualCacheable { + cache.storeCachedResponse(actualCacheable, for: task) + } + } + } else { + cache.storeCachedResponse(cacheable, for: task) + } + } + } + switch session.behaviour(for: task) { case .taskDelegate(let delegate): if let downloadDelegate = delegate as? URLSessionDownloadDelegate, let downloadTask = task as? URLSessionDownloadTask { session.delegateQueue.addOperation { - downloadDelegate.urlSession(session, downloadTask: downloadTask, didFinishDownloadingTo: `protocol`.properties[URLProtocol._PropertyKey.temporaryFileURL] as! URL) + downloadDelegate.urlSession(session, downloadTask: downloadTask, didFinishDownloadingTo: urlProtocol.properties[URLProtocol._PropertyKey.temporaryFileURL] as! URL) } } session.delegateQueue.addOperation { @@ -608,7 +733,7 @@ extension _ProtocolClient : URLProtocolClient { } case .dataCompletionHandler(let completion): session.delegateQueue.addOperation { - completion(`protocol`.properties[URLProtocol._PropertyKey.responseData] as? Data ?? Data(), task.response, nil) + completion(urlProtocol.properties[URLProtocol._PropertyKey.responseData] as? Data ?? Data(), task.response, nil) task.state = .completed session.workQueue.async { session.taskRegistry.remove(task) @@ -616,14 +741,14 @@ extension _ProtocolClient : URLProtocolClient { } case .downloadCompletionHandler(let completion): session.delegateQueue.addOperation { - completion(`protocol`.properties[URLProtocol._PropertyKey.temporaryFileURL] as? URL, task.response, nil) + completion(urlProtocol.properties[URLProtocol._PropertyKey.temporaryFileURL] as? URL, task.response, nil) task.state = .completed session.workQueue.async { session.taskRegistry.remove(task) } } } - task._protocol = nil + task._invalidateProtocol() } func urlProtocol(_ protocol: URLProtocol, didCancel challenge: URLAuthenticationChallenge) { @@ -643,7 +768,11 @@ extension _ProtocolClient : URLProtocolClient { fatalError("\(authScheme) is not supported") } handler(task, disposition, credential) - task._protocol = _HTTPURLProtocol(task: task, cachedResponse: nil, client: nil) + + task._protocolLock.performLocked { + task._protocolStorage = .existing(_HTTPURLProtocol(task: task, cachedResponse: nil, client: nil)) + } + task.resume() } } @@ -655,6 +784,16 @@ extension _ProtocolClient : URLProtocolClient { `protocol`.properties[.responseData] = data guard let task = `protocol`.task else { fatalError() } guard let session = task.session as? URLSession else { fatalError() } + + switch cachePolicy { + case .allowed: fallthrough + case .allowedInMemoryOnly: + cacheableData?.append(data) + + case .notAllowed: + break + } + switch session.behaviour(for: task) { case .taskDelegate(let delegate): let dataDelegate = delegate as? URLSessionDataDelegate @@ -704,12 +843,10 @@ extension _ProtocolClient : URLProtocolClient { } } } - task._protocol = nil + task._invalidateProtocol() } - func urlProtocol(_ protocol: URLProtocol, cachedResponseIsValid cachedResponse: CachedURLResponse) { - NSUnimplemented() - } + func urlProtocol(_ protocol: URLProtocol, cachedResponseIsValid cachedResponse: CachedURLResponse) {} func urlProtocol(_ protocol: URLProtocol, wasRedirectedTo request: URLRequest, redirectResponse: URLResponse) { NSUnimplemented() diff --git a/Foundation/URLSession/http/HTTPURLProtocol.swift b/Foundation/URLSession/http/HTTPURLProtocol.swift index b281086de3..26b5993b2e 100644 --- a/Foundation/URLSession/http/HTTPURLProtocol.swift +++ b/Foundation/URLSession/http/HTTPURLProtocol.swift @@ -60,6 +60,202 @@ internal class _HTTPURLProtocol: _NativeProtocol { return .abort } } + + // This implements a RFC 7234 cache with the following: + // - this is a private cache; + // - we conform to the specification for the Vary header by not caching responses that have a Vary header. + // - we do not implement a concept of staleness; we are unwilling to serve stale requests from the cache, and we are aggressively going to prune any storage that is used by stored stale data. + + struct CacheControlDirectives { + var maxAge: UInt? + var sharedMaxAge: UInt? + var noCache: Bool = false + var noStore: Bool = false + + init(headerValue: String) { + func isWithArgument(_ part: String, named: String, converter: (String) -> T?) -> T? { + if part.hasPrefix("\(named)=") { + let split = part.components(separatedBy: "=") + if split.count == 2 { + let argument = split[1] + if argument.first == "\"" && argument.last == "\"" { + if argument.count >= 2 { + return converter(String(argument[argument.index(after: argument.startIndex) ..< argument.index(before: argument.endIndex)])) + } else { + return nil + } + } else { + return converter(argument) + } + } + } + + return nil + } + + let parts = headerValue.components(separatedBy: ",").map { $0.trimmingCharacters(in: .whitespaces).lowercased(with: NSLocale.system) } + + for part in parts { + if part == "no-cache" { + noCache = true + } else if part == "no-store" { + noStore = true + } else if let maxAge = isWithArgument(part, named: "max-age", converter: { UInt($0) }) { + self.maxAge = maxAge + } else if let sharedMaxAge = isWithArgument(part, named: "s-maxage", converter: { UInt($0) }) { + self.sharedMaxAge = sharedMaxAge + } + } + } + } + + override func canCache(_ response: CachedURLResponse) -> Bool { + guard let httpRequest = task?.currentRequest else { return false } + guard let httpResponse = response.response as? HTTPURLResponse else { return false } + + let now = Date() + + // Figure out the date we should start counting expiration from. + let expirationStart: Date + + if let dateString = httpResponse.allHeaderFields["Date"] as? String, + let date = _HTTPURLProtocol.dateFormatter.date(from: dateString) { + expirationStart = min(date, response.date) // Do not accept a date in the future of the point where we stored it, or of now if we haven't stored it yet. That is: a Date header can only make a response expire _faster_ than if it was issued now, and can't be used to prolong its age. + } else { + expirationStart = response.date + } + + // We opt not to cache any requests or responses that contain authorization headers. + if httpResponse.allHeaderFields["WWW-Authenticate"] != nil || + httpResponse.allHeaderFields["Proxy-Authenticate"] != nil || + httpRequest.allHTTPHeaderFields?["Authorization"] != nil || + httpRequest.allHTTPHeaderFields?["Proxy-Authorization"] != nil { + return false + } + + // HTTP Methods: https://tools.ietf.org/html/rfc7231#section-4.2.3 + switch httpRequest.httpMethod { + case "GET": + break + case "HEAD": + if response.data.isEmpty { + break + } else { + return false + } + default: + return false + } + + // Cache-Control: https://tools.ietf.org/html/rfc7234#section-5.2 + var hasCacheControl = false + var hasMaxAge = false + if let cacheControl = httpResponse.allHeaderFields["Cache-Control"] as? String { + let directives = CacheControlDirectives(headerValue: cacheControl) + + if directives.noCache || directives.noStore { + return false + } + + // We should not cache a response that has already expired. (This is also the expiration check for canRespondFromCaching(using:) below.) + if let maxAge = directives.maxAge { + hasMaxAge = true + + let expiration = expirationStart + TimeInterval(maxAge) + if now >= expiration { + // Do not cache an expired response. + return false + } + } + + // This is not a shared cache, but per , + // if a response has Cache-Control: s-maxage="…" set, we MUST ignore the Expires field below. + if directives.sharedMaxAge != nil { + hasMaxAge = true + } + + hasCacheControl = true + } + + // Pragma: https://tools.ietf.org/html/rfc7234#section-5.4 + if !hasCacheControl, let pragma = httpResponse.allHeaderFields["Cache-Control"] as? String { + let parts = pragma.components(separatedBy: ",").map { $0.trimmingCharacters(in: .whitespaces).lowercased(with: NSLocale.system) } + if parts.contains("no-cache") { + return false + } + } + + // HTTP status codes: https://tools.ietf.org/html/rfc7231#section-6.1 + switch httpResponse.statusCode { + case 200: fallthrough + case 203: fallthrough + case 204: fallthrough + case 206: fallthrough + case 300: fallthrough + case 301: fallthrough + case 404: fallthrough + case 405: fallthrough + case 410: fallthrough + case 414: fallthrough + case 501: + break + + default: + return false + } + + // Vary: https://tools.ietf.org/html/rfc7231#section-7.1.4 + /* "1. To inform cache recipients that they MUST NOT use this response + to satisfy a later request unless the later request has the same + values for the listed fields as the original request (Section 4.1 + of [RFC7234]). In other words, Vary expands the cache key + required to match a new request to the stored cache entry." + + If we do not store this response, we will never use it to satisfy a later request, including a later request for which it would be incorrect. + */ + if httpResponse.allHeaderFields["Vary"] != nil { + return false + } + + // Expires: + // We should not cache a response that has already expired. (This is also the expiration check for canRespondFromCaching(using:) below.) + // We MUST ignore this if we have Cache-Control: max-age or s-maxage. + if !hasMaxAge, let expires = httpResponse.allHeaderFields["Expires"] as? String { + guard let expiration = _HTTPURLProtocol.dateFormatter.date(from: expires) else { + // From the spec: + /* "A cache recipient MUST interpret invalid date formats, especially the + value "0", as representing a time in the past (i.e., 'already + expired')." + */ + return false + } + + if now >= expiration { + // Do not cache an expired response. + return false + } + } + + return true + } + + static let dateFormatter: DateFormatter = { + let x = DateFormatter() + x.locale = NSLocale.system + x.dateFormat = "EEE',' dd' 'MMM' 'yyyy HH':'mm':'ss zzz" + return x + }() + + override func canRespondFromCache(using response: CachedURLResponse) -> Bool { + // If somehow cached a response that shouldn't have been, we should remove it. + guard canCache(response) else { + // Calling super removes it from the cache and returns false, which is the default. + return super.canRespondFromCache(using: response) + } + + // Expiration checks are done in canCache(…). + return true + } /// Set options on the easy handle to match the given request. /// diff --git a/TestFoundation/TestURLCache.swift b/TestFoundation/TestURLCache.swift new file mode 100644 index 0000000000..11864f2e2b --- /dev/null +++ b/TestFoundation/TestURLCache.swift @@ -0,0 +1,297 @@ +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2019 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// + +class TestURLCache : XCTestCase { + + let aBit = 2 * 1024 /* 2 KB */ + let lots = 200 * 1024 * 1024 /* 200 MB */ + + func testStorageRoundtrip() throws { + let cache = try self.cache(memoryCapacity: lots, diskCapacity: lots) + + let (request, response) = try cachePair(for: "https://google.com/", ofSize: aBit, storagePolicy: .allowed) + cache.storeCachedResponse(response, for: request) + + let storedResponse = cache.cachedResponse(for: request) + XCTAssertEqual(response, storedResponse) + } + + func testStoragePolicy() throws { + do { + let cache = try self.cache(memoryCapacity: lots, diskCapacity: lots) + + let (request, response) = try cachePair(for: "https://google.com/", ofSize: aBit, storagePolicy: .allowed) + cache.storeCachedResponse(response, for: request) + + XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: writableTestDirectoryURL.path).count, 1) + XCTAssertNotNil(cache.cachedResponse(for: request)) + } + + try FileManager.default.removeItem(at: writableTestDirectoryURL) + + do { + let cache = try self.cache(memoryCapacity: lots, diskCapacity: lots) + + let (request, response) = try cachePair(for: "https://google.com/", ofSize: aBit, storagePolicy: .allowedInMemoryOnly) + cache.storeCachedResponse(response, for: request) + + XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: writableTestDirectoryURL.path).count, 0) + XCTAssertNotNil(cache.cachedResponse(for: request)) + } + + try FileManager.default.removeItem(at: writableTestDirectoryURL) + + do { + let cache = try self.cache(memoryCapacity: lots, diskCapacity: lots) + + let (request, response) = try cachePair(for: "https://google.com/", ofSize: aBit, storagePolicy: .notAllowed) + cache.storeCachedResponse(response, for: request) + + XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: writableTestDirectoryURL.path).count, 0) + XCTAssertNil(cache.cachedResponse(for: request)) + } + + try FileManager.default.removeItem(at: writableTestDirectoryURL) + } + + func testNoDiskUsageIfDisabled() throws { + let cache = try self.cache(memoryCapacity: lots, diskCapacity: 0) + let (request, response) = try cachePair(for: "https://google.com/", ofSize: aBit) + cache.storeCachedResponse(response, for: request) + + XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: writableTestDirectoryURL.path).count, 0) + XCTAssertNotNil(cache.cachedResponse(for: request)) + } + + func testShrinkingDiskCapacityEvictsItems() throws { + let cache = try self.cache(memoryCapacity: lots, diskCapacity: lots) + + let urls = [ "https://apple.com/", + "https://google.com/", + "https://facebook.com/" ] + + for (request, response) in try urls.map({ try cachePair(for: $0, ofSize: aBit) }) { + cache.storeCachedResponse(response, for: request) + } + + XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: writableTestDirectoryURL.path).count, 3) + for url in urls { + XCTAssertNotNil(cache.cachedResponse(for: URLRequest(url: URL(string: url)!))) + } + + cache.diskCapacity = 0 + XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: writableTestDirectoryURL.path).count, 0) + for url in urls { + XCTAssertNotNil(cache.cachedResponse(for: URLRequest(url: URL(string: url)!))) + } + } + + func testNoMemoryUsageIfDisabled() throws { + let cache = try self.cache(memoryCapacity: 0, diskCapacity: lots) + let (request, response) = try cachePair(for: "https://google.com/", ofSize: aBit) + cache.storeCachedResponse(response, for: request) + + XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: writableTestDirectoryURL.path).count, 1) + XCTAssertNotNil(cache.cachedResponse(for: request)) + + // Ensure that the fulfillment doesn't come from memory: + try FileManager.default.removeItem(at: writableTestDirectoryURL) + try FileManager.default.createDirectory(at: writableTestDirectoryURL, withIntermediateDirectories: true) + + XCTAssertNil(cache.cachedResponse(for: request)) + } + + func testShrinkingMemoryCapacityEvictsItems() throws { + let cache = try self.cache(memoryCapacity: lots, diskCapacity: lots) + + let urls = [ "https://apple.com/", + "https://google.com/", + "https://facebook.com/" ] + + for (request, response) in try urls.map({ try cachePair(for: $0, ofSize: aBit) }) { + cache.storeCachedResponse(response, for: request) + } + + // Ensure these can be fulfilled from memory: + try FileManager.default.removeItem(at: writableTestDirectoryURL) + try FileManager.default.createDirectory(at: writableTestDirectoryURL, withIntermediateDirectories: true) + + for url in urls { + XCTAssertNotNil(cache.cachedResponse(for: URLRequest(url: URL(string: url)!))) + } + + // And evict all: + cache.memoryCapacity = 0 + + for url in urls { + XCTAssertNil(cache.cachedResponse(for: URLRequest(url: URL(string: url)!))) + } + } + + func testRemovingOne() throws { + let cache = try self.cache(memoryCapacity: lots, diskCapacity: lots) + + let urls = [ "https://apple.com/", + "https://google.com/", + "https://facebook.com/" ] + + for (request, response) in try urls.map({ try cachePair(for: $0, ofSize: aBit) }) { + cache.storeCachedResponse(response, for: request) + } + + let request = URLRequest(url: URL(string: urls[0])!) + cache.removeCachedResponse(for: request) + + XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: writableTestDirectoryURL.path).count, 2) + + var first = true + for request in urls.map({ URLRequest(url: URL(string: $0)!) }) { + if first { + XCTAssertNil(cache.cachedResponse(for: request)) + } else { + XCTAssertNotNil(cache.cachedResponse(for: request)) + } + + first = false + } + } + + func testRemovingAll() throws { + let cache = try self.cache(memoryCapacity: lots, diskCapacity: lots) + + let urls = [ "https://apple.com/", + "https://google.com/", + "https://facebook.com/" ] + + for (request, response) in try urls.map({ try cachePair(for: $0, ofSize: aBit) }) { + cache.storeCachedResponse(response, for: request) + } + + XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: writableTestDirectoryURL.path).count, 3) + + cache.removeAllCachedResponses() + + XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: writableTestDirectoryURL.path).count, 0) + + for request in urls.map({ URLRequest(url: URL(string: $0)!) }) { + XCTAssertNil(cache.cachedResponse(for: request)) + } + } + + func testRemovingSince() throws { + let cache = try self.cache(memoryCapacity: lots, diskCapacity: lots) + + let urls = [ "https://apple.com/", + "https://google.com/", + "https://facebook.com/" ] + + var first = true + for (request, response) in try urls.map({ try cachePair(for: $0, ofSize: aBit) }) { + cache.storeCachedResponse(response, for: request) + if first { + Thread.sleep(forTimeInterval: 5.0) + first = false + } + } + + cache.removeCachedResponses(since: Date(timeIntervalSinceNow: -3.5)) + + XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: writableTestDirectoryURL.path).count, 1) + + first = true + for request in urls.map({ URLRequest(url: URL(string: $0)!) }) { + if first { + XCTAssertNotNil(cache.cachedResponse(for: request)) + } else { + XCTAssertNil(cache.cachedResponse(for: request)) + } + + first = false + } + } + + func testStoringTwiceOnlyHasOneEntry() throws { + let cache = try self.cache(memoryCapacity: lots, diskCapacity: lots) + + let url = "https://apple.com/" + let (requestA, responseA) = try cachePair(for: url, ofSize: aBit, startingWith: 1) + cache.storeCachedResponse(responseA, for: requestA) + + Thread.sleep(forTimeInterval: 3.0) // Enough to make the timestamp move forward. + + let (requestB, responseB) = try cachePair(for: url, ofSize: aBit, startingWith: 2) + cache.storeCachedResponse(responseB, for: requestB) + + XCTAssertEqual(try FileManager.default.contentsOfDirectory(atPath: writableTestDirectoryURL.path).count, 1) + + let response = cache.cachedResponse(for: requestB) + XCTAssertNotNil(response) + XCTAssertEqual((try response.unwrapped()).data, responseB.data) + } + + // ----- + + static var allTests: [(String, (TestURLCache) -> () throws -> Void)] { + return [ + ("testStorageRoundtrip", testStorageRoundtrip), + ("testStoragePolicy", testStoragePolicy), + ("testNoDiskUsageIfDisabled", testNoDiskUsageIfDisabled), + ("testShrinkingDiskCapacityEvictsItems", testShrinkingDiskCapacityEvictsItems), + ("testNoMemoryUsageIfDisabled", testNoMemoryUsageIfDisabled), + ("testShrinkingMemoryCapacityEvictsItems", testShrinkingMemoryCapacityEvictsItems), + ("testRemovingOne", testRemovingOne), + ("testRemovingAll", testRemovingAll), + ("testRemovingSince", testRemovingSince), + ("testStoringTwiceOnlyHasOneEntry", testStoringTwiceOnlyHasOneEntry), + ] + } + + // ----- + + func cache(memoryCapacity: Int = 0, diskCapacity: Int = 0) throws -> URLCache { + try FileManager.default.createDirectory(at: writableTestDirectoryURL, withIntermediateDirectories: true) + return URLCache(memoryCapacity: memoryCapacity, diskCapacity: diskCapacity, diskPath: writableTestDirectoryURL.path) + } + + func cachePair(for urlString: String, ofSize size: Int, storagePolicy: URLCache.StoragePolicy = .allowed, startingWith: UInt8 = 0) throws -> (URLRequest, CachedURLResponse) { + let url = try URL(string: urlString).unwrapped() + let request = URLRequest(url: url) + let response = try HTTPURLResponse(url: url, statusCode: 200, httpVersion: "1.1", headerFields: [:]).unwrapped() + + var data = Data(count: size) + if data.count > 0 { + data[0] = startingWith + } + + return (request, CachedURLResponse(response: response, data: data, storagePolicy: storagePolicy)) + } + + var writableTestDirectoryURL: URL! + + override func setUp() { + super.setUp() + + let pid = ProcessInfo.processInfo.processIdentifier + writableTestDirectoryURL = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent("org.swift.TestFoundation.TestURLCache.\(pid)") + } + + override func tearDown() { + if let directoryURL = writableTestDirectoryURL, + (try? FileManager.default.attributesOfItem(atPath: directoryURL.path)) != nil { + do { + try FileManager.default.removeItem(at: directoryURL) + } catch { + NSLog("Could not remove test directory at URL \(directoryURL): \(error)") + } + } + + super.tearDown() + } + +} diff --git a/TestFoundation/main.swift b/TestFoundation/main.swift index 7af6609999..7dba39d35d 100644 --- a/TestFoundation/main.swift +++ b/TestFoundation/main.swift @@ -82,6 +82,7 @@ var allTestCases = [ testCase(TestTimer.allTests), testCase(TestTimeZone.allTests), testCase(TestURL.allTests), + testCase(TestURLCache.allTests), testCase(TestURLComponents.allTests), testCase(TestURLCredential.allTests), testCase(TestURLProtectionSpace.allTests),