From 73923a2dd17b4d4b4682150fbce55f53155f1db9 Mon Sep 17 00:00:00 2001 From: Karthik Date: Fri, 8 Mar 2019 13:16:10 -0800 Subject: [PATCH 1/6] URLCache: init method and first time sqlite database setup implemented * init method implemented * URLCache.shared singleton object created with 4MB of memory space and 20MB of disk space * Directory and database file Cache.db file created under local directory * Sqlite Tables and Indices created in database * Unit tests added for URLCache to verify directory, file, tables and indices --- Foundation.xcodeproj/project.pbxproj | 4 + Foundation/URLCache.swift | 167 ++++++++++++++++++++++++++- TestFoundation/TestURLCache.swift | 91 +++++++++++++++ TestFoundation/main.swift | 1 + 4 files changed, 260 insertions(+), 3 deletions(-) create mode 100644 TestFoundation/TestURLCache.swift diff --git a/Foundation.xcodeproj/project.pbxproj b/Foundation.xcodeproj/project.pbxproj index 70d733b68e..6ec28e2c21 100644 --- a/Foundation.xcodeproj/project.pbxproj +++ b/Foundation.xcodeproj/project.pbxproj @@ -74,6 +74,7 @@ 15FF00CC22934AD7004AD205 /* libCFURLSessionInterface.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 15FF00CA229348F2004AD205 /* libCFURLSessionInterface.a */; }; 15FF00CE22934B78004AD205 /* module.map in Headers */ = {isa = PBXBuildFile; fileRef = 15FF00CD22934B49004AD205 /* module.map */; settings = {ATTRIBUTES = (Public, ); }; }; 231503DB1D8AEE5D0061694D /* TestDecimal.swift in Sources */ = {isa = PBXBuildFile; fileRef = 231503DA1D8AEE5D0061694D /* TestDecimal.swift */; }; + 25EB1806223334D30053EE59 /* TestURLCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = 25EB1805223334D30053EE59 /* TestURLCache.swift */; }; 294E3C1D1CC5E19300E4F44C /* TestNSAttributedString.swift in Sources */ = {isa = PBXBuildFile; fileRef = 294E3C1C1CC5E19300E4F44C /* TestNSAttributedString.swift */; }; 2EBE67A51C77BF0E006583D5 /* TestDateFormatter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2EBE67A31C77BF05006583D5 /* TestDateFormatter.swift */; }; 3E55A2331F52463B00082000 /* TestUnit.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3E55A2321F52463B00082000 /* TestUnit.swift */; }; @@ -646,6 +647,7 @@ 15FF00CD22934B49004AD205 /* module.map */ = {isa = PBXFileReference; lastKnownFileType = "sourcecode.module-map"; name = module.map; path = CoreFoundation/URL.subproj/module.map; sourceTree = SOURCE_ROOT; }; 22B9C1E01C165D7A00DECFF9 /* TestDate.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = TestDate.swift; sourceTree = ""; }; 231503DA1D8AEE5D0061694D /* TestDecimal.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = TestDecimal.swift; sourceTree = ""; }; + 25EB1805223334D30053EE59 /* TestURLCache.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TestURLCache.swift; sourceTree = ""; }; 294E3C1C1CC5E19300E4F44C /* TestNSAttributedString.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = TestNSAttributedString.swift; sourceTree = ""; }; 2EBE67A31C77BF05006583D5 /* TestDateFormatter.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = TestDateFormatter.swift; sourceTree = ""; }; 3E55A2321F52463B00082000 /* TestUnit.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TestUnit.swift; sourceTree = ""; }; @@ -1777,6 +1779,7 @@ 03B6F5831F15F339004F25AF /* TestURLProtocol.swift */, 3E55A2321F52463B00082000 /* TestUnit.swift */, 7D8BD738225ED1480057CF37 /* TestMeasurement.swift */, + 25EB1805223334D30053EE59 /* TestURLCache.swift */, ); name = Tests; sourceTree = ""; @@ -2834,6 +2837,7 @@ 5B13B33E1C582D4C00651CE2 /* TestProcessInfo.swift in Sources */, 5B13B33F1C582D4C00651CE2 /* TestPropertyListSerialization.swift in Sources */, 5B13B32C1C582D4C00651CE2 /* TestDate.swift in Sources */, + 25EB1806223334D30053EE59 /* TestURLCache.swift in Sources */, C7DE1FCC21EEE67200174F35 /* TestUUID.swift in Sources */, 231503DB1D8AEE5D0061694D /* TestDecimal.swift in Sources */, 7900433C1CACD33E00ECCBF1 /* TestNSPredicate.swift in Sources */, diff --git a/Foundation/URLCache.swift b/Foundation/URLCache.swift index 7afb0c2432..22a0f824bd 100644 --- a/Foundation/URLCache.swift +++ b/Foundation/URLCache.swift @@ -12,6 +12,7 @@ import SwiftFoundation #else import Foundation #endif +import SQLite3 /*! @enum URLCache.StoragePolicy @@ -128,6 +129,26 @@ open class CachedURLResponse : NSObject, NSSecureCoding, NSCopying { open class URLCache : NSObject { + private static let sharedSyncQ = DispatchQueue(label: "org.swift.URLCache.sharedSyncQ") + + private static var sharedCache: URLCache? { + willSet { + URLCache.sharedCache?.syncQ.sync { + URLCache.sharedCache?._databaseClient?.close() + URLCache.sharedCache?.flushDatabase() + } + } + didSet { + URLCache.sharedCache?.syncQ.sync { + URLCache.sharedCache?.setupCacheDatabaseIfNotExist() + } + } + } + + private let syncQ = DispatchQueue(label: "org.swift.URLCache.syncQ") + private let _baseDiskPath: String? + private var _databaseClient: _CacheSQLiteClient? + /*! @method sharedURLCache @abstract Returns the shared URLCache instance. @@ -147,10 +168,22 @@ open class URLCache : NSObject { */ open class var shared: URLCache { get { - NSUnimplemented() + return sharedSyncQ.sync { + if let cache = sharedCache { + return cache + } else { + let fourMegaByte = 4 * 1024 * 1024 + let twentyMegaByte = 20 * 1024 * 1024 + let cacheDirectoryPath = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?.path ?? "\(NSHomeDirectory())/Library/Caches/" + let path = "\(cacheDirectoryPath)\(Bundle.main.bundleIdentifier ?? UUID().uuidString)" + let cache = URLCache(memoryCapacity: fourMegaByte, diskCapacity: twentyMegaByte, diskPath: path) + sharedCache = cache + return cache + } + } } set { - NSUnimplemented() + sharedSyncQ.sync { sharedCache = newValue } } } @@ -167,7 +200,13 @@ open class URLCache : NSObject { @result an initialized URLCache, with the given capacity, backed by disk. */ - public init(memoryCapacity: Int, diskCapacity: Int, diskPath path: String?) { NSUnimplemented() } + public init(memoryCapacity: Int, diskCapacity: Int, diskPath path: String?) { + self.memoryCapacity = memoryCapacity + self.diskCapacity = diskCapacity + self._baseDiskPath = path + + super.init() + } /*! @method cachedResponseForRequest: @@ -249,6 +288,18 @@ open class URLCache : NSObject { @result the current usage of the on-disk cache of the receiver. */ open var currentDiskUsage: Int { NSUnimplemented() } + + private func flushDatabase() { + guard let path = _baseDiskPath else { return } + + do { + let dbPath = path.appending("/Cache.db") + try FileManager.default.removeItem(atPath: dbPath) + } catch { + fatalError("Unable to flush database for URLCache: \(error.localizedDescription)") + } + } + } extension URLCache { @@ -256,3 +307,113 @@ extension URLCache { public func getCachedResponse(for dataTask: URLSessionDataTask, completionHandler: (CachedURLResponse?) -> Void) { NSUnimplemented() } public func removeCachedResponse(for dataTask: URLSessionDataTask) { NSUnimplemented() } } + +extension URLCache { + + private func setupCacheDatabaseIfNotExist() { + guard let path = _baseDiskPath else { return } + + if !FileManager.default.fileExists(atPath: path) { + do { + try FileManager.default.createDirectory(atPath: path, withIntermediateDirectories: true) + } catch { + fatalError("Unable to create directories for URLCache: \(error.localizedDescription)") + } + } + + // Close the currently opened database connection(if any), before creating/replacing the db file + _databaseClient?.close() + + let dbPath = path.appending("/Cache.db") + if !FileManager.default.createFile(atPath: dbPath, contents: nil, attributes: nil) { + fatalError("Unable to setup database for URLCache") + } + + _databaseClient = _CacheSQLiteClient(databasePath: dbPath) + if _databaseClient == nil { + _databaseClient?.close() + flushDatabase() + fatalError("Unable to setup database for URLCache") + } + + if !createTables() { + _databaseClient?.close() + flushDatabase() + fatalError("Unable to setup database for URLCache: Tables not created") + } + + if !createIndicesForTables() { + _databaseClient?.close() + flushDatabase() + fatalError("Unable to setup database for URLCache: Indices not created for tables") + } + } + + private func createTables() -> Bool { + guard _databaseClient != nil else { + fatalError("Cannot create table before database setup") + } + + let tableSQLs = [ + "CREATE TABLE cfurl_cache_response(entry_ID INTEGER PRIMARY KEY, version INTEGER, hash_value VARCHAR, storage_policy INTEGER, request_key VARCHAR, time_stamp DATETIME, partition VARCHAR)", + "CREATE TABLE cfurl_cache_receiver_data(entry_ID INTEGER PRIMARY KEY, isDataOnFS INTEGER, receiver_data BLOB)", + "CREATE TABLE cfurl_cache_blob_data(entry_ID INTEGER PRIMARY KEY, response_object BLOB, request_object BLOB, proto_props BLOB, user_info BLOB)", + "CREATE TABLE cfurl_cache_schema_version(schema_version INTEGER)" + ] + + for sql in tableSQLs { + if let isSuccess = _databaseClient?.execute(sql: sql), !isSuccess { + return false + } + } + + return true + } + + private func createIndicesForTables() -> Bool { + guard _databaseClient != nil else { + fatalError("Cannot create table before database setup") + } + + let indicesSQLs = [ + "CREATE INDEX proto_props_index ON cfurl_cache_blob_data(entry_ID)", + "CREATE INDEX receiver_data_index ON cfurl_cache_receiver_data(entry_ID)", + "CREATE INDEX request_key_index ON cfurl_cache_response(request_key)", + "CREATE INDEX time_stamp_index ON cfurl_cache_response(time_stamp)" + ] + + for sql in indicesSQLs { + if let isSuccess = _databaseClient?.execute(sql: sql), !isSuccess { + return false + } + } + + return true + } + +} + +fileprivate struct _CacheSQLiteClient { + + private var database: OpaquePointer? + + init?(databasePath: String) { + if sqlite3_open_v2(databasePath, &database, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, nil) != SQLITE_OK { + return nil + } + } + + func execute(sql: String) -> Bool { + guard let db = database else { return false } + + return sqlite3_exec(db, sql, nil, nil, nil) == SQLITE_OK + } + + mutating func close() { + guard let db = database else { return } + + sqlite3_close_v2(db) + database = nil + } + +} diff --git a/TestFoundation/TestURLCache.swift b/TestFoundation/TestURLCache.swift new file mode 100644 index 0000000000..b290969ec6 --- /dev/null +++ b/TestFoundation/TestURLCache.swift @@ -0,0 +1,91 @@ +// +// TestURLCache.swift +// TestFoundation +// +// Created by Karthikkeyan Bala Sundaram on 3/8/19. +// Copyright © 2019 Apple. All rights reserved. +// + +import SQLite3 + +class TestURLCache: XCTestCase { + + static var allTests: [(String, (TestURLCache) -> () throws -> Void)] { + return [ + ("test_cacheFileAndDirectorySetup", test_cacheFileAndDirectorySetup), + ("test_cacheDatabaseTables", test_cacheDatabaseTables), + ("test_cacheDatabaseIndices", test_cacheDatabaseIndices), + ] + } + + private var cacheDirectoryPath: String { + if let path = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?.path { + return "\(path)/org.swift.TestFoundation" + } else { + return "\(NSHomeDirectory())/Library/Caches/org.swift.TestFoundation" + } + } + + private var cacheDatabasePath: String { + return "\(cacheDirectoryPath)/Cache.db" + } + + func test_cacheFileAndDirectorySetup() { + let _ = URLCache.shared + + XCTAssertTrue(FileManager.default.fileExists(atPath: cacheDirectoryPath)) + XCTAssertTrue(FileManager.default.fileExists(atPath: cacheDatabasePath)) + } + + func test_cacheDatabaseTables() { + let _ = URLCache.shared + + var db: OpaquePointer? = nil + let openDBResult = sqlite3_open_v2(cacheDatabasePath, &db, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, nil) + XCTAssertTrue(openDBResult == SQLITE_OK, "Unable to open database") + + var statement: OpaquePointer? = nil + let prepareResult = sqlite3_prepare_v2(db!, "select tbl_name from sqlite_master where type='table'", -1, &statement, nil) + XCTAssertTrue(prepareResult == SQLITE_OK, "Unable to prepare list tables statement") + + var tables = ["cfurl_cache_response": false, "cfurl_cache_receiver_data": false, "cfurl_cache_blob_data": false, "cfurl_cache_schema_version": false] + while sqlite3_step(statement!) == SQLITE_ROW { + let tableName = String(cString: sqlite3_column_text(statement!, 0)) + tables[tableName] = true + } + + let tablesNotExist = tables.filter({ !$0.value }) + if tablesNotExist.count == tables.count { + XCTFail("No tables created for URLCache") + } + + XCTAssertTrue(tablesNotExist.count == 0, "Table(s) not created: \(tablesNotExist.map({ $0.key }).joined(separator: ", "))") + sqlite3_close_v2(db!) + } + + func test_cacheDatabaseIndices() { + let _ = URLCache.shared + + var db: OpaquePointer? = nil + let openDBResult = sqlite3_open_v2(cacheDatabasePath, &db, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, nil) + XCTAssertTrue(openDBResult == SQLITE_OK, "Unable to open database") + + var statement: OpaquePointer? = nil + let prepareResult = sqlite3_prepare_v2(db!, "select name from sqlite_master where type='index'", -1, &statement, nil) + XCTAssertTrue(prepareResult == SQLITE_OK, "Unable to prepare list tables statement") + + var indices = ["proto_props_index": false, "receiver_data_index": false, "request_key_index": false, "time_stamp_index": false] + while sqlite3_step(statement!) == SQLITE_ROW { + let name = String(cString: sqlite3_column_text(statement!, 0)) + indices[name] = true + } + + let indicesNotExist = indices.filter({ !$0.value }) + if indicesNotExist.count == indices.count { + XCTFail("No index created for URLCache") + } + + XCTAssertTrue(indicesNotExist.count == 0, "Indices not created: \(indicesNotExist.map({ $0.key }).joined(separator: ", "))") + } + +} diff --git a/TestFoundation/main.swift b/TestFoundation/main.swift index 1464f1cdab..089b60b656 100644 --- a/TestFoundation/main.swift +++ b/TestFoundation/main.swift @@ -81,6 +81,7 @@ var allTestCases = [ testCase(TestTimer.allTests), testCase(TestTimeZone.allTests), testCase(TestURL.allTests), + testCase(TestURLCache.allTests), testCase(TestURLComponents.allTests), testCase(TestURLCredential.allTests), testCase(TestURLProtectionSpace.allTests), From 54b26765fbd9b72cb1f2eee0ec6d4168fa5fdc09 Mon Sep 17 00:00:00 2001 From: Karthik Date: Wed, 13 Mar 2019 15:47:03 -0700 Subject: [PATCH 2/6] Database connection closed at the end of test_cacheDatabaseIndices unit test --- TestFoundation/TestURLCache.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/TestFoundation/TestURLCache.swift b/TestFoundation/TestURLCache.swift index b290969ec6..dcfe10fb24 100644 --- a/TestFoundation/TestURLCache.swift +++ b/TestFoundation/TestURLCache.swift @@ -86,6 +86,7 @@ class TestURLCache: XCTestCase { } XCTAssertTrue(indicesNotExist.count == 0, "Indices not created: \(indicesNotExist.map({ $0.key }).joined(separator: ", "))") + sqlite3_close_v2(db!) } } From 02436cf5d8020774f0754b3a89fbce628e74672b Mon Sep 17 00:00:00 2001 From: Karthik Date: Fri, 26 Apr 2019 12:13:51 -0700 Subject: [PATCH 3/6] URLCache store and fetch methods implemented using NSKeyedArchiver and NSKeyedUnarchiver --- Foundation/URLCache.swift | 220 ++++++++++++------------------ TestFoundation/TestURLCache.swift | 87 +++--------- 2 files changed, 108 insertions(+), 199 deletions(-) diff --git a/Foundation/URLCache.swift b/Foundation/URLCache.swift index 22a0f824bd..24c49cc13b 100644 --- a/Foundation/URLCache.swift +++ b/Foundation/URLCache.swift @@ -12,7 +12,6 @@ import SwiftFoundation #else import Foundation #endif -import SQLite3 /*! @enum URLCache.StoragePolicy @@ -130,24 +129,10 @@ open class CachedURLResponse : NSObject, NSSecureCoding, NSCopying { open class URLCache : NSObject { private static let sharedSyncQ = DispatchQueue(label: "org.swift.URLCache.sharedSyncQ") - - private static var sharedCache: URLCache? { - willSet { - URLCache.sharedCache?.syncQ.sync { - URLCache.sharedCache?._databaseClient?.close() - URLCache.sharedCache?.flushDatabase() - } - } - didSet { - URLCache.sharedCache?.syncQ.sync { - URLCache.sharedCache?.setupCacheDatabaseIfNotExist() - } - } - } + private static var sharedCache: URLCache? private let syncQ = DispatchQueue(label: "org.swift.URLCache.syncQ") - private let _baseDiskPath: String? - private var _databaseClient: _CacheSQLiteClient? + private var persistence: CachePersistence? /*! @method sharedURLCache @@ -174,19 +159,31 @@ open class URLCache : NSObject { } else { let fourMegaByte = 4 * 1024 * 1024 let twentyMegaByte = 20 * 1024 * 1024 - let cacheDirectoryPath = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?.path ?? "\(NSHomeDirectory())/Library/Caches/" - let path = "\(cacheDirectoryPath)\(Bundle.main.bundleIdentifier ?? UUID().uuidString)" - let cache = URLCache(memoryCapacity: fourMegaByte, diskCapacity: twentyMegaByte, diskPath: path) + let bundleIdentifier = Bundle.main.bundleIdentifier ?? UUID().uuidString + let cacheDirectoryPath = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?.path ?? "\(NSHomeDirectory())/Library/Caches" + let diskPath = cacheDirectoryPath + "/" + bundleIdentifier + let cache = URLCache(memoryCapacity: fourMegaByte, diskCapacity: twentyMegaByte, diskPath: diskPath) sharedCache = cache + cache.persistence?.setupCacheDirectory() return cache } } } set { - sharedSyncQ.sync { sharedCache = newValue } + guard newValue !== sharedCache else { return } + + sharedSyncQ.sync { + // Remove previous data before resetting new URLCache instance + URLCache.sharedCache?.persistence?.deleteAllCaches() + sharedCache = newValue + newValue.persistence?.setupCacheDirectory() + } } } + private var allCaches: [String: CachedURLResponse] = [:] + private var cacheSizeInMemory: Int = 0 + /*! @method initWithMemoryCapacity:diskCapacity:diskPath: @abstract Initializes an URLCache with the given capacity and @@ -203,8 +200,11 @@ open class URLCache : NSObject { public init(memoryCapacity: Int, diskCapacity: Int, diskPath path: String?) { self.memoryCapacity = memoryCapacity self.diskCapacity = diskCapacity - self._baseDiskPath = path - + + if let _path = path { + self.persistence = CachePersistence(path: _path) + } + super.init() } @@ -277,7 +277,9 @@ open class URLCache : NSObject { usage of the in-memory cache. @result the current usage of the in-memory cache of the receiver. */ - open var currentMemoryUsage: Int { NSUnimplemented() } + open var currentMemoryUsage: Int { + return self.syncQ.sync { self.cacheSizeInMemory } + } /*! @method currentDiskUsage @@ -287,19 +289,10 @@ open class URLCache : NSObject { usage of the on-disk cache. @result the current usage of the on-disk cache of the receiver. */ - open var currentDiskUsage: Int { NSUnimplemented() } - - private func flushDatabase() { - guard let path = _baseDiskPath else { return } - - do { - let dbPath = path.appending("/Cache.db") - try FileManager.default.removeItem(atPath: dbPath) - } catch { - fatalError("Unable to flush database for URLCache: \(error.localizedDescription)") - } + open var currentDiskUsage: Int { + return self.syncQ.sync { self.persistence?.cacheSizeInDesk ?? 0 } } - + } extension URLCache { @@ -308,112 +301,79 @@ extension URLCache { public func removeCachedResponse(for dataTask: URLSessionDataTask) { NSUnimplemented() } } -extension URLCache { - - private func setupCacheDatabaseIfNotExist() { - guard let path = _baseDiskPath else { return } - - if !FileManager.default.fileExists(atPath: path) { - do { - try FileManager.default.createDirectory(atPath: path, withIntermediateDirectories: true) - } catch { - fatalError("Unable to create directories for URLCache: \(error.localizedDescription)") +fileprivate struct CachePersistence { + + let path: String + + var cacheSizeInDesk: Int { + do { + let subFiles = try FileManager.default.subpathsOfDirectory(atPath: path) + var total: Int = 0 + for fileName in subFiles { + let attributes = try FileManager.default.attributesOfItem(atPath: path.appending(fileName)) + total += (attributes[.size] as? Int) ?? 0 } - } - - // Close the currently opened database connection(if any), before creating/replacing the db file - _databaseClient?.close() - - let dbPath = path.appending("/Cache.db") - if !FileManager.default.createFile(atPath: dbPath, contents: nil, attributes: nil) { - fatalError("Unable to setup database for URLCache") - } - - _databaseClient = _CacheSQLiteClient(databasePath: dbPath) - if _databaseClient == nil { - _databaseClient?.close() - flushDatabase() - fatalError("Unable to setup database for URLCache") - } - - if !createTables() { - _databaseClient?.close() - flushDatabase() - fatalError("Unable to setup database for URLCache: Tables not created") - } - - if !createIndicesForTables() { - _databaseClient?.close() - flushDatabase() - fatalError("Unable to setup database for URLCache: Indices not created for tables") + return total + } catch { + return 0 } } - - private func createTables() -> Bool { - guard _databaseClient != nil else { - fatalError("Cannot create table before database setup") - } - - let tableSQLs = [ - "CREATE TABLE cfurl_cache_response(entry_ID INTEGER PRIMARY KEY, version INTEGER, hash_value VARCHAR, storage_policy INTEGER, request_key VARCHAR, time_stamp DATETIME, partition VARCHAR)", - "CREATE TABLE cfurl_cache_receiver_data(entry_ID INTEGER PRIMARY KEY, isDataOnFS INTEGER, receiver_data BLOB)", - "CREATE TABLE cfurl_cache_blob_data(entry_ID INTEGER PRIMARY KEY, response_object BLOB, request_object BLOB, proto_props BLOB, user_info BLOB)", - "CREATE TABLE cfurl_cache_schema_version(schema_version INTEGER)" - ] - - for sql in tableSQLs { - if let isSuccess = _databaseClient?.execute(sql: sql), !isSuccess { - return false + + func setupCacheDirectory() { + do { + if FileManager.default.fileExists(atPath: path) { + try FileManager.default.removeItem(atPath: path) } + + try FileManager.default.createDirectory(atPath: path, withIntermediateDirectories: true) + } catch { + fatalError("Unable to create directories for URLCache: \(error.localizedDescription)") } - - return true } - - private func createIndicesForTables() -> Bool { - guard _databaseClient != nil else { - fatalError("Cannot create table before database setup") - } - - let indicesSQLs = [ - "CREATE INDEX proto_props_index ON cfurl_cache_blob_data(entry_ID)", - "CREATE INDEX receiver_data_index ON cfurl_cache_receiver_data(entry_ID)", - "CREATE INDEX request_key_index ON cfurl_cache_response(request_key)", - "CREATE INDEX time_stamp_index ON cfurl_cache_response(time_stamp)" - ] - - for sql in indicesSQLs { - if let isSuccess = _databaseClient?.execute(sql: sql), !isSuccess { - return false - } + + func saveCachedResponse(_ response: CachedURLResponse, for request: URLRequest) { + let archivedData = NSKeyedArchiver.archivedData(withRootObject: response) + do { + let cacheFileURL = urlForFileIdentifier(request.cacheFileIdentifier) + try archivedData.write(to: cacheFileURL) + } catch { + fatalError("Unable to save cache data: \(error.localizedDescription)") } - - return true } - -} -fileprivate struct _CacheSQLiteClient { - - private var database: OpaquePointer? - - init?(databasePath: String) { - if sqlite3_open_v2(databasePath, &database, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, nil) != SQLITE_OK { + func cachedResponse(for request: URLRequest) -> CachedURLResponse? { + let url = urlForFileIdentifier(request.cacheFileIdentifier) + guard let data = try? Data(contentsOf: url), + let response = NSKeyedUnarchiver.unarchiveObject(with: data) as? CachedURLResponse else { return nil } + + return response } - - func execute(sql: String) -> Bool { - guard let db = database else { return false } - - return sqlite3_exec(db, sql, nil, nil, nil) == SQLITE_OK + + func deleteAllCaches() { + do { + try FileManager.default.removeItem(atPath: path) + } catch { + fatalError("Unable to flush database for URLCache: \(error.localizedDescription)") + } } - - mutating func close() { - guard let db = database else { return } - - sqlite3_close_v2(db) - database = nil + + private func urlForFileIdentifier(_ identifier: String) -> URL { + return URL(fileURLWithPath: path + "/" + identifier) } - + +} + +fileprivate extension URLRequest { + + var cacheFileIdentifier: String { + guard let urlString = url?.absoluteString else { + fatalError("Unable to create cache identifier for request: \(self)") + } + + let method = httpMethod ?? "GET" + return urlString + method + } + } diff --git a/TestFoundation/TestURLCache.swift b/TestFoundation/TestURLCache.swift index dcfe10fb24..95bcfdcab3 100644 --- a/TestFoundation/TestURLCache.swift +++ b/TestFoundation/TestURLCache.swift @@ -1,23 +1,20 @@ +// This source file is part of the Swift.org open source project // -// TestURLCache.swift -// TestFoundation +// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception // -// Created by Karthikkeyan Bala Sundaram on 3/8/19. -// Copyright © 2019 Apple. All rights reserved. +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors // -import SQLite3 - class TestURLCache: XCTestCase { static var allTests: [(String, (TestURLCache) -> () throws -> Void)] { return [ ("test_cacheFileAndDirectorySetup", test_cacheFileAndDirectorySetup), - ("test_cacheDatabaseTables", test_cacheDatabaseTables), - ("test_cacheDatabaseIndices", test_cacheDatabaseIndices), ] } - + private var cacheDirectoryPath: String { if let path = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?.path { return "\(path)/org.swift.TestFoundation" @@ -25,68 +22,20 @@ class TestURLCache: XCTestCase { return "\(NSHomeDirectory())/Library/Caches/org.swift.TestFoundation" } } - - private var cacheDatabasePath: String { - return "\(cacheDirectoryPath)/Cache.db" - } - + func test_cacheFileAndDirectorySetup() { + // Test default directory let _ = URLCache.shared - XCTAssertTrue(FileManager.default.fileExists(atPath: cacheDirectoryPath)) - XCTAssertTrue(FileManager.default.fileExists(atPath: cacheDatabasePath)) - } - - func test_cacheDatabaseTables() { - let _ = URLCache.shared - - var db: OpaquePointer? = nil - let openDBResult = sqlite3_open_v2(cacheDatabasePath, &db, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, nil) - XCTAssertTrue(openDBResult == SQLITE_OK, "Unable to open database") - - var statement: OpaquePointer? = nil - let prepareResult = sqlite3_prepare_v2(db!, "select tbl_name from sqlite_master where type='table'", -1, &statement, nil) - XCTAssertTrue(prepareResult == SQLITE_OK, "Unable to prepare list tables statement") - - var tables = ["cfurl_cache_response": false, "cfurl_cache_receiver_data": false, "cfurl_cache_blob_data": false, "cfurl_cache_schema_version": false] - while sqlite3_step(statement!) == SQLITE_ROW { - let tableName = String(cString: sqlite3_column_text(statement!, 0)) - tables[tableName] = true - } - - let tablesNotExist = tables.filter({ !$0.value }) - if tablesNotExist.count == tables.count { - XCTFail("No tables created for URLCache") - } - - XCTAssertTrue(tablesNotExist.count == 0, "Table(s) not created: \(tablesNotExist.map({ $0.key }).joined(separator: ", "))") - sqlite3_close_v2(db!) - } - - func test_cacheDatabaseIndices() { - let _ = URLCache.shared - - var db: OpaquePointer? = nil - let openDBResult = sqlite3_open_v2(cacheDatabasePath, &db, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, nil) - XCTAssertTrue(openDBResult == SQLITE_OK, "Unable to open database") - - var statement: OpaquePointer? = nil - let prepareResult = sqlite3_prepare_v2(db!, "select name from sqlite_master where type='index'", -1, &statement, nil) - XCTAssertTrue(prepareResult == SQLITE_OK, "Unable to prepare list tables statement") - - var indices = ["proto_props_index": false, "receiver_data_index": false, "request_key_index": false, "time_stamp_index": false] - while sqlite3_step(statement!) == SQLITE_ROW { - let name = String(cString: sqlite3_column_text(statement!, 0)) - indices[name] = true - } - - let indicesNotExist = indices.filter({ !$0.value }) - if indicesNotExist.count == indices.count { - XCTFail("No index created for URLCache") - } - - XCTAssertTrue(indicesNotExist.count == 0, "Indices not created: \(indicesNotExist.map({ $0.key }).joined(separator: ", "))") - sqlite3_close_v2(db!) + + let fourMegaByte = 4 * 1024 * 1024 + let twentyMegaByte = 20 * 1024 * 1024 + + // Test with a custom directory + let newPath = cacheDirectoryPath + ".test_cacheFileAndDirectorySetup/" + URLCache.shared = URLCache(memoryCapacity: fourMegaByte, diskCapacity: twentyMegaByte, diskPath: newPath) + XCTAssertTrue(FileManager.default.fileExists(atPath: newPath)) + XCTAssertFalse(FileManager.default.fileExists(atPath: cacheDirectoryPath)) } - + } From 5de096c78f5c7f2268279f9fe233b46b783c5080 Mon Sep 17 00:00:00 2001 From: Karthik Date: Sat, 27 Apr 2019 18:23:49 -0700 Subject: [PATCH 4/6] Underscore added as prefix for privare type CachePersistence as per API guidelines. --- Foundation/URLCache.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Foundation/URLCache.swift b/Foundation/URLCache.swift index 24c49cc13b..290d2aefca 100644 --- a/Foundation/URLCache.swift +++ b/Foundation/URLCache.swift @@ -132,7 +132,7 @@ open class URLCache : NSObject { private static var sharedCache: URLCache? private let syncQ = DispatchQueue(label: "org.swift.URLCache.syncQ") - private var persistence: CachePersistence? + private var persistence: _CachePersistence? /*! @method sharedURLCache @@ -202,7 +202,7 @@ open class URLCache : NSObject { self.diskCapacity = diskCapacity if let _path = path { - self.persistence = CachePersistence(path: _path) + self.persistence = _CachePersistence(path: _path) } super.init() @@ -301,7 +301,7 @@ extension URLCache { public func removeCachedResponse(for dataTask: URLSessionDataTask) { NSUnimplemented() } } -fileprivate struct CachePersistence { +fileprivate struct _CachePersistence { let path: String From ec3880622b6aada5593e1c0b7f318a01e4720272 Mon Sep 17 00:00:00 2001 From: Karthik Date: Sat, 25 May 2019 16:17:12 -0700 Subject: [PATCH 5/6] Old cache directory keep in disk when new URLCache object assigned --- Foundation/URLCache.swift | 72 ++++++++++++++++++------------- TestFoundation/TestURLCache.swift | 11 ++--- 2 files changed, 47 insertions(+), 36 deletions(-) diff --git a/Foundation/URLCache.swift b/Foundation/URLCache.swift index 290d2aefca..bb79394573 100644 --- a/Foundation/URLCache.swift +++ b/Foundation/URLCache.swift @@ -157,12 +157,14 @@ open class URLCache : NSObject { if let cache = sharedCache { return cache } else { + guard var cacheDirectoryUrl = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first else { + fatalError("Unable to find cache directory") + } + let fourMegaByte = 4 * 1024 * 1024 let twentyMegaByte = 20 * 1024 * 1024 - let bundleIdentifier = Bundle.main.bundleIdentifier ?? UUID().uuidString - let cacheDirectoryPath = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?.path ?? "\(NSHomeDirectory())/Library/Caches" - let diskPath = cacheDirectoryPath + "/" + bundleIdentifier - let cache = URLCache(memoryCapacity: fourMegaByte, diskCapacity: twentyMegaByte, diskPath: diskPath) + cacheDirectoryUrl.appendPathComponent(ProcessInfo.processInfo.processName) + let cache = URLCache(memoryCapacity: fourMegaByte, diskCapacity: twentyMegaByte, diskPath: cacheDirectoryUrl.path) sharedCache = cache cache.persistence?.setupCacheDirectory() return cache @@ -173,8 +175,6 @@ open class URLCache : NSObject { guard newValue !== sharedCache else { return } sharedSyncQ.sync { - // Remove previous data before resetting new URLCache instance - URLCache.sharedCache?.persistence?.deleteAllCaches() sharedCache = newValue newValue.persistence?.setupCacheDirectory() } @@ -201,7 +201,7 @@ open class URLCache : NSObject { self.memoryCapacity = memoryCapacity self.diskCapacity = diskCapacity - if let _path = path { + if diskCapacity > 0, let _path = path { self.persistence = _CachePersistence(path: _path) } @@ -290,7 +290,7 @@ open class URLCache : NSObject { @result the current usage of the on-disk cache of the receiver. */ open var currentDiskUsage: Int { - return self.syncQ.sync { self.persistence?.cacheSizeInDesk ?? 0 } + return self.syncQ.sync { self.persistence?.cacheSizeInDisk ?? 0 } } } @@ -305,7 +305,9 @@ fileprivate struct _CachePersistence { let path: String - var cacheSizeInDesk: Int { + // FIXME: Create a stored property + // Update this value as the cache added and evicted + var cacheSizeInDisk: Int { do { let subFiles = try FileManager.default.subpathsOfDirectory(atPath: path) var total: Int = 0 @@ -321,10 +323,6 @@ fileprivate struct _CachePersistence { func setupCacheDirectory() { do { - if FileManager.default.fileExists(atPath: path) { - try FileManager.default.removeItem(atPath: path) - } - try FileManager.default.createDirectory(atPath: path, withIntermediateDirectories: true) } catch { fatalError("Unable to create directories for URLCache: \(error.localizedDescription)") @@ -334,15 +332,21 @@ fileprivate struct _CachePersistence { func saveCachedResponse(_ response: CachedURLResponse, for request: URLRequest) { let archivedData = NSKeyedArchiver.archivedData(withRootObject: response) do { - let cacheFileURL = urlForFileIdentifier(request.cacheFileIdentifier) - try archivedData.write(to: cacheFileURL) + if let fileIdentifier = request.cacheFileIdentifier { + let cacheFileURL = urlForFileIdentifier(fileIdentifier) + try archivedData.write(to: cacheFileURL) + } } catch { fatalError("Unable to save cache data: \(error.localizedDescription)") } } func cachedResponse(for request: URLRequest) -> CachedURLResponse? { - let url = urlForFileIdentifier(request.cacheFileIdentifier) + guard let fileIdentifier = request.cacheFileIdentifier else { + return nil + } + + let url = urlForFileIdentifier(fileIdentifier) guard let data = try? Data(contentsOf: url), let response = NSKeyedUnarchiver.unarchiveObject(with: data) as? CachedURLResponse else { return nil @@ -351,29 +355,35 @@ fileprivate struct _CachePersistence { return response } - func deleteAllCaches() { - do { - try FileManager.default.removeItem(atPath: path) - } catch { - fatalError("Unable to flush database for URLCache: \(error.localizedDescription)") - } - } - private func urlForFileIdentifier(_ identifier: String) -> URL { - return URL(fileURLWithPath: path + "/" + identifier) + return URL(fileURLWithPath: path).appendingPathComponent(identifier) } } -fileprivate extension URLRequest { +extension URLRequest { + + fileprivate var cacheFileIdentifier: String? { + guard let scheme = self.url?.scheme, scheme == "http" || scheme == "https", + let method = httpMethod, !method.isEmpty, + let urlString = url?.absoluteString else { + return nil + } + + var hashString = "\(scheme)_\(method)_\(urlString)" + if let userAgent = self.allHTTPHeaderFields?["User-Agent"], !userAgent.isEmpty { + hashString.append(contentsOf: "_\(userAgent)") + } + + if let acceptLanguage = self.allHTTPHeaderFields?["Accept-Language"], !acceptLanguage.isEmpty { + hashString.append(contentsOf: "_\(acceptLanguage)") + } - var cacheFileIdentifier: String { - guard let urlString = url?.absoluteString else { - fatalError("Unable to create cache identifier for request: \(self)") + if let range = self.allHTTPHeaderFields?["Range"], !range.isEmpty { + hashString.append(contentsOf: "_\(range)") } - let method = httpMethod ?? "GET" - return urlString + method + return String(format: "%X", hashString.hashValue) } } diff --git a/TestFoundation/TestURLCache.swift b/TestFoundation/TestURLCache.swift index 95bcfdcab3..ad36187847 100644 --- a/TestFoundation/TestURLCache.swift +++ b/TestFoundation/TestURLCache.swift @@ -16,11 +16,12 @@ class TestURLCache: XCTestCase { } private var cacheDirectoryPath: String { - if let path = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?.path { - return "\(path)/org.swift.TestFoundation" - } else { - return "\(NSHomeDirectory())/Library/Caches/org.swift.TestFoundation" + guard var cacheDirectoryUrl = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first else { + fatalError("Unable to find cache directory") } + + cacheDirectoryUrl.appendPathComponent(ProcessInfo.processInfo.processName) + return cacheDirectoryUrl.path } func test_cacheFileAndDirectorySetup() { @@ -35,7 +36,7 @@ class TestURLCache: XCTestCase { let newPath = cacheDirectoryPath + ".test_cacheFileAndDirectorySetup/" URLCache.shared = URLCache(memoryCapacity: fourMegaByte, diskCapacity: twentyMegaByte, diskPath: newPath) XCTAssertTrue(FileManager.default.fileExists(atPath: newPath)) - XCTAssertFalse(FileManager.default.fileExists(atPath: cacheDirectoryPath)) + XCTAssertTrue(FileManager.default.fileExists(atPath: cacheDirectoryPath)) } } From b5dafd26d2556da2391a73b51261a09440d6ad23 Mon Sep 17 00:00:00 2001 From: Karthik Date: Fri, 31 May 2019 02:09:37 +0530 Subject: [PATCH 6/6] Secure archive and unarchive used for cache url response and cacheFileIdentifier hash mechanism updated. --- Foundation/URLCache.swift | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/Foundation/URLCache.swift b/Foundation/URLCache.swift index bb79394573..8b188c3a34 100644 --- a/Foundation/URLCache.swift +++ b/Foundation/URLCache.swift @@ -201,6 +201,7 @@ open class URLCache : NSObject { self.memoryCapacity = memoryCapacity self.diskCapacity = diskCapacity + // As per the function of URLCache, `diskCapacity` of `0` is assumed as no-persistence. if diskCapacity > 0, let _path = path { self.persistence = _CachePersistence(path: _path) } @@ -322,17 +323,13 @@ fileprivate struct _CachePersistence { } func setupCacheDirectory() { - do { - try FileManager.default.createDirectory(atPath: path, withIntermediateDirectories: true) - } catch { - fatalError("Unable to create directories for URLCache: \(error.localizedDescription)") - } + try? FileManager.default.createDirectory(atPath: path, withIntermediateDirectories: true) } func saveCachedResponse(_ response: CachedURLResponse, for request: URLRequest) { - let archivedData = NSKeyedArchiver.archivedData(withRootObject: response) do { - if let fileIdentifier = request.cacheFileIdentifier { + if let archivedData = try? NSKeyedArchiver.archivedData(withRootObject: response, requiringSecureCoding: true), + let fileIdentifier = request.cacheFileIdentifier { let cacheFileURL = urlForFileIdentifier(fileIdentifier) try archivedData.write(to: cacheFileURL) } @@ -348,7 +345,7 @@ fileprivate struct _CachePersistence { let url = urlForFileIdentifier(fileIdentifier) guard let data = try? Data(contentsOf: url), - let response = NSKeyedUnarchiver.unarchiveObject(with: data) as? CachedURLResponse else { + let response = try? NSKeyedUnarchiver.unarchivedObject(ofClasses:[CachedURLResponse.self], from: data) as? CachedURLResponse else { return nil } @@ -367,23 +364,28 @@ extension URLRequest { guard let scheme = self.url?.scheme, scheme == "http" || scheme == "https", let method = httpMethod, !method.isEmpty, let urlString = url?.absoluteString else { - return nil + return nil } - var hashString = "\(scheme)_\(method)_\(urlString)" + var hashString = "\(abs(method.hashValue))-\(abs(urlString.hashValue))" + if let userAgent = self.allHTTPHeaderFields?["User-Agent"], !userAgent.isEmpty { - hashString.append(contentsOf: "_\(userAgent)") + hashString.append("\(abs(userAgent.hashValue))") } if let acceptLanguage = self.allHTTPHeaderFields?["Accept-Language"], !acceptLanguage.isEmpty { - hashString.append(contentsOf: "_\(acceptLanguage)") + hashString.append("-\(abs(acceptLanguage.hashValue))") } if let range = self.allHTTPHeaderFields?["Range"], !range.isEmpty { - hashString.append(contentsOf: "_\(range)") + hashString.append("-\(abs(range.hashValue))") + } + + if let data = self.httpBody, !data.isEmpty { + hashString.append("-\(abs(data.hashValue))") } - return String(format: "%X", hashString.hashValue) + return hashString } }