diff --git a/Firestore/Example/Tests/Local/FSTLRUGarbageCollectorTests.mm b/Firestore/Example/Tests/Local/FSTLRUGarbageCollectorTests.mm index b90d91c01ee..86e5c315026 100644 --- a/Firestore/Example/Tests/Local/FSTLRUGarbageCollectorTests.mm +++ b/Firestore/Example/Tests/Local/FSTLRUGarbageCollectorTests.mm @@ -639,6 +639,28 @@ - (void)testRemoveTargetsThenGC { }); [_persistence shutdown]; } + +- (void)testGetsSize { + if ([self isTestBaseClass]) return; + + [self newTestResources]; + + size_t initialSize = [_gc byteSize]; + + _persistence.run("fill cache", [&]() { + // Simulate a bunch of ack'd mutations + for (int i = 0; i < 50; i++) { + FSTDocument *doc = [self cacheADocumentInTransaction]; + [self markDocumentEligibleForGCInTransaction:doc.key]; + } + }); + + size_t finalSize = [_gc byteSize]; + XCTAssertGreaterThan(finalSize, initialSize); + + [_persistence shutdown]; +} + @end NS_ASSUME_NONNULL_END diff --git a/Firestore/Example/Tests/Local/FSTPersistenceTestHelpers.mm b/Firestore/Example/Tests/Local/FSTPersistenceTestHelpers.mm index 599f8b9d5e4..c544ff6f05a 100644 --- a/Firestore/Example/Tests/Local/FSTPersistenceTestHelpers.mm +++ b/Firestore/Example/Tests/Local/FSTPersistenceTestHelpers.mm @@ -38,6 +38,14 @@ @implementation FSTPersistenceTestHelpers ++ (FSTLocalSerializer *)localSerializer { + // This owns the DatabaseIds since we do not have FirestoreClient instance to own them. + static DatabaseId database_id{"p", "d"}; + + FSTSerializerBeta *remoteSerializer = [[FSTSerializerBeta alloc] initWithDatabaseID:&database_id]; + return [[FSTLocalSerializer alloc] initWithRemoteSerializer:remoteSerializer]; +} + + (Path)levelDBDir { Path dir = util::TempDir().AppendUtf8("FSTPersistenceTestHelpers"); @@ -53,12 +61,7 @@ + (Path)levelDBDir { } + (FSTLevelDB *)levelDBPersistenceWithDir:(Path)dir { - // This owns the DatabaseIds since we do not have FirestoreClient instance to own them. - static DatabaseId database_id{"p", "d"}; - - FSTSerializerBeta *remoteSerializer = [[FSTSerializerBeta alloc] initWithDatabaseID:&database_id]; - FSTLocalSerializer *serializer = - [[FSTLocalSerializer alloc] initWithRemoteSerializer:remoteSerializer]; + FSTLocalSerializer *serializer = [self localSerializer]; FSTLevelDB *db = [[FSTLevelDB alloc] initWithDirectory:std::move(dir) serializer:serializer]; Status status = [db start]; if (!status.ok()) { @@ -85,7 +88,9 @@ + (FSTMemoryPersistence *)eagerGCMemoryPersistence { } + (FSTMemoryPersistence *)lruMemoryPersistence { - FSTMemoryPersistence *persistence = [FSTMemoryPersistence persistenceWithLRUGC]; + FSTLocalSerializer *serializer = [self localSerializer]; + FSTMemoryPersistence *persistence = + [FSTMemoryPersistence persistenceWithLRUGCAndSerializer:serializer]; Status status = [persistence start]; if (!status.ok()) { [NSException raise:NSInternalInconsistencyException diff --git a/Firestore/Source/Local/FSTLRUGarbageCollector.h b/Firestore/Source/Local/FSTLRUGarbageCollector.h index b4250f272b9..39f41f3dd1e 100644 --- a/Firestore/Source/Local/FSTLRUGarbageCollector.h +++ b/Firestore/Source/Local/FSTLRUGarbageCollector.h @@ -61,6 +61,8 @@ extern const firebase::firestore::model::ListenSequenceNumber kFSTListenSequence (firebase::firestore::model::ListenSequenceNumber)sequenceNumber liveQueries:(NSDictionary *)liveQueries; +- (size_t)byteSize; + /** Access to the underlying LRU Garbage collector instance. */ @property(strong, nonatomic, readonly) FSTLRUGarbageCollector *gc; @@ -103,4 +105,6 @@ extern const firebase::firestore::model::ListenSequenceNumber kFSTListenSequence - (int)removeOrphanedDocumentsThroughSequenceNumber: (firebase::firestore::model::ListenSequenceNumber)sequenceNumber; +- (size_t)byteSize; + @end diff --git a/Firestore/Source/Local/FSTLRUGarbageCollector.mm b/Firestore/Source/Local/FSTLRUGarbageCollector.mm index f1a00de4bef..7f9b6b93d0c 100644 --- a/Firestore/Source/Local/FSTLRUGarbageCollector.mm +++ b/Firestore/Source/Local/FSTLRUGarbageCollector.mm @@ -119,4 +119,8 @@ - (int)removeOrphanedDocumentsThroughSequenceNumber:(ListenSequenceNumber)sequen return [_delegate removeOrphanedDocumentsThroughSequenceNumber:sequenceNumber]; } +- (size_t)byteSize { + return [_delegate byteSize]; +} + @end diff --git a/Firestore/Source/Local/FSTLevelDB.mm b/Firestore/Source/Local/FSTLevelDB.mm index 68891b57d1f..8501c44b53b 100644 --- a/Firestore/Source/Local/FSTLevelDB.mm +++ b/Firestore/Source/Local/FSTLevelDB.mm @@ -77,6 +77,15 @@ static const char *kReservedPathComponent = "firestore"; +@interface FSTLevelDB () + +- (size_t)byteSize; + +@property(nonatomic, assign, getter=isStarted) BOOL started; +@property(nonatomic, strong, readonly) FSTLocalSerializer *serializer; + +@end + /** * Provides LRU functionality for leveldb persistence. * @@ -234,12 +243,9 @@ - (void)limboDocumentUpdated:(const DocumentKey &)key { [self writeSentinelForKey:key]; } -@end - -@interface FSTLevelDB () - -@property(nonatomic, assign, getter=isStarted) BOOL started; -@property(nonatomic, strong, readonly) FSTLocalSerializer *serializer; +- (size_t)byteSize { + return [_db byteSize]; +} @end @@ -290,6 +296,19 @@ - (instancetype)initWithDirectory:(Path)directory serializer:(FSTLocalSerializer return self; } +- (size_t)byteSize { + off_t count = 0; + auto iter = util::DirectoryIterator::Create(_directory); + for (; iter->Valid(); iter->Next()) { + off_t fileSize = util::FileSize(iter->file()).ValueOrDie(); + count += fileSize; + } + HARD_ASSERT(iter->status().ok(), "Failed to iterate leveldb directory: %s", + iter->status().error_message().c_str()); + HARD_ASSERT(count <= SIZE_MAX, "Overflowed counting bytes cached"); + return count; +} + - (const std::set &)users { return _users; } diff --git a/Firestore/Source/Local/FSTMemoryMutationQueue.h b/Firestore/Source/Local/FSTMemoryMutationQueue.h index 3df8a47eeb6..3a7808d0ebb 100644 --- a/Firestore/Source/Local/FSTMemoryMutationQueue.h +++ b/Firestore/Source/Local/FSTMemoryMutationQueue.h @@ -21,6 +21,8 @@ NS_ASSUME_NONNULL_BEGIN +@class FSTLocalSerializer; + @interface FSTMemoryMutationQueue : NSObject - (instancetype)initWithPersistence:(FSTMemoryPersistence *)persistence NS_DESIGNATED_INITIALIZER; @@ -32,6 +34,8 @@ NS_ASSUME_NONNULL_BEGIN */ - (BOOL)containsKey:(const firebase::firestore::model::DocumentKey &)key; +- (size_t)byteSizeWithSerializer:(FSTLocalSerializer *)serializer; + @end NS_ASSUME_NONNULL_END diff --git a/Firestore/Source/Local/FSTMemoryMutationQueue.mm b/Firestore/Source/Local/FSTMemoryMutationQueue.mm index 37efb7fcae3..c4a570eaf83 100644 --- a/Firestore/Source/Local/FSTMemoryMutationQueue.mm +++ b/Firestore/Source/Local/FSTMemoryMutationQueue.mm @@ -16,8 +16,11 @@ #import "Firestore/Source/Local/FSTMemoryMutationQueue.h" +#import + #include +#import "Firestore/Protos/objc/firestore/local/Mutation.pbobjc.h" #import "Firestore/Source/Core/FSTQuery.h" #import "Firestore/Source/Local/FSTDocumentReference.h" #import "Firestore/Source/Local/FSTMemoryPersistence.h" @@ -466,6 +469,14 @@ - (NSUInteger)indexOfExistingBatchID:(BatchId)batchID action:(NSString *)action return (NSUInteger)index; } +- (size_t)byteSizeWithSerializer:(FSTLocalSerializer *)serializer { + size_t count = 0; + for (FSTMutationBatch *batch in self.queue) { + count += [[[serializer encodedMutationBatch:batch] data] length]; + }; + return count; +} + @end NS_ASSUME_NONNULL_END diff --git a/Firestore/Source/Local/FSTMemoryPersistence.h b/Firestore/Source/Local/FSTMemoryPersistence.h index f980921f80c..94ee875f155 100644 --- a/Firestore/Source/Local/FSTMemoryPersistence.h +++ b/Firestore/Source/Local/FSTMemoryPersistence.h @@ -17,6 +17,7 @@ #import #import "Firestore/Source/Local/FSTLRUGarbageCollector.h" +#import "Firestore/Source/Local/FSTLocalSerializer.h" #import "Firestore/Source/Local/FSTPersistence.h" #include "Firestore/core/src/firebase/firestore/model/document_key.h" #include "Firestore/core/src/firebase/firestore/model/types.h" @@ -31,7 +32,7 @@ NS_ASSUME_NONNULL_BEGIN + (instancetype)persistenceWithEagerGC; -+ (instancetype)persistenceWithLRUGC; ++ (instancetype)persistenceWithLRUGCAndSerializer:(FSTLocalSerializer *)serializer; @end @@ -50,7 +51,8 @@ NS_ASSUME_NONNULL_BEGIN @interface FSTMemoryLRUReferenceDelegate : NSObject -- (instancetype)initWithPersistence:(FSTMemoryPersistence *)persistence; +- (instancetype)initWithPersistence:(FSTMemoryPersistence *)persistence + serializer:(FSTLocalSerializer *)serializer; - (BOOL)isPinnedAtSequenceNumber:(firebase::firestore::model::ListenSequenceNumber)upperBound document:(const firebase::firestore::model::DocumentKey &)key; diff --git a/Firestore/Source/Local/FSTMemoryPersistence.mm b/Firestore/Source/Local/FSTMemoryPersistence.mm index 1cc92b112e7..0d0dcd50d77 100644 --- a/Firestore/Source/Local/FSTMemoryPersistence.mm +++ b/Firestore/Source/Local/FSTMemoryPersistence.mm @@ -44,6 +44,10 @@ @interface FSTMemoryPersistence () +- (FSTMemoryQueryCache *)queryCache; + +- (FSTMemoryRemoteDocumentCache *)remoteDocumentCache; + @property(nonatomic, readonly) MutationQueues &mutationQueues; @property(nonatomic, assign, getter=isStarted) BOOL started; @@ -79,10 +83,10 @@ + (instancetype)persistenceWithEagerGC { return persistence; } -+ (instancetype)persistenceWithLRUGC { ++ (instancetype)persistenceWithLRUGCAndSerializer:(FSTLocalSerializer *)serializer { FSTMemoryPersistence *persistence = [[FSTMemoryPersistence alloc] init]; persistence.referenceDelegate = - [[FSTMemoryLRUReferenceDelegate alloc] initWithPersistence:persistence]; + [[FSTMemoryLRUReferenceDelegate alloc] initWithPersistence:persistence serializer:serializer]; return persistence; } @@ -136,7 +140,7 @@ - (ListenSequenceNumber)currentSequenceNumber { return queue; } -- (id)queryCache { +- (FSTMemoryQueryCache *)queryCache { return _queryCache; } @@ -155,9 +159,11 @@ @implementation FSTMemoryLRUReferenceDelegate { FSTLRUGarbageCollector *_gc; FSTListenSequence *_listenSequence; ListenSequenceNumber _currentSequenceNumber; + FSTLocalSerializer *_serializer; } -- (instancetype)initWithPersistence:(FSTMemoryPersistence *)persistence { +- (instancetype)initWithPersistence:(FSTMemoryPersistence *)persistence + serializer:(FSTLocalSerializer *)serializer { if (self = [super init]) { _persistence = persistence; _gc = @@ -167,6 +173,7 @@ - (instancetype)initWithPersistence:(FSTMemoryPersistence *)persistence { ListenSequenceNumber highestSequenceNumber = _persistence.queryCache.highestListenSequenceNumber; _listenSequence = [[FSTListenSequence alloc] initStartingAfter:highestSequenceNumber]; + _serializer = serializer; } return self; } @@ -274,6 +281,20 @@ - (BOOL)isPinnedAtSequenceNumber:(ListenSequenceNumber)upperBound return NO; } +- (size_t)byteSize { + // Note that this method is only used for testing because this delegate is only + // used for testing. The algorithm here (loop through everything, serialize it + // and count bytes) is inefficient and inexact, but won't run in production. + size_t count = 0; + count += [_persistence.queryCache byteSizeWithSerializer:_serializer]; + count += [_persistence.remoteDocumentCache byteSizeWithSerializer:_serializer]; + const MutationQueues &queues = [_persistence mutationQueues]; + for (auto it = queues.begin(); it != queues.end(); ++it) { + count += [it->second byteSizeWithSerializer:_serializer]; + } + return count; +} + @end @implementation FSTMemoryEagerReferenceDelegate { diff --git a/Firestore/Source/Local/FSTMemoryQueryCache.h b/Firestore/Source/Local/FSTMemoryQueryCache.h index 126ce59adcd..66cbdb91cc1 100644 --- a/Firestore/Source/Local/FSTMemoryQueryCache.h +++ b/Firestore/Source/Local/FSTMemoryQueryCache.h @@ -20,6 +20,7 @@ NS_ASSUME_NONNULL_BEGIN +@class FSTLocalSerializer; @class FSTMemoryPersistence; /** @@ -32,6 +33,8 @@ NS_ASSUME_NONNULL_BEGIN - (instancetype)init NS_UNAVAILABLE; +- (size_t)byteSizeWithSerializer:(FSTLocalSerializer *)serializer; + @end NS_ASSUME_NONNULL_END diff --git a/Firestore/Source/Local/FSTMemoryQueryCache.mm b/Firestore/Source/Local/FSTMemoryQueryCache.mm index 8229475489b..16d4055d213 100644 --- a/Firestore/Source/Local/FSTMemoryQueryCache.mm +++ b/Firestore/Source/Local/FSTMemoryQueryCache.mm @@ -16,8 +16,11 @@ #import "Firestore/Source/Local/FSTMemoryQueryCache.h" +#import + #include +#import "Firestore/Protos/objc/firestore/local/Target.pbobjc.h" #import "Firestore/Source/Core/FSTQuery.h" #import "Firestore/Source/Local/FSTMemoryPersistence.h" #import "Firestore/Source/Local/FSTQueryData.h" @@ -168,6 +171,15 @@ - (BOOL)containsKey:(const firebase::firestore::model::DocumentKey &)key { return [self.references containsKey:key]; } +- (size_t)byteSizeWithSerializer:(FSTLocalSerializer *)serializer { + __block size_t count = 0; + [self.queries + enumerateKeysAndObjectsUsingBlock:^(FSTQuery *key, FSTQueryData *queryData, BOOL *stop) { + count += [[[serializer encodedQueryData:queryData] data] length]; + }]; + return count; +} + @end NS_ASSUME_NONNULL_END diff --git a/Firestore/Source/Local/FSTMemoryRemoteDocumentCache.h b/Firestore/Source/Local/FSTMemoryRemoteDocumentCache.h index 33e201afdba..9b0292f2e6f 100644 --- a/Firestore/Source/Local/FSTMemoryRemoteDocumentCache.h +++ b/Firestore/Source/Local/FSTMemoryRemoteDocumentCache.h @@ -22,6 +22,7 @@ NS_ASSUME_NONNULL_BEGIN +@class FSTLocalSerializer; @class FSTMemoryLRUReferenceDelegate; @interface FSTMemoryRemoteDocumentCache : NSObject @@ -31,6 +32,8 @@ NS_ASSUME_NONNULL_BEGIN - (int)removeOrphanedDocuments:(FSTMemoryLRUReferenceDelegate *)referenceDelegate throughSequenceNumber:(firebase::firestore::model::ListenSequenceNumber)upperBound; +- (size_t)byteSizeWithSerializer:(FSTLocalSerializer *)serializer; + @end NS_ASSUME_NONNULL_END diff --git a/Firestore/Source/Local/FSTMemoryRemoteDocumentCache.mm b/Firestore/Source/Local/FSTMemoryRemoteDocumentCache.mm index 8e9d25d5232..b46327e5410 100644 --- a/Firestore/Source/Local/FSTMemoryRemoteDocumentCache.mm +++ b/Firestore/Source/Local/FSTMemoryRemoteDocumentCache.mm @@ -16,6 +16,8 @@ #import "Firestore/Source/Local/FSTMemoryRemoteDocumentCache.h" +#import +#import "Firestore/Protos/objc/firestore/local/MaybeDocument.pbobjc.h" #import "Firestore/Source/Core/FSTQuery.h" #import "Firestore/Source/Local/FSTMemoryPersistence.h" #import "Firestore/Source/Model/FSTDocument.h" @@ -28,6 +30,19 @@ NS_ASSUME_NONNULL_BEGIN +/** + * Returns an estimate of the number of bytes used to store the given + * document key in memory. This is only an estimate and includes the size + * of the segments of the path, but not any object overhead or path separators. + */ +static size_t FSTDocumentKeyByteSize(FSTDocumentKey *key) { + size_t count = 0; + for (auto it = key.path.begin(); it != key.path.end(); it++) { + count += (*it).size(); + } + return count; +} + @interface FSTMemoryRemoteDocumentCache () /** Underlying cache of documents. */ @@ -95,6 +110,16 @@ - (int)removeOrphanedDocuments:(FSTMemoryLRUReferenceDelegate *)referenceDelegat return count; } +- (size_t)byteSizeWithSerializer:(FSTLocalSerializer *)serializer { + __block size_t count = 0; + [self.docs + enumerateKeysAndObjectsUsingBlock:^(FSTDocumentKey *key, FSTMaybeDocument *doc, BOOL *stop) { + count += FSTDocumentKeyByteSize(key); + count += [[[serializer encodedMaybeDocument:doc] data] length]; + }]; + return count; +} + @end NS_ASSUME_NONNULL_END