diff --git a/Firestore/CHANGELOG.md b/Firestore/CHANGELOG.md index 4880cc91e12..8aacca5ff1b 100644 --- a/Firestore/CHANGELOG.md +++ b/Firestore/CHANGELOG.md @@ -1,4 +1,7 @@ # Unreleased +- [changed] Firestore now limits the number of concurrent document lookups it + will perform when resolving inconsistencies in the local cache + (https://github.com/firebase/firebase-js-sdk/issues/2683). # v1.11.3 - [changed] Internal changes. diff --git a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm index 32767dc4cd3..feca1cdfddb 100644 --- a/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm +++ b/Firestore/Example/Tests/SpecTests/FSTSpecTests.mm @@ -19,8 +19,11 @@ #import #include +#include +#include #include #include +#include #include #include #include @@ -152,16 +155,24 @@ ByteString MakeResumeToken(NSString *specString) { return MakeByteString([specString dataUsingEncoding:NSUTF8StringEncoding]); } -NSString *ToDocumentListString(const std::map &map) { +NSString *ToDocumentListString(const std::set &keys) { std::vector strings; - strings.reserve(map.size()); - for (const auto &kv : map) { - strings.push_back(kv.first.ToString()); + strings.reserve(keys.size()); + for (const auto &key : keys) { + strings.push_back(key.ToString()); } std::sort(strings.begin(), strings.end()); return MakeNSString(absl::StrJoin(strings, ", ")); } +NSString *ToDocumentListString(const std::map &map) { + std::set keys; + for (const auto &kv : map) { + keys.insert(kv.first); + } + return ToDocumentListString(keys); +} + NSString *ToTargetIdListString(const ActiveTargetMap &map) { std::vector targetIds; targetIds.reserve(map.size()); @@ -181,6 +192,7 @@ @interface FSTSpecTests () @implementation FSTSpecTests { BOOL _gcEnabled; + size_t _maxConcurrentLimboResolutions; BOOL _networkEnabled; FSTUserDataConverter *_converter; } @@ -212,12 +224,20 @@ - (void)setUpForSpecWithConfig:(NSDictionary *)config { // Store GCEnabled so we can re-use it in doRestart. NSNumber *GCEnabled = config[@"useGarbageCollection"]; _gcEnabled = [GCEnabled boolValue]; + NSNumber *maxConcurrentLimboResolutions = config[@"maxConcurrentLimboResolutions"]; + _maxConcurrentLimboResolutions = (maxConcurrentLimboResolutions == nil) + ? std::numeric_limits::max() + : maxConcurrentLimboResolutions.unsignedIntValue; NSNumber *numClients = config[@"numClients"]; if (numClients) { XCTAssertEqualObjects(numClients, @1, @"The iOS client does not support multi-client tests"); } std::unique_ptr persistence = [self persistenceWithGCEnabled:_gcEnabled]; - self.driver = [[FSTSyncEngineTestDriver alloc] initWithPersistence:std::move(persistence)]; + self.driver = + [[FSTSyncEngineTestDriver alloc] initWithPersistence:std::move(persistence) + initialUser:User::Unauthenticated() + outstandingWrites:{} + maxConcurrentLimboResolutions:_maxConcurrentLimboResolutions]; [self.driver start]; } @@ -522,9 +542,11 @@ - (void)doRestart { [self.driver shutdown]; std::unique_ptr persistence = [self persistenceWithGCEnabled:_gcEnabled]; - self.driver = [[FSTSyncEngineTestDriver alloc] initWithPersistence:std::move(persistence) - initialUser:currentUser - outstandingWrites:outstandingWrites]; + self.driver = + [[FSTSyncEngineTestDriver alloc] initWithPersistence:std::move(persistence) + initialUser:currentUser + outstandingWrites:outstandingWrites + maxConcurrentLimboResolutions:_maxConcurrentLimboResolutions]; [self.driver start]; } @@ -689,9 +711,18 @@ - (void)validateExpectedState:(nullable NSDictionary *)expectedState { for (NSString *name in docNames) { expectedActiveLimboDocuments = expectedActiveLimboDocuments.insert(FSTTestDocKey(name)); } - // Update the expected limbo documents + // Update the expected active limbo documents [self.driver setExpectedActiveLimboDocuments:std::move(expectedActiveLimboDocuments)]; } + if (expectedState[@"enqueuedLimboDocs"]) { + DocumentKeySet expectedEnqueuedLimboDocuments; + NSArray *docNames = expectedState[@"enqueuedLimboDocs"]; + for (NSString *name in docNames) { + expectedEnqueuedLimboDocuments = expectedEnqueuedLimboDocuments.insert(FSTTestDocKey(name)); + } + // Update the expected enqueued limbo documents + [self.driver setExpectedEnqueuedLimboDocuments:std::move(expectedEnqueuedLimboDocuments)]; + } if (expectedState[@"activeTargets"]) { __block ActiveTargetMap expectedActiveTargets; [expectedState[@"activeTargets"] @@ -719,7 +750,8 @@ - (void)validateExpectedState:(nullable NSDictionary *)expectedState { // Always validate the we received the expected number of callbacks. [self validateUserCallbacks:expectedState]; // Always validate that the expected limbo docs match the actual limbo docs. - [self validateLimboDocuments]; + [self validateActiveLimboDocuments]; + [self validateEnqueuedLimboDocuments]; // Always validate that the expected active targets match the actual active targets. [self validateActiveTargets]; } @@ -744,9 +776,9 @@ - (void)validateUserCallbacks:(nullable NSDictionary *)expected { } } -- (void)validateLimboDocuments { +- (void)validateActiveLimboDocuments { // Make a copy so it can modified while checking against the expected limbo docs. - std::map actualLimboDocs = self.driver.currentLimboDocuments; + std::map actualLimboDocs = self.driver.activeLimboDocumentResolutions; // Validate that each active limbo doc has an expected active target for (const auto &kv : actualLimboDocs) { @@ -767,6 +799,31 @@ - (void)validateLimboDocuments { ToDocumentListString(actualLimboDocs)); } +- (void)validateEnqueuedLimboDocuments { + std::set actualLimboDocs; + for (const auto &key : self.driver.enqueuedLimboDocumentResolutions) { + actualLimboDocs.insert(key); + } + std::set expectedLimboDocs; + for (const auto &key : self.driver.expectedEnqueuedLimboDocuments) { + expectedLimboDocs.insert(key); + } + + for (const auto &key : actualLimboDocs) { + XCTAssertTrue(expectedLimboDocs.find(key) != expectedLimboDocs.end(), + @"Found enqueued limbo doc %s, but it was not in the set of " + @"expected enqueued limbo documents (%@)", + key.ToString().c_str(), ToDocumentListString(expectedLimboDocs)); + } + + for (const auto &key : expectedLimboDocs) { + XCTAssertTrue(actualLimboDocs.find(key) != actualLimboDocs.end(), + @"Expected doc %s to be enqueued for limbo resolution, " + @"but it was not in the queue (%@)", + key.ToString().c_str(), ToDocumentListString(actualLimboDocs)); + } +} + - (void)validateActiveTargets { if (!_networkEnabled) { return; diff --git a/Firestore/Example/Tests/SpecTests/FSTSyncEngineTestDriver.h b/Firestore/Example/Tests/SpecTests/FSTSyncEngineTestDriver.h index 26556d6ba0d..29eba629bd8 100644 --- a/Firestore/Example/Tests/SpecTests/FSTSyncEngineTestDriver.h +++ b/Firestore/Example/Tests/SpecTests/FSTSyncEngineTestDriver.h @@ -16,6 +16,8 @@ #import +#include +#include #include #include #include @@ -115,12 +117,6 @@ typedef std::unordered_map *, */ @interface FSTSyncEngineTestDriver : NSObject -/** - * Initializes the underlying FSTSyncEngine with the given local persistence implementation and - * garbage collection policy. - */ -- (instancetype)initWithPersistence:(std::unique_ptr)persistence; - /** * Initializes the underlying FSTSyncEngine with the given local persistence implementation and * a set of existing outstandingWrites (useful when your Persistence object has persisted @@ -129,7 +125,7 @@ typedef std::unordered_map *, - (instancetype)initWithPersistence:(std::unique_ptr)persistence initialUser:(const auth::User &)initialUser outstandingWrites:(const FSTOutstandingWriteQueues &)outstandingWrites - NS_DESIGNATED_INITIALIZER; + maxConcurrentLimboResolutions:(size_t)maxConcurrentLimboResolutions NS_DESIGNATED_INITIALIZER; - (instancetype)init NS_UNAVAILABLE; @@ -289,8 +285,11 @@ typedef std::unordered_map *, */ - (NSArray *)capturedRejectedWritesSinceLastCall; -/** The current set of documents in limbo. */ -- (std::map)currentLimboDocuments; +/** The current set of documents in limbo with active targets. */ +- (std::map)activeLimboDocumentResolutions; + +/** The current set of documents in limbo that are enqueued for resolution. */ +- (std::deque)enqueuedLimboDocumentResolutions; /** The expected set of documents in limbo with an active target. */ - (const model::DocumentKeySet &)expectedActiveLimboDocuments; @@ -298,6 +297,12 @@ typedef std::unordered_map *, /** Sets the expected set of documents in limbo with an active target. */ - (void)setExpectedActiveLimboDocuments:(model::DocumentKeySet)docs; +/** The expected set of documents in limbo that are enqueued for resolution. */ +- (const model::DocumentKeySet &)expectedEnqueuedLimboDocuments; + +/** Sets the expected set of documents in limbo that are enqueued for resolution. */ +- (void)setExpectedEnqueuedLimboDocuments:(model::DocumentKeySet)docs; + /** * The writes that have been sent to the FSTSyncEngine via writeUserMutation: but not yet * acknowledged by calling receiveWriteAck/Error:. They are tracked per-user. diff --git a/Firestore/Example/Tests/SpecTests/FSTSyncEngineTestDriver.mm b/Firestore/Example/Tests/SpecTests/FSTSyncEngineTestDriver.mm index 0a1f15ba694..2501c44553b 100644 --- a/Firestore/Example/Tests/SpecTests/FSTSyncEngineTestDriver.mm +++ b/Firestore/Example/Tests/SpecTests/FSTSyncEngineTestDriver.mm @@ -18,6 +18,7 @@ #import +#include #include #include #include @@ -154,6 +155,8 @@ @interface FSTSyncEngineTestDriver () @end @implementation FSTSyncEngineTestDriver { + size_t _maxConcurrentLimboResolutions; + std::unique_ptr _persistence; std::unique_ptr _localStore; @@ -173,6 +176,7 @@ @implementation FSTSyncEngineTestDriver { // ivar is declared as mutable. std::unordered_map *, HashUser> _outstandingWrites; DocumentKeySet _expectedActiveLimboDocuments; + DocumentKeySet _expectedEnqueuedLimboDocuments; /** A dictionary for tracking the listens on queries. */ std::unordered_map> _queryListeners; @@ -188,16 +192,13 @@ @implementation FSTSyncEngineTestDriver { int _snapshotsInSyncEvents; } -- (instancetype)initWithPersistence:(std::unique_ptr)persistence { - return [self initWithPersistence:std::move(persistence) - initialUser:User::Unauthenticated() - outstandingWrites:{}]; -} - - (instancetype)initWithPersistence:(std::unique_ptr)persistence initialUser:(const User &)initialUser - outstandingWrites:(const FSTOutstandingWriteQueues &)outstandingWrites { + outstandingWrites:(const FSTOutstandingWriteQueues &)outstandingWrites + maxConcurrentLimboResolutions:(size_t)maxConcurrentLimboResolutions { if (self = [super init]) { + _maxConcurrentLimboResolutions = maxConcurrentLimboResolutions; + // Do a deep copy. for (const auto &pair : outstandingWrites) { _outstandingWrites[pair.first] = [pair.second mutableCopy]; @@ -219,7 +220,8 @@ - (instancetype)initWithPersistence:(std::unique_ptr)persistence [self](OnlineState onlineState) { _syncEngine->HandleOnlineStateChange(onlineState); }); ; - _syncEngine = absl::make_unique(_localStore.get(), _remoteStore.get(), initialUser); + _syncEngine = absl::make_unique(_localStore.get(), _remoteStore.get(), initialUser, + _maxConcurrentLimboResolutions); _remoteStore->set_sync_engine(_syncEngine.get()); _eventManager.Init(_syncEngine.get()); @@ -251,6 +253,14 @@ - (void)setExpectedActiveLimboDocuments:(DocumentKeySet)docs { _expectedActiveLimboDocuments = std::move(docs); } +- (const DocumentKeySet &)expectedEnqueuedLimboDocuments { + return _expectedEnqueuedLimboDocuments; +} + +- (void)setExpectedEnqueuedLimboDocuments:(DocumentKeySet)docs { + _expectedEnqueuedLimboDocuments = std::move(docs); +} + - (void)drainQueue { _workerQueue->EnqueueBlocking([] {}); } @@ -472,8 +482,12 @@ - (void)receiveWatchStreamError:(int)errorCode userInfo:(NSDictionary)currentLimboDocuments { - return _syncEngine->GetCurrentLimboDocuments(); +- (std::map)activeLimboDocumentResolutions { + return _syncEngine->GetActiveLimboDocumentResolutions(); +} + +- (std::deque)enqueuedLimboDocumentResolutions { + return _syncEngine->GetEnqueuedLimboDocumentResolutions(); } - (const std::unordered_map &)activeTargets { diff --git a/Firestore/Example/Tests/SpecTests/json/limbo_spec_test.json b/Firestore/Example/Tests/SpecTests/json/limbo_spec_test.json index d1bf542d769..7fb00f29b81 100644 --- a/Firestore/Example/Tests/SpecTests/json/limbo_spec_test.json +++ b/Firestore/Example/Tests/SpecTests/json/limbo_spec_test.json @@ -4656,8 +4656,6 @@ "describeName": "Limbo Documents:", "itName": "Limbo resolution throttling when a limbo listen is rejected.", "tags": [ - "no-android", - "no-ios" ], "config": { "maxConcurrentLimboResolutions": 1, @@ -4992,8 +4990,6 @@ "describeName": "Limbo Documents:", "itName": "Limbo resolution throttling with all results at once from watch", "tags": [ - "no-android", - "no-ios" ], "config": { "maxConcurrentLimboResolutions": 2, @@ -5571,8 +5567,6 @@ "describeName": "Limbo Documents:", "itName": "Limbo resolution throttling with existence filter mismatch", "tags": [ - "no-android", - "no-ios" ], "config": { "maxConcurrentLimboResolutions": 2, @@ -6185,8 +6179,6 @@ "describeName": "Limbo Documents:", "itName": "Limbo resolution throttling with results one at a time from watch", "tags": [ - "no-android", - "no-ios" ], "config": { "maxConcurrentLimboResolutions": 2, diff --git a/Firestore/core/src/firebase/firestore/core/firestore_client.cc b/Firestore/core/src/firebase/firestore/core/firestore_client.cc index 2c81439360e..aac7768ea92 100644 --- a/Firestore/core/src/firebase/firestore/core/firestore_client.cc +++ b/Firestore/core/src/firebase/firestore/core/firestore_client.cc @@ -100,6 +100,8 @@ using util::StatusOrCallback; using util::ThrowIllegalState; using util::TimerId; +static const size_t kMaxConcurrentLimboResolutions = 100; + std::shared_ptr FirestoreClient::Create( const DatabaseInfo& database_info, const api::Settings& settings, @@ -202,8 +204,9 @@ void FirestoreClient::Initialize(const User& user, const Settings& settings) { weak_this.lock()->sync_engine_->HandleOnlineStateChange(online_state); }); - sync_engine_ = absl::make_unique(local_store_.get(), - remote_store_.get(), user); + sync_engine_ = + absl::make_unique(local_store_.get(), remote_store_.get(), + user, kMaxConcurrentLimboResolutions); event_manager_ = absl::make_unique(sync_engine_.get()); diff --git a/Firestore/core/src/firebase/firestore/core/sync_engine.cc b/Firestore/core/src/firebase/firestore/core/sync_engine.cc index 4dd994e167d..083db0076d6 100644 --- a/Firestore/core/src/firebase/firestore/core/sync_engine.cc +++ b/Firestore/core/src/firebase/firestore/core/sync_engine.cc @@ -82,11 +82,13 @@ bool ErrorIsInteresting(const Status& error) { SyncEngine::SyncEngine(LocalStore* local_store, remote::RemoteStore* remote_store, - const auth::User& initial_user) + const auth::User& initial_user, + size_t max_concurrent_limbo_resolutions) : local_store_(local_store), remote_store_(remote_store), current_user_(initial_user), - target_id_generator_(TargetIdGenerator::SyncEngineTargetIdGenerator()) { + target_id_generator_(TargetIdGenerator::SyncEngineTargetIdGenerator()), + max_concurrent_limbo_resolutions_(max_concurrent_limbo_resolutions) { } void SyncEngine::AssertCallbackExists(absl::string_view source) { @@ -264,8 +266,8 @@ void SyncEngine::ApplyRemoteEvent(const RemoteEvent& remote_event) { for (const auto& entry : remote_event.target_changes()) { TargetId target_id = entry.first; const TargetChange& change = entry.second; - auto it = limbo_resolutions_by_target_.find(target_id); - if (it == limbo_resolutions_by_target_.end()) { + auto it = active_limbo_resolutions_by_target_.find(target_id); + if (it == active_limbo_resolutions_by_target_.end()) { continue; } @@ -300,13 +302,14 @@ void SyncEngine::ApplyRemoteEvent(const RemoteEvent& remote_event) { void SyncEngine::HandleRejectedListen(TargetId target_id, Status error) { AssertCallbackExists("HandleRejectedListen"); - auto it = limbo_resolutions_by_target_.find(target_id); - if (it != limbo_resolutions_by_target_.end()) { + auto it = active_limbo_resolutions_by_target_.find(target_id); + if (it != active_limbo_resolutions_by_target_.end()) { DocumentKey limbo_key = it->second.key; // Since this query failed, we won't want to manually unlisten to it. // So go ahead and remove it from bookkeeping. - limbo_targets_by_key_.erase(limbo_key); - limbo_resolutions_by_target_.erase(target_id); + active_limbo_targets_by_key_.erase(limbo_key); + active_limbo_resolutions_by_target_.erase(target_id); + PumpEnqueuedLimboResolutions(); // TODO(dimond): Retry on transient errors? @@ -395,8 +398,8 @@ void SyncEngine::HandleOnlineStateChange(model::OnlineState online_state) { } DocumentKeySet SyncEngine::GetRemoteKeys(TargetId target_id) const { - auto it = limbo_resolutions_by_target_.find(target_id); - if (it != limbo_resolutions_by_target_.end() && + auto it = active_limbo_resolutions_by_target_.find(target_id); + if (it != active_limbo_resolutions_by_target_.end() && it->second.document_received) { return DocumentKeySet{it->second.key}; } else { @@ -525,32 +528,42 @@ void SyncEngine::UpdateTrackedLimboDocuments( void SyncEngine::TrackLimboChange(const LimboDocumentChange& limbo_change) { const DocumentKey& key = limbo_change.key(); - - if (limbo_targets_by_key_.find(key) == limbo_targets_by_key_.end()) { + if (active_limbo_targets_by_key_.find(key) == + active_limbo_targets_by_key_.end()) { LOG_DEBUG("New document in limbo: %s", key.ToString()); + enqueued_limbo_resolutions_.push_back(key); + PumpEnqueuedLimboResolutions(); + } +} +void SyncEngine::PumpEnqueuedLimboResolutions() { + while (!enqueued_limbo_resolutions_.empty() && + active_limbo_targets_by_key_.size() < + max_concurrent_limbo_resolutions_) { + DocumentKey key = enqueued_limbo_resolutions_.front(); + enqueued_limbo_resolutions_.pop_front(); TargetId limbo_target_id = target_id_generator_.NextId(); - Query query(key.path()); - TargetData target_data(query.ToTarget(), limbo_target_id, - kIrrelevantSequenceNumber, - QueryPurpose::LimboResolution); - limbo_resolutions_by_target_.emplace(limbo_target_id, LimboResolution{key}); - remote_store_->Listen(target_data); - limbo_targets_by_key_[key] = limbo_target_id; + active_limbo_resolutions_by_target_.emplace(limbo_target_id, + LimboResolution{key}); + active_limbo_targets_by_key_.emplace(key, limbo_target_id); + remote_store_->Listen(TargetData(Query(key.path()).ToTarget(), + limbo_target_id, kIrrelevantSequenceNumber, + QueryPurpose::LimboResolution)); } } void SyncEngine::RemoveLimboTarget(const DocumentKey& key) { - auto it = limbo_targets_by_key_.find(key); - if (it == limbo_targets_by_key_.end()) { + auto it = active_limbo_targets_by_key_.find(key); + if (it == active_limbo_targets_by_key_.end()) { // This target already got removed, because the query failed. return; } TargetId limbo_target_id = it->second; remote_store_->StopListening(limbo_target_id); - limbo_targets_by_key_.erase(key); - limbo_resolutions_by_target_.erase(limbo_target_id); + active_limbo_targets_by_key_.erase(key); + active_limbo_resolutions_by_target_.erase(limbo_target_id); + PumpEnqueuedLimboResolutions(); } } // namespace core diff --git a/Firestore/core/src/firebase/firestore/core/sync_engine.h b/Firestore/core/src/firebase/firestore/core/sync_engine.h index 1c031a15ca3..48e8272a2e3 100644 --- a/Firestore/core/src/firebase/firestore/core/sync_engine.h +++ b/Firestore/core/src/firebase/firestore/core/sync_engine.h @@ -17,6 +17,8 @@ #ifndef FIRESTORE_CORE_SRC_FIREBASE_FIRESTORE_CORE_SYNC_ENGINE_H_ #define FIRESTORE_CORE_SRC_FIREBASE_FIRESTORE_CORE_SYNC_ENGINE_H_ +#include +#include #include #include #include @@ -90,7 +92,8 @@ class SyncEngine : public remote::RemoteStoreCallback, public QueryEventSource { public: SyncEngine(local::LocalStore* local_store, remote::RemoteStore* remote_store, - const auth::User& initial_user); + const auth::User& initial_user, + size_t max_concurrent_limbo_resolutions); // Implements `QueryEventSource`. void SetCallback(SyncEngineCallback* callback) override { @@ -145,10 +148,16 @@ class SyncEngine : public remote::RemoteStoreCallback, public QueryEventSource { model::DocumentKeySet GetRemoteKeys(model::TargetId target_id) const override; // For tests only - std::map GetCurrentLimboDocuments() - const { + std::map + GetActiveLimboDocumentResolutions() const { // Return defensive copy - return limbo_targets_by_key_; + return active_limbo_targets_by_key_; + } + + // For tests only + std::deque GetEnqueuedLimboDocumentResolutions() const { + // Return defensive copy + return enqueued_limbo_resolutions_; } private: @@ -231,6 +240,19 @@ class SyncEngine : public remote::RemoteStoreCallback, public QueryEventSource { void TrackLimboChange(const LimboDocumentChange& limbo_change); + /** + * Starts listens for documents in limbo that are enqueued for resolution, + * subject to a maximum number of concurrent resolutions. + * + * The maximum number of concurrent limbo resolutions is defined in + * max_concurrent_limbo_resolutions_. + * + * Without bounding the number of concurrent resolutions, the server can fail + * with "resource exhausted" errors which can lead to pathological client + * behavior as seen in https://github.com/firebase/firebase-js-sdk/issues/2683 + */ + void PumpEnqueuedLimboResolutions(); + void NotifyUser(model::BatchId batch_id, util::Status status); /** @@ -273,19 +295,26 @@ class SyncEngine : public remote::RemoteStoreCallback, public QueryEventSource { /** Queries mapped to Targets, indexed by target ID. */ std::unordered_map> queries_by_target_; + const size_t max_concurrent_limbo_resolutions_; + + /** + * The keys of documents that are in limbo for which we haven't yet started a + * limbo resolution query. + */ + std::deque enqueued_limbo_resolutions_; + /** - * When a document is in limbo, we create a special listen to resolve it. This - * maps the DocumentKey of each limbo document to the TargetId of the listen - * resolving it. + * Keeps track of the target ID for each document that is in limbo with an + * active target. */ - std::map limbo_targets_by_key_; + std::map active_limbo_targets_by_key_; /** - * Basically the inverse of limbo_targets_by_key_, a map of target ID to a - * LimboResolution (which includes the DocumentKey as well as whether we've - * received a document for the target). + * Keeps track of the information about an active limbo resolution for each + * active target ID that was started for the purpose of limbo resolution. */ - std::map limbo_resolutions_by_target_; + std::map + active_limbo_resolutions_by_target_; /** Used to track any documents that are currently in limbo. */ local::ReferenceSet limbo_document_refs_;