Skip to content

Commit ecb26bf

Browse files
authored
Migrate LevelDB data out of ~/Documents (#4609)
This moves iOS and macOS user data from `NSDocumentDirectory` to `NSApplicationSupportDirectory`, adding a pre-open migration step that: * Moves old data into the new location * Cleans up any resulting empty directories in the old Documents folder * Stops before removing the Documents folder itself (see tests) Note that other platforms don't need a migration: * tvOS was already storing documents in `NSCachesDirectory` * Linux, Windows, and other UNIX haven't been publicly released or aren't changing. See individual commits for details. Most notably this adds some additional filesystem manipulation routines to our (sadly) growing library. Fixes #843.
1 parent 1bc3255 commit ecb26bf

33 files changed

+1113
-208
lines changed

Firestore/CHANGELOG.md

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,12 @@
44
improvements. This benefit is reduced if changes accumulate while the query
55
is inactive. Queries that use the `limit()` API may not always benefit,
66
depending on the accumulated changes.
7+
- [changed] Changed the location of Firestore's locally stored data from the
8+
Documents folder to Library/Application Support, hiding it from users of apps
9+
that share their files with the iOS Files app. **Important**: After a user's
10+
data is migrated, downgrading to an older version of the SDK will cause the
11+
user to appear to lose data, since older versions of the SDK can't read data
12+
from the new location (#843).
713

814
# v1.9.0
915
- [feature] Added a `limit(toLast:)` query operator, which returns the last
@@ -43,7 +49,7 @@
4349
experience.
4450

4551
# v1.6.1
46-
- [fixed] Fix a race condition that could cause a segmentation fault during
52+
- [fixed] Fixed a race condition that could cause a segmentation fault during
4753
client initialization.
4854

4955
# v1.6.0

Firestore/Example/Firestore.xcodeproj/project.pbxproj

Lines changed: 46 additions & 16 deletions
Large diffs are not rendered by default.

Firestore/Example/Tests/Util/FSTIntegrationTestCase.mm

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
#include "Firestore/core/src/firebase/firestore/auth/credentials_provider.h"
4141
#include "Firestore/core/src/firebase/firestore/auth/empty_credentials_provider.h"
4242
#include "Firestore/core/src/firebase/firestore/auth/user.h"
43-
#include "Firestore/core/src/firebase/firestore/local/leveldb_persistence.h"
43+
#include "Firestore/core/src/firebase/firestore/local/leveldb_opener.h"
4444
#include "Firestore/core/src/firebase/firestore/model/database_id.h"
4545
#include "Firestore/core/src/firebase/firestore/remote/grpc_connection.h"
4646
#include "Firestore/core/src/firebase/firestore/util/async_queue.h"
@@ -50,7 +50,7 @@
5050
#include "Firestore/core/src/firebase/firestore/util/string_apple.h"
5151
#include "Firestore/core/test/firebase/firestore/testutil/app_testing.h"
5252
#include "Firestore/core/test/firebase/firestore/testutil/async_testing.h"
53-
#include "Firestore/core/test/firebase/firestore/util/status_testing.h"
53+
#include "Firestore/core/test/firebase/firestore/testutil/status_testing.h"
5454
#include "absl/memory/memory.h"
5555

5656
namespace testutil = firebase::firestore::testutil;
@@ -60,7 +60,8 @@
6060
using firebase::firestore::auth::CredentialsProvider;
6161
using firebase::firestore::auth::EmptyCredentialsProvider;
6262
using firebase::firestore::auth::User;
63-
using firebase::firestore::local::LevelDbPersistence;
63+
using firebase::firestore::core::DatabaseInfo;
64+
using firebase::firestore::local::LevelDbOpener;
6465
using firebase::firestore::model::DatabaseId;
6566
using firebase::firestore::testutil::AppForUnitTesting;
6667
using firebase::firestore::testutil::AsyncQueueForTesting;
@@ -148,10 +149,12 @@ - (void)clearPersistenceOnce {
148149

149150
@synchronized([FSTIntegrationTestCase class]) {
150151
if (clearedPersistence) return;
151-
StatusOr<Path> maybe_dir = LevelDbPersistence::AppDataDirectory();
152-
ASSERT_OK(maybe_dir);
152+
DatabaseInfo dbInfo;
153+
LevelDbOpener opener(dbInfo);
154+
StatusOr<Path> maybeLevelDBDir = opener.FirestoreAppDataDir();
155+
ASSERT_OK(maybeLevelDBDir.status());
156+
Path levelDBDir = std::move(maybeLevelDBDir).ValueOrDie();
153157

154-
Path levelDBDir = maybe_dir.ValueOrDie();
155158
Status status = fs->RecursivelyRemove(levelDBDir);
156159
ASSERT_OK(status);
157160

Firestore/core/src/firebase/firestore/core/firestore_client.cc

Lines changed: 5 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include "Firestore/core/src/firebase/firestore/core/event_manager.h"
2727
#include "Firestore/core/src/firebase/firestore/core/view.h"
2828
#include "Firestore/core/src/firebase/firestore/local/index_free_query_engine.h"
29+
#include "Firestore/core/src/firebase/firestore/local/leveldb_opener.h"
2930
#include "Firestore/core/src/firebase/firestore/local/leveldb_persistence.h"
3031
#include "Firestore/core/src/firebase/firestore/local/local_serializer.h"
3132
#include "Firestore/core/src/firebase/firestore/local/memory_persistence.h"
@@ -60,7 +61,7 @@ using auth::CredentialsProvider;
6061
using auth::User;
6162
using firestore::Error;
6263
using local::IndexFreeQueryEngine;
63-
using local::LevelDbPersistence;
64+
using local::LevelDbOpener;
6465
using local::LocalSerializer;
6566
using local::LocalStore;
6667
using local::LruParams;
@@ -155,18 +156,10 @@ void FirestoreClient::Initialize(const User& user, const Settings& settings) {
155156
// more work) since external write/listen operations could get queued to run
156157
// before that subsequent work completes.
157158
if (settings.persistence_enabled()) {
158-
auto maybe_data_dir = LevelDbPersistence::AppDataDirectory();
159-
HARD_ASSERT(maybe_data_dir.ok(),
160-
"Failed to find the App data directory for the current user.");
159+
LevelDbOpener opener(database_info_);
161160

162-
Path dir = LevelDbPersistence::StorageDirectory(
163-
database_info_, maybe_data_dir.ValueOrDie());
164-
165-
Serializer remote_serializer{database_info_.database_id()};
166-
167-
auto created = LevelDbPersistence::Create(
168-
std::move(dir), LocalSerializer{std::move(remote_serializer)},
169-
LruParams::WithCacheSize(settings.cache_size_bytes()));
161+
auto created =
162+
opener.Create(LruParams::WithCacheSize(settings.cache_size_bytes()));
170163
// If leveldb fails to start then just throw up our hands: the error is
171164
// unrecoverable. There's nothing an end-user can do and nearly all
172165
// failures indicate the developer is doing something grossly wrong so we

Firestore/core/src/firebase/firestore/local/CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ cc_library(
5151
leveldb_migrations.h
5252
leveldb_mutation_queue.cc
5353
leveldb_mutation_queue.h
54+
leveldb_opener.cc
55+
leveldb_opener.h
5456
leveldb_persistence.cc
5557
leveldb_persistence.h
5658
leveldb_remote_document_cache.cc
Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
/*
2+
* Copyright 2020 Google
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#include "Firestore/core/src/firebase/firestore/local/leveldb_opener.h"
18+
19+
#include <string>
20+
#include <utility>
21+
22+
#include "Firestore/core/src/firebase/firestore/core/database_info.h"
23+
#include "Firestore/core/src/firebase/firestore/local/leveldb_persistence.h"
24+
#include "Firestore/core/src/firebase/firestore/local/local_serializer.h"
25+
#include "Firestore/core/src/firebase/firestore/remote/serializer.h"
26+
#include "Firestore/core/src/firebase/firestore/util/filesystem.h"
27+
#include "Firestore/core/src/firebase/firestore/util/log.h"
28+
#include "Firestore/core/src/firebase/firestore/util/path.h"
29+
#include "Firestore/core/src/firebase/firestore/util/statusor.h"
30+
#include "Firestore/core/src/firebase/firestore/util/string_format.h"
31+
#include "absl/strings/match.h"
32+
33+
namespace firebase {
34+
namespace firestore {
35+
namespace local {
36+
namespace {
37+
38+
using core::DatabaseInfo;
39+
using remote::Serializer;
40+
using util::Filesystem;
41+
using util::Path;
42+
using util::Status;
43+
using util::StatusOr;
44+
using util::StringFormat;
45+
46+
constexpr const char* kReservedPathComponent = "firestore";
47+
48+
Status FromCause(const std::string& message, const Status& cause) {
49+
if (cause.ok()) return cause;
50+
51+
return Status(cause.code(), message).CausedBy(cause);
52+
}
53+
54+
} // namespace
55+
56+
LevelDbOpener::LevelDbOpener(DatabaseInfo database_info, Filesystem* fs)
57+
: database_info_(std::move(database_info)),
58+
fs_(fs ? fs : Filesystem::Default()) {
59+
}
60+
61+
LevelDbOpener::LevelDbOpener(DatabaseInfo database_info,
62+
Path firestore_app_data_dir)
63+
: database_info_(std::move(database_info)),
64+
app_data_dir_(std::move(firestore_app_data_dir)),
65+
fs_(Filesystem::Default()) {
66+
}
67+
68+
util::StatusOr<std::unique_ptr<LevelDbPersistence>> LevelDbOpener::Create(
69+
const LruParams& lru_params) {
70+
auto maybe_dir = PrepareDataDir();
71+
if (!maybe_dir.ok()) return maybe_dir.status();
72+
Path db_data_dir = maybe_dir.ValueOrDie();
73+
74+
LOG_DEBUG("Using %s for LevelDB storage", db_data_dir.ToUtf8String());
75+
76+
Serializer remote_serializer(database_info_.database_id());
77+
LocalSerializer local_serializer(std::move(remote_serializer));
78+
79+
return LevelDbPersistence::Create(db_data_dir, std::move(local_serializer),
80+
lru_params);
81+
}
82+
83+
StatusOr<Path> LevelDbOpener::LevelDbDataDir() {
84+
StatusOr<Path> maybe_dir = FirestoreAppDataDir();
85+
if (!maybe_dir.ok()) return maybe_dir;
86+
return StorageDir(maybe_dir.ValueOrDie());
87+
}
88+
89+
StatusOr<Path> LevelDbOpener::PrepareDataDir() {
90+
StatusOr<Path> maybe_dir = LevelDbDataDir();
91+
if (!maybe_dir.ok()) return maybe_dir;
92+
Path db_data_dir = std::move(maybe_dir).ValueOrDie();
93+
94+
// Check for the preferred location. If it exists, we're done.
95+
Status dir_status = fs_->IsDirectory(db_data_dir);
96+
if (dir_status.ok()) {
97+
return db_data_dir;
98+
} else if (dir_status.code() != Error::NotFound) {
99+
return dir_status;
100+
}
101+
102+
// The preferred dir doesn't exist so check for the legacy location. If it
103+
// exists, migrate.
104+
maybe_dir = FirestoreLegacyAppDataDir();
105+
Path legacy_db_data_dir;
106+
if (maybe_dir.ok()) {
107+
legacy_db_data_dir = StorageDir(std::move(maybe_dir).ValueOrDie());
108+
dir_status = fs_->IsDirectory(legacy_db_data_dir);
109+
} else {
110+
dir_status = maybe_dir.status();
111+
}
112+
113+
if (dir_status.ok()) {
114+
// The legacy directory does exist, so migrate
115+
return MigrateDataDir(legacy_db_data_dir, db_data_dir);
116+
117+
} else if (dir_status.code() != Error::NotFound &&
118+
dir_status.code() != Error::Unimplemented) {
119+
return dir_status;
120+
}
121+
122+
// Either we couldn't find the legacy directory or this platform has no legacy
123+
// directory so create the new directory.
124+
Status created = fs_->RecursivelyCreateDir(db_data_dir);
125+
if (!created.ok()) {
126+
std::string message =
127+
StringFormat("Could not create LevelDB data directory %s",
128+
db_data_dir.ToUtf8String());
129+
130+
return FromCause(message, created);
131+
}
132+
133+
return db_data_dir;
134+
}
135+
136+
StatusOr<Path> LevelDbOpener::FirestoreAppDataDir() {
137+
if (app_data_dir_.empty()) {
138+
auto maybe_dir = fs_->AppDataDir(kReservedPathComponent);
139+
if (!maybe_dir.ok()) {
140+
return FromCause(
141+
"Failed to find the App data directory for the current user",
142+
maybe_dir.status());
143+
}
144+
app_data_dir_ = std::move(maybe_dir).ValueOrDie();
145+
}
146+
return app_data_dir_;
147+
}
148+
149+
StatusOr<Path> LevelDbOpener::FirestoreLegacyAppDataDir() {
150+
if (legacy_app_data_dir_.empty()) {
151+
auto maybe_dir = fs_->LegacyDocumentsDir(kReservedPathComponent);
152+
if (!maybe_dir.ok()) {
153+
return FromCause(
154+
"Failed to find the Documents directory for the current user",
155+
maybe_dir.status());
156+
}
157+
legacy_app_data_dir_ = std::move(maybe_dir).ValueOrDie();
158+
}
159+
return legacy_app_data_dir_;
160+
}
161+
162+
Path LevelDbOpener::StorageDir(const Path& base_path) {
163+
// Use two different path formats:
164+
//
165+
// * persistence_key / project_id . database_id / name
166+
// * persistence_key / project_id / name
167+
//
168+
// project_ids are DNS-compatible names and cannot contain dots so there's
169+
// no danger of collisions.
170+
std::string project_key = database_info_.database_id().project_id();
171+
if (!database_info_.database_id().IsDefaultDatabase()) {
172+
absl::StrAppend(&project_key, ".",
173+
database_info_.database_id().database_id());
174+
}
175+
176+
// Reserve one additional path component to allow multiple physical databases
177+
return Path::JoinUtf8(base_path, database_info_.persistence_key(),
178+
project_key, "main");
179+
}
180+
181+
StatusOr<Path> LevelDbOpener::MigrateDataDir(
182+
const firebase::firestore::util::Path& legacy_db_data_dir,
183+
const firebase::firestore::util::Path& db_data_dir) {
184+
// At this point the legacy location exists and the preferred location doesn't
185+
// so just move into place.
186+
LOG_DEBUG(
187+
"Migrating LevelDB storage from legacy location: %s\nMigrating to: %s",
188+
legacy_db_data_dir.ToUtf8String(), db_data_dir.ToUtf8String());
189+
190+
Path db_data_parent = db_data_dir.Dirname();
191+
Status created = fs_->RecursivelyCreateDir(db_data_parent);
192+
if (!created.ok()) {
193+
std::string message =
194+
StringFormat("Could not create LevelDB data directory %s",
195+
db_data_parent.ToUtf8String());
196+
LOG_ERROR("Migration failed: %s. Existing data unchanged.", message);
197+
return FromCause(message, created);
198+
}
199+
200+
Status renamed = fs_->Rename(legacy_db_data_dir, db_data_dir);
201+
if (!renamed.ok()) {
202+
std::string message = StringFormat(
203+
"Failed to migrate LevelDB data from %s to %s",
204+
legacy_db_data_dir.ToUtf8String(), db_data_dir.ToUtf8String());
205+
LOG_ERROR("Migration failed: %s. Existing data unchanged.", message);
206+
return FromCause(message, renamed);
207+
}
208+
209+
RecursivelyCleanupLegacyDirs(legacy_db_data_dir);
210+
return db_data_dir;
211+
}
212+
213+
void LevelDbOpener::RecursivelyCleanupLegacyDirs(Path legacy_dir) {
214+
// The legacy_dir must be within the container_dir.
215+
HARD_ASSERT(!legacy_app_data_dir_.empty());
216+
HARD_ASSERT(absl::StartsWith(legacy_dir.ToUtf8String(),
217+
legacy_app_data_dir_.ToUtf8String()));
218+
219+
// The container directory contains a trailing "firestore" component
220+
HARD_ASSERT(absl::EndsWith(legacy_app_data_dir_.ToUtf8String(),
221+
kReservedPathComponent));
222+
223+
Path parent_most = legacy_app_data_dir_.Dirname();
224+
for (; legacy_dir != parent_most; legacy_dir = legacy_dir.Dirname()) {
225+
Status is_dir = fs_->IsDirectory(legacy_dir);
226+
if (is_dir.ok()) {
227+
if (util::IsEmptyDir(legacy_dir)) {
228+
Status removed = fs_->RemoveDir(legacy_dir);
229+
if (!removed.ok()) {
230+
LOG_WARN("Could not remove directory %s: %s",
231+
legacy_dir.ToUtf8String(), removed.ToString());
232+
break;
233+
}
234+
}
235+
236+
} else if (is_dir.code() != Error::NotFound) {
237+
LOG_WARN("Could not remove directory %s: %s", legacy_dir.ToUtf8String(),
238+
is_dir.ToString());
239+
break;
240+
}
241+
}
242+
}
243+
244+
} // namespace local
245+
} // namespace firestore
246+
} // namespace firebase

0 commit comments

Comments
 (0)