Migrate LevelDB data out of ~/Documents

Firestore/CHANGELOG.md

@@ -4,6 +4,12 @@
   improvements. This benefit is reduced if changes accumulate while the query
   is inactive. Queries that use the `limit()` API may not always benefit,
   depending on the accumulated changes.
+- [changed] Changed the location of Firestore's locally stored data from the
+  Documents folder to Library/Application Support, hiding it from users of apps
+  that share their files with the iOS Files app. **Important**: After a user's
+  data is migrated, downgrading to an older version of the SDK will cause the
+  user to appear to lose data, since older versions of the SDK can't read data
+  from the new location (#843).
 
 # v1.9.0
 - [feature] Added a `limit(toLast:)` query operator, which returns the last
@@ -43,7 +49,7 @@
   experience.
 
 # v1.6.1
-- [fixed] Fix a race condition that could cause a segmentation fault during
+- [fixed] Fixed a race condition that could cause a segmentation fault during
   client initialization.
 
 # v1.6.0

Firestore/Example/Tests/Util/FSTIntegrationTestCase.mm

@@ -40,7 +40,7 @@
 #include "Firestore/core/src/firebase/firestore/auth/credentials_provider.h"
 #include "Firestore/core/src/firebase/firestore/auth/empty_credentials_provider.h"
 #include "Firestore/core/src/firebase/firestore/auth/user.h"
-#include "Firestore/core/src/firebase/firestore/local/leveldb_persistence.h"
+#include "Firestore/core/src/firebase/firestore/local/leveldb_opener.h"
 #include "Firestore/core/src/firebase/firestore/model/database_id.h"
 #include "Firestore/core/src/firebase/firestore/remote/grpc_connection.h"
 #include "Firestore/core/src/firebase/firestore/util/async_queue.h"
@@ -50,7 +50,7 @@
 #include "Firestore/core/src/firebase/firestore/util/string_apple.h"
 #include "Firestore/core/test/firebase/firestore/testutil/app_testing.h"
 #include "Firestore/core/test/firebase/firestore/testutil/async_testing.h"
-#include "Firestore/core/test/firebase/firestore/util/status_testing.h"
+#include "Firestore/core/test/firebase/firestore/testutil/status_testing.h"
 #include "absl/memory/memory.h"
 
 namespace testutil = firebase::firestore::testutil;
@@ -60,7 +60,8 @@
 using firebase::firestore::auth::CredentialsProvider;
 using firebase::firestore::auth::EmptyCredentialsProvider;
 using firebase::firestore::auth::User;
-using firebase::firestore::local::LevelDbPersistence;
+using firebase::firestore::core::DatabaseInfo;
+using firebase::firestore::local::LevelDbOpener;
 using firebase::firestore::model::DatabaseId;
 using firebase::firestore::testutil::AppForUnitTesting;
 using firebase::firestore::testutil::AsyncQueueForTesting;
@@ -148,10 +149,12 @@ - (void)clearPersistenceOnce {
 
   @synchronized([FSTIntegrationTestCase class]) {
     if (clearedPersistence) return;
-    StatusOr<Path> maybe_dir = LevelDbPersistence::AppDataDirectory();
-    ASSERT_OK(maybe_dir);
+    DatabaseInfo dbInfo;
+    LevelDbOpener opener(dbInfo);
+    StatusOr<Path> maybeLevelDBDir = opener.FirestoreAppDataDir();
+    ASSERT_OK(maybeLevelDBDir.status());
+    Path levelDBDir = std::move(maybeLevelDBDir).ValueOrDie();
 
-    Path levelDBDir = maybe_dir.ValueOrDie();
     Status status = fs->RecursivelyRemove(levelDBDir);
     ASSERT_OK(status);
 

Firestore/core/src/firebase/firestore/core/firestore_client.cc

@@ -26,6 +26,7 @@
 #include "Firestore/core/src/firebase/firestore/core/event_manager.h"
 #include "Firestore/core/src/firebase/firestore/core/view.h"
 #include "Firestore/core/src/firebase/firestore/local/index_free_query_engine.h"
+#include "Firestore/core/src/firebase/firestore/local/leveldb_opener.h"
 #include "Firestore/core/src/firebase/firestore/local/leveldb_persistence.h"
 #include "Firestore/core/src/firebase/firestore/local/local_serializer.h"
 #include "Firestore/core/src/firebase/firestore/local/memory_persistence.h"
@@ -60,7 +61,7 @@ using auth::CredentialsProvider;
 using auth::User;
 using firestore::Error;
 using local::IndexFreeQueryEngine;
-using local::LevelDbPersistence;
+using local::LevelDbOpener;
 using local::LocalSerializer;
 using local::LocalStore;
 using local::LruParams;
@@ -155,18 +156,10 @@ void FirestoreClient::Initialize(const User& user, const Settings& settings) {
   // more work) since external write/listen operations could get queued to run
   // before that subsequent work completes.
   if (settings.persistence_enabled()) {
-    auto maybe_data_dir = LevelDbPersistence::AppDataDirectory();
-    HARD_ASSERT(maybe_data_dir.ok(),
-                "Failed to find the App data directory for the current user.");
+    LevelDbOpener opener(database_info_);
 
-    Path dir = LevelDbPersistence::StorageDirectory(
-        database_info_, maybe_data_dir.ValueOrDie());
-
-    Serializer remote_serializer{database_info_.database_id()};
-
-    auto created = LevelDbPersistence::Create(
-        std::move(dir), LocalSerializer{std::move(remote_serializer)},
-        LruParams::WithCacheSize(settings.cache_size_bytes()));
+    auto created =
+        opener.Create(LruParams::WithCacheSize(settings.cache_size_bytes()));
     // If leveldb fails to start then just throw up our hands: the error is
     // unrecoverable. There's nothing an end-user can do and nearly all
     // failures indicate the developer is doing something grossly wrong so we

Firestore/core/src/firebase/firestore/local/CMakeLists.txt

@@ -51,6 +51,8 @@ cc_library(
     leveldb_migrations.h
     leveldb_mutation_queue.cc
     leveldb_mutation_queue.h
+    leveldb_opener.cc
+    leveldb_opener.h
     leveldb_persistence.cc
     leveldb_persistence.h
     leveldb_query_cache.cc

Firestore/core/src/firebase/firestore/local/leveldb_opener.cc

@@ -0,0 +1,246 @@
+/*
+ * Copyright 2020 Google
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Firestore/core/src/firebase/firestore/local/leveldb_opener.h"
+
+#include <string>
+#include <utility>
+
+#include "Firestore/core/src/firebase/firestore/core/database_info.h"
+#include "Firestore/core/src/firebase/firestore/local/leveldb_persistence.h"
+#include "Firestore/core/src/firebase/firestore/local/local_serializer.h"
+#include "Firestore/core/src/firebase/firestore/remote/serializer.h"
+#include "Firestore/core/src/firebase/firestore/util/filesystem.h"
+#include "Firestore/core/src/firebase/firestore/util/log.h"
+#include "Firestore/core/src/firebase/firestore/util/path.h"
+#include "Firestore/core/src/firebase/firestore/util/statusor.h"
+#include "Firestore/core/src/firebase/firestore/util/string_format.h"
+#include "absl/strings/match.h"
+
+namespace firebase {
+namespace firestore {
+namespace local {
+namespace {
+
+using core::DatabaseInfo;
+using remote::Serializer;
+using util::Filesystem;
+using util::Path;
+using util::Status;
+using util::StatusOr;
+using util::StringFormat;
+
+constexpr const char* kReservedPathComponent = "firestore";
+
+Status FromCause(const std::string& message, const Status& cause) {
+  if (cause.ok()) return cause;
+
+  return Status(cause.code(), message).CausedBy(cause);
+}
+
+}  // namespace
+
+LevelDbOpener::LevelDbOpener(DatabaseInfo database_info, Filesystem* fs)
+    : database_info_(std::move(database_info)),
+      fs_(fs ? fs : Filesystem::Default()) {
+}
+
+LevelDbOpener::LevelDbOpener(DatabaseInfo database_info,
+                             Path firestore_app_data_dir)
+    : database_info_(std::move(database_info)),
+      app_data_dir_(std::move(firestore_app_data_dir)),
+      fs_(Filesystem::Default()) {
+}
+
+util::StatusOr<std::unique_ptr<LevelDbPersistence>> LevelDbOpener::Create(
+    const LruParams& lru_params) {
+  auto maybe_dir = PrepareDataDir();
+  if (!maybe_dir.ok()) return maybe_dir.status();
+  Path db_data_dir = maybe_dir.ValueOrDie();
+
+  LOG_DEBUG("Using %s for LevelDB storage", db_data_dir.ToUtf8String());
+
+  Serializer remote_serializer(database_info_.database_id());
+  LocalSerializer local_serializer(std::move(remote_serializer));
+
+  return LevelDbPersistence::Create(db_data_dir, std::move(local_serializer),
+                                    lru_params);
+}
+
+StatusOr<Path> LevelDbOpener::LevelDbDataDir() {
+  StatusOr<Path> maybe_dir = FirestoreAppDataDir();
+  if (!maybe_dir.ok()) return maybe_dir;
+  return StorageDir(maybe_dir.ValueOrDie());
+}
+
+StatusOr<Path> LevelDbOpener::PrepareDataDir() {
+  StatusOr<Path> maybe_dir = LevelDbDataDir();
+  if (!maybe_dir.ok()) return maybe_dir;
+  Path db_data_dir = std::move(maybe_dir).ValueOrDie();
+
+  // Check for the preferred location. If it exists, we're done.
+  Status dir_status = fs_->IsDirectory(db_data_dir);
+  if (dir_status.ok()) {
+    return db_data_dir;
+  } else if (dir_status.code() != Error::NotFound) {
+    return dir_status;
+  }
+
+  // The preferred dir doesn't exist so check for the legacy location. If it
+  // exists, migrate.
+  maybe_dir = FirestoreLegacyAppDataDir();
+  Path legacy_db_data_dir;
+  if (maybe_dir.ok()) {
+    legacy_db_data_dir = StorageDir(std::move(maybe_dir).ValueOrDie());
+    dir_status = fs_->IsDirectory(legacy_db_data_dir);
+  } else {
+    dir_status = maybe_dir.status();
+  }
+
+  if (dir_status.ok()) {
+    // The legacy directory does exist, so migrate
+    return MigrateDataDir(legacy_db_data_dir, db_data_dir);
+
+  } else if (dir_status.code() != Error::NotFound &&
+             dir_status.code() != Error::Unimplemented) {
+    return dir_status;
+  }
+
+  // Either we couldn't find the legacy directory or this platform has no legacy
+  // directory so create the new directory.
+  Status created = fs_->RecursivelyCreateDir(db_data_dir);
+  if (!created.ok()) {
+    std::string message =
+        StringFormat("Could not create LevelDB data directory %s",
+                     db_data_dir.ToUtf8String());
+
+    return FromCause(message, created);
+  }
+
+  return db_data_dir;
+}
+
+StatusOr<Path> LevelDbOpener::FirestoreAppDataDir() {
+  if (app_data_dir_.empty()) {
+    auto maybe_dir = fs_->AppDataDir(kReservedPathComponent);
+    if (!maybe_dir.ok()) {
+      return FromCause(
+          "Failed to find the App data directory for the current user",
+          maybe_dir.status());
+    }
+    app_data_dir_ = std::move(maybe_dir).ValueOrDie();
+  }
+  return app_data_dir_;
+}
+
+StatusOr<Path> LevelDbOpener::FirestoreLegacyAppDataDir() {
+  if (legacy_app_data_dir_.empty()) {
+    auto maybe_dir = fs_->LegacyDocumentsDir(kReservedPathComponent);
+    if (!maybe_dir.ok()) {
+      return FromCause(
+          "Failed to find the Documents directory for the current user",
+          maybe_dir.status());
+    }
+    legacy_app_data_dir_ = std::move(maybe_dir).ValueOrDie();
+  }
+  return legacy_app_data_dir_;
+}
+
+Path LevelDbOpener::StorageDir(const Path& base_path) {
+  // Use two different path formats:
+  //
+  //   * persistence_key / project_id . database_id / name
+  //   * persistence_key / project_id / name
+  //
+  // project_ids are DNS-compatible names and cannot contain dots so there's
+  // no danger of collisions.
+  std::string project_key = database_info_.database_id().project_id();
+  if (!database_info_.database_id().IsDefaultDatabase()) {
+    absl::StrAppend(&project_key, ".",
+                    database_info_.database_id().database_id());
+  }
+
+  // Reserve one additional path component to allow multiple physical databases
+  return Path::JoinUtf8(base_path, database_info_.persistence_key(),
+                        project_key, "main");
+}
+
+StatusOr<Path> LevelDbOpener::MigrateDataDir(
+    const firebase::firestore::util::Path& legacy_db_data_dir,
+    const firebase::firestore::util::Path& db_data_dir) {
+  // At this point the legacy location exists and the preferred location doesn't
+  // so just move into place.
+  LOG_DEBUG(
+      "Migrating LevelDB storage from legacy location: %s\nMigrating to: %s",
+      legacy_db_data_dir.ToUtf8String(), db_data_dir.ToUtf8String());
+
+  Path db_data_parent = db_data_dir.Dirname();
+  Status created = fs_->RecursivelyCreateDir(db_data_parent);
+  if (!created.ok()) {
+    std::string message =
+        StringFormat("Could not create LevelDB data directory %s",
+                     db_data_parent.ToUtf8String());
+    LOG_ERROR("Migration failed: %s. Existing data unchanged.", message);
+    return FromCause(message, created);
+  }
+
+  Status renamed = fs_->Rename(legacy_db_data_dir, db_data_dir);
+  if (!renamed.ok()) {
+    std::string message = StringFormat(
+        "Failed to migrate LevelDB data from %s to %s",
+        legacy_db_data_dir.ToUtf8String(), db_data_dir.ToUtf8String());
+    LOG_ERROR("Migration failed: %s. Existing data unchanged.", message);
+    return FromCause(message, renamed);
+  }
+
+  RecursivelyCleanupLegacyDirs(legacy_db_data_dir);
+  return db_data_dir;
+}
+
+void LevelDbOpener::RecursivelyCleanupLegacyDirs(Path legacy_dir) {
+  // The legacy_dir must be within the container_dir.
+  HARD_ASSERT(!legacy_app_data_dir_.empty());
+  HARD_ASSERT(absl::StartsWith(legacy_dir.ToUtf8String(),
+                               legacy_app_data_dir_.ToUtf8String()));
+
+  // The container directory contains a trailing "firestore" component
+  HARD_ASSERT(absl::EndsWith(legacy_app_data_dir_.ToUtf8String(),
+                             kReservedPathComponent));
+
+  Path parent_most = legacy_app_data_dir_.Dirname();
+  for (; legacy_dir != parent_most; legacy_dir = legacy_dir.Dirname()) {
+    Status is_dir = fs_->IsDirectory(legacy_dir);
+    if (is_dir.ok()) {
+      if (util::IsEmptyDir(legacy_dir)) {
+        Status removed = fs_->RemoveDir(legacy_dir);
+        if (!removed.ok()) {
+          LOG_WARN("Could not remove directory %s: %s",
+                   legacy_dir.ToUtf8String(), removed.ToString());
+          break;
+        }
+      }
+
+    } else if (is_dir.code() != Error::NotFound) {
+      LOG_WARN("Could not remove directory %s: %s", legacy_dir.ToUtf8String(),
+               is_dir.ToString());
+      break;
+    }
+  }
+}
+
+}  // namespace local
+}  // namespace firestore
+}  // namespace firebase