Don't deserialize full Document proto for Query execution

Author: schmidt-sebastian

packages/firestore/CHANGELOG.md

@@ -1,5 +1,6 @@
-
 # Unreleased
+- [changed] Improved performance for queries with filters that only return a
+  small subset of the documents in a collection.
 - [fixed] Fixed an internal assertion that was triggered when an update
    with a `FieldValue.serverTimestamp()` and an update with a
   `FieldValue.increment()` were pending for the same document.

packages/firestore/src/api/database.ts

@@ -1311,7 +1311,7 @@ export class DocumentSnapshot implements firestore.DocumentSnapshot {
     return !this._document
       ? undefined
       : this.convertObject(
-          this._document.data,
+          this._document.data(),
           FieldValueOptions.fromSnapshotOptions(
             options,
             this._firestore._areTimestampsInSnapshotsEnabled()
@@ -1326,9 +1326,9 @@ export class DocumentSnapshot implements firestore.DocumentSnapshot {
     validateBetweenNumberOfArgs('DocumentSnapshot.get', arguments, 1, 2);
     options = validateSnapshotOptions('DocumentSnapshot.get', options);
     if (this._document) {
-      const value = this._document.data.field(
-        fieldPathFromArgument('DocumentSnapshot.get', fieldPath)
-      );
+      const value = this._document
+        .data()
+        .field(fieldPathFromArgument('DocumentSnapshot.get', fieldPath));
       if (value !== null) {
         return this.convertValue(
           value,

packages/firestore/src/core/query.ts

@@ -759,7 +759,7 @@ export class Bound {
       } else {
         const docValue = doc.field(orderByComponent.field);
         assert(
-          docValue !== undefined,
+          docValue !== null,
           'Field should exist since document matched the orderBy already.'
         );
         comparison = component.compareTo(docValue!);

packages/firestore/src/core/view.ts

@@ -169,7 +169,7 @@ export class View {
 
         // Calculate change
         if (oldDoc && newDoc) {
-          const docsEqual = oldDoc.data.isEqual(newDoc.data);
+          const docsEqual = oldDoc.data().isEqual(newDoc.data());
           if (!docsEqual) {
             if (!this.shouldWaitForSyncedDocument(oldDoc, newDoc)) {
               changeSet.track({

packages/firestore/src/model/document.ts

@@ -16,13 +16,14 @@
  */
 
 import { SnapshotVersion } from '../core/snapshot_version';
-import { fail } from '../util/assert';
+import { assert, fail } from '../util/assert';
 
 import { DocumentKey } from './document_key';
 import { FieldValue, JsonObject, ObjectValue } from './field_value';
 import { FieldPath } from './path';
 
 import * as api from '../protos/firestore_proto_api';
+import * as obj from '../util/obj';
 
 export interface DocumentOptions {
   hasLocalMutations?: boolean;
@@ -59,43 +60,111 @@ export class Document extends MaybeDocument {
   readonly hasLocalMutations: boolean;
   readonly hasCommittedMutations: boolean;
 
+  /**
+   * A cache of canonicalized FieldPaths to FieldValues that have already been
+   * deserialized in `getField()`.
+   */
+  private fieldValueCache?: Map<string, FieldValue | null>;
+
   constructor(
     key: DocumentKey,
     version: SnapshotVersion,
-    readonly data: ObjectValue,
     options: DocumentOptions,
-    /**
-     * Memoized serialized form of the document for optimization purposes (avoids repeated
-     * serialization). Might be undefined.
-     */
-    readonly proto?: api.Document
+    private objectValue?: ObjectValue,
+    readonly proto?: api.Document,
+    private readonly converter?: (value: api.Value) => FieldValue
   ) {
     super(key, version);
     this.hasLocalMutations = !!options.hasLocalMutations;
     this.hasCommittedMutations = !!options.hasCommittedMutations;
   }
 
   field(path: FieldPath): FieldValue | null {
-    return this.data.field(path);
+    if (this.objectValue) {
+      return this.objectValue.field(path);
+    } else {
+      assert(
+        this.proto !== undefined && this.converter !== undefined,
+        'Expected proto and converter to be defined.'
+      );
+
+      if (!this.fieldValueCache) {
+        // TODO(b/136090445): Remove the cache when `getField` is no longer
+        // called during Query ordering.
+        this.fieldValueCache = new Map<string, FieldValue>();
+      }
+
+      const canonicalPath = path.canonicalString();
+
+      let fieldValue = this.fieldValueCache.get(canonicalPath);
+
+      if (fieldValue === undefined) {
+        // Instead of deserializing the full Document proto, we only
+        // deserialize the value at the requested field path. This speeds up
+        // Query execution as query filters can discard documents based on a
+        // single field.
+        let protoValue: api.Value | undefined = this.proto!.fields[
+          path.firstSegment()
+        ];
+        for (let i = 1; protoValue !== undefined && i < path.length; ++i) {
+          if (protoValue.mapValue) {
+            protoValue = protoValue.mapValue.fields[path.get(i)];
+          } else {
+            protoValue = undefined;
+          }
+        }
+
+        if (protoValue === undefined) {
+          fieldValue = null;
+        } else {
+          fieldValue = this.converter!(protoValue);
+        }
+
+        this.fieldValueCache.set(canonicalPath, fieldValue);
+      }
+
+      return fieldValue!;
+    }
   }
 
   fieldValue(path: FieldPath): unknown {
     const field = this.field(path);
     return field ? field.value() : undefined;
   }
 
+  data(): ObjectValue {
+    if (!this.objectValue) {
+      assert(
+        this.proto !== undefined && this.converter !== undefined,
+        'Expected proto and converter to be defined.'
+      );
+
+      let result = ObjectValue.EMPTY;
+      obj.forEach(this.proto!.fields, (key: string, value: api.Value) => {
+        result = result.set(new FieldPath([key]), this.converter!(value));
+      });
+      this.objectValue = result;
+
+      // Once objectValue is computed, values inside the fieldValueCache are no
+      // longer accessed.
+      this.fieldValueCache = undefined;
+    }
+
+    return this.objectValue;
+  }
+
   value(): JsonObject<unknown> {
-    return this.data.value();
+    return this.data().value();
   }
 
   isEqual(other: MaybeDocument | null | undefined): boolean {
     return (
       other instanceof Document &&
       this.key.isEqual(other.key) &&
       this.version.isEqual(other.version) &&
-      this.data.isEqual(other.data) &&
       this.hasLocalMutations === other.hasLocalMutations &&
-      this.hasCommittedMutations === other.hasCommittedMutations
+      this.hasCommittedMutations === other.hasCommittedMutations &&
+      this.data().isEqual(other.data())
     );
   }
 

packages/firestore/src/model/mutation.ts

@@ -353,9 +353,14 @@ export class SetMutation extends Mutation {
     // have held.
 
     const version = mutationResult.version;
-    return new Document(this.key, version, this.value, {
-      hasCommittedMutations: true
-    });
+    return new Document(
+      this.key,
+      version,
+      {
+        hasCommittedMutations: true
+      },
+      this.value
+    );
   }
 
   applyToLocalView(
@@ -370,9 +375,14 @@ export class SetMutation extends Mutation {
     }
 
     const version = Mutation.getPostMutationVersion(maybeDoc);
-    return new Document(this.key, version, this.value, {
-      hasLocalMutations: true
-    });
+    return new Document(
+      this.key,
+      version,
+      {
+        hasLocalMutations: true
+      },
+      this.value
+    );
   }
 
   extractBaseValue(maybeDoc: MaybeDocument | null): null {
@@ -434,9 +444,14 @@ export class PatchMutation extends Mutation {
     }
 
     const newData = this.patchDocument(maybeDoc);
-    return new Document(this.key, mutationResult.version, newData, {
-      hasCommittedMutations: true
-    });
+    return new Document(
+      this.key,
+      mutationResult.version,
+      {
+        hasCommittedMutations: true
+      },
+      newData
+    );
   }
 
   applyToLocalView(
@@ -452,9 +467,14 @@ export class PatchMutation extends Mutation {
 
     const version = Mutation.getPostMutationVersion(maybeDoc);
     const newData = this.patchDocument(maybeDoc);
-    return new Document(this.key, version, newData, {
-      hasLocalMutations: true
-    });
+    return new Document(
+      this.key,
+      version,
+      {
+        hasLocalMutations: true
+      },
+      newData
+    );
   }
 
   extractBaseValue(maybeDoc: MaybeDocument | null): null {
@@ -478,7 +498,7 @@ export class PatchMutation extends Mutation {
   private patchDocument(maybeDoc: MaybeDocument | null): ObjectValue {
     let data: ObjectValue;
     if (maybeDoc instanceof Document) {
-      data = maybeDoc.data;
+      data = maybeDoc.data();
     } else {
       data = ObjectValue.EMPTY;
     }
@@ -550,10 +570,15 @@ export class TransformMutation extends Mutation {
     );
 
     const version = mutationResult.version;
-    const newData = this.transformObject(doc.data, transformResults);
-    return new Document(this.key, version, newData, {
-      hasCommittedMutations: true
-    });
+    const newData = this.transformObject(doc.data(), transformResults);
+    return new Document(
+      this.key,
+      version,
+      {
+        hasCommittedMutations: true
+      },
+      newData
+    );
   }
 
   applyToLocalView(
@@ -573,10 +598,15 @@ export class TransformMutation extends Mutation {
       maybeDoc,
       baseDoc
     );
-    const newData = this.transformObject(doc.data, transformResults);
-    return new Document(this.key, doc.version, newData, {
-      hasLocalMutations: true
-    });
+    const newData = this.transformObject(doc.data(), transformResults);
+    return new Document(
+      this.key,
+      doc.version,
+      {
+        hasLocalMutations: true
+      },
+      newData
+    );
   }
 
   extractBaseValue(maybeDoc: MaybeDocument | null): ObjectValue | null {

packages/firestore/src/remote/serializer.ts

@@ -519,7 +519,7 @@ export class JsonProtoSerializer {
     );
     return {
       name: this.toName(document.key),
-      fields: this.toFields(document.data),
+      fields: this.toFields(document.data()),
       updateTime: this.toTimestamp(document.version.toTimestamp())
     };
   }
@@ -528,11 +528,15 @@ export class JsonProtoSerializer {
     document: api.Document,
     hasCommittedMutations?: boolean
   ): Document {
+    const key = this.fromName(document.name!);
+    const version = this.fromVersion(document.updateTime!);
     return new Document(
-      this.fromName(document.name!),
-      this.fromVersion(document.updateTime!),
-      this.fromFields(document.fields || {}),
-      { hasCommittedMutations: !!hasCommittedMutations }
+      key,
+      version,
+      { hasCommittedMutations: !!hasCommittedMutations },
+      undefined,
+      document,
+      v => this.fromValue(v)
     );
   }
 
@@ -577,8 +581,9 @@ export class JsonProtoSerializer {
     assertPresent(doc.found!.updateTime, 'doc.found.updateTime');
     const key = this.fromName(doc.found!.name!);
     const version = this.fromVersion(doc.found!.updateTime!);
-    const fields = this.fromFields(doc.found!.fields || {});
-    return new Document(key, version, fields, {}, doc.found!);
+    return new Document(key, version, {}, undefined, doc.found!, v =>
+      this.fromValue(v)
+    );
   }
 
   private fromMissing(result: api.BatchGetDocumentsResponse): NoDocument {
@@ -639,7 +644,7 @@ export class JsonProtoSerializer {
           documentChange: {
             document: {
               name: this.toName(doc.key),
-              fields: this.toFields(doc.data),
+              fields: this.toFields(doc.data()),
               updateTime: this.toVersion(doc.version)
             },
             targetIds: watchChange.updatedTargetIds,
@@ -718,15 +723,13 @@ export class JsonProtoSerializer {
       const entityChange = change.documentChange!;
       const key = this.fromName(entityChange.document!.name!);
       const version = this.fromVersion(entityChange.document!.updateTime!);
-      const fields = this.fromFields(entityChange.document!.fields || {});
-      // The document may soon be re-serialized back to protos in order to store it in local
-      // persistence. Memoize the encoded form to avoid encoding it again.
       const doc = new Document(
         key,
         version,
-        fields,
         {},
-        entityChange.document!
+        undefined,
+        entityChange.document!,
+        v => this.fromValue(v)
       );
       const updatedTargetIds = entityChange.targetIds || [];
       const removedTargetIds = entityChange.removedTargetIds || [];

packages/firestore/test/unit/local/lru_garbage_collector.test.ts

@@ -243,8 +243,8 @@ function genericLruGarbageCollectorTests(
     return new Document(
       key,
       SnapshotVersion.fromMicroseconds(1000),
-      wrapObject({ foo: 3, bar: false }),
-      {}
+      {},
+      wrapObject({ foo: 3, bar: false })
     );
   }
 
@@ -774,8 +774,8 @@ function genericLruGarbageCollectorTests(
         const doc = new Document(
           middleDocToUpdate,
           SnapshotVersion.fromMicroseconds(2000),
-          wrapObject({ foo: 4, bar: true }),
-          {}
+          {},
+          wrapObject({ foo: 4, bar: true })
         );
         return saveDocument(txn, doc).next(() => {
           return updateTargetInTransaction(txn, middleTarget);