diff --git a/packages/firestore/src/exp/snapshot.ts b/packages/firestore/src/exp/snapshot.ts
index c7a1f499527..576238af9a8 100644
--- a/packages/firestore/src/exp/snapshot.ts
+++ b/packages/firestore/src/exp/snapshot.ts
@@ -278,7 +278,7 @@ export class DocumentSnapshot<
       return this._converter.fromFirestore(snapshot, options);
     } else {
       return this._userDataWriter.convertValue(
-        this._document.data.toProto(),
+        this._document.data.value,
         options.serverTimestamps
       ) as T;
     }
diff --git a/packages/firestore/src/lite/snapshot.ts b/packages/firestore/src/lite/snapshot.ts
index 01ab14d976b..297f046eb44 100644
--- a/packages/firestore/src/lite/snapshot.ts
+++ b/packages/firestore/src/lite/snapshot.ts
@@ -174,9 +174,7 @@ export class DocumentSnapshot<T = DocumentData> {
       );
       return this._converter.fromFirestore(snapshot);
     } else {
-      return this._userDataWriter.convertValue(
-        this._document.data.toProto()
-      ) as T;
+      return this._userDataWriter.convertValue(this._document.data.value) as T;
     }
   }
 
diff --git a/packages/firestore/src/lite/user_data_writer.ts b/packages/firestore/src/lite/user_data_writer.ts
index 3f48d9094cf..2b30b970573 100644
--- a/packages/firestore/src/lite/user_data_writer.ts
+++ b/packages/firestore/src/lite/user_data_writer.ts
@@ -92,7 +92,7 @@ export abstract class AbstractUserDataWriter {
     serverTimestampBehavior: ServerTimestampBehavior
   ): DocumentData {
     const result: DocumentData = {};
-    forEach(mapValue.fields || {}, (key, value) => {
+    forEach(mapValue.fields, (key, value) => {
       result[key] = this.convertValue(value, serverTimestampBehavior);
     });
     return result;
diff --git a/packages/firestore/src/local/local_store_impl.ts b/packages/firestore/src/local/local_store_impl.ts
index 8dae27bc990..ba5782753d6 100644
--- a/packages/firestore/src/local/local_store_impl.ts
+++ b/packages/firestore/src/local/local_store_impl.ts
@@ -16,7 +16,7 @@
  */
 
 import { User } from '../auth/user';
-import { BundledDocuments, NamedQuery, BundleConverter } from '../core/bundle';
+import { BundleConverter, BundledDocuments, NamedQuery } from '../core/bundle';
 import { newQueryForPath, Query, queryToTarget } from '../core/query';
 import { SnapshotVersion } from '../core/snapshot_version';
 import { canonifyTarget, Target, targetEquals } from '../core/target';
@@ -330,7 +330,7 @@ export function localStoreWriteLocally(
                 new PatchMutation(
                   mutation.key,
                   baseValue,
-                  extractFieldMask(baseValue.toProto().mapValue!),
+                  extractFieldMask(baseValue.value.mapValue),
                   Precondition.exists(true)
                 )
               );
diff --git a/packages/firestore/src/local/memory_persistence.ts b/packages/firestore/src/local/memory_persistence.ts
index 6437955c035..740ac5114c7 100644
--- a/packages/firestore/src/local/memory_persistence.ts
+++ b/packages/firestore/src/local/memory_persistence.ts
@@ -474,7 +474,7 @@ export class MemoryLruDelegate implements ReferenceDelegate, LruDelegate {
   documentSize(document: Document): number {
     let documentSize = document.key.toString().length;
     if (document.isFoundDocument()) {
-      documentSize += estimateByteSize(document.data.toProto());
+      documentSize += estimateByteSize(document.data.value);
     }
     return documentSize;
   }
diff --git a/packages/firestore/src/model/document.ts b/packages/firestore/src/model/document.ts
index 951d2d9c9d6..fa9dd8af977 100644
--- a/packages/firestore/src/model/document.ts
+++ b/packages/firestore/src/model/document.ts
@@ -331,7 +331,7 @@ export class MutableDocument implements Document {
   toString(): string {
     return (
       `Document(${this.key}, ${this.version}, ${JSON.stringify(
-        this.data.toProto()
+        this.data.value
       )}, ` +
       `{documentType: ${this.documentType}}), ` +
       `{documentState: ${this.documentState}})`
diff --git a/packages/firestore/src/model/object_value.ts b/packages/firestore/src/model/object_value.ts
index faa879044da..d5cb273eb9d 100644
--- a/packages/firestore/src/model/object_value.ts
+++ b/packages/firestore/src/model/object_value.ts
@@ -25,45 +25,21 @@ import { forEach } from '../util/obj';
 import { FieldMask } from './field_mask';
 import { FieldPath } from './path';
 import { isServerTimestamp } from './server_timestamps';
-import { TypeOrder } from './type_order';
-import { isMapValue, typeOrder, valueEquals } from './values';
+import { deepClone, isMapValue, valueEquals } from './values';
 
 export interface JsonObject<T> {
   [name: string]: T;
 }
-
-/**
- * An Overlay, which contains an update to apply. Can either be Value proto, a
- * map of Overlay values (to represent additional nesting at the given key) or
- * `null` (to represent field deletes).
- */
-type Overlay = Map<string, Overlay> | ProtoValue | null;
-
 /**
  * An ObjectValue represents a MapValue in the Firestore Proto and offers the
  * ability to add and remove fields (via the ObjectValueBuilder).
  */
 export class ObjectValue {
-  /**
-   * The immutable Value proto for this object. Local mutations are stored in
-   * `overlayMap` and only applied when `buildProto()` is invoked.
-   */
-  private partialValue: { mapValue: ProtoMapValue };
-
-  /**
-   * A nested map that contains the accumulated changes that haven't yet been
-   * applied to `partialValue`. Values can either be `Value` protos, Map<String,
-   * Object> values (to represent additional nesting) or `null` (to represent
-   * field deletes).
-   */
-  private overlayMap = new Map<string, Overlay>();
-
-  constructor(proto: { mapValue: ProtoMapValue }) {
+  constructor(readonly value: { mapValue: ProtoMapValue }) {
     debugAssert(
-      !isServerTimestamp(proto),
+      !isServerTimestamp(value),
       'ServerTimestamps should be converted to ServerTimestampValue'
     );
-    this.partialValue = proto;
   }
 
   static empty(): ObjectValue {
@@ -77,12 +53,19 @@ export class ObjectValue {
    * @returns The value at the path or null if the path is not set.
    */
   field(path: FieldPath): ProtoValue | null {
-    return ObjectValue.extractNestedValue(this.buildProto(), path);
-  }
-
-  /** Returns the full protobuf representation. */
-  toProto(): { mapValue: ProtoMapValue } {
-    return this.field(FieldPath.emptyPath()) as { mapValue: ProtoMapValue };
+    if (path.isEmpty()) {
+      return this.value;
+    } else {
+      let currentLevel: ProtoValue = this.value;
+      for (let i = 0; i < path.length - 1; ++i) {
+        currentLevel = (currentLevel.mapValue!.fields || {})[path.get(i)];
+        if (!isMapValue(currentLevel)) {
+          return null;
+        }
+      }
+      currentLevel = (currentLevel.mapValue!.fields! || {})[path.lastSegment()];
+      return currentLevel || null;
+    }
   }
 
   /**
@@ -96,7 +79,8 @@ export class ObjectValue {
       !path.isEmpty(),
       'Cannot set field for empty path on ObjectValue'
     );
-    this.setOverlay(path, value);
+    const fieldsMap = this.getFieldsMap(path.popLast());
+    fieldsMap[path.lastSegment()] = deepClone(value);
   }
 
   /**
@@ -105,13 +89,30 @@ export class ObjectValue {
    * @param data - A map of fields to values (or null for deletes).
    */
   setAll(data: Map<FieldPath, ProtoValue | null>): void {
-    data.forEach((value, fieldPath) => {
+    let parent = FieldPath.emptyPath();
+
+    let upserts: { [key: string]: ProtoValue } = {};
+    let deletes: string[] = [];
+
+    data.forEach((value, path) => {
+      if (!parent.isImmediateParentOf(path)) {
+        // Insert the accumulated changes at this parent location
+        const fieldsMap = this.getFieldsMap(parent);
+        this.applyChanges(fieldsMap, upserts, deletes);
+        upserts = {};
+        deletes = [];
+        parent = path.popLast();
+      }
+
       if (value) {
-        this.set(fieldPath, value);
+        upserts[path.lastSegment()] = deepClone(value);
       } else {
-        this.delete(fieldPath);
+        deletes.push(path.lastSegment());
       }
     });
+
+    const fieldsMap = this.getFieldsMap(parent);
+    this.applyChanges(fieldsMap, upserts, deletes);
   }
 
   /**
@@ -125,138 +126,58 @@ export class ObjectValue {
       !path.isEmpty(),
       'Cannot delete field for empty path on ObjectValue'
     );
-    this.setOverlay(path, null);
+    const nestedValue = this.field(path.popLast());
+    if (isMapValue(nestedValue) && nestedValue.mapValue.fields) {
+      delete nestedValue.mapValue.fields[path.lastSegment()];
+    }
   }
 
   isEqual(other: ObjectValue): boolean {
-    return valueEquals(this.buildProto(), other.buildProto());
+    return valueEquals(this.value, other.value);
   }
 
   /**
-   * Adds `value` to the overlay map at `path`. Creates nested map entries if
-   * needed.
+   * Returns the map that contains the leaf element of `path`. If the parent
+   * entry does not yet exist, or if it is not a map, a new map will be created.
    */
-  private setOverlay(path: FieldPath, value: ProtoValue | null): void {
-    let currentLevel = this.overlayMap;
+  private getFieldsMap(path: FieldPath): Record<string, ProtoValue> {
+    let current = this.value;
 
-    for (let i = 0; i < path.length - 1; ++i) {
-      const currentSegment = path.get(i);
-      let currentValue = currentLevel.get(currentSegment);
-
-      if (currentValue instanceof Map) {
-        // Re-use a previously created map
-        currentLevel = currentValue;
-      } else if (
-        currentValue &&
-        typeOrder(currentValue) === TypeOrder.ObjectValue
-      ) {
-        // Convert the existing Protobuf MapValue into a map
-        currentValue = new Map<string, Overlay>(
-          Object.entries(currentValue.mapValue!.fields || {})
-        );
-        currentLevel.set(currentSegment, currentValue);
-        currentLevel = currentValue;
-      } else {
-        // Create an empty map to represent the current nesting level
-        currentValue = new Map<string, Overlay>();
-        currentLevel.set(currentSegment, currentValue);
-        currentLevel = currentValue;
-      }
+    if (!current.mapValue!.fields) {
+      current.mapValue = { fields: {} };
     }
 
-    currentLevel.set(path.lastSegment(), value);
-  }
-
-  /**
-   * Applies any overlays from `currentOverlays` that exist at `currentPath`
-   * and returns the merged data at `currentPath` (or null if there were no
-   * changes).
-   *
-   * @param currentPath - The path at the current nesting level. Can be set to
-   * FieldValue.emptyPath() to represent the root.
-   * @param currentOverlays - The overlays at the current nesting level in the
-   * same format as `overlayMap`.
-   * @returns The merged data at `currentPath` or null if no modifications
-   * were applied.
-   */
-  private applyOverlay(
-    currentPath: FieldPath,
-    currentOverlays: Map<string, Overlay>
-  ): { mapValue: ProtoMapValue } | null {
-    let modified = false;
-
-    const existingValue = ObjectValue.extractNestedValue(
-      this.partialValue,
-      currentPath
-    );
-    const resultAtPath = isMapValue(existingValue)
-      ? // If there is already data at the current path, base our
-        // modifications on top of the existing data.
-        { ...existingValue.mapValue.fields }
-      : {};
-
-    currentOverlays.forEach((value, pathSegment) => {
-      if (value instanceof Map) {
-        const nested = this.applyOverlay(currentPath.child(pathSegment), value);
-        if (nested != null) {
-          resultAtPath[pathSegment] = nested;
-          modified = true;
-        }
-      } else if (value !== null) {
-        resultAtPath[pathSegment] = value;
-        modified = true;
-      } else if (resultAtPath.hasOwnProperty(pathSegment)) {
-        delete resultAtPath[pathSegment];
-        modified = true;
+    for (let i = 0; i < path.length; ++i) {
+      let next = current.mapValue!.fields![path.get(i)];
+      if (!isMapValue(next) || !next.mapValue.fields) {
+        next = { mapValue: { fields: {} } };
+        current.mapValue!.fields![path.get(i)] = next;
       }
-    });
+      current = next as { mapValue: ProtoMapValue };
+    }
 
-    return modified ? { mapValue: { fields: resultAtPath } } : null;
+    return current.mapValue!.fields!;
   }
 
   /**
-   * Builds the Protobuf that backs this ObjectValue.
-   *
-   * This method applies any outstanding modifications and memoizes the result.
-   * Further invocations are based on this memoized result.
+   * Modifies `fieldsMap` by adding, replacing or deleting the specified
+   * entries.
    */
-  private buildProto(): { mapValue: ProtoMapValue } {
-    const mergedResult = this.applyOverlay(
-      FieldPath.emptyPath(),
-      this.overlayMap
-    );
-    if (mergedResult != null) {
-      this.partialValue = mergedResult;
-      this.overlayMap.clear();
-    }
-    return this.partialValue;
-  }
-
-  private static extractNestedValue(
-    proto: ProtoValue,
-    path: FieldPath
-  ): ProtoValue | null {
-    if (path.isEmpty()) {
-      return proto;
-    } else {
-      let value: ProtoValue = proto;
-      for (let i = 0; i < path.length - 1; ++i) {
-        if (!value.mapValue!.fields) {
-          return null;
-        }
-        value = value.mapValue!.fields[path.get(i)];
-        if (!isMapValue(value)) {
-          return null;
-        }
-      }
-
-      value = (value.mapValue!.fields || {})[path.lastSegment()];
-      return value || null;
+  private applyChanges(
+    fieldsMap: Record<string, ProtoValue>,
+    inserts: { [key: string]: ProtoValue },
+    deletes: string[]
+  ): void {
+    forEach(inserts, (key, val) => (fieldsMap[key] = val));
+    for (const field of deletes) {
+      delete fieldsMap[field];
     }
   }
 
   clone(): ObjectValue {
-    return new ObjectValue(this.buildProto());
+    return new ObjectValue(
+      deepClone(this.value) as { mapValue: ProtoMapValue }
+    );
   }
 }
 
@@ -265,7 +186,7 @@ export class ObjectValue {
  */
 export function extractFieldMask(value: ProtoMapValue): FieldMask {
   const fields: FieldPath[] = [];
-  forEach(value!.fields || {}, (key, value) => {
+  forEach(value!.fields, (key, value) => {
     const currentPath = new FieldPath([key]);
     if (isMapValue(value)) {
       const nestedMask = extractFieldMask(value.mapValue!);
diff --git a/packages/firestore/src/model/values.ts b/packages/firestore/src/model/values.ts
index 04764b6caf1..720d65137d4 100644
--- a/packages/firestore/src/model/values.ts
+++ b/packages/firestore/src/model/values.ts
@@ -479,7 +479,7 @@ export function estimateByteSize(value: Value): number {
 
 function estimateMapByteSize(mapValue: MapValue): number {
   let size = 0;
-  forEach(mapValue.fields || {}, (key, val) => {
+  forEach(mapValue.fields, (key, val) => {
     size += key.length + estimateByteSize(val);
   });
   return size;
@@ -554,3 +554,27 @@ export function isMapValue(
 ): value is { mapValue: MapValue } {
   return !!value && 'mapValue' in value;
 }
+
+/** Creates a deep copy of `source`. */
+export function deepClone(source: Value): Value {
+  if (source.geoPointValue) {
+    return { geoPointValue: { ...source.geoPointValue } };
+  } else if (source.timestampValue) {
+    return { timestampValue: { ...normalizeTimestamp(source.timestampValue) } };
+  } else if (source.mapValue) {
+    const target: Value = { mapValue: { fields: {} } };
+    forEach(
+      source.mapValue.fields,
+      (key, val) => (target.mapValue!.fields![key] = deepClone(val))
+    );
+    return target;
+  } else if (source.arrayValue) {
+    const target: Value = { arrayValue: { values: [] } };
+    for (let i = 0; i < (source.arrayValue.values || []).length; ++i) {
+      target.arrayValue!.values![i] = deepClone(source.arrayValue.values![i]);
+    }
+    return target;
+  } else {
+    return { ...source };
+  }
+}
diff --git a/packages/firestore/src/remote/serializer.ts b/packages/firestore/src/remote/serializer.ts
index 06045a21900..7f4b71e8f2b 100644
--- a/packages/firestore/src/remote/serializer.ts
+++ b/packages/firestore/src/remote/serializer.ts
@@ -379,7 +379,7 @@ export function toMutationDocument(
 ): ProtoDocument {
   return {
     name: toName(serializer, key),
-    fields: fields.toProto().mapValue.fields
+    fields: fields.value.mapValue.fields
   };
 }
 
@@ -393,7 +393,7 @@ export function toDocument(
   );
   return {
     name: toName(serializer, document.key),
-    fields: document.data.toProto().mapValue.fields,
+    fields: document.data.value.mapValue.fields,
     updateTime: toTimestamp(serializer, document.version.toTimestamp())
   };
 }
diff --git a/packages/firestore/src/util/obj.ts b/packages/firestore/src/util/obj.ts
index 5081aed1867..53e05f48c3f 100644
--- a/packages/firestore/src/util/obj.ts
+++ b/packages/firestore/src/util/obj.ts
@@ -32,7 +32,7 @@ export function objectSize(obj: object): number {
 }
 
 export function forEach<V>(
-  obj: Dict<V>,
+  obj: Dict<V> | undefined,
   fn: (key: string, val: V) => void
 ): void {
   for (const key in obj) {
diff --git a/packages/firestore/test/unit/model/document.test.ts b/packages/firestore/test/unit/model/document.test.ts
index 431f77d0ca2..cfb93d15e6f 100644
--- a/packages/firestore/test/unit/model/document.test.ts
+++ b/packages/firestore/test/unit/model/document.test.ts
@@ -34,7 +34,7 @@ describe('Document', () => {
     const document = doc('rooms/Eros', 1, data);
 
     const value = document.data;
-    expect(value.toProto()).to.deep.equal(
+    expect(value.value).to.deep.equal(
       wrap({
         desc: 'Discuss all the project related stuff',
         owner: 'Jonny'
diff --git a/packages/firestore/test/unit/model/object_value.test.ts b/packages/firestore/test/unit/model/object_value.test.ts
index 32e1036cb22..d2f3140a191 100644
--- a/packages/firestore/test/unit/model/object_value.test.ts
+++ b/packages/firestore/test/unit/model/object_value.test.ts
@@ -170,7 +170,7 @@ describe('ObjectValue', () => {
       'map.nested.d',
       'emptymap'
     );
-    const actualMask = extractFieldMask(objValue.toProto().mapValue);
+    const actualMask = extractFieldMask(objValue.value.mapValue);
     expect(actualMask.isEqual(expectedMask)).to.be.true;
   });
 
diff --git a/packages/firestore/test/unit/remote/serializer.helper.ts b/packages/firestore/test/unit/remote/serializer.helper.ts
index 56ccfd9aefc..f06509fb7fd 100644
--- a/packages/firestore/test/unit/remote/serializer.helper.ts
+++ b/packages/firestore/test/unit/remote/serializer.helper.ts
@@ -797,7 +797,7 @@ export function serializerTest(
       const d = doc('foo/bar', 42, { a: 5, b: 'b' });
       const proto = {
         name: toName(s, d.key),
-        fields: d.data.toProto().mapValue.fields,
+        fields: d.data.value.mapValue.fields,
         updateTime: toVersion(s, d.version)
       };
       const serialized = toDocument(s, d);
diff --git a/packages/firestore/test/unit/specs/bundle_spec.test.ts b/packages/firestore/test/unit/specs/bundle_spec.test.ts
index 5d0ac79eb3c..3efa1be4d63 100644
--- a/packages/firestore/test/unit/specs/bundle_spec.test.ts
+++ b/packages/firestore/test/unit/specs/bundle_spec.test.ts
@@ -22,8 +22,8 @@ import { LimitType } from '../../../src/protos/firestore_bundle_proto';
 import { toVersion } from '../../../src/remote/serializer';
 import {
   doc,
-  query,
   filter,
+  query,
   TestSnapshotVersion,
   version,
   wrapObject
@@ -76,7 +76,7 @@ function bundleWithDocumentAndQuery(
       testDoc.key,
       toVersion(JSON_SERIALIZER, version(testDoc.createTime)),
       toVersion(JSON_SERIALIZER, version(testDoc.updateTime!)),
-      wrapObject(testDoc.content!).toProto().mapValue.fields!
+      wrapObject(testDoc.content!).value.mapValue.fields!
     );
   }
   return builder.build(
diff --git a/packages/firestore/test/unit/specs/spec_builder.ts b/packages/firestore/test/unit/specs/spec_builder.ts
index 26993992107..9bba02260f3 100644
--- a/packages/firestore/test/unit/specs/spec_builder.ts
+++ b/packages/firestore/test/unit/specs/spec_builder.ts
@@ -17,12 +17,12 @@
 
 import { UserDataWriter } from '../../../src/api/database';
 import {
+  hasLimitToFirst,
+  hasLimitToLast,
+  newQueryForPath,
   Query,
   queryEquals,
-  newQueryForPath,
-  queryToTarget,
-  hasLimitToLast,
-  hasLimitToFirst
+  queryToTarget
 } from '../../../src/core/query';
 import {
   canonifyTarget,
@@ -1044,7 +1044,7 @@ export class SpecBuilder {
         key: SpecBuilder.keyToSpec(doc.key),
         version: doc.version.toMicroseconds(),
         value: userDataWriter.convertValue(
-          doc.data.toProto()
+          doc.data.value
         ) as JsonObject<unknown>,
         options: {
           hasLocalMutations: doc.hasLocalMutations,
diff --git a/packages/firestore/test/util/spec_test_helpers.ts b/packages/firestore/test/util/spec_test_helpers.ts
index 389112f3fd6..c721d0e0675 100644
--- a/packages/firestore/test/util/spec_test_helpers.ts
+++ b/packages/firestore/test/util/spec_test_helpers.ts
@@ -56,7 +56,7 @@ export function encodeWatchChange(
         documentChange: {
           document: {
             name: toName(serializer, doc.key),
-            fields: doc?.data?.toProto().mapValue.fields,
+            fields: doc?.data.value.mapValue.fields,
             updateTime: toVersion(serializer, doc.version)
           },
           targetIds: watchChange.updatedTargetIds,