Adding Spec Test Benchmark suite

Author: schmidt-sebastian

packages/firestore/test/unit/specs/describe_spec.ts

@@ -31,7 +31,9 @@ const NO_WEB_TAG = 'no-web';
 const NO_ANDROID_TAG = 'no-android';
 const NO_IOS_TAG = 'no-ios';
 const NO_LRU = 'no-lru';
+const BENCHMARK_TAG = 'benchmark';
 const KNOWN_TAGS = [
+  BENCHMARK_TAG,
   EXCLUSIVE_TAG,
   PERSISTENCE_TAG,
   NO_WEB_TAG,
@@ -40,6 +42,9 @@ const KNOWN_TAGS = [
   NO_LRU
 ];
 
+// TOOD: Make this configurable with mocha options.
+const RUN_BENCHMARK_TESTS = false;
+
 const WEB_SPEC_TEST_FILTER = (tags: string[]) =>
   tags.indexOf(NO_WEB_TAG) === -1;
 
@@ -127,6 +132,8 @@ export function specTest(
         runner = it.only;
       } else if (!WEB_SPEC_TEST_FILTER(tags)) {
         runner = it.skip;
+      } else if (tags.indexOf(BENCHMARK_TAG) >= 0 && !RUN_BENCHMARK_TESTS) {
+        runner = it.skip;
       } else if (usePersistence && tags.indexOf('no-lru') !== -1) {
         // spec should have a comment explaining why it is being skipped.
         runner = it.skip;
@@ -135,8 +142,14 @@ export function specTest(
       }
       const mode = usePersistence ? '(Persistence)' : '(Memory)';
       const fullName = `${mode} ${name}`;
-      runner(fullName, () => {
-        return spec.runAsTest(fullName, usePersistence);
+      runner(fullName, async () => {
+        const start = Date.now();
+        await spec.runAsTest(fullName, usePersistence);
+        const end = Date.now();
+        if (tags.indexOf(BENCHMARK_TAG) >= 0) {
+          // tslint:disable-next-line:no-console
+          console.log(`Runtime: ${end - start} ms.`);
+        }
       });
     }
   } else {

packages/firestore/test/unit/specs/perf_spec.test.ts

@@ -0,0 +1,216 @@
+/**
+ * Copyright 2017 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Query } from '../../../src/core/query';
+import { doc, orderBy, path } from '../../util/helpers';
+
+import { describeSpec, specTest } from './describe_spec';
+import { spec } from './spec_builder';
+
+const STEP_COUNT = 10;
+
+describeSpec('Performance Tests:', ['benchmark'], () => {
+  specTest('Insert a new document', [], () => {
+    let steps = spec().withGCEnabled(false);
+    for (let i = 0; i < STEP_COUNT; ++i) {
+      steps = steps.userSets(`collection/{i}`, { doc: i }).writeAcks(i);
+    }
+    return steps;
+  });
+
+  specTest('Insert a new document and wait for snapshot', [], () => {
+    let currentVersion = 1;
+    let steps = spec().withGCEnabled(false);
+
+    for (let i = 0; i < STEP_COUNT; ++i) {
+      const query = Query.atPath(path(`collection/${i}`));
+      const docLocal = doc(
+        `collection/${i}`,
+        0,
+        { doc: i },
+        { hasLocalMutations: true }
+      );
+      const docRemote = doc(`collection/${i}`, ++currentVersion, { doc: i });
+
+      steps = steps
+        .userListens(query)
+        .userSets(`collection/${i}`, { doc: i })
+        .expectEvents(query, {
+          added: [docLocal],
+          fromCache: true,
+          hasPendingWrites: true
+        })
+        .writeAcks(++currentVersion)
+        .watchAcksFull(query, ++currentVersion, docRemote)
+        .expectEvents(query, { metadata: [docRemote] })
+        .userUnlistens(query)
+        .watchRemoves(query);
+    }
+    return steps;
+  });
+
+  specTest('Watch has cached mutations', [], () => {
+    const cachedDocumentCount = 100;
+
+    const query = Query.atPath(path(`collection`)).addOrderBy(orderBy('v'));
+
+    let steps = spec().withGCEnabled(false);
+
+    const docs = [];
+
+    for (let i = 0; i < cachedDocumentCount; ++i) {
+      steps.userSets(`collection/${i}`, { v: i });
+      docs.push(
+        doc(`collection/${i}`, 0, { v: i }, { hasLocalMutations: true })
+      );
+    }
+
+    for (let i = 1; i <= STEP_COUNT; ++i) {
+      steps = steps
+        .userListens(query)
+        .expectEvents(query, {
+          added: docs,
+          fromCache: true,
+          hasPendingWrites: true
+        })
+        .userUnlistens(query);
+    }
+
+    return steps;
+  });
+
+  specTest('Update a single document', [], () => {
+    let steps = spec().withGCEnabled(false);
+    steps = steps.userSets(`collection/doc`, { v: 0 });
+    for (let i = 1; i <= STEP_COUNT; ++i) {
+      steps = steps.userPatches(`collection/doc`, { v: i }).writeAcks(i);
+    }
+    return steps;
+  });
+
+  specTest('Update a single document and wait for snapshot', [], () => {
+    const query = Query.atPath(path(`collection/doc`));
+
+    let currentVersion = 1;
+    let steps = spec().withGCEnabled(false);
+
+    let docLocal = doc(
+      `collection/doc`,
+      0,
+      { v: 0 },
+      { hasLocalMutations: true }
+    );
+    let docRemote = doc(`collection/doc`, ++currentVersion, { v: 0 });
+    let lastRemoteVersion = currentVersion;
+
+    steps = steps
+      .userListens(query)
+      .userSets(`collection/doc`, { v: 0 })
+      .expectEvents(query, {
+        added: [docLocal],
+        fromCache: true,
+        hasPendingWrites: true
+      })
+      .writeAcks(++currentVersion)
+      .watchAcksFull(query, ++currentVersion, docRemote)
+      .expectEvents(query, { metadata: [docRemote] });
+
+    for (let i = 1; i <= STEP_COUNT; ++i) {
+      docLocal = doc(
+        `collection/doc`,
+        lastRemoteVersion,
+        { v: i },
+        { hasLocalMutations: true }
+      );
+      docRemote = doc(`collection/doc`, ++currentVersion, { v: i });
+      lastRemoteVersion = currentVersion;
+
+      steps = steps
+        .userPatches(`collection/doc`, { v: i })
+        .expectEvents(query, { modified: [docLocal], hasPendingWrites: true })
+        .writeAcks(++currentVersion)
+        .watchSends({ affects: [query] }, docRemote)
+        .watchSnapshots(++currentVersion)
+        .expectEvents(query, { metadata: [docRemote] });
+    }
+    return steps;
+  });
+
+  specTest('Watch sends 100 documents', [], () => {
+    const documentsPerStep = 100;
+
+    const query = Query.atPath(path(`collection`)).addOrderBy(orderBy('v'));
+
+    let currentVersion = 1;
+    let steps = spec().withGCEnabled(false);
+
+    steps = steps
+      .userListens(query)
+      .watchAcksFull(query, currentVersion)
+      .expectEvents(query, {});
+
+    for (let i = 1; i <= STEP_COUNT; ++i) {
+      const docs = [];
+
+      for (let j = 0; j < documentsPerStep; ++j) {
+        docs.push(
+          doc(`collection/${j}`, ++currentVersion, { v: currentVersion })
+        );
+      }
+
+      const changeType = i === 1 ? 'added' : 'modified';
+
+      steps = steps
+        .watchSends({ affects: [query] }, ...docs)
+        .watchSnapshots(++currentVersion)
+        .expectEvents(query, { [changeType]: docs });
+    }
+
+    return steps;
+  });
+
+  specTest('Watch has cached results', [], () => {
+    const documentsPerStep = 100;
+
+    let currentVersion = 1;
+    let steps = spec().withGCEnabled(false);
+
+    for (let i = 1; i <= STEP_COUNT; ++i) {
+      const collPath = `collection/${i}/coll`;
+      const query = Query.atPath(path(collPath)).addOrderBy(orderBy('v'));
+
+      const docs = [];
+      for (let j = 0; j < documentsPerStep; ++j) {
+        docs.push(doc(`${collPath}/${j}`, ++currentVersion, { v: j }));
+      }
+
+      steps = steps
+        .userListens(query)
+        .watchAcksFull(query, ++currentVersion, ...docs)
+        .expectEvents(query, { added: docs })
+        .userUnlistens(query)
+        .watchRemoves(query)
+        .userListens(query, 'resume-token-' + currentVersion)
+        .expectEvents(query, { added: docs, fromCache: true })
+        .watchAcksFull(query, ++currentVersion)
+        .expectEvents(query, {})
+        .userUnlistens(query)
+        .watchRemoves(query);
+    }
+
+    return steps;
+  });
+});