Revive Node.JS Support for Cloud Firestore (fixes #221). (#319)

* Fix miscellaneous node / GRPC bit rot.
  * Re-add grpc dependency (upgraded).
  * Avoid protobufjs dependency by loading protos via grpc.
  * Remove crazy grpc stream error handling nonsense that is no longer
    necessary after grpc/grpc#9101
  * Clean up grpc_connection logging (and consistently use util.inspect()).
* Fix WebChannel / GRPC Connection compatibility issues.
  * Add an explicit mapping from "RPC name" (e.g. "BatchGetDocuments") to the
    REST url path (e.g. "batchGet") for WebChannel, and for GRPC just assume
    the first letter should be lowercased (e.g. "batchGetDocuments").
  * Split Connection.invoke() into invokeRPC() and invokeStreamingRPC(), with
    the latter accepting a stream of results and aggregating them into an
    array (needed to support BatchGetDocuments RPC).
* Fix serializer issues
  * Query limits are an 'Int32Value' but we were serializing them as a normal
    int which GRPC / protobufjs didn't like.
  * Several proto "oneof tags" were outdated.
* Add build steps to copy protos into npm package.
* Run integration tests for Node.JS
  * Added to 'test:node' script in package.json and in .vscode/launch.json
  * Include index.ts for browser and index.node.ts for node so the appropriate
    PlatformSupport gets registered.
* Misc cleanup
  * Remove unused MockPlatform since we use the normal NodePlatform now.
  * Remove 'google-auth-library' CredentialsProvider that we used to use for
    node.js (before we were integrated with FirebaseAuth).
  * Fixed several tests that were hitting node.js warnings about unhandled
    promise failures.
  * mocha commmand-line args:
    * "--compilers ts:ts-node/register" was deprecated in favor of
      "--require ts-node/register"
    * Remove "--retries 5" when running mocha tests from VS Code.
    * Consistently use "--require" instead of "-r"
    * Add "--exit" when running from VS Code.

Author: mikelehen

.vscode/launch.json

@@ -11,11 +11,27 @@
       "program": "${workspaceRoot}/packages/firestore/node_modules/.bin/_mocha",
       "cwd": "${workspaceRoot}/packages/firestore",
       "args": [
-        "--compilers", "ts:ts-node/register",
-        "-r", "src/platform_node/node_init.ts",
-        "--retries", "5",
+        "--require", "ts-node/register",
+        "--require", "index.node.ts",
         "--timeout", "5000",
-        "test/{,!(integration|browser)/**/}*.test.ts"
+        "test/{,!(browser|integration)/**/}*.test.ts",
+        "--exit"
+      ],
+      "sourceMaps": true,
+      "protocol": "inspector"
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Firestore Integration Tests (Node)",
+      "program": "${workspaceRoot}/packages/firestore/node_modules/.bin/_mocha",
+      "cwd": "${workspaceRoot}/packages/firestore",
+      "args": [
+        "--require", "ts-node/register",
+        "--require", "index.node.ts",
+        "--timeout", "5000",
+        "test/{,!(browser|unit)/**/}*.test.ts",
+        "--exit"
       ],
       "sourceMaps": true,
       "protocol": "inspector"
@@ -25,7 +41,7 @@
       "request": "launch",
       "name": "Firestore Unit Tests (Browser)",
       "program": "${workspaceRoot}/packages/firestore/node_modules/.bin/karma",
-      "cwd": "${workspaceRoot}/packages/firestore",      
+      "cwd": "${workspaceRoot}/packages/firestore",
       "args": [
         "start",
         "--auto-watch",
@@ -38,7 +54,7 @@
       "request": "launch",
       "name": "Firestore Integration Tests (Browser)",
       "program": "${workspaceRoot}/packages/firestore/node_modules/.bin/karma",
-      "cwd": "${workspaceRoot}/packages/firestore",      
+      "cwd": "${workspaceRoot}/packages/firestore",
       "args": [
         "start",
         "--auto-watch",

packages/firestore/gulpfile.js

@@ -17,9 +17,19 @@
 const gulp = require('gulp');
 const tools = require('../../tools/build');
 
+function copyProtos(dest) {
+  return function copyProtos() {
+    return gulp
+      .src([__dirname + '/src/protos/**/*.proto'])
+      .pipe(gulp.dest(dest));
+  };
+}
+
 const buildModule = gulp.parallel([
   tools.buildCjs(__dirname),
-  tools.buildEsm(__dirname)
+  copyProtos('dist/cjs/src/protos'),
+  tools.buildEsm(__dirname),
+  copyProtos('dist/esm/src/protos')
 ]);
 
 const setupWatcher = () => {

packages/firestore/package.json

@@ -7,7 +7,7 @@
     "test": "run-p test:browser test:node",
     "test:browser": "karma start --single-run",
     "test:browser:debug": "karma start --browsers=Chrome --auto-watch",
-    "test:node": "nyc --reporter lcovonly -- mocha 'test/{,!(integration|browser)/**/}*.test.ts' --compilers ts:ts-node/register -r src/platform_node/node_init.ts --retries 5 --timeout 5000 --exit",
+    "test:node": "nyc --reporter lcovonly -- mocha 'test/{,!(browser)/**/}*.test.ts' --require ts-node/register --require index.node.ts --retries 5 --timeout 5000 --exit",
     "prepare": "gulp build"
   },
   "main": "dist/cjs/index.node.js",
@@ -18,7 +18,8 @@
   "module": "dist/esm/index.js",
   "license": "Apache-2.0",
   "dependencies": {
-    "@firebase/webchannel-wrapper": "^0.2.4"
+    "@firebase/webchannel-wrapper": "^0.2.4",
+    "grpc": "^1.7.1"
   },
   "peerDependencies": {
     "@firebase/app": "^0.1.0"
@@ -28,7 +29,6 @@
     "@types/mocha": "^2.2.44",
     "@types/sinon": "^2.3.7",
     "chai": "^4.1.1",
-    "grpc": "^1.6.6",
     "gulp": "gulpjs/gulp#4.0",
     "karma": "^1.7.0",
     "karma-chrome-launcher": "^2.2.0",

packages/firestore/src/api/credentials.ts

@@ -29,11 +29,6 @@ export interface FirstPartyCredentialsSettings {
   sessionIndex: string;
 }
 
-export interface GoogleAuthCredentialsSettings {
-  type: 'google-auth';
-  client: GoogleAuthClient;
-}
-
 export interface ProviderCredentialsSettings {
   type: 'provider';
   client: CredentialsProvider;
@@ -42,7 +37,6 @@ export interface ProviderCredentialsSettings {
 /** Settings for private credentials */
 export type CredentialsSettings =
   | FirstPartyCredentialsSettings
-  | GoogleAuthCredentialsSettings
   | ProviderCredentialsSettings;
 
 export type TokenType = 'OAuth' | 'FirstParty';
@@ -238,45 +232,6 @@ export class FirebaseCredentialsProvider implements CredentialsProvider {
   }
 }
 
-// Wrap a google-auth-library client as a CredentialsProvider.
-// NOTE: grpc-connection can natively accept a google-auth-library
-// client via createFromGoogleCredential(), but we opt to plumb the tokens
-// through our CredentialsProvider interface, at least for now.
-export class GoogleCredentialsProvider implements CredentialsProvider {
-  constructor(private authClient: GoogleAuthClient) {}
-
-  getToken(forceRefresh: boolean): Promise<Token | null> {
-    return new Promise<Token | null>((resolve, reject) => {
-      // TODO(b/32935141): ideally this would be declared as an extern
-      this.authClient['getAccessToken'](
-        (error: AnyJs, tokenLiteral: string) => {
-          if (error) {
-            reject(error);
-          } else {
-            resolve(new OAuthToken(tokenLiteral, User.GOOGLE_CREDENTIALS));
-          }
-        }
-      );
-    });
-  }
-
-  // NOTE: A google-auth-library client represents an immutable "user", so
-  // once we fire the initial event, it'll never change.
-  setUserChangeListener(listener: UserListener): void {
-    // Fire with initial uid.
-    listener(User.GOOGLE_CREDENTIALS);
-  }
-
-  removeUserChangeListener(): void {}
-}
-
-/**
- * Very incomplete typing for an auth client from
- * https://github.com/google/google-auth-library-nodejs/
- */
-export interface GoogleAuthClient {
-  getAccessToken(callback: (error?: Error, token?: string) => void): void;
-}
 // TODO(b/32935141): Ideally gapi type would be declared as an extern
 // tslint:disable-next-line:no-any
 export type Gapi = any;
@@ -348,9 +303,6 @@ export function makeCredentialsProvider(credentials?: CredentialsSettings) {
   }
 
   switch (credentials.type) {
-    case 'google-auth':
-      return new GoogleCredentialsProvider(credentials.client);
-
     case 'gapi':
       return new FirstPartyCredentialsProvider(
         credentials.client,

packages/firestore/src/local/garbage_source.ts

@@ -37,7 +37,7 @@ export interface GarbageSource {
    * This can be used by garbage collectors to double-check if a key exists in
    * this collection when it was released elsewhere.
    *
-   * PORTING NODE: This is used in contexts where PersistenceTransaction is
+   * PORTING NOTE: This is used in contexts where PersistenceTransaction is
    * known not to be needed, in this case we just pass in null. Therefore
    * any implementations must gaurd against null values.
    */

packages/firestore/src/platform/platform.ts

@@ -19,6 +19,7 @@ import { ProtoByteString } from '../core/types';
 import { Connection } from '../remote/connection';
 import { JsonProtoSerializer } from '../remote/serializer';
 import { fail } from '../util/assert';
+import { AnyJs } from '../util/misc';
 
 /**
  * Provides a common interface to load anything platform dependent, e.g.
@@ -30,6 +31,9 @@ export interface Platform {
   loadConnection(databaseInfo: DatabaseInfo): Promise<Connection>;
   newSerializer(databaseId: DatabaseId): JsonProtoSerializer;
 
+  /** Formats an object as a JSON string, suitable for logging. */
+  formatJSON(value: AnyJs): string;
+
   /** Converts a Base64 encoded string to a binary string. */
   atob(encoded: string): string;
 

packages/firestore/src/platform_browser/browser_platform.ts

@@ -20,6 +20,7 @@ import { Connection } from '../remote/connection';
 import { JsonProtoSerializer } from '../remote/serializer';
 
 import { WebChannelConnection } from './webchannel_connection';
+import { AnyJs } from '../util/misc';
 
 export class BrowserPlatform implements Platform {
   readonly base64Available: boolean;
@@ -38,6 +39,10 @@ export class BrowserPlatform implements Platform {
     return new JsonProtoSerializer(databaseId, { useProto3Json: true });
   }
 
+  formatJSON(value: AnyJs): string {
+    return JSON.stringify(value);
+  }
+
   atob(encoded: string): string {
     return atob(encoded);
   }

packages/firestore/src/platform_browser/webchannel_connection.ts

@@ -38,8 +38,15 @@ import { Rejecter, Resolver } from '../util/promise';
 
 const LOG_TAG = 'Connection';
 
+const RPC_STREAM_SERVICE = 'google.firestore.v1beta1.Firestore';
 const RPC_URL_VERSION = 'v1beta1';
 
+/** Maps RPC names to the corresponding REST endpoint name. */
+const RPC_NAME_REST_MAPPING = {
+  BatchGetDocuments: 'batchGet',
+  Commit: 'commit'
+};
+
 // TODO(b/38203344): The SDK_VERSION is set independently from Firebase because
 // we are doing out-of-band releases. Once we release as part of Firebase, we
 // should use the Firebase version instead.
@@ -52,24 +59,6 @@ export class WebChannelConnection implements Connection {
   private readonly baseUrl: string;
   private readonly pool: XhrIoPool;
 
-  /**
-   * Mapping from RPC name to service providing the RPC.
-   * For streaming RPCs only.
-   */
-  private static RPC_STREAM_SERVICE_MAPPING: { [rpc: string]: string } = {
-    Write: 'google.firestore.v1beta1.Firestore',
-    Listen: 'google.firestore.v1beta1.Firestore'
-  };
-
-  /**
-   * Mapping from RPC name to actual RPC name in URLs.
-   * For streaming RPCs only.
-   */
-  private static RPC_STREAM_NAME_MAPPING: { [rpc: string]: string } = {
-    Write: 'Write',
-    Listen: 'Listen'
-  };
-
   constructor(info: DatabaseInfo) {
     this.databaseId = info.databaseId;
     this.pool = new XhrIoPool();
@@ -100,7 +89,7 @@ export class WebChannelConnection implements Connection {
       `databases/${this.databaseId.database}`;
   }
 
-  invoke(rpcName: string, request: any, token: Token | null): Promise<any> {
+  invokeRPC(rpcName: string, request: any, token: Token | null): Promise<any> {
     const url = this.makeUrl(rpcName);
 
     return new Promise((resolve: Resolver<any>, reject: Rejecter) => {
@@ -178,18 +167,23 @@ export class WebChannelConnection implements Connection {
     });
   }
 
+  invokeStreamingRPC(
+    rpcName: string,
+    request: any,
+    token: Token | null
+  ): Promise<any[]> {
+    // The REST API automatically aggregates all of the streamed results, so we
+    // can just use the normal invoke() method.
+    return this.invokeRPC(rpcName, request, token);
+  }
+
   openStream(rpcName: string, token: Token | null): Stream<any, any> {
-    const rpcService = WebChannelConnection.RPC_STREAM_SERVICE_MAPPING[rpcName];
-    const rpcUrlName = WebChannelConnection.RPC_STREAM_NAME_MAPPING[rpcName];
-    if (!rpcService || !rpcUrlName) {
-      fail('Unknown RPC name: ' + rpcName);
-    }
     const urlParts = [
       this.baseUrl,
       '/',
-      rpcService,
+      RPC_STREAM_SERVICE,
       '/',
-      rpcUrlName,
+      rpcName,
       '/channel'
     ];
     const webchannelTransport = createWebChannelTransport();
@@ -343,6 +337,8 @@ export class WebChannelConnection implements Connection {
 
   // visible for testing
   makeUrl(rpcName: string): string {
+    const urlRpcName = RPC_NAME_REST_MAPPING[rpcName];
+    assert(urlRpcName !== undefined, 'Unknown REST mapping for: ' + rpcName);
     const url = [this.baseUrl, '/', RPC_URL_VERSION];
     url.push('/projects/');
     url.push(this.databaseId.projectId);
@@ -352,7 +348,7 @@ export class WebChannelConnection implements Connection {
     url.push('/documents');
 
     url.push(':');
-    url.push(rpcName);
+    url.push(urlRpcName);
     return url.join('');
   }
 }