chore(telemetry): add ability to inject tags from env and track websocket updates (#14711)

* chore(telemetry) add ability to inject tags from env
Add tracking for websocket pushed live data updates

* Try to get activePath from the websocket handshake.

* Add missing nullcheck

* Make the 3rd parameter to be an opts object for named parameter
experience and plan ahead for future extension needs

* Remove debug print

Author: jamo

packages/gatsby-telemetry/package.json

@@ -20,6 +20,7 @@
     "resolve-cwd": "^2.0.0",
     "source-map": "^0.5.7",
     "stack-trace": "^0.0.10",
+    "lodash": "^4.17.10",
     "stack-utils": "1.0.2",
     "uuid": "3.3.2"
   },

packages/gatsby-telemetry/src/index.js

@@ -16,7 +16,7 @@ const tick = _ => {
 }
 
 module.exports = {
-  trackCli: (input, tags) => instance.captureEvent(input, tags),
+  trackCli: (input, tags, opts) => instance.captureEvent(input, tags, opts),
   trackError: (input, tags) => instance.captureError(input, tags),
   trackBuildError: (input, tags) => instance.captureBuildError(input, tags),
   setDefaultTags: tags => instance.decorateAll(tags),

packages/gatsby-telemetry/src/telemetry.js

@@ -8,6 +8,7 @@ const { basename, join, sep } = require(`path`)
 const { execSync } = require(`child_process`)
 const isDocker = require(`is-docker`)
 const showAnalyticsNotification = require(`./showAnalyticsNotification`)
+const lodash = require(`lodash`)
 
 module.exports = class AnalyticsTracker {
   store = new EventStorage()
@@ -18,16 +19,29 @@ module.exports = class AnalyticsTracker {
   trackingEnabled // lazy
   componentVersion
   sessionId = uuid()
+
   constructor() {
     try {
       this.componentVersion = require(`../package.json`).version
       this.installedGatsbyVersion = this.getGatsbyVersion()
       this.gatsbyCliVersion = this.getGatsbyCliVersion()
+      this.defaultTags = this.getTagsFromEnv()
     } catch (e) {
       // ignore
     }
   }
 
+  getTagsFromEnv() {
+    if (process.env.GATSBY_TELEMETRY_TAGS) {
+      try {
+        return JSON.parse(process.env.GATSBY_TELEMETRY_TAGS)
+      } catch (_) {
+        // ignore
+      }
+    }
+    return {}
+  }
+
   getGatsbyVersion() {
     const packageInfo = require(join(
       process.cwd(),
@@ -60,7 +74,7 @@ module.exports = class AnalyticsTracker {
     }
     return undefined
   }
-  captureEvent(type = ``, tags = {}) {
+  captureEvent(type = ``, tags = {}, opts = { debounce: false }) {
     if (!this.isTrackingEnabled()) {
       return
     }
@@ -71,9 +85,21 @@ module.exports = class AnalyticsTracker {
     }
 
     const decoration = this.metadataCache[type]
-    delete this.metadataCache[type]
     const eventType = `${baseEventType}_${type}`
-    this.buildAndStoreEvent(eventType, Object.assign(tags, decoration))
+
+    if (opts.debounce) {
+      const debounceTime = 5 * 1000
+      const now = Date.now()
+      const debounceKey = JSON.stringify({ type, decoration })
+      const last = this.debouncer[debounceKey] || 0
+      if (now - last < debounceTime) {
+        return
+      }
+      this.debouncer[debounceKey] = now
+    }
+
+    delete this.metadataCache[type]
+    this.buildAndStoreEvent(eventType, lodash.merge({}, tags, decoration))
   }
 
   captureError(type, tags = {}) {
@@ -89,7 +115,7 @@ module.exports = class AnalyticsTracker {
       tags.error = sanitizeErrors(tags.error)
     }
 
-    this.buildAndStoreEvent(eventType, Object.assign(tags, decoration))
+    this.buildAndStoreEvent(eventType, lodash.merge({}, tags, decoration))
   }
 
   captureBuildError(type, tags = {}) {
@@ -105,15 +131,14 @@ module.exports = class AnalyticsTracker {
       tags.error = sanitizeErrors(tags.error)
     }
 
-    this.buildAndStoreEvent(eventType, Object.assign(tags, decoration))
+    this.buildAndStoreEvent(eventType, lodash.merge({}, tags, decoration))
   }
 
   buildAndStoreEvent(eventType, tags) {
     const event = {
       installedGatsbyVersion: this.installedGatsbyVersion,
       gatsbyCliVersion: this.gatsbyCliVersion,
-      ...this.defaultTags,
-      ...tags, // The schema must include these
+      ...lodash.merge({}, this.defaultTags, tags), // The schema must include these
       eventType,
       sessionId: this.sessionId,
       time: new Date(),

packages/gatsby/src/utils/websocket-manager.js

@@ -4,6 +4,9 @@ const path = require(`path`)
 const { store } = require(`../redux`)
 const fs = require(`fs`)
 const pageDataUtil = require(`../utils/page-data`)
+const telemetry = require(`gatsby-telemetry`)
+const url = require(`url`)
+const { createHash } = require(`crypto`)
 
 type QueryResult = {
   id: string,
@@ -37,6 +40,20 @@ const getCachedPageData = async (
   }
 }
 
+const hashPaths = paths => {
+  if (!paths) {
+    return undefined
+  }
+  return paths.map(path => {
+    if (!path) {
+      return undefined
+    }
+    return createHash(`sha256`)
+      .update(path)
+      .digest(`hex`)
+  })
+}
+
 /**
  * Get cached StaticQuery results for components that Gatsby didn't run query yet.
  * @param {QueryResultsMap} resultsMap Already stored results for queries that don't need to be read from files.
@@ -99,6 +116,7 @@ class WebsocketManager {
     this.emitPageData = this.emitPageData.bind(this)
     this.emitStaticQueryData = this.emitStaticQueryData.bind(this)
     this.emitError = this.emitError.bind(this)
+    this.connectedClients = 0
   }
 
   init({ server, directory }) {
@@ -117,6 +135,20 @@ class WebsocketManager {
 
     this.websocket.on(`connection`, s => {
       let activePath = null
+      if (
+        s &&
+        s.handshake &&
+        s.handshake.headers &&
+        s.handshake.headers.referer
+      ) {
+        const path = url.parse(s.handshake.headers.referer).path
+        if (path) {
+          activePath = path
+          this.activePaths.add(path)
+        }
+      }
+
+      this.connectedClients += 1
       // Send already existing static query results
       this.staticQueryResults.forEach(result => {
         this.websocket.send({
@@ -160,6 +192,17 @@ class WebsocketManager {
           why: `getDataForPath`,
           payload: this.pageResults.get(path),
         })
+
+        telemetry.trackCli(
+          `WEBSOCKET_PAGE_DATA_UPDATE`,
+          {
+            siteMeasurements: {
+              clientsCount: this.connectedClients,
+              paths: hashPaths(Array.from(this.activePaths)),
+            },
+          },
+          { debounce: true }
+        )
       }
 
       s.on(`getDataForPath`, getDataForPath)
@@ -172,6 +215,7 @@ class WebsocketManager {
 
       s.on(`disconnect`, s => {
         leaveRoom(activePath)
+        this.connectedClients -= 1
       })
 
       s.on(`unregisterPath`, path => {
@@ -190,13 +234,33 @@ class WebsocketManager {
     this.staticQueryResults.set(data.id, data)
     if (this.isInitialised) {
       this.websocket.send({ type: `staticQueryResult`, payload: data })
+      telemetry.trackCli(
+        `WEBSOCKET_EMIT_STATIC_PAGE_DATA_UPDATE`,
+        {
+          siteMeasurements: {
+            clientsCount: this.connectedClients,
+            paths: hashPaths(Array.from(this.activePaths)),
+          },
+        },
+        { debounce: true }
+      )
     }
   }
 
   emitPageData(data: QueryResult) {
     this.pageResults.set(data.id, data)
     if (this.isInitialised) {
       this.websocket.send({ type: `pageQueryResult`, payload: data })
+      telemetry.trackCli(
+        `WEBSOCKET_EMIT_PAGE_DATA_UPDATE`,
+        {
+          siteMeasurements: {
+            clientsCount: this.connectedClients,
+            paths: hashPaths(Array.from(this.activePaths)),
+          },
+        },
+        { debounce: true }
+      )
     }
   }
   emitError(id: string, message?: string) {