diff --git a/packages/app/src/heartbeatService.test.ts b/packages/app/src/heartbeatService.test.ts index 0e664dee4ce..ad0d453b706 100644 --- a/packages/app/src/heartbeatService.test.ts +++ b/packages/app/src/heartbeatService.test.ts @@ -17,7 +17,11 @@ import { expect } from 'chai'; import '../test/setup'; -import { HeartbeatServiceImpl } from './heartbeatService'; +import { + countBytes, + HeartbeatServiceImpl, + extractHeartbeatsForHeader +} from './heartbeatService'; import { Component, ComponentType, @@ -28,7 +32,7 @@ import { FirebaseApp } from './public-types'; import * as firebaseUtil from '@firebase/util'; import { SinonStub, stub, useFakeTimers } from 'sinon'; import * as indexedDb from './indexeddb'; -import { isIndexedDBAvailable } from '@firebase/util'; +import { base64Encode, isIndexedDBAvailable } from '@firebase/util'; declare module '@firebase/component' { interface NameServiceMapping { @@ -39,6 +43,24 @@ declare module '@firebase/component' { const USER_AGENT_STRING_1 = 'vs1/1.2.3 vs2/2.3.4'; const USER_AGENT_STRING_2 = 'different/1.2.3'; +function generateUserAgentString(pairs: number): string { + let uaString = ''; + for (let i = 0; i < pairs; i++) { + uaString += `test-platform/${i % 10}.${i % 10}.${i % 10}`; + } + return uaString; +} + +function generateDates(count: number): string[] { + let currentTimestamp = Date.now(); + const dates = []; + for (let i = 0; i < count; i++) { + dates.push(new Date(currentTimestamp).toISOString().slice(0, 10)); + currentTimestamp += 24 * 60 * 60 * 1000; + } + return dates; +} + describe('HeartbeatServiceImpl', () => { describe('If IndexedDB has no entries', () => { let heartbeatService: HeartbeatServiceImpl; @@ -80,29 +102,32 @@ describe('HeartbeatServiceImpl', () => { expect(heartbeatService._heartbeatsCache?.length).to.equal(1); const heartbeat1 = heartbeatService._heartbeatsCache?.[0]; expect(heartbeat1?.userAgent).to.equal(USER_AGENT_STRING_1); - expect(heartbeat1?.dates[0]).to.equal('1970-01-01'); + expect(heartbeat1?.date).to.equal('1970-01-01'); expect(writeStub).to.be.calledWith([heartbeat1]); }); it(`triggerHeartbeat() doesn't store another heartbeat on the same day`, async () => { + expect(heartbeatService._heartbeatsCache?.length).to.equal(1); await heartbeatService.triggerHeartbeat(); - const heartbeat1 = heartbeatService._heartbeatsCache?.[0]; - expect(heartbeat1?.dates.length).to.equal(1); + expect(heartbeatService._heartbeatsCache?.length).to.equal(1); }); it(`triggerHeartbeat() does store another heartbeat on a different day`, async () => { + expect(heartbeatService._heartbeatsCache?.length).to.equal(1); clock.tick(24 * 60 * 60 * 1000); await heartbeatService.triggerHeartbeat(); - const heartbeat1 = heartbeatService._heartbeatsCache?.[0]; - expect(heartbeat1?.dates.length).to.equal(2); - expect(heartbeat1?.dates[1]).to.equal('1970-01-02'); + expect(heartbeatService._heartbeatsCache?.length).to.equal(2); + expect(heartbeatService._heartbeatsCache?.[1].date).to.equal( + '1970-01-02' + ); }); it(`triggerHeartbeat() stores another entry for a different user agent`, async () => { userAgentString = USER_AGENT_STRING_2; + expect(heartbeatService._heartbeatsCache?.length).to.equal(2); clock.tick(2 * 24 * 60 * 60 * 1000); await heartbeatService.triggerHeartbeat(); - expect(heartbeatService._heartbeatsCache?.length).to.equal(2); - const heartbeat2 = heartbeatService._heartbeatsCache?.[1]; - expect(heartbeat2?.dates.length).to.equal(1); - expect(heartbeat2?.dates[0]).to.equal('1970-01-03'); + expect(heartbeatService._heartbeatsCache?.length).to.equal(3); + expect(heartbeatService._heartbeatsCache?.[2].date).to.equal( + '1970-01-03' + ); }); it('getHeartbeatHeaders() gets stored heartbeats and clears heartbeats', async () => { const deleteStub = stub(heartbeatService._storage, 'deleteAll'); @@ -127,9 +152,14 @@ describe('HeartbeatServiceImpl', () => { let writeStub: SinonStub; let userAgentString = USER_AGENT_STRING_1; const mockIndexedDBHeartbeats = [ + // Chosen so one will exceed 30 day limit and one will not. { userAgent: 'old-user-agent', - dates: ['1969-01-01', '1969-01-02'] + date: '1969-12-01' + }, + { + userAgent: 'old-user-agent', + date: '1969-12-31' } ]; before(() => { @@ -179,18 +209,19 @@ describe('HeartbeatServiceImpl', () => { expect(heartbeatService._heartbeatsCache).to.deep.equal([]); } }); - it(`triggerHeartbeat() writes new heartbeats without removing old ones`, async () => { + it(`triggerHeartbeat() writes new heartbeats and retains old ones newer than 30 days`, async () => { userAgentString = USER_AGENT_STRING_2; clock.tick(3 * 24 * 60 * 60 * 1000); await heartbeatService.triggerHeartbeat(); if (isIndexedDBAvailable()) { expect(writeStub).to.be.calledWith([ - ...mockIndexedDBHeartbeats, - { userAgent: USER_AGENT_STRING_2, dates: ['1970-01-04'] } + // The first entry exceeds the 30 day retention limit. + mockIndexedDBHeartbeats[1], + { userAgent: USER_AGENT_STRING_2, date: '1970-01-04' } ]); } else { expect(writeStub).to.be.calledWith([ - { userAgent: USER_AGENT_STRING_2, dates: ['1970-01-04'] } + { userAgent: USER_AGENT_STRING_2, date: '1970-01-04' } ]); } }); @@ -201,8 +232,7 @@ describe('HeartbeatServiceImpl', () => { ); if (isIndexedDBAvailable()) { expect(heartbeatHeaders).to.include('old-user-agent'); - expect(heartbeatHeaders).to.include('1969-01-01'); - expect(heartbeatHeaders).to.include('1969-01-02'); + expect(heartbeatHeaders).to.include('1969-12-31'); } expect(heartbeatHeaders).to.include(USER_AGENT_STRING_2); expect(heartbeatHeaders).to.include('1970-01-04'); @@ -213,4 +243,78 @@ describe('HeartbeatServiceImpl', () => { expect(deleteStub).to.be.called; }); }); + + describe('countBytes()', () => { + it('counts how many bytes there will be in a stringified, encoded header', () => { + const heartbeats = [ + { userAgent: generateUserAgentString(1), dates: generateDates(1) }, + { userAgent: generateUserAgentString(3), dates: generateDates(2) } + ]; + let size: number = 0; + const headerString = base64Encode( + JSON.stringify({ version: 2, heartbeats }) + ); + // Use independent methods to validate our byte count method matches. + // We don't use this measurement method in the app because user + // environments are much more unpredictable while we know the + // tests will run in either a standard headless browser or Node. + if (typeof Blob !== 'undefined') { + const blob = new Blob([headerString]); + size = blob.size; + } else if (typeof Buffer !== 'undefined') { + const buffer = Buffer.from(headerString); + size = buffer.byteLength; + } + expect(countBytes(heartbeats)).to.equal(size); + }); + }); + + describe('_extractHeartbeatsForHeader()', () => { + it('returns empty heartbeatsToKeep if it cannot get under maxSize', () => { + const heartbeats = [ + { userAgent: generateUserAgentString(1), date: '2022-01-01' } + ]; + const { unsentEntries, heartbeatsToSend } = extractHeartbeatsForHeader( + heartbeats, + 5 + ); + expect(heartbeatsToSend.length).to.equal(0); + expect(unsentEntries).to.deep.equal(heartbeats); + }); + it('splits heartbeats array', () => { + const heartbeats = [ + { userAgent: generateUserAgentString(20), date: '2022-01-01' }, + { userAgent: generateUserAgentString(4), date: '2022-01-02' } + ]; + const sizeWithHeartbeat0Only = countBytes([ + { userAgent: heartbeats[0].userAgent, dates: [heartbeats[0].date] } + ]); + const { unsentEntries, heartbeatsToSend } = extractHeartbeatsForHeader( + heartbeats, + sizeWithHeartbeat0Only + 1 + ); + expect(heartbeatsToSend.length).to.equal(1); + expect(unsentEntries.length).to.equal(1); + }); + it('splits the first heartbeat if needed', () => { + const uaString = generateUserAgentString(20); + const heartbeats = [ + { userAgent: uaString, date: '2022-01-01' }, + { userAgent: uaString, date: '2022-01-02' }, + { userAgent: uaString, date: '2022-01-03' } + ]; + const sizeWithHeartbeat0Only = countBytes([ + { userAgent: heartbeats[0].userAgent, dates: [heartbeats[0].date] } + ]); + const { unsentEntries, heartbeatsToSend } = extractHeartbeatsForHeader( + heartbeats, + sizeWithHeartbeat0Only + 1 + ); + expect(heartbeatsToSend.length).to.equal(1); + expect(unsentEntries.length).to.equal(2); + expect(heartbeatsToSend[0].dates.length + unsentEntries.length).to.equal( + heartbeats.length + ); + }); + }); }); diff --git a/packages/app/src/heartbeatService.ts b/packages/app/src/heartbeatService.ts index 838e1e31338..88ada8c9cf9 100644 --- a/packages/app/src/heartbeatService.ts +++ b/packages/app/src/heartbeatService.ts @@ -30,9 +30,14 @@ import { FirebaseApp } from './public-types'; import { HeartbeatsByUserAgent, HeartbeatService, - HeartbeatStorage + HeartbeatStorage, + SingleDateHeartbeat } from './types'; +const MAX_HEADER_BYTES = 1024; +// 30 days +const STORED_HEARTBEAT_RETENTION_MAX_MILLIS = 30 * 24 * 60 * 60 * 1000; + export class HeartbeatServiceImpl implements HeartbeatService { /** * The persistence layer for heartbeats @@ -43,11 +48,13 @@ export class HeartbeatServiceImpl implements HeartbeatService { /** * In-memory cache for heartbeats, used by getHeartbeatsHeader() to generate * the header string. + * Stores one record per date. This will be consolidated into the standard + * format of one record per user agent string before being sent as a header. * Populated from indexedDB when the controller is instantiated and should * be kept in sync with indexedDB. * Leave public for easier testing. */ - _heartbeatsCache: HeartbeatsByUserAgent[] | null = null; + _heartbeatsCache: SingleDateHeartbeat[] | null = null; /** * the initialization promise for populating heartbeatCache. @@ -55,7 +62,7 @@ export class HeartbeatServiceImpl implements HeartbeatService { * (hearbeatsCache == null), it should wait for this promise * Leave public for easier testing. */ - _heartbeatsCachePromise: Promise; + _heartbeatsCachePromise: Promise; constructor(private readonly container: ComponentContainer) { const app = this.container.getProvider('app').getImmediate(); this._storage = new HeartbeatStorageImpl(app); @@ -82,28 +89,28 @@ export class HeartbeatServiceImpl implements HeartbeatService { const userAgent = platformLogger.getPlatformInfoString(); const date = getUTCDateString(); if (this._heartbeatsCache === null) { - await this._heartbeatsCachePromise; + this._heartbeatsCache = await this._heartbeatsCachePromise; } - let heartbeatsEntry = this._heartbeatsCache!.find( - heartbeats => heartbeats.userAgent === userAgent - ); - if (heartbeatsEntry) { - if (heartbeatsEntry.dates.includes(date)) { - // Only one per day. - return; - } else { - // Modify in-place in this.heartbeatsCache - heartbeatsEntry.dates.push(date); - } + if ( + this._heartbeatsCache.some( + singleDateHeartbeat => singleDateHeartbeat.date === date + ) + ) { + // Do not store a heartbeat if one is already stored for this day. + return; } else { - // There is no entry for this Firebase user agent. Create one. - heartbeatsEntry = { - userAgent, - dates: [date] - }; - this._heartbeatsCache!.push(heartbeatsEntry); + // There is no entry for this date. Create one. + this._heartbeatsCache.push({ date, userAgent }); } - return this._storage.overwrite(this._heartbeatsCache!); + // Remove entries older than 30 days. + this._heartbeatsCache = this._heartbeatsCache.filter( + singleDateHeartbeat => { + const hbTimestamp = new Date(singleDateHeartbeat.date).valueOf(); + const now = Date.now(); + return now - hbTimestamp <= STORED_HEARTBEAT_RETENTION_MAX_MILLIS; + } + ); + return this._storage.overwrite(this._heartbeatsCache); } /** @@ -120,19 +127,81 @@ export class HeartbeatServiceImpl implements HeartbeatService { if (this._heartbeatsCache === null) { return ''; } + // Extract as many heartbeats from the cache as will fit under the size limit. + const { heartbeatsToSend, unsentEntries } = extractHeartbeatsForHeader( + this._heartbeatsCache + ); const headerString = base64Encode( - JSON.stringify({ version: 2, heartbeats: this._heartbeatsCache }) + JSON.stringify({ version: 2, heartbeats: heartbeatsToSend }) ); - this._heartbeatsCache = null; - // Do not wait for this, to reduce latency. - void this._storage.deleteAll(); + if (unsentEntries.length > 0) { + // Store any unsent entries if they exist. + this._heartbeatsCache = unsentEntries; + // This seems more likely than deleteAll (below) to lead to some odd state + // since the cache isn't empty and this will be called again on the next request, + // and is probably safest if we await it. + await this._storage.overwrite(this._heartbeatsCache); + } else { + this._heartbeatsCache = null; + // Do not wait for this, to reduce latency. + void this._storage.deleteAll(); + } return headerString; } } function getUTCDateString(): string { const today = new Date(); - return today.toISOString().substring(0,10); + // Returns date format 'YYYY-MM-DD' + return today.toISOString().substring(0, 10); +} + +export function extractHeartbeatsForHeader( + heartbeatsCache: SingleDateHeartbeat[], + maxSize = MAX_HEADER_BYTES +): { + heartbeatsToSend: HeartbeatsByUserAgent[]; + unsentEntries: SingleDateHeartbeat[]; +} { + // Heartbeats grouped by user agent in the standard format to be sent in + // the header. + const heartbeatsToSend: HeartbeatsByUserAgent[] = []; + // Single date format heartbeats that are not sent. + let unsentEntries = heartbeatsCache.slice(); + for (const singleDateHeartbeat of heartbeatsCache) { + // Look for an existing entry with the same user agent. + const heartbeatEntry = heartbeatsToSend.find( + hb => hb.userAgent === singleDateHeartbeat.userAgent + ); + if (!heartbeatEntry) { + // If no entry for this user agent exists, create one. + heartbeatsToSend.push({ + userAgent: singleDateHeartbeat.userAgent, + dates: [singleDateHeartbeat.date] + }); + if (countBytes(heartbeatsToSend) > maxSize) { + // If the header would exceed max size, remove the added heartbeat + // entry and stop adding to the header. + heartbeatsToSend.pop(); + break; + } + } else { + heartbeatEntry.dates.push(singleDateHeartbeat.date); + // If the header would exceed max size, remove the added date + // and stop adding to the header. + if (countBytes(heartbeatsToSend) > maxSize) { + heartbeatEntry.dates.pop(); + break; + } + } + // Pop unsent entry from queue. (Skipped if adding the entry exceeded + // quota and the loop breaks early.) + unsentEntries = unsentEntries.slice(1); + } + return { + heartbeatsToSend, + unsentEntries + }; } export class HeartbeatStorageImpl implements HeartbeatStorage { @@ -152,7 +221,7 @@ export class HeartbeatStorageImpl implements HeartbeatStorage { /** * Read all heartbeats. */ - async read(): Promise { + async read(): Promise { const canUseIndexedDB = await this._canUseIndexedDBPromise; if (!canUseIndexedDB) { return []; @@ -162,7 +231,7 @@ export class HeartbeatStorageImpl implements HeartbeatStorage { } } // overwrite the storage with the provided heartbeats - async overwrite(heartbeats: HeartbeatsByUserAgent[]): Promise { + async overwrite(heartbeats: SingleDateHeartbeat[]): Promise { const canUseIndexedDB = await this._canUseIndexedDBPromise; if (!canUseIndexedDB) { return; @@ -171,7 +240,7 @@ export class HeartbeatStorageImpl implements HeartbeatStorage { } } // add heartbeats - async add(heartbeats: HeartbeatsByUserAgent[]): Promise { + async add(heartbeats: SingleDateHeartbeat[]): Promise { const canUseIndexedDB = await this._canUseIndexedDBPromise; if (!canUseIndexedDB) { return; @@ -183,7 +252,7 @@ export class HeartbeatStorageImpl implements HeartbeatStorage { } } // delete heartbeats - async delete(heartbeats: HeartbeatsByUserAgent[]): Promise { + async delete(heartbeats: SingleDateHeartbeat[]): Promise { const canUseIndexedDB = await this._canUseIndexedDBPromise; if (!canUseIndexedDB) { return; @@ -206,3 +275,16 @@ export class HeartbeatStorageImpl implements HeartbeatStorage { } } } + +/** + * Calculate bytes of a HeartbeatsByUserAgent array after being wrapped + * in a platform logging header JSON object, stringified, and converted + * to base 64. + */ +export function countBytes(heartbeatsCache: HeartbeatsByUserAgent[]): number { + // base64 has a restricted set of characters, all of which should be 1 byte. + return base64Encode( + // heartbeatsCache wrapper properties + JSON.stringify({ version: 2, heartbeats: heartbeatsCache }) + ).length; +} diff --git a/packages/app/src/types.ts b/packages/app/src/types.ts index e6d2a0dfe2c..2b47c967382 100644 --- a/packages/app/src/types.ts +++ b/packages/app/src/types.ts @@ -46,19 +46,24 @@ export interface HeartbeatsByUserAgent { dates: string[]; } +export interface SingleDateHeartbeat { + userAgent: string; + date: string; +} + export interface HeartbeatStorage { // overwrite the storage with the provided heartbeats - overwrite(heartbeats: HeartbeatsByUserAgent[]): Promise; + overwrite(heartbeats: SingleDateHeartbeat[]): Promise; // add heartbeats - add(heartbeats: HeartbeatsByUserAgent[]): Promise; + add(heartbeats: SingleDateHeartbeat[]): Promise; // delete heartbeats - delete(heartbeats: HeartbeatsByUserAgent[]): Promise; + delete(heartbeats: SingleDateHeartbeat[]): Promise; // delete all heartbeats deleteAll(): Promise; // read all heartbeats - read(): Promise; + read(): Promise; } export interface HeartbeatsInIndexedDB { - heartbeats: HeartbeatsByUserAgent[]; + heartbeats: SingleDateHeartbeat[]; }