Skip to content

Commit b3e7e1b

Browse files
hsubox76hsubox76
committed
WIP heartbeat size limit
1 parent 7fffc57 commit b3e7e1b

File tree

3 files changed

+236
-7
lines changed

3 files changed

+236
-7
lines changed

‎packages/app/src/heartbeatService.ts

Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import {
2121
isIndexedDBAvailable,
2222
validateIndexedDBOpenable
2323
} from '@firebase/util';
24+
import { countBytes, splitHeartbeatsCache } from './heartbeatSize';
2425
import {
2526
deleteHeartbeatsFromIndexedDB,
2627
readHeartbeatsFromIndexedDB,
@@ -33,6 +34,8 @@ import {
3334
HeartbeatStorage
3435
} from './types';
3536

37+
const HEADER_SIZE_LIMIT_BYTES = 1000;
38+
3639
export class HeartbeatServiceImpl implements HeartbeatService {
3740
/**
3841
* The persistence layer for heartbeats
@@ -120,19 +123,42 @@ export class HeartbeatServiceImpl implements HeartbeatService {
120123
if (this._heartbeatsCache === null) {
121124
return '';
122125
}
123-
const headerString = base64Encode(
124-
JSON.stringify({ version: 2, heartbeats: this._heartbeatsCache })
125-
);
126-
this._heartbeatsCache = null;
127-
// Do not wait for this, to reduce latency.
128-
void this._storage.deleteAll();
126+
// Count size of _heartbeatsCache after being converted into a base64
127+
// header string.
128+
const base64Bytes = countBytes(this._heartbeatsCache);
129+
// If it exceeds the limit, split out the oldest portion under the
130+
// limit to return. Put the rest back into _heartbeatsCache.
131+
let headerString = '';
132+
if (base64Bytes > HEADER_SIZE_LIMIT_BYTES) {
133+
const { heartbeatsToSend, heartbeatsToKeep } = splitHeartbeatsCache(
134+
this._heartbeatsCache,
135+
HEADER_SIZE_LIMIT_BYTES
136+
);
137+
headerString = base64Encode(
138+
JSON.stringify({ version: 2, heartbeats: heartbeatsToSend })
139+
);
140+
// Write the portion not sent back to memory, and then to indexedDB.
141+
this._heartbeatsCache = heartbeatsToKeep;
142+
// This is more likely than deleteAll() to cause some mixed up state
143+
// problems if we don't wait for execution to finish.
144+
await this._storage.overwrite(this._heartbeatsCache);
145+
} else {
146+
// If _heartbeatsCache does not exceed the size limit, send all the
147+
// data in a header and delete memory and indexedDB caches.
148+
headerString = base64Encode(
149+
JSON.stringify({ version: 2, heartbeats: this._heartbeatsCache })
150+
);
151+
this._heartbeatsCache = null;
152+
// Do not wait for this, to reduce latency.
153+
void this._storage.deleteAll();
154+
}
129155
return headerString;
130156
}
131157
}
132158

133159
function getUTCDateString(): string {
134160
const today = new Date();
135-
return today.toISOString().substring(0,10);
161+
return today.toISOString().substring(0, 10);
136162
}
137163

138164
export class HeartbeatStorageImpl implements HeartbeatStorage {

‎packages/app/src/heartbeatSize.test.ts

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/**
2+
* @license
3+
* Copyright 2021 Google LLC
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
import { expect } from 'chai';
19+
import '../test/setup';
20+
import {
21+
countBytes,
22+
countHeartbeatBytes,
23+
splitHeartbeatsCache
24+
} from './heartbeatSize';
25+
26+
function generateUserAgentString(pairs: number): string {
27+
let uaString = '';
28+
for (let i = 0; i < pairs; i++) {
29+
uaString += `test-platform/${i % 10}.${i % 10}.${i % 10}`;
30+
}
31+
return uaString;
32+
}
33+
34+
function generateDates(count: number): string[] {
35+
let currentTimestamp = Date.now();
36+
const dates = [];
37+
for (let i = 0; i < count; i++) {
38+
dates.push(new Date(currentTimestamp).toISOString().slice(0, 10));
39+
currentTimestamp += 24 * 60 * 60 * 1000;
40+
}
41+
return dates;
42+
}
43+
44+
describe.only('splitHeartbeatsCache()', () => {
45+
it('returns empty heartbeatsToKeep if it cannot get under maxSize', () => {
46+
const heartbeats = [
47+
{ userAgent: generateUserAgentString(1), dates: generateDates(1) }
48+
];
49+
const { heartbeatsToKeep, heartbeatsToSend } = splitHeartbeatsCache(
50+
heartbeats,
51+
5
52+
);
53+
expect(heartbeatsToSend.length).to.equal(0);
54+
expect(heartbeatsToKeep).to.deep.equal(heartbeats);
55+
});
56+
it('splits heartbeats array', () => {
57+
const heartbeats = [
58+
{ userAgent: generateUserAgentString(20), dates: generateDates(8) },
59+
{ userAgent: generateUserAgentString(4), dates: generateDates(10) }
60+
];
61+
const heartbeat1Size = countHeartbeatBytes(heartbeats[0]);
62+
const { heartbeatsToKeep, heartbeatsToSend } = splitHeartbeatsCache(
63+
heartbeats,
64+
heartbeat1Size + 1
65+
);
66+
expect(heartbeatsToSend.length).to.equal(1);
67+
expect(heartbeatsToKeep.length).to.equal(1);
68+
});
69+
it('splits the first heartbeat if needed', () => {
70+
const heartbeats = [
71+
{ userAgent: generateUserAgentString(20), dates: generateDates(50) },
72+
{ userAgent: generateUserAgentString(4), dates: generateDates(10) }
73+
];
74+
const heartbeat1Size = countHeartbeatBytes(heartbeats[0]);
75+
const { heartbeatsToKeep, heartbeatsToSend } = splitHeartbeatsCache(
76+
heartbeats,
77+
heartbeat1Size - 50
78+
);
79+
expect(heartbeatsToSend.length).to.equal(1);
80+
expect(heartbeatsToKeep.length).to.equal(2);
81+
expect(
82+
heartbeatsToSend[0].dates.length + heartbeatsToKeep[0].dates.length
83+
).to.equal(heartbeats[0].dates.length);
84+
expect(heartbeatsToSend[0].userAgent).to.equal(
85+
heartbeatsToKeep[0].userAgent
86+
);
87+
});
88+
});

‎packages/app/src/heartbeatSize.ts

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
/**
2+
* @license
3+
* Copyright 2022 Google LLC
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
import { HeartbeatsByUserAgent } from './types';
19+
20+
const BASE64_SIZE_MULTIPLIER = 4 / 3;
21+
const BYTES_PER_DATE = 12 * BASE64_SIZE_MULTIPLIER;
22+
23+
function getByteLength(str: string): number {
24+
let byteLength = 0;
25+
for (let i = 0; i < str.length; i++) {
26+
const c = str.charCodeAt(i);
27+
byteLength +=
28+
(c & 0xf800) === 0xd800
29+
? 2 // Code point is half of a surrogate pair
30+
: c < 1 << 7
31+
? 1
32+
: c < 1 << 11
33+
? 2
34+
: 3;
35+
}
36+
return byteLength;
37+
}
38+
39+
/**
40+
* Calculate bytes of a single HeartbeatsByUserAgent object after
41+
* being stringified and converted to base64.
42+
*/
43+
export function countHeartbeatBytes(heartbeat: HeartbeatsByUserAgent): number {
44+
return getByteLength(JSON.stringify(heartbeat)) * BASE64_SIZE_MULTIPLIER;
45+
}
46+
47+
/**
48+
* Calculate bytes of a HeartbeatsByUserAgent array after being wrapped
49+
* in a platform logging header JSON object, stringified, and converted
50+
* to base 64.
51+
*/
52+
export function countBytes(heartbeatsCache: HeartbeatsByUserAgent[]): number {
53+
// heartbeatsCache wrapper properties
54+
let count =
55+
getByteLength(JSON.stringify({ version: 2, heartbeats: [] })) *
56+
BASE64_SIZE_MULTIPLIER;
57+
for (const heartbeat of heartbeatsCache) {
58+
count += countHeartbeatBytes(heartbeat);
59+
}
60+
return count;
61+
}
62+
63+
/**
64+
* Split a HeartbeatsByUserAgent array into 2 arrays, one that fits
65+
* under `maxSize`, to be sent as a header, and the remainder. If
66+
* the first heartbeat in the array is too big by itself, it will
67+
* split that heartbeat into two by splitting its `dates` array.
68+
*/
69+
export function splitHeartbeatsCache(
70+
heartbeatsCache: HeartbeatsByUserAgent[],
71+
maxSize: number
72+
): {
73+
heartbeatsToSend: HeartbeatsByUserAgent[];
74+
heartbeatsToKeep: HeartbeatsByUserAgent[];
75+
} {
76+
let totalBytes = 0;
77+
const heartbeatsToSend = [];
78+
const heartbeatsToKeep = [...heartbeatsCache];
79+
for (const heartbeat of heartbeatsCache) {
80+
totalBytes += countHeartbeatBytes(heartbeat);
81+
if (totalBytes > maxSize) {
82+
if (heartbeatsToSend.length === 0) {
83+
// The first heartbeat is too large and needs to be split or we have
84+
// nothing to send.
85+
const heartbeatBytes = countHeartbeatBytes(heartbeat);
86+
const bytesOverLimit = heartbeatBytes - maxSize;
87+
const datesToRemove = Math.ceil(bytesOverLimit / BYTES_PER_DATE);
88+
if (datesToRemove >= heartbeat.dates.length) {
89+
// If no amount of removing dates can get this heartbeat under
90+
// the limit (unlikely scenario), nothing can be sent.
91+
break;
92+
}
93+
const heartbeatToSend = {
94+
...heartbeat,
95+
dates: heartbeat.dates.slice(0, -datesToRemove)
96+
};
97+
const heartbeatToKeep = {
98+
...heartbeat,
99+
dates: heartbeat.dates.slice(-datesToRemove)
100+
};
101+
heartbeatsToSend.push(heartbeatToSend);
102+
heartbeatsToKeep[0] = heartbeatToKeep;
103+
} else {
104+
break;
105+
}
106+
} else {
107+
heartbeatsToSend.push(heartbeat);
108+
heartbeatsToKeep.shift();
109+
}
110+
}
111+
return {
112+
heartbeatsToSend,
113+
heartbeatsToKeep
114+
};
115+
}

0 commit comments

Comments
 (0)