diff --git a/packages/firestore/src/core/firestore_client.ts b/packages/firestore/src/core/firestore_client.ts index 2c2c0af1771..a58786c6685 100644 --- a/packages/firestore/src/core/firestore_client.ts +++ b/packages/firestore/src/core/firestore_client.ts @@ -41,7 +41,8 @@ import { Document } from '../model/document'; import { DocumentKey } from '../model/document_key'; import { Mutation } from '../model/mutation'; import { toByteStreamReader } from '../platform/byte_stream_reader'; -import { newSerializer, newTextEncoder } from '../platform/serializer'; +import { newSerializer } from '../platform/serializer'; +import { newTextEncoder } from '../platform/text_serializer'; import { Datastore } from '../remote/datastore'; import { canUseNetwork, diff --git a/packages/firestore/src/model/path.ts b/packages/firestore/src/model/path.ts index ecdb9ed086c..e5e55e1b12c 100644 --- a/packages/firestore/src/model/path.ts +++ b/packages/firestore/src/model/path.ts @@ -163,6 +163,11 @@ abstract class BasePath> { return this.segments.slice(this.offset, this.limit()); } + // TODO(Mila): Use database info and toString() to get full path instead. + toFullPath(): string { + return this.segments.join('/'); + } + static comparator>( p1: BasePath, p2: BasePath diff --git a/packages/firestore/src/platform/browser/serializer.ts b/packages/firestore/src/platform/browser/serializer.ts index 722f40e605f..5e009d89f60 100644 --- a/packages/firestore/src/platform/browser/serializer.ts +++ b/packages/firestore/src/platform/browser/serializer.ts @@ -22,17 +22,3 @@ import { JsonProtoSerializer } from '../../remote/serializer'; export function newSerializer(databaseId: DatabaseId): JsonProtoSerializer { return new JsonProtoSerializer(databaseId, /* useProto3Json= */ true); } - -/** - * An instance of the Platform's 'TextEncoder' implementation. - */ -export function newTextEncoder(): TextEncoder { - return new TextEncoder(); -} - -/** - * An instance of the Platform's 'TextDecoder' implementation. - */ -export function newTextDecoder(): TextDecoder { - return new TextDecoder('utf-8'); -} diff --git a/packages/firestore/src/platform/browser/text_serializer.ts b/packages/firestore/src/platform/browser/text_serializer.ts new file mode 100644 index 00000000000..6a53021f7c0 --- /dev/null +++ b/packages/firestore/src/platform/browser/text_serializer.ts @@ -0,0 +1,30 @@ +/** + * @license + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * An instance of the Platform's 'TextEncoder' implementation. + */ +export function newTextEncoder(): TextEncoder { + return new TextEncoder(); +} + +/** + * An instance of the Platform's 'TextDecoder' implementation. + */ +export function newTextDecoder(): TextDecoder { + return new TextDecoder('utf-8'); +} diff --git a/packages/firestore/src/platform/browser_lite/text_serializer.ts b/packages/firestore/src/platform/browser_lite/text_serializer.ts new file mode 100644 index 00000000000..a9231f63914 --- /dev/null +++ b/packages/firestore/src/platform/browser_lite/text_serializer.ts @@ -0,0 +1,18 @@ +/** + * @license + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from '../browser/text_serializer'; diff --git a/packages/firestore/src/platform/node/serializer.ts b/packages/firestore/src/platform/node/serializer.ts index 1f61010902b..e19dbc8b1db 100644 --- a/packages/firestore/src/platform/node/serializer.ts +++ b/packages/firestore/src/platform/node/serializer.ts @@ -16,25 +16,9 @@ */ /** Return the Platform-specific serializer monitor. */ -import { TextDecoder, TextEncoder } from 'util'; - import { DatabaseId } from '../../core/database_info'; import { JsonProtoSerializer } from '../../remote/serializer'; export function newSerializer(databaseId: DatabaseId): JsonProtoSerializer { return new JsonProtoSerializer(databaseId, /* useProto3Json= */ false); } - -/** - * An instance of the Platform's 'TextEncoder' implementation. - */ -export function newTextEncoder(): TextEncoder { - return new TextEncoder(); -} - -/** - * An instance of the Platform's 'TextDecoder' implementation. - */ -export function newTextDecoder(): TextDecoder { - return new TextDecoder('utf-8'); -} diff --git a/packages/firestore/src/platform/node/text_serializer.ts b/packages/firestore/src/platform/node/text_serializer.ts new file mode 100644 index 00000000000..cc7852e5f4b --- /dev/null +++ b/packages/firestore/src/platform/node/text_serializer.ts @@ -0,0 +1,32 @@ +/** + * @license + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TextDecoder, TextEncoder } from 'util'; + +/** + * An instance of the Platform's 'TextEncoder' implementation. + */ +export function newTextEncoder(): TextEncoder { + return new TextEncoder(); +} + +/** + * An instance of the Platform's 'TextDecoder' implementation. + */ +export function newTextDecoder(): TextDecoder { + return new TextDecoder('utf-8'); +} diff --git a/packages/firestore/src/platform/node_lite/text_serializer.ts b/packages/firestore/src/platform/node_lite/text_serializer.ts new file mode 100644 index 00000000000..efc3610847d --- /dev/null +++ b/packages/firestore/src/platform/node_lite/text_serializer.ts @@ -0,0 +1,18 @@ +/** + * @license + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from '../browser_lite/text_serializer'; diff --git a/packages/firestore/src/platform/rn/serializer.ts b/packages/firestore/src/platform/rn/serializer.ts index 2b168a0dffa..c5ab7bf2bb5 100644 --- a/packages/firestore/src/platform/rn/serializer.ts +++ b/packages/firestore/src/platform/rn/serializer.ts @@ -15,8 +15,4 @@ * limitations under the License. */ -export { - newSerializer, - newTextEncoder, - newTextDecoder -} from '../browser/serializer'; +export { newSerializer } from '../browser/serializer'; diff --git a/packages/firestore/src/platform/rn/text_serializer.ts b/packages/firestore/src/platform/rn/text_serializer.ts new file mode 100644 index 00000000000..8edf69e42f2 --- /dev/null +++ b/packages/firestore/src/platform/rn/text_serializer.ts @@ -0,0 +1,18 @@ +/** + * @license + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { newTextEncoder, newTextDecoder } from '../browser/text_serializer'; diff --git a/packages/firestore/src/platform/rn_lite/text_serializer.ts b/packages/firestore/src/platform/rn_lite/text_serializer.ts new file mode 100644 index 00000000000..efc3610847d --- /dev/null +++ b/packages/firestore/src/platform/rn_lite/text_serializer.ts @@ -0,0 +1,18 @@ +/** + * @license + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from '../browser_lite/text_serializer'; diff --git a/packages/firestore/src/platform/serializer.ts b/packages/firestore/src/platform/serializer.ts index 89e8fe650d1..47b659f5664 100644 --- a/packages/firestore/src/platform/serializer.ts +++ b/packages/firestore/src/platform/serializer.ts @@ -15,15 +15,9 @@ * limitations under the License. */ -import { isNode, isReactNative } from '@firebase/util'; - import { DatabaseId } from '../core/database_info'; import { JsonProtoSerializer } from '../remote/serializer'; -import * as browser from './browser/serializer'; -import * as node from './node/serializer'; -import * as rn from './rn/serializer'; - // This file is only used under ts-node. // eslint-disable-next-line @typescript-eslint/no-require-imports const platform = require(`./${process.env.TEST_PLATFORM ?? 'node'}/serializer`); @@ -31,29 +25,3 @@ const platform = require(`./${process.env.TEST_PLATFORM ?? 'node'}/serializer`); export function newSerializer(databaseId: DatabaseId): JsonProtoSerializer { return platform.newSerializer(databaseId); } - -/** - * An instance of the Platform's 'TextEncoder' implementation. - */ -export function newTextEncoder(): TextEncoder { - if (isNode()) { - return node.newTextEncoder(); - } else if (isReactNative()) { - return rn.newTextEncoder(); - } else { - return browser.newTextEncoder(); - } -} - -/** - * An instance of the Platform's 'TextDecoder' implementation. - */ -export function newTextDecoder(): TextDecoder { - if (isNode()) { - return node.newTextDecoder() as TextDecoder; - } else if (isReactNative()) { - return rn.newTextDecoder(); - } else { - return browser.newTextDecoder(); - } -} diff --git a/packages/firestore/src/platform/text_serializer.ts b/packages/firestore/src/platform/text_serializer.ts new file mode 100644 index 00000000000..72ebba86c3c --- /dev/null +++ b/packages/firestore/src/platform/text_serializer.ts @@ -0,0 +1,48 @@ +/** + * @license + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isNode, isReactNative } from '@firebase/util'; + +import * as browser from './browser/text_serializer'; +import * as node from './node/text_serializer'; +import * as rn from './rn/text_serializer'; + +/** + * An instance of the Platform's 'TextEncoder' implementation. + */ +export function newTextEncoder(): TextEncoder { + if (isNode()) { + return node.newTextEncoder(); + } else if (isReactNative()) { + return rn.newTextEncoder(); + } else { + return browser.newTextEncoder(); + } +} + +/** + * An instance of the Platform's 'TextDecoder' implementation. + */ +export function newTextDecoder(): TextDecoder { + if (isNode()) { + return node.newTextDecoder() as TextDecoder; + } else if (isReactNative()) { + return rn.newTextDecoder(); + } else { + return browser.newTextDecoder(); + } +} diff --git a/packages/firestore/src/remote/bloom_filter.ts b/packages/firestore/src/remote/bloom_filter.ts index 041848ef35b..4c0e4ef6b7e 100644 --- a/packages/firestore/src/remote/bloom_filter.ts +++ b/packages/firestore/src/remote/bloom_filter.ts @@ -16,8 +16,7 @@ */ import { Md5, Integer } from '@firebase/webchannel-wrapper'; -import { newTextEncoder } from '../platform/serializer'; -import { debugAssert } from '../util/assert'; +import { newTextEncoder } from '../platform/text_serializer'; const MAX_64_BIT_UNSIGNED_INTEGER = new Integer([0xffffffff, 0xffffffff], 0); @@ -43,43 +42,49 @@ function get64BitUints(Bytes: Uint8Array): [Integer, Integer] { } export class BloomFilter { - readonly size: number; - private readonly sizeInInteger: Integer; + readonly bitCount: number; + private readonly bitCountInInteger: Integer; constructor( - private readonly bitmap: Uint8Array, - padding: number, - private readonly hashCount: number + readonly bitmap: Uint8Array, + readonly padding: number, + readonly hashCount: number ) { - debugAssert(padding >= 0 && padding < 8, `Invalid padding: ${padding}`); - if (bitmap.length > 0) { - debugAssert(this.hashCount > 0, `Invalid hash count: ${hashCount}`); - } else { + if (padding < 0 || padding >= 8) { + throw new BloomFilterError(`Invalid padding: ${padding}`); + } + + if (hashCount < 0) { + throw new BloomFilterError(`Invalid hash count: ${hashCount}`); + } + + if (bitmap.length > 0 && this.hashCount === 0) { // Only empty bloom filter can have 0 hash count. - debugAssert(this.hashCount >= 0, `Invalid hash count: ${hashCount}`); + throw new BloomFilterError(`Invalid hash count: ${hashCount}`); + } + if (bitmap.length === 0 && padding !== 0) { // Empty bloom filter should have 0 padding. - debugAssert( - padding === 0, + throw new BloomFilterError( `Invalid padding when bitmap length is 0: ${padding}` ); } - this.size = bitmap.length * 8 - padding; - // Set the size in Integer to avoid repeated calculation in mightContain(). - this.sizeInInteger = Integer.fromNumber(this.size); + this.bitCount = bitmap.length * 8 - padding; + // Set the bit count in Integer to avoid repetition in mightContain(). + this.bitCountInInteger = Integer.fromNumber(this.bitCount); } // Calculate the ith hash value based on the hashed 64bit integers, // and calculate its corresponding bit index in the bitmap to be checked. - private getBitIndex(num1: Integer, num2: Integer, index: number): number { + private getBitIndex(num1: Integer, num2: Integer, hashIndex: number): number { // Calculate hashed value h(i) = h1 + (i * h2). - let hashValue = num1.add(num2.multiply(Integer.fromNumber(index))); + let hashValue = num1.add(num2.multiply(Integer.fromNumber(hashIndex))); // Wrap if hash value overflow 64bit. if (hashValue.compare(MAX_64_BIT_UNSIGNED_INTEGER) === 1) { hashValue = new Integer([hashValue.getBits(0), hashValue.getBits(1)], 0); } - return hashValue.modulo(this.sizeInInteger).toNumber(); + return hashValue.modulo(this.bitCountInInteger).toNumber(); } // Return whether the bit on the given index in the bitmap is set to 1. @@ -91,12 +96,10 @@ export class BloomFilter { } mightContain(value: string): boolean { - // Empty bitmap and empty value should always return false on membership - // check. - if (this.size === 0 || value === '') { + // Empty bitmap should always return false on membership check. + if (this.bitCount === 0) { return false; } - const md5HashedValue = getMd5HashValue(value); const [hash1, hash2] = get64BitUints(md5HashedValue); for (let i = 0; i < this.hashCount; i++) { @@ -107,4 +110,40 @@ export class BloomFilter { } return true; } + + /** Create bloom filter for testing purposes only. */ + static create( + bitCount: number, + hashCount: number, + contains: string[] + ): BloomFilter { + const padding = bitCount % 8 === 0 ? 0 : 8 - (bitCount % 8); + const bitmap = new Uint8Array(Math.ceil(bitCount / 8)); + const bloomFilter = new BloomFilter(bitmap, padding, hashCount); + contains.forEach(item => bloomFilter.insert(item)); + return bloomFilter; + } + + private insert(value: string): void { + if (this.bitCount === 0) { + return; + } + + const md5HashedValue = getMd5HashValue(value); + const [hash1, hash2] = get64BitUints(md5HashedValue); + for (let i = 0; i < this.hashCount; i++) { + const index = this.getBitIndex(hash1, hash2, i); + this.setBit(index); + } + } + + private setBit(index: number): void { + const indexOfByte = Math.floor(index / 8); + const offset = index % 8; + this.bitmap[indexOfByte] |= 0x01 << offset; + } +} + +export class BloomFilterError extends Error { + readonly name = 'BloomFilterError'; } diff --git a/packages/firestore/src/remote/watch_change.ts b/packages/firestore/src/remote/watch_change.ts index ab80ef0b962..596f5b93609 100644 --- a/packages/firestore/src/remote/watch_change.ts +++ b/packages/firestore/src/remote/watch_change.ts @@ -27,14 +27,16 @@ import { } from '../model/collections'; import { MutableDocument } from '../model/document'; import { DocumentKey } from '../model/document_key'; +import { normalizeByteString } from '../model/normalize'; import { debugAssert, fail, hardAssert } from '../util/assert'; import { ByteString } from '../util/byte_string'; import { FirestoreError } from '../util/error'; -import { logDebug } from '../util/log'; +import { logDebug, logWarn } from '../util/log'; import { primitiveComparator } from '../util/misc'; import { SortedMap } from '../util/sorted_map'; import { SortedSet } from '../util/sorted_set'; +import { BloomFilter, BloomFilterError } from './bloom_filter'; import { ExistenceFilter } from './existence_filter'; import { RemoteEvent, TargetChange } from './remote_event'; @@ -409,16 +411,103 @@ export class WatchChangeAggregator { } } else { const currentSize = this.getCurrentDocumentCountForTarget(targetId); + // Existence filter mismatch. Mark the documents as being in limbo, and + // raise a snapshot with `isFromCache:true`. if (currentSize !== expectedCount) { - // Existence filter mismatch: We reset the mapping and raise a new - // snapshot with `isFromCache:true`. - this.resetTarget(targetId); - this.pendingTargetResets = this.pendingTargetResets.add(targetId); + // Apply bloom filter to identify and mark removed documents. + const bloomFilterApplied = this.applyBloomFilter( + watchChange.existenceFilter, + targetId, + currentSize + ); + if (!bloomFilterApplied) { + // If bloom filter application fails, we reset the mapping and + // trigger re-run of the query. + this.resetTarget(targetId); + this.pendingTargetResets = this.pendingTargetResets.add(targetId); + } } } } } + /** Returns whether a bloom filter removed the deleted documents successfully. */ + private applyBloomFilter( + existenceFilter: ExistenceFilter, + targetId: number, + currentCount: number + ): boolean { + const unchangedNames = existenceFilter.unchangedNames; + const expectedCount = existenceFilter.count; + + if (!unchangedNames || !unchangedNames.bits) { + return false; + } + + const { + bits: { bitmap = '', padding = 0 }, + hashCount = 0 + } = unchangedNames; + + // TODO(Mila): Remove this validation, add try catch to normalizeByteString. + if (typeof bitmap === 'string') { + const isValidBitmap = this.isValidBase64String(bitmap); + if (!isValidBitmap) { + logWarn('Invalid base64 string. Applying bloom filter failed.'); + return false; + } + } + + const normalizedBitmap = normalizeByteString(bitmap).toUint8Array(); + + let bloomFilter: BloomFilter; + try { + // BloomFilter throws error if the inputs are invalid. + bloomFilter = new BloomFilter(normalizedBitmap, padding, hashCount); + } catch (err) { + if (err instanceof BloomFilterError) { + logWarn('BloomFilter error: ', err); + } else { + logWarn('Applying bloom filter failed: ', err); + } + return false; + } + + const removedDocumentCount = this.filterRemovedDocuments( + bloomFilter, + targetId + ); + + return expectedCount === currentCount - removedDocumentCount; + } + + // TODO(Mila): Move the validation into normalizeByteString. + private isValidBase64String(value: string): boolean { + const regExp = new RegExp( + '^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$' + ); + return regExp.test(value); + } + + /** + * Filter out removed documents based on bloom filter membership result and + * return number of documents removed. + */ + private filterRemovedDocuments( + bloomFilter: BloomFilter, + targetId: number + ): number { + const existingKeys = this.metadataProvider.getRemoteKeysForTarget(targetId); + let removalCount = 0; + existingKeys.forEach(key => { + if (!bloomFilter.mightContain(key.path.toFullPath())) { + this.removeDocumentFromTarget(targetId, key, /*updatedDocument=*/ null); + removalCount++; + } + }); + return removalCount; + } + /** * Converts the currently accumulated state into a remote event at the * provided snapshot version. Resets the accumulated changes before returning. diff --git a/packages/firestore/src/util/bundle_reader_impl.ts b/packages/firestore/src/util/bundle_reader_impl.ts index 3f08218d2fc..05875fce1af 100644 --- a/packages/firestore/src/util/bundle_reader_impl.ts +++ b/packages/firestore/src/util/bundle_reader_impl.ts @@ -15,7 +15,7 @@ * limitations under the License. */ -import { newTextDecoder } from '../platform/serializer'; +import { newTextDecoder } from '../platform/text_serializer'; import { BundleMetadata } from '../protos/firestore_bundle_proto'; import { JsonProtoSerializer } from '../remote/serializer'; diff --git a/packages/firestore/test/unit/core/webchannel_wrapper.test.ts b/packages/firestore/test/unit/core/webchannel_wrapper.test.ts index 9d1f924735c..3f3272aea48 100644 --- a/packages/firestore/test/unit/core/webchannel_wrapper.test.ts +++ b/packages/firestore/test/unit/core/webchannel_wrapper.test.ts @@ -22,7 +22,7 @@ import { Md5, Integer } from '@firebase/webchannel-wrapper'; import { expect } from 'chai'; -import { newTextEncoder } from '../../../src/platform/serializer'; +import { newTextEncoder } from '../../../src/platform/text_serializer'; describe('Md5', () => { // The precomputed MD5 digests of the 3-character strings "abc" and "def". diff --git a/packages/firestore/test/unit/remote/bloom_filter.test.ts b/packages/firestore/test/unit/remote/bloom_filter.test.ts index d33c4521a58..fd0b66e7d68 100644 --- a/packages/firestore/test/unit/remote/bloom_filter.test.ts +++ b/packages/firestore/test/unit/remote/bloom_filter.test.ts @@ -16,15 +16,15 @@ */ import { expect } from 'chai'; +import { normalizeByteString } from '../../../src/model/normalize'; import { BloomFilter } from '../../../src/remote/bloom_filter'; -import { ByteString } from '../../../src/util/byte_string'; import * as TEST_DATA from './bloom_filter_golden_test_data'; describe('BloomFilter', () => { it('can instantiate an empty bloom filter', () => { const bloomFilter = new BloomFilter(new Uint8Array(0), 0, 0); - expect(bloomFilter.size).to.equal(0); + expect(bloomFilter.bitCount).to.equal(0); }); it('should throw error if empty bloom filter inputs are invalid', () => { @@ -46,14 +46,14 @@ describe('BloomFilter', () => { const bloomFilter6 = new BloomFilter(new Uint8Array(1), 6, 1); const bloomFilter7 = new BloomFilter(new Uint8Array(1), 7, 1); - expect(bloomFilter0.size).to.equal(8); - expect(bloomFilter1.size).to.equal(7); - expect(bloomFilter2.size).to.equal(6); - expect(bloomFilter3.size).to.equal(5); - expect(bloomFilter4.size).to.equal(4); - expect(bloomFilter5.size).to.equal(3); - expect(bloomFilter6.size).to.equal(2); - expect(bloomFilter7.size).to.equal(1); + expect(bloomFilter0.bitCount).to.equal(8); + expect(bloomFilter1.bitCount).to.equal(7); + expect(bloomFilter2.bitCount).to.equal(6); + expect(bloomFilter3.bitCount).to.equal(5); + expect(bloomFilter4.bitCount).to.equal(4); + expect(bloomFilter5.bitCount).to.equal(3); + expect(bloomFilter6.bitCount).to.equal(2); + expect(bloomFilter7.bitCount).to.equal(1); }); it('should throw error if padding is invalid', () => { @@ -86,19 +86,15 @@ describe('BloomFilter', () => { it('mightContain in empty bloom filter should always return false', () => { const bloomFilter = new BloomFilter(new Uint8Array(0), 0, 0); + expect(bloomFilter.mightContain('')).to.be.false; expect(bloomFilter.mightContain('abc')).to.be.false; - expect(bloomFilter.mightContain('def')).to.be.false; }); - it('mightContain should always return false for empty string', () => { - const emptyBloomFilter = new BloomFilter(new Uint8Array(0), 0, 0); - const nonEmptyBloomFilter = new BloomFilter( - new Uint8Array([255, 255, 255]), - 1, - 16 - ); - expect(emptyBloomFilter.mightContain('')).to.be.false; - expect(nonEmptyBloomFilter.mightContain('')).to.be.false; + it('mightContain on empty string might return false positive result', () => { + const bloomFilter1 = new BloomFilter(new Uint8Array([1]), 1, 1); + const bloomFilter2 = new BloomFilter(new Uint8Array([255]), 0, 16); + expect(bloomFilter1.mightContain('')).to.be.false; + expect(bloomFilter2.mightContain('')).to.be.true; }); /** @@ -139,7 +135,7 @@ describe('BloomFilter', () => { } = bloomFilterInputs; const { membershipTestResults } = expectedResult; - const byteArray = ByteString.fromBase64String(bitmap).toUint8Array(); + const byteArray = normalizeByteString(bitmap).toUint8Array(); const bloomFilter = new BloomFilter(byteArray, padding, hashCount); for (let i = 0; i < membershipTestResults.length; i++) { const expectedMembershipResult = membershipTestResults[i] === '1'; diff --git a/packages/firestore/test/unit/specs/existence_filter_spec.test.ts b/packages/firestore/test/unit/specs/existence_filter_spec.test.ts index 32346c31af5..080dcd08b5a 100644 --- a/packages/firestore/test/unit/specs/existence_filter_spec.test.ts +++ b/packages/firestore/test/unit/specs/existence_filter_spec.test.ts @@ -17,7 +17,13 @@ import { newQueryForPath } from '../../../src/core/query'; import { Code } from '../../../src/util/error'; -import { deletedDoc, doc, query } from '../../util/helpers'; +import { + deletedDoc, + doc, + filter, + generateBloomFilterProto, + query +} from '../../util/helpers'; import { describeSpec, specTest } from './describe_spec'; import { spec } from './spec_builder'; @@ -35,22 +41,6 @@ describeSpec('Existence Filters:', [], () => { .watchSnapshots(2000); }); - // This test is only to make sure watchFilters can accept bloom filter. - // TODO:(mila) update the tests when bloom filter logic is implemented. - specTest('Existence filter with bloom filter match', [], () => { - const query1 = query('collection'); - const doc1 = doc('collection/1', 1000, { v: 1 }); - return spec() - .userListens(query1) - .watchAcksFull(query1, 1000, doc1) - .expectEvents(query1, { added: [doc1] }) - .watchFilters([query1], [doc1.key], { - bits: { bitmap: 'a', padding: 1 }, - hashCount: 1 - }) - .watchSnapshots(2000); - }); - specTest('Existence filter match after pending update', [], () => { const query1 = query('collection'); const doc1 = doc('collection/1', 2000, { v: 2 }); @@ -128,36 +118,6 @@ describeSpec('Existence Filters:', [], () => { ); }); - // This test is only to make sure watchFilters can accept bloom filter. - // TODO:(mila) update the tests when bloom filter logic is implemented. - specTest('Existence filter mismatch triggers bloom filter', [], () => { - const query1 = query('collection'); - const doc1 = doc('collection/1', 1000, { v: 1 }); - const doc2 = doc('collection/2', 1000, { v: 2 }); - return ( - spec() - .userListens(query1) - .watchAcksFull(query1, 1000, doc1, doc2) - .expectEvents(query1, { added: [doc1, doc2] }) - .watchFilters([query1], [doc1.key], { - bits: { bitmap: 'a', padding: 1 }, - hashCount: 3 - }) // in the next sync doc2 was deleted - .watchSnapshots(2000) - // query is now marked as "inconsistent" because of filter mismatch - .expectEvents(query1, { fromCache: true }) - .expectActiveTargets({ query: query1, resumeToken: '' }) - .watchRemoves(query1) // Acks removal of query - .watchAcksFull(query1, 2000, doc1) - .expectLimboDocs(doc2.key) // doc2 is now in limbo - .ackLimbo(2000, deletedDoc('collection/2', 2000)) - .expectLimboDocs() // doc2 is no longer in limbo - .expectEvents(query1, { - removed: [doc2] - }) - ); - }); - specTest('Existence filter mismatch will drop resume token', [], () => { const query1 = query('collection'); const doc1 = doc('collection/1', 1000, { v: 1 }); @@ -286,4 +246,341 @@ describeSpec('Existence Filters:', [], () => { ); } ); + + /** + * Test existence filter with bloom filter. + */ + specTest( + 'Full re-query is skipped when bloom filter can identify documents deleted', + [], + () => { + const query1 = query('collection'); + const docA = doc('collection/a', 1000, { v: 1 }); + const docB = doc('collection/b', 1000, { v: 2 }); + const bloomFilterProto = generateBloomFilterProto({ + contains: [docA], + notContains: [docB] + }); + return ( + spec() + .userListens(query1) + .watchAcksFull(query1, 1000, docA, docB) + .expectEvents(query1, { added: [docA, docB] }) + // DocB is deleted in the next sync. + .watchFilters([query1], [docA.key], bloomFilterProto) + .watchSnapshots(2000) + // BloomFilter identify docB is deleted, skip full query and put docB + // into limbo directly. + .expectEvents(query1, { fromCache: true }) + .expectLimboDocs(docB.key) // DocB is now in limbo. + .ackLimbo(2000, deletedDoc('collection/b', 2000)) + .expectLimboDocs() // DocB is no longer in limbo. + .expectEvents(query1, { + removed: [docB] + }) + ); + } + ); + + specTest( + 'Full re-query is triggered when bloom filter can not identify documents deleted', + [], + () => { + const query1 = query('collection'); + const docA = doc('collection/a', 1000, { v: 1 }); + const docB = doc('collection/b', 1000, { v: 2 }); + const docC = doc('collection/c', 1000, { v: 2 }); + const bloomFilterProto = generateBloomFilterProto({ + contains: [docA, docB], + notContains: [docC] + }); + return ( + spec() + .userListens(query1) + .watchAcksFull(query1, 1000, docA, docB, docC) + .expectEvents(query1, { added: [docA, docB, docC] }) + // DocB and DocC are deleted in the next sync. + .watchFilters([query1], [docA.key], bloomFilterProto) + .watchSnapshots(2000) + // BloomFilter correctly identifies docC is deleted, but yields false + // positive results for docB. Re-run query is triggered. + .expectEvents(query1, { fromCache: true }) + .expectActiveTargets({ query: query1, resumeToken: '' }) + ); + } + ); + + specTest( + 'Bloom filter can process special characters in document name', + [], + () => { + const query1 = query('collection'); + const docA = doc('collection/ÀÒ∑', 1000, { v: 1 }); + const docB = doc('collection/À∑Ò', 1000, { v: 1 }); + const bloomFilterProto = generateBloomFilterProto({ + contains: [docA], + notContains: [docB] + }); + + return ( + spec() + .userListens(query1) + .watchAcksFull(query1, 1000, docA, docB) + .expectEvents(query1, { added: [docA, docB] }) + // DocB is deleted in the next sync. + .watchFilters([query1], [docA.key], bloomFilterProto) + .watchSnapshots(2000) + // BloomFilter identifies docB is deleted, skip full query and put + // docB into limbo directly. + .expectEvents(query1, { fromCache: true }) + .expectLimboDocs(docB.key) // DocB is now in limbo. + ); + } + ); + + specTest( + 'Bloom filter fills in default values for undefined padding and hashCount', + [], + () => { + const query1 = query('collection'); + const docA = doc('collection/a', 1000, { v: 1 }); + const docB = doc('collection/b', 1000, { v: 2 }); + + const bloomFilterProto = generateBloomFilterProto({ + contains: [docA], + notContains: [docB] + }); + // Omit padding and hashCount. Default value 0 should be used. + delete bloomFilterProto.hashCount; + delete bloomFilterProto.bits!.padding; + + return ( + spec() + .userListens(query1) + .watchAcksFull(query1, 1000, docA, docB) + .expectEvents(query1, { added: [docA, docB] }) + // DocB is deleted in the next sync. + .watchFilters([query1], [docA.key], bloomFilterProto) + .watchSnapshots(2000) + // Re-run query is triggered. + .expectEvents(query1, { fromCache: true }) + .expectActiveTargets({ query: query1, resumeToken: '' }) + ); + } + ); + + specTest( + 'Full re-query is triggered when bloom filter bitmap is invalid', + [], + () => { + const query1 = query('collection'); + const docA = doc('collection/a', 1000, { v: 1 }); + const docB = doc('collection/b', 1000, { v: 1 }); + + const bloomFilterProto = generateBloomFilterProto({ + contains: [docA], + notContains: [docB] + }); + // Set bitmap to invalid base64 string. + bloomFilterProto.bits!.bitmap = 'INVALID_BASE_64'; + + return ( + spec() + .userListens(query1) + .watchAcksFull(query1, 1000, docA, docB) + .expectEvents(query1, { added: [docA, docB] }) + // DocB is deleted in the next sync. + .watchFilters([query1], [docA.key], bloomFilterProto) + .watchSnapshots(2000) + // Re-run query is triggered. + .expectEvents(query1, { fromCache: true }) + .expectActiveTargets({ query: query1, resumeToken: '' }) + ); + } + ); + + specTest( + 'Full re-query is triggered when bloom filter hashCount is invalid', + [], + () => { + const query1 = query('collection'); + const docA = doc('collection/a', 1000, { v: 1 }); + const docB = doc('collection/b', 1000, { v: 1 }); + + const bloomFilterProto = generateBloomFilterProto({ + contains: [docA], + notContains: [docB] + }); + // Set hashCount to negative number. + bloomFilterProto.hashCount = -1; + + return ( + spec() + .userListens(query1) + .watchAcksFull(query1, 1000, docA, docB) + .expectEvents(query1, { added: [docA, docB] }) + // DocB is deleted in the next sync. + .watchFilters([query1], [docA.key], bloomFilterProto) + .watchSnapshots(2000) + // Re-run query is triggered. + .expectEvents(query1, { fromCache: true }) + .expectActiveTargets({ query: query1, resumeToken: '' }) + ); + } + ); + + specTest('Full re-query is triggered when bloom filter is empty', [], () => { + const query1 = query('collection'); + const docA = doc('collection/a', 1000, { v: 1 }); + const docB = doc('collection/b', 1000, { v: 1 }); + + //Generate an empty bloom filter. + const bloomFilterProto = generateBloomFilterProto({ + contains: [], + notContains: [], + bitCount: 0, + hashCount: 0 + }); + + return ( + spec() + .userListens(query1) + .watchAcksFull(query1, 1000, docA, docB) + .expectEvents(query1, { added: [docA, docB] }) + // DocB is deleted in the next sync. + .watchFilters([query1], [docA.key], bloomFilterProto) + .watchSnapshots(2000) + // Re-run query is triggered. + .expectEvents(query1, { fromCache: true }) + .expectActiveTargets({ query: query1, resumeToken: '' }) + ); + }); + + specTest('Same documents can have different bloom filters', [], () => { + const query1 = query('collection', filter('v', '<=', 2)); + const query2 = query('collection', filter('v', '>=', 2)); + + const docA = doc('collection/a', 1000, { v: 1 }); + const docB = doc('collection/b', 1000, { v: 2 }); + const docC = doc('collection/c', 1000, { v: 3 }); + + const bloomFilterProto1 = generateBloomFilterProto({ + contains: [docB], + notContains: [docA, docC], + bitCount: 5, + hashCount: 2 + }); + const bloomFilterProto2 = generateBloomFilterProto({ + contains: [docB], + notContains: [docA, docC], + bitCount: 4, + hashCount: 1 + }); + return ( + spec() + .userListens(query1) + .watchAcksFull(query1, 1000, docA, docB) + .expectEvents(query1, { added: [docA, docB] }) + .userListens(query2) + .expectEvents(query2, { added: [docB], fromCache: true }) + .watchAcksFull(query2, 1001, docB, docC) + .expectEvents(query2, { added: [docC] }) + + // DocA is deleted in the next sync for query1. + .watchFilters([query1], [docB.key], bloomFilterProto1) + .watchSnapshots(2000) + // BloomFilter identify docA is deleted, skip full query. + .expectEvents(query1, { fromCache: true }) + .expectLimboDocs(docA.key) // DocA is now in limbo. + + // DocC is deleted in the next sync for query2. + .watchFilters([query2], [docB.key], bloomFilterProto2) + .watchSnapshots(3000) + // BloomFilter identify docC is deleted, skip full query. + .expectEvents(query2, { fromCache: true }) + .expectLimboDocs(docA.key, docC.key) // DocC is now in limbo. + ); + }); + + specTest('Bloom filter is handled at global snapshot', [], () => { + const query1 = query('collection'); + const docA = doc('collection/a', 1000, { v: 1 }); + const docB = doc('collection/b', 2000, { v: 2 }); + const docC = doc('collection/c', 3000, { v: 3 }); + + const bloomFilterProto = generateBloomFilterProto({ + contains: [docA], + notContains: [docB] + }); + + return ( + spec() + .userListens(query1) + .watchAcksFull(query1, 1000, docA, docB) + .expectEvents(query1, { added: [docA, docB] }) + // Send a mismatching existence filter with one document, but don't + // send a new global snapshot. We should not see an event until we + // receive the snapshot. + .watchFilters([query1], [docA.key], bloomFilterProto) + .watchSends({ affects: [query1] }, docC) + .watchSnapshots(2000) + .expectEvents(query1, { added: [docC], fromCache: true }) + // Re-run of the query1 is skipped, docB is in limbo. + .expectLimboDocs(docB.key) + ); + }); + + specTest('Bloom filter limbo resolution is denied', [], () => { + const query1 = query('collection'); + const docA = doc('collection/a', 1000, { v: 1 }); + const docB = doc('collection/b', 1000, { v: 1 }); + const bloomFilterProto = generateBloomFilterProto({ + contains: [docA], + notContains: [docB] + }); + return spec() + .userListens(query1) + .watchAcksFull(query1, 1000, docA, docB) + .expectEvents(query1, { added: [docA, docB] }) + .watchFilters([query1], [docA.key], bloomFilterProto) + .watchSnapshots(2000) + .expectEvents(query1, { fromCache: true }) + .expectLimboDocs(docB.key) // DocB is now in limbo. + .watchRemoves( + newQueryForPath(docB.key.path), + new RpcError(Code.PERMISSION_DENIED, 'no') + ) + .expectLimboDocs() // DocB is no longer in limbo. + .expectEvents(query1, { + removed: [docB] + }); + }); + + specTest('Bloom filter with large size works as expected', [], () => { + const query1 = query('collection'); + const docs = []; + for (let i = 0; i < 100; i++) { + docs.push(doc(`collection/doc${i}`, 1000, { v: 1 })); + } + const docKeys = docs.map(item => item.key); + + const bloomFilterProto = generateBloomFilterProto({ + contains: docs.slice(0, 50), + notContains: docs.slice(50), + bitCount: 1000, + hashCount: 16 + }); + return ( + spec() + .userListens(query1) + .watchAcksFull(query1, 1000, ...docs) + .expectEvents(query1, { added: docs }) + // Doc0 to doc49 are deleted in the next sync. + .watchFilters([query1], docKeys.slice(0, 50), bloomFilterProto) + .watchSnapshots(2000) + // BloomFilter correctly identifies docs that deleted, skip full query. + .expectEvents(query1, { fromCache: true }) + .expectLimboDocs(...docKeys.slice(50)) + ); + }); }); diff --git a/packages/firestore/test/unit/specs/limbo_spec.test.ts b/packages/firestore/test/unit/specs/limbo_spec.test.ts index 4451fc5a80a..b69094b4eaa 100644 --- a/packages/firestore/test/unit/specs/limbo_spec.test.ts +++ b/packages/firestore/test/unit/specs/limbo_spec.test.ts @@ -22,7 +22,14 @@ import { } from '../../../src/core/query'; import { TimerId } from '../../../src/util/async_queue'; import { Code } from '../../../src/util/error'; -import { deletedDoc, doc, filter, orderBy, query } from '../../util/helpers'; +import { + deletedDoc, + doc, + filter, + generateBloomFilterProto, + orderBy, + query +} from '../../util/helpers'; import { describeSpec, specTest } from './describe_spec'; import { client, spec } from './spec_builder'; @@ -915,6 +922,66 @@ describeSpec('Limbo Documents:', [], () => { } ); + specTest( + 'Limbo resolution throttling with bloom filter application', + [], + () => { + const query1 = query('collection'); + const docA1 = doc('collection/a1', 1000, { key: 'a1' }); + const docA2 = doc('collection/a2', 1000, { key: 'a2' }); + const docA3 = doc('collection/a3', 1000, { key: 'a3' }); + const docB1 = doc('collection/b1', 1000, { key: 'b1' }); + const docB2 = doc('collection/b2', 1000, { key: 'b2' }); + const docB3 = doc('collection/b3', 1000, { key: 'b3' }); + const bloomFilterProto = generateBloomFilterProto({ + contains: [docB1, docB2, docB3], + notContains: [docA1, docA2, docA3] + }); + + // Verify that limbo resolution throttling works as expected with bloom filter. + return ( + spec() + .withMaxConcurrentLimboResolutions(2) + .userListens(query1) + .watchAcksFull(query1, 1000, docA1, docA2, docA3) + .expectEvents(query1, { added: [docA1, docA2, docA3] }) + // Simulate that the client loses network connection. + .disableNetwork() + .expectEvents(query1, { fromCache: true }) + .enableNetwork() + .restoreListen(query1, 'resume-token-1000') + .watchAcks(query1) + // While this client was disconnected, another client deleted all the + // docAs replaced them with docBs. If Watch has to re-run the + // underlying query when this client re-listens, Watch won't be able + // to tell that docAs were deleted and will only send us existing + // documents that changed since the resume token. This will cause it + // to just send the docBs with an existence filter with a count of 3. + .watchSends({ affects: [query1] }, docB1, docB2, docB3) + .watchFilters( + [query1], + [docB1.key, docB2.key, docB3.key], + bloomFilterProto + ) + .watchSnapshots(1001) + .expectEvents(query1, { + added: [docB1, docB2, docB3], + fromCache: true + }) + // The view now contains the docAs and the docBs (6 documents), but + // the existence filter indicated only 3 should match. There is an + // existence filter mismatch. Bloom filter checks membership of the + // docs, and filters out docAs, while docBs returns true. Number of + // existing docs matches the expected count, so skip the re-query. + .watchCurrents(query1, 'resume-token-1002') + .watchSnapshots(1002) + // The docAs are now in limbo; the client begins limbo resolution. + .expectLimboDocs(docA1.key, docA2.key) + .expectEnqueuedLimboDocs(docA3.key) + ); + } + ); + specTest( 'A limbo resolution for a document should not be started if one is already active', [], diff --git a/packages/firestore/test/unit/specs/spec_test_runner.ts b/packages/firestore/test/unit/specs/spec_test_runner.ts index 107459ebce2..137f51d6d54 100644 --- a/packages/firestore/test/unit/specs/spec_test_runner.ts +++ b/packages/firestore/test/unit/specs/spec_test_runner.ts @@ -90,7 +90,7 @@ import { Mutation } from '../../../src/model/mutation'; import { JsonObject } from '../../../src/model/object_value'; import { encodeBase64 } from '../../../src/platform/base64'; import { toByteStreamReader } from '../../../src/platform/byte_stream_reader'; -import { newTextEncoder } from '../../../src/platform/serializer'; +import { newTextEncoder } from '../../../src/platform/text_serializer'; import * as api from '../../../src/protos/firestore_proto_api'; import { ExistenceFilter } from '../../../src/remote/existence_filter'; import { @@ -1180,7 +1180,13 @@ abstract class TestRunner { }); } - expect(actual.view!.docChanges).to.deep.equal(expectedChanges); + const actualChangesSorted = Array.from(actual.view!.docChanges).sort( + (a, b) => primitiveComparator(a.doc, b.doc) + ); + const expectedChangesSorted = Array.from(expectedChanges).sort((a, b) => + primitiveComparator(a.doc, b.doc) + ); + expect(actualChangesSorted).to.deep.equal(expectedChangesSorted); expect(actual.view!.hasPendingWrites).to.equal( expected.hasPendingWrites, diff --git a/packages/firestore/test/unit/util/bundle.test.ts b/packages/firestore/test/unit/util/bundle.test.ts index 9f225138491..b32ca0842b7 100644 --- a/packages/firestore/test/unit/util/bundle.test.ts +++ b/packages/firestore/test/unit/util/bundle.test.ts @@ -18,7 +18,7 @@ import { expect, use } from 'chai'; import chaiAsPromised from 'chai-as-promised'; import { toByteStreamReader } from '../../../src/platform/byte_stream_reader'; -import { newTextEncoder } from '../../../src/platform/serializer'; +import { newTextEncoder } from '../../../src/platform/text_serializer'; import { BundleReader, SizedBundleElement diff --git a/packages/firestore/test/unit/util/bundle_data.ts b/packages/firestore/test/unit/util/bundle_data.ts index 03bc8b35f4c..92cfb8c30cd 100644 --- a/packages/firestore/test/unit/util/bundle_data.ts +++ b/packages/firestore/test/unit/util/bundle_data.ts @@ -22,10 +22,8 @@ import { queryWithLimit } from '../../../src/core/query'; import { DocumentKey } from '../../../src/model/document_key'; -import { - newSerializer, - newTextEncoder -} from '../../../src/platform/serializer'; +import { newSerializer } from '../../../src/platform/serializer'; +import { newTextEncoder } from '../../../src/platform/text_serializer'; import { BundleElement, LimitType as BundleLimitType diff --git a/packages/firestore/test/util/helpers.ts b/packages/firestore/test/util/helpers.ts index 4d62273a616..f4adb5d0434 100644 --- a/packages/firestore/test/util/helpers.ts +++ b/packages/firestore/test/util/helpers.ts @@ -85,6 +85,7 @@ import { SetMutation, FieldTransform } from '../../src/model/mutation'; +import { normalizeByteString } from '../../src/model/normalize'; import { JsonObject, ObjectValue } from '../../src/model/object_value'; import { FieldPath, ResourcePath } from '../../src/model/path'; import { decodeBase64, encodeBase64 } from '../../src/platform/base64'; @@ -94,6 +95,7 @@ import { LimitType as ProtoLimitType } from '../../src/protos/firestore_bundle_proto'; import * as api from '../../src/protos/firestore_proto_api'; +import { BloomFilter } from '../../src/remote/bloom_filter'; import { ExistenceFilter } from '../../src/remote/existence_filter'; import { RemoteEvent, TargetChange } from '../../src/remote/remote_event'; import { @@ -1073,3 +1075,44 @@ export function computeCombinations(input: T[]): T[][] { }; return computeNonEmptyCombinations(input).concat([[]]); } + +/** + * Helper method to generate bloom filter proto value for mocking watch + * existence filter response. + */ +export function generateBloomFilterProto(config: { + contains: MutableDocument[]; + notContains: MutableDocument[]; + hashCount?: number; + bitCount?: number; +}): api.BloomFilter { + const DOCUMENT_PREFIX = + 'projects/test-project/databases/(default)/documents/'; + + const { contains, notContains, hashCount = 10, bitCount = 100 } = config; + + if (bitCount === 0 && contains.length !== 0) { + throw new Error('To contain strings, number of bits cannot be 0.'); + } + const bloomFilter = BloomFilter.create( + bitCount, + hashCount, + contains.map(item => DOCUMENT_PREFIX + item.key) + ); + + notContains.forEach(item => { + if (bloomFilter.mightContain(DOCUMENT_PREFIX + item.key)) { + throw new Error( + 'Cannot generate desired bloom filter. Please adjust the hashCount ' + + 'and/or number of bits.' + ); + } + }); + return { + bits: { + bitmap: normalizeByteString(bloomFilter.bitmap).toBase64(), + padding: bloomFilter.padding + }, + hashCount: bloomFilter.hashCount + }; +}