diff --git a/.changeset/cool-games-care.md b/.changeset/cool-games-care.md new file mode 100644 index 00000000000..ca5eb47ae4c --- /dev/null +++ b/.changeset/cool-games-care.md @@ -0,0 +1,6 @@ +--- +'@firebase/firestore': patch +'firebase': patch +--- + +Implemented internal logic to auto-create client-side indexes diff --git a/packages/firestore/src/api.ts b/packages/firestore/src/api.ts index 0e871303cb8..9f9ac38749a 100644 --- a/packages/firestore/src/api.ts +++ b/packages/firestore/src/api.ts @@ -202,6 +202,13 @@ export { setIndexConfiguration } from './api/index_configuration'; +export { + PersistentCacheIndexManager, + getPersistentCacheIndexManager, + enablePersistentCacheIndexAutoCreation, + disablePersistentCacheIndexAutoCreation +} from './api/persistent_cache_index_manager'; + /** * Internal exports */ diff --git a/packages/firestore/src/api/persistent_cache_index_manager.ts b/packages/firestore/src/api/persistent_cache_index_manager.ts new file mode 100644 index 00000000000..96751fee074 --- /dev/null +++ b/packages/firestore/src/api/persistent_cache_index_manager.ts @@ -0,0 +1,140 @@ +/** + * @license + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + firestoreClientSetPersistentCacheIndexAutoCreationEnabled, + FirestoreClient +} from '../core/firestore_client'; +import { cast } from '../util/input_validation'; +import { logDebug, logWarn } from '../util/log'; + +import { ensureFirestoreConfigured, Firestore } from './database'; + +/** + * A `PersistentCacheIndexManager` which you can config persistent cache indexes + * used for local query execution. + * + * To use, call `getPersistentCacheIndexManager()` to get an instance. + * + * TODO(CSI) Remove @internal to make the API publicly available. + * @internal + */ +export class PersistentCacheIndexManager { + readonly type: 'PersistentCacheIndexManager' = 'PersistentCacheIndexManager'; + + /** @hideconstructor */ + constructor(readonly _client: FirestoreClient) {} +} + +/** + * Returns the PersistentCache Index Manager used by the given `Firestore` + * object. + * + * @return The `PersistentCacheIndexManager` instance, or `null` if local + * persistent storage is not in use. + * + * TODO(CSI) Remove @internal to make the API publicly available. + * @internal + */ +export function getPersistentCacheIndexManager( + firestore: Firestore +): PersistentCacheIndexManager | null { + firestore = cast(firestore, Firestore); + + const cachedInstance = persistentCacheIndexManagerByFirestore.get(firestore); + if (cachedInstance) { + return cachedInstance; + } + + const client = ensureFirestoreConfigured(firestore); + if (client._uninitializedComponentsProvider?._offlineKind !== 'persistent') { + return null; + } + + const instance = new PersistentCacheIndexManager(client); + persistentCacheIndexManagerByFirestore.set(firestore, instance); + return instance; +} + +/** + * Enables SDK to create persistent cache indexes automatically for local query + * execution when SDK believes cache indexes can help improves performance. + * + * This feature is disabled by default. + * + * TODO(CSI) Remove @internal to make the API publicly available. + * @internal + */ +export function enablePersistentCacheIndexAutoCreation( + indexManager: PersistentCacheIndexManager +): void { + setPersistentCacheIndexAutoCreationEnabled(indexManager, true); +} + +/** + * Stops creating persistent cache indexes automatically for local query + * execution. The indexes which have been created by calling + * `enablePersistentCacheIndexAutoCreation()` still take effect. + * + * TODO(CSI) Remove @internal to make the API publicly available. + * @internal + */ +export function disablePersistentCacheIndexAutoCreation( + indexManager: PersistentCacheIndexManager +): void { + setPersistentCacheIndexAutoCreationEnabled(indexManager, false); +} + +function setPersistentCacheIndexAutoCreationEnabled( + indexManager: PersistentCacheIndexManager, + isEnabled: boolean +): void { + indexManager._client.verifyNotTerminated(); + + const promise = firestoreClientSetPersistentCacheIndexAutoCreationEnabled( + indexManager._client, + isEnabled + ); + + promise + .then(_ => + logDebug( + `setting persistent cache index auto creation ` + + `isEnabled=${isEnabled} succeeded` + ) + ) + .catch(error => + logWarn( + `setting persistent cache index auto creation ` + + `isEnabled=${isEnabled} failed`, + error + ) + ); +} + +/** + * Maps `Firestore` instances to their corresponding + * `PersistentCacheIndexManager` instances. + * + * Use a `WeakMap` so that the mapping will be automatically dropped when the + * `Firestore` instance is garbage collected. This emulates a private member + * as described in https://goo.gle/454yvug. + */ +const persistentCacheIndexManagerByFirestore = new WeakMap< + Firestore, + PersistentCacheIndexManager +>(); diff --git a/packages/firestore/src/core/firestore_client.ts b/packages/firestore/src/core/firestore_client.ts index df6127f978e..6b7950825f6 100644 --- a/packages/firestore/src/core/firestore_client.ts +++ b/packages/firestore/src/core/firestore_client.ts @@ -29,7 +29,8 @@ import { localStoreExecuteQuery, localStoreGetNamedQuery, localStoreHandleUserChange, - localStoreReadDocument + localStoreReadDocument, + localStoreSetIndexAutoCreationEnabled } from '../local/local_store_impl'; import { Persistence } from '../local/persistence'; import { Document } from '../model/document'; @@ -828,3 +829,15 @@ export function firestoreClientSetIndexConfiguration( ); }); } + +export function firestoreClientSetPersistentCacheIndexAutoCreationEnabled( + client: FirestoreClient, + isEnabled: boolean +): Promise { + return client.asyncQueue.enqueue(async () => { + return localStoreSetIndexAutoCreationEnabled( + await getLocalStore(client), + isEnabled + ); + }); +} diff --git a/packages/firestore/src/local/index_manager.ts b/packages/firestore/src/local/index_manager.ts index 39d958349d4..78bc47a9471 100644 --- a/packages/firestore/src/local/index_manager.ts +++ b/packages/firestore/src/local/index_manager.ts @@ -41,6 +41,19 @@ export const enum IndexType { FULL } +export function displayNameForIndexType(indexType: IndexType): string { + switch (indexType) { + case IndexType.NONE: + return 'NONE'; + case IndexType.PARTIAL: + return 'PARTIAL'; + case IndexType.FULL: + return 'FULL'; + default: + return `[unknown IndexType: ${indexType}]`; + } +} + /** * Represents a set of indexes that are used to execute queries efficiently. * @@ -92,6 +105,12 @@ export interface IndexManager { index: FieldIndex ): PersistencePromise; + /** Creates a full matched field index which serves the given target. */ + createTargetIndexes( + transaction: PersistenceTransaction, + target: Target + ): PersistencePromise; + /** * Returns a list of field indexes that correspond to the specified collection * group. diff --git a/packages/firestore/src/local/indexeddb_index_manager.ts b/packages/firestore/src/local/indexeddb_index_manager.ts index a776c95c1da..f7e5991a6be 100644 --- a/packages/firestore/src/local/indexeddb_index_manager.ts +++ b/packages/firestore/src/local/indexeddb_index_manager.ts @@ -252,6 +252,26 @@ export class IndexedDbIndexManager implements IndexManager { ); } + createTargetIndexes( + transaction: PersistenceTransaction, + target: Target + ): PersistencePromise { + return PersistencePromise.forEach( + this.getSubTargets(target), + (subTarget: Target) => { + return this.getIndexType(transaction, subTarget).next(type => { + if (type === IndexType.NONE || type === IndexType.PARTIAL) { + const targetIndexMatcher = new TargetIndexMatcher(subTarget); + return this.addFieldIndex( + transaction, + targetIndexMatcher.buildTargetIndex() + ); + } + }); + } + ); + } + getDocumentsMatchingTarget( transaction: PersistenceTransaction, target: Target diff --git a/packages/firestore/src/local/indexeddb_remote_document_cache.ts b/packages/firestore/src/local/indexeddb_remote_document_cache.ts index c3af0655cc4..b3d4658d53d 100644 --- a/packages/firestore/src/local/indexeddb_remote_document_cache.ts +++ b/packages/firestore/src/local/indexeddb_remote_document_cache.ts @@ -55,6 +55,7 @@ import { } from './local_serializer'; import { PersistencePromise } from './persistence_promise'; import { PersistenceTransaction } from './persistence_transaction'; +import { QueryContext } from './query_context'; import { RemoteDocumentCache } from './remote_document_cache'; import { RemoteDocumentChangeBuffer } from './remote_document_change_buffer'; import { SimpleDbStore } from './simple_db'; @@ -279,7 +280,8 @@ class IndexedDbRemoteDocumentCacheImpl implements IndexedDbRemoteDocumentCache { transaction: PersistenceTransaction, query: Query, offset: IndexOffset, - mutatedDocs: OverlayMap + mutatedDocs: OverlayMap, + context?: QueryContext ): PersistencePromise { const collection = query.path; const startKey = [ @@ -300,6 +302,7 @@ class IndexedDbRemoteDocumentCacheImpl implements IndexedDbRemoteDocumentCache { return remoteDocumentsStore(transaction) .loadAll(IDBKeyRange.bound(startKey, endKey, true)) .next(dbRemoteDocs => { + context?.incrementDocumentReadCount(dbRemoteDocs.length); let results = mutableDocumentMap(); for (const dbRemoteDoc of dbRemoteDocs) { const document = this.maybeDecodeDocument( diff --git a/packages/firestore/src/local/local_documents_view.ts b/packages/firestore/src/local/local_documents_view.ts index 78802e443bf..fa64ed76eb2 100644 --- a/packages/firestore/src/local/local_documents_view.ts +++ b/packages/firestore/src/local/local_documents_view.ts @@ -60,6 +60,7 @@ import { MutationQueue } from './mutation_queue'; import { OverlayedDocument } from './overlayed_document'; import { PersistencePromise } from './persistence_promise'; import { PersistenceTransaction } from './persistence_transaction'; +import { QueryContext } from './query_context'; import { RemoteDocumentCache } from './remote_document_cache'; /** @@ -355,11 +356,14 @@ export class LocalDocumentsView { * @param transaction - The persistence transaction. * @param query - The query to match documents against. * @param offset - Read time and key to start scanning by (exclusive). + * @param context - A optional tracker to keep a record of important details + * during database local query execution. */ getDocumentsMatchingQuery( transaction: PersistenceTransaction, query: Query, - offset: IndexOffset + offset: IndexOffset, + context?: QueryContext ): PersistencePromise { if (isDocumentQuery(query)) { return this.getDocumentsMatchingDocumentQuery(transaction, query.path); @@ -367,13 +371,15 @@ export class LocalDocumentsView { return this.getDocumentsMatchingCollectionGroupQuery( transaction, query, - offset + offset, + context ); } else { return this.getDocumentsMatchingCollectionQuery( transaction, query, - offset + offset, + context ); } } @@ -472,7 +478,8 @@ export class LocalDocumentsView { private getDocumentsMatchingCollectionGroupQuery( transaction: PersistenceTransaction, query: Query, - offset: IndexOffset + offset: IndexOffset, + context?: QueryContext ): PersistencePromise { debugAssert( query.path.isEmpty(), @@ -493,7 +500,8 @@ export class LocalDocumentsView { return this.getDocumentsMatchingCollectionQuery( transaction, collectionQuery, - offset + offset, + context ).next(r => { r.forEach((key, doc) => { results = results.insert(key, doc); @@ -506,7 +514,8 @@ export class LocalDocumentsView { private getDocumentsMatchingCollectionQuery( transaction: PersistenceTransaction, query: Query, - offset: IndexOffset + offset: IndexOffset, + context?: QueryContext ): PersistencePromise { // Query the remote documents and overlay mutations. let overlays: OverlayMap; @@ -518,7 +527,8 @@ export class LocalDocumentsView { transaction, query, offset, - overlays + overlays, + context ); }) .next(remoteDocuments => { diff --git a/packages/firestore/src/local/local_store_impl.ts b/packages/firestore/src/local/local_store_impl.ts index e3bb8dd3fdb..11459a1716c 100644 --- a/packages/firestore/src/local/local_store_impl.ts +++ b/packages/firestore/src/local/local_store_impl.ts @@ -1085,7 +1085,7 @@ export function localStoreExecuteQuery( return localStoreImpl.persistence.runTransaction( 'Execute query', - 'readonly', + 'readwrite', // Use readwrite instead of readonly so indexes can be created txn => { return localStoreGetTargetData(localStoreImpl, txn, queryToTarget(query)) .next(targetData => { @@ -1526,3 +1526,38 @@ export async function localStoreConfigureFieldIndexes( .next(() => PersistencePromise.waitFor(promises)) ); } + +export function localStoreSetIndexAutoCreationEnabled( + localStore: LocalStore, + isEnabled: boolean +): void { + const localStoreImpl = debugCast(localStore, LocalStoreImpl); + localStoreImpl.queryEngine.indexAutoCreationEnabled = isEnabled; +} + +/** + * Test-only hooks into the SDK for use exclusively by tests. + */ +export class TestingHooks { + private constructor() { + throw new Error('creating instances is not supported'); + } + + static setIndexAutoCreationSettings( + localStore: LocalStore, + settings: { + indexAutoCreationMinCollectionSize?: number; + relativeIndexReadCostPerDocument?: number; + } + ): void { + const localStoreImpl = debugCast(localStore, LocalStoreImpl); + if (settings.indexAutoCreationMinCollectionSize !== undefined) { + localStoreImpl.queryEngine.indexAutoCreationMinCollectionSize = + settings.indexAutoCreationMinCollectionSize; + } + if (settings.relativeIndexReadCostPerDocument !== undefined) { + localStoreImpl.queryEngine.relativeIndexReadCostPerDocument = + settings.relativeIndexReadCostPerDocument; + } + } +} diff --git a/packages/firestore/src/local/memory_index_manager.ts b/packages/firestore/src/local/memory_index_manager.ts index 025bc566ab1..5a363118767 100644 --- a/packages/firestore/src/local/memory_index_manager.ts +++ b/packages/firestore/src/local/memory_index_manager.ts @@ -66,6 +66,14 @@ export class MemoryIndexManager implements IndexManager { return PersistencePromise.resolve(); } + createTargetIndexes( + transaction: PersistenceTransaction, + target: Target + ): PersistencePromise { + // Field indices are not supported with memory persistence. + return PersistencePromise.resolve(); + } + getDocumentsMatchingTarget( transaction: PersistenceTransaction, target: Target diff --git a/packages/firestore/src/local/query_context.ts b/packages/firestore/src/local/query_context.ts new file mode 100644 index 00000000000..1d9891720f9 --- /dev/null +++ b/packages/firestore/src/local/query_context.ts @@ -0,0 +1,35 @@ +/** + * @license + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A tracker to keep a record of important details during database local query + * execution. + */ +export class QueryContext { + /** + * Counts the number of documents passed through during local query execution. + */ + private _documentReadCount = 0; + + get documentReadCount(): number { + return this._documentReadCount; + } + + incrementDocumentReadCount(amount: number): void { + this._documentReadCount += amount; + } +} diff --git a/packages/firestore/src/local/query_engine.ts b/packages/firestore/src/local/query_engine.ts index 7728e9d7aa7..a0adb7ed95a 100644 --- a/packages/firestore/src/local/query_engine.ts +++ b/packages/firestore/src/local/query_engine.ts @@ -46,6 +46,18 @@ import { IndexManager, IndexType } from './index_manager'; import { LocalDocumentsView } from './local_documents_view'; import { PersistencePromise } from './persistence_promise'; import { PersistenceTransaction } from './persistence_transaction'; +import { QueryContext } from './query_context'; + +const DEFAULT_INDEX_AUTO_CREATION_MIN_COLLECTION_SIZE = 100; + +/** + * This cost represents the evaluation result of + * (([index, docKey] + [docKey, docContent]) per document in the result set) + * / ([docKey, docContent] per documents in full collection scan) coming from + * experiment [enter PR experiment URL here]. + * TODO: Enter PR experiment URL above. + */ +const DEFAULT_RELATIVE_INDEX_READ_COST_PER_DOCUMENT = 2; /** * The Firestore query engine. @@ -90,6 +102,18 @@ export class QueryEngine { private indexManager!: IndexManager; private initialized = false; + indexAutoCreationEnabled = false; + + /** + * SDK only decides whether it should create index when collection size is + * larger than this. + */ + indexAutoCreationMinCollectionSize = + DEFAULT_INDEX_AUTO_CREATION_MIN_COLLECTION_SIZE; + + relativeIndexReadCostPerDocument = + DEFAULT_RELATIVE_INDEX_READ_COST_PER_DOCUMENT; + /** Sets the document view to query against. */ initialize( localDocuments: LocalDocumentsView, @@ -109,20 +133,103 @@ export class QueryEngine { ): PersistencePromise { debugAssert(this.initialized, 'initialize() not called'); + // Stores the result from executing the query; using this object is more + // convenient than passing the result between steps of the persistence + // transaction and improves readability comparatively. + const queryResult: { result: DocumentMap | null } = { result: null }; + return this.performQueryUsingIndex(transaction, query) - .next(result => - result - ? result - : this.performQueryUsingRemoteKeys( - transaction, - query, - remoteKeys, - lastLimboFreeSnapshotVersion - ) - ) - .next(result => - result ? result : this.executeFullCollectionScan(transaction, query) + .next(result => { + queryResult.result = result; + }) + .next(() => { + if (queryResult.result) { + return; + } + return this.performQueryUsingRemoteKeys( + transaction, + query, + remoteKeys, + lastLimboFreeSnapshotVersion + ).next(result => { + queryResult.result = result; + }); + }) + .next(() => { + if (queryResult.result) { + return; + } + const context = new QueryContext(); + return this.executeFullCollectionScan(transaction, query, context).next( + result => { + queryResult.result = result; + if (this.indexAutoCreationEnabled) { + return this.createCacheIndexes( + transaction, + query, + context, + result.size + ); + } + } + ); + }) + .next(() => queryResult.result!); + } + + createCacheIndexes( + transaction: PersistenceTransaction, + query: Query, + context: QueryContext, + resultSize: number + ): PersistencePromise { + if (context.documentReadCount < this.indexAutoCreationMinCollectionSize) { + if (getLogLevel() <= LogLevel.DEBUG) { + logDebug( + 'QueryEngine', + 'SDK will not create cache indexes for query:', + stringifyQuery(query), + 'since it only creates cache indexes for collection contains', + 'more than or equal to', + this.indexAutoCreationMinCollectionSize, + 'documents' + ); + } + return PersistencePromise.resolve(); + } + + if (getLogLevel() <= LogLevel.DEBUG) { + logDebug( + 'QueryEngine', + 'Query:', + stringifyQuery(query), + 'scans', + context.documentReadCount, + 'local documents and returns', + resultSize, + 'documents as results.' + ); + } + + if ( + context.documentReadCount > + this.relativeIndexReadCostPerDocument * resultSize + ) { + if (getLogLevel() <= LogLevel.DEBUG) { + logDebug( + 'QueryEngine', + 'The SDK decides to create cache indexes for query:', + stringifyQuery(query), + 'as using cache indexes may help improve performance.' + ); + } + return this.indexManager.createTargetIndexes( + transaction, + queryToTarget(query) ); + } + + return PersistencePromise.resolve(); } /** @@ -221,18 +328,18 @@ export class QueryEngine { query: Query, remoteKeys: DocumentKeySet, lastLimboFreeSnapshotVersion: SnapshotVersion - ): PersistencePromise { + ): PersistencePromise { if (queryMatchesAllDocuments(query)) { // Queries that match all documents don't benefit from using // key-based lookups. It is more efficient to scan all documents in a // collection, rather than to perform individual lookups. - return this.executeFullCollectionScan(transaction, query); + return PersistencePromise.resolve(null); } // Queries that have never seen a snapshot without limbo free documents // should also be run as a full collection scan. if (lastLimboFreeSnapshotVersion.isEqual(SnapshotVersion.min())) { - return this.executeFullCollectionScan(transaction, query); + return PersistencePromise.resolve(null); } return this.localDocumentsView!.getDocuments(transaction, remoteKeys).next( @@ -247,7 +354,7 @@ export class QueryEngine { lastLimboFreeSnapshotVersion ) ) { - return this.executeFullCollectionScan(transaction, query); + return PersistencePromise.resolve(null); } if (getLogLevel() <= LogLevel.DEBUG) { @@ -269,7 +376,7 @@ export class QueryEngine { lastLimboFreeSnapshotVersion, INITIAL_LARGEST_BATCH_ID ) - ); + ).next(results => results); } ); } @@ -343,7 +450,8 @@ export class QueryEngine { private executeFullCollectionScan( transaction: PersistenceTransaction, - query: Query + query: Query, + context: QueryContext ): PersistencePromise { if (getLogLevel() <= LogLevel.DEBUG) { logDebug( @@ -356,7 +464,8 @@ export class QueryEngine { return this.localDocumentsView!.getDocumentsMatchingQuery( transaction, query, - IndexOffset.min() + IndexOffset.min(), + context ); } diff --git a/packages/firestore/src/local/remote_document_cache.ts b/packages/firestore/src/local/remote_document_cache.ts index a66f1b4a253..15fcecdc836 100644 --- a/packages/firestore/src/local/remote_document_cache.ts +++ b/packages/firestore/src/local/remote_document_cache.ts @@ -28,6 +28,7 @@ import { IndexOffset } from '../model/field_index'; import { IndexManager } from './index_manager'; import { PersistencePromise } from './persistence_promise'; import { PersistenceTransaction } from './persistence_transaction'; +import { QueryContext } from './query_context'; import { RemoteDocumentChangeBuffer } from './remote_document_change_buffer'; /** @@ -70,13 +71,16 @@ export interface RemoteDocumentCache { * * @param query - The query to match documents against. * @param offset - The offset to start the scan at (exclusive). + * @param context - A optional tracker to keep a record of important details + * during database local query execution. * @returns The set of matching documents. */ getDocumentsMatchingQuery( transaction: PersistenceTransaction, query: Query, offset: IndexOffset, - mutatedDocs: OverlayMap + mutatedDocs: OverlayMap, + context?: QueryContext ): PersistencePromise; /** diff --git a/packages/firestore/src/model/target_index_matcher.ts b/packages/firestore/src/model/target_index_matcher.ts index d3bd95d473c..e274ed7ba05 100644 --- a/packages/firestore/src/model/target_index_matcher.ts +++ b/packages/firestore/src/model/target_index_matcher.ts @@ -19,14 +19,17 @@ import { FieldFilter, Operator } from '../core/filter'; import { Direction, OrderBy } from '../core/order_by'; import { Target } from '../core/target'; import { debugAssert, hardAssert } from '../util/assert'; +import { SortedSet } from '../util/sorted_set'; import { FieldIndex, fieldIndexGetArraySegment, fieldIndexGetDirectionalSegments, IndexKind, - IndexSegment + IndexSegment, + IndexState } from './field_index'; +import { FieldPath } from './path'; /** * A light query planner for Firestore. @@ -179,6 +182,69 @@ export class TargetIndexMatcher { return true; } + /** Returns a full matched field index for this target. */ + buildTargetIndex(): FieldIndex { + // We want to make sure only one segment created for one field. For example, + // in case like a == 3 and a > 2, Index {a ASCENDING} will only be created + // once. + let uniqueFields = new SortedSet(FieldPath.comparator); + const segments: IndexSegment[] = []; + + for (const filter of this.equalityFilters) { + if (filter.field.isKeyField()) { + continue; + } + const isArrayOperator = + filter.op === Operator.ARRAY_CONTAINS || + filter.op === Operator.ARRAY_CONTAINS_ANY; + if (isArrayOperator) { + segments.push(new IndexSegment(filter.field, IndexKind.CONTAINS)); + } else { + if (uniqueFields.has(filter.field)) { + continue; + } + uniqueFields = uniqueFields.add(filter.field); + segments.push(new IndexSegment(filter.field, IndexKind.ASCENDING)); + } + } + + // Note: We do not explicitly check `this.inequalityFilter` but rather rely + // on the target defining an appropriate "order by" to ensure that the + // required index segment is added. The query engine would reject a query + // with an inequality filter that lacks the required order-by clause. + for (const orderBy of this.orderBys) { + // Stop adding more segments if we see a order-by on key. Typically this + // is the default implicit order-by which is covered in the index_entry + // table as a separate column. If it is not the default order-by, the + // generated index will be missing some segments optimized for order-bys, + // which is probably fine. + if (orderBy.field.isKeyField()) { + continue; + } + + if (uniqueFields.has(orderBy.field)) { + continue; + } + uniqueFields = uniqueFields.add(orderBy.field); + + segments.push( + new IndexSegment( + orderBy.field, + orderBy.dir === Direction.ASCENDING + ? IndexKind.ASCENDING + : IndexKind.DESCENDING + ) + ); + } + + return new FieldIndex( + FieldIndex.UNKNOWN_ID, + this.collectionId, + segments, + IndexState.empty() + ); + } + private hasMatchingEqualityFilter(segment: IndexSegment): boolean { for (const filter of this.equalityFilters) { if (this.matchesFilter(filter, segment)) { diff --git a/packages/firestore/src/util/async_queue.ts b/packages/firestore/src/util/async_queue.ts index bffd4433a39..c430320a4c4 100644 --- a/packages/firestore/src/util/async_queue.ts +++ b/packages/firestore/src/util/async_queue.ts @@ -115,6 +115,10 @@ export class DelayedOperation implements PromiseLike { this.deferred.promise.catch(err => {}); } + get promise(): Promise { + return this.deferred.promise; + } + /** * Creates and returns a DelayedOperation that has been scheduled to be * executed on the provided asyncQueue after the provided delayMs. diff --git a/packages/firestore/test/integration/api/persistent_cache_index_manager.test.ts b/packages/firestore/test/integration/api/persistent_cache_index_manager.test.ts new file mode 100644 index 00000000000..5d09c948a64 --- /dev/null +++ b/packages/firestore/test/integration/api/persistent_cache_index_manager.test.ts @@ -0,0 +1,152 @@ +/** + * @license + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from 'chai'; + +import { + disablePersistentCacheIndexAutoCreation, + doc, + enablePersistentCacheIndexAutoCreation, + getDoc, + getDocs, + getDocsFromCache, + getPersistentCacheIndexManager, + PersistentCacheIndexManager, + query, + terminate, + where +} from '../util/firebase_export'; +import { + apiDescribe, + partitionedTestDocs, + withTestCollection, + withTestDb +} from '../util/helpers'; + +apiDescribe('PersistentCacheIndexManager', persistence => { + describe('getPersistentCacheIndexManager()', () => { + it('should return non-null if, and only if, IndexedDB persistence is enabled', () => + withTestDb(persistence, async db => { + const indexManager = getPersistentCacheIndexManager(db); + if (persistence.storage === 'indexeddb') { + expect(indexManager).to.be.instanceof(PersistentCacheIndexManager); + } else { + expect(indexManager).to.be.null; + } + })); + + it('should always return the same object', () => + withTestDb(persistence, async db => { + const indexManager1 = getPersistentCacheIndexManager(db); + const indexManager2 = getPersistentCacheIndexManager(db); + expect(indexManager1).to.equal(indexManager2); + })); + + it('should fail if invoked after terminate()', () => + withTestDb(persistence, async db => { + terminate(db).catch(e => expect.fail(`terminate() failed: ${e}`)); + expect(() => getPersistentCacheIndexManager(db)).to.throw( + 'The client has already been terminated.' + ); + })); + }); + + // Skip the rest of the tests since they require `PersistentCacheIndexManager` + // support, which is only available with indexeddb persistence. + if (persistence.storage !== 'indexeddb') { + return; + } + + describe('enable/disable persistent index auto creation', () => { + it('enable on new instance should succeed', () => + withTestDb(persistence, async db => { + const indexManager = getPersistentCacheIndexManager(db)!; + enablePersistentCacheIndexAutoCreation(indexManager); + })); + + it('disable on new instance should succeed', () => + withTestDb(persistence, async db => { + const indexManager = getPersistentCacheIndexManager(db)!; + disablePersistentCacheIndexAutoCreation(indexManager); + })); + + it('enable when already enabled should succeed', async () => + withTestDb(persistence, async db => { + const documentRef = doc(db, 'a/b'); + const indexManager = getPersistentCacheIndexManager(db)!; + enablePersistentCacheIndexAutoCreation(indexManager); + await getDoc(documentRef); // flush the async queue + enablePersistentCacheIndexAutoCreation(indexManager); + enablePersistentCacheIndexAutoCreation(indexManager); + })); + + it('disable when already disabled should succeed', async () => + withTestDb(persistence, async db => { + const documentRef = doc(db, 'a/b'); + const indexManager = getPersistentCacheIndexManager(db)!; + disablePersistentCacheIndexAutoCreation(indexManager); + await getDoc(documentRef); // flush the async queue + disablePersistentCacheIndexAutoCreation(indexManager); + disablePersistentCacheIndexAutoCreation(indexManager); + })); + + it('enabling after terminate() should throw', () => + withTestDb(persistence, async db => { + const indexManager = getPersistentCacheIndexManager(db)!; + terminate(db).catch(e => expect.fail(`terminate() failed: ${e}`)); + expect(() => + enablePersistentCacheIndexAutoCreation(indexManager) + ).to.throw('The client has already been terminated.'); + })); + + it('disabling after terminate() should throw', () => + withTestDb(persistence, async db => { + const indexManager = getPersistentCacheIndexManager(db)!; + terminate(db).catch(e => expect.fail(`terminate() failed: ${e}`)); + expect(() => + disablePersistentCacheIndexAutoCreation(indexManager) + ).to.throw('The client has already been terminated.'); + })); + + it('query returns correct results when index is auto-created', () => { + const testDocs = partitionedTestDocs({ + matching: { documentData: { match: true }, documentCount: 1 }, + nonmatching: { documentData: { match: false }, documentCount: 100 } + }); + return withTestCollection(persistence, testDocs, async (coll, db) => { + const indexManager = getPersistentCacheIndexManager(db)!; + enablePersistentCacheIndexAutoCreation(indexManager); + + // Populate the local cache with the entire collection's contents. + await getDocs(coll); + + // Run a query that matches only one of the documents in the collection; + // this should cause an index to be auto-created. + const query_ = query(coll, where('match', '==', true)); + const snapshot1 = await getDocsFromCache(query_); + expect(snapshot1.size).to.equal(1); + + // Run the query that matches only one of the documents again, which + // should _still_ return the one and only document that matches. Since + // the public API surface does not reveal whether an index was used, + // there isn't anything else that can be verified. + const snapshot2 = await getDocsFromCache(query_); + expect(snapshot2.size).to.equal(1); + }); + }); + }); +}); diff --git a/packages/firestore/test/integration/util/helpers.ts b/packages/firestore/test/integration/util/helpers.ts index 3b27e5e7c47..9f97ec6bf79 100644 --- a/packages/firestore/test/integration/util/helpers.ts +++ b/packages/firestore/test/integration/util/helpers.ts @@ -483,3 +483,37 @@ export function withTestCollectionSettings( } ); } + +/** + * Creates a `docs` argument suitable for specifying to `withTestCollection()` + * that defines subsets of documents with different document data. + * + * This can be useful for pre-populating a collection with some documents that + * match a query and others that do _not_ match that query. + * + * Each key of the given `partitions` object will be considered a partition + * "name". The returned object will specify `documentCount` documents with the + * `documentData` whose document IDs are prefixed with the partition "name". + */ +export function partitionedTestDocs(partitions: { + [partitionName: string]: { + documentData: DocumentData; + documentCount: number; + }; +}): { [key: string]: DocumentData } { + const testDocs: { [key: string]: DocumentData } = {}; + + for (const partitionName in partitions) { + // Make lint happy (see https://eslint.org/docs/latest/rules/guard-for-in). + if (!Object.prototype.hasOwnProperty.call(partitions, partitionName)) { + continue; + } + const partition = partitions[partitionName]; + for (let i = 0; i < partition.documentCount; i++) { + const documentId = `${partitionName}_${`${i}`.padStart(4, '0')}`; + testDocs[documentId] = partition.documentData; + } + } + + return testDocs; +} diff --git a/packages/firestore/test/unit/local/counting_query_engine.ts b/packages/firestore/test/unit/local/counting_query_engine.ts index d407abfd60a..deaef12a829 100644 --- a/packages/firestore/test/unit/local/counting_query_engine.ts +++ b/packages/firestore/test/unit/local/counting_query_engine.ts @@ -105,14 +105,16 @@ export class CountingQueryEngine extends QueryEngine { transaction, query, sinceReadTime, - overlays + overlays, + context ) => { return subject .getDocumentsMatchingQuery( transaction, query, sinceReadTime, - overlays + overlays, + context ) .next(result => { this.documentsReadByCollection += result.size; diff --git a/packages/firestore/test/unit/local/index_manager.test.ts b/packages/firestore/test/unit/local/index_manager.test.ts index f256312399d..1385064e4a0 100644 --- a/packages/firestore/test/unit/local/index_manager.test.ts +++ b/packages/firestore/test/unit/local/index_manager.test.ts @@ -30,7 +30,10 @@ import { queryWithLimit, queryWithStartAt } from '../../../src/core/query'; -import { IndexType } from '../../../src/local/index_manager'; +import { + displayNameForIndexType, + IndexType +} from '../../../src/local/index_manager'; import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence'; import { Persistence } from '../../../src/local/persistence'; import { documentMap } from '../../../src/model/collections'; @@ -51,6 +54,7 @@ import { filter, key, orderBy, + orFilter, path, query, version, @@ -60,6 +64,26 @@ import { import * as persistenceHelpers from './persistence_test_helpers'; import { TestIndexManager } from './test_index_manager'; +describe('index_manager.ts top-level functions', () => { + describe('displayNameForIndexType()', () => { + it('IndexType.NONE', () => + expect(displayNameForIndexType(IndexType.NONE)).to.equal('NONE')); + + it('IndexType.FULL', () => + expect(displayNameForIndexType(IndexType.FULL)).to.equal('FULL')); + + it('IndexType.PARTIAL', () => + expect(displayNameForIndexType(IndexType.PARTIAL)).to.equal('PARTIAL')); + + it('invalid IndexType', () => + // @ts-expect-error: specifying a string to displayNameForIndexType() + // causes a TypeScript compiler error, but is handled gracefully. + expect(displayNameForIndexType('zzyzx')).to.equal( + '[unknown IndexType: zzyzx]' + )); + }); +}); + describe('MemoryIndexManager', async () => { genericIndexManagerTests(persistenceHelpers.testMemoryEagerPersistence); }); @@ -1660,14 +1684,80 @@ describe('IndexedDbIndexManager', async () => { await validateIsFullIndex(query15); }); + it('createTargetIndexes() creates full indexes for each sub-target', async () => { + const query_ = queryWithAddedFilter( + query('coll'), + orFilter(filter('a', '==', 1), filter('b', '==', 2), filter('c', '==', 3)) + ); + const subQuery1 = queryWithAddedFilter(query('coll'), filter('a', '==', 1)); + const subQuery2 = queryWithAddedFilter(query('coll'), filter('b', '==', 2)); + const subQuery3 = queryWithAddedFilter(query('coll'), filter('c', '==', 3)); + await validateIsNoneIndex(query_); + await validateIsNoneIndex(subQuery1); + await validateIsNoneIndex(subQuery2); + await validateIsNoneIndex(subQuery3); + + await indexManager.createTargetIndexes(queryToTarget(query_)); + + await validateIsFullIndex(query_); + await validateIsFullIndex(subQuery1); + await validateIsFullIndex(subQuery2); + await validateIsFullIndex(subQuery3); + }); + + it('createTargetIndexes() upgrades a partial index to a full index', async () => { + const query_ = queryWithAddedFilter( + queryWithAddedFilter(query('coll'), filter('a', '==', 1)), + filter('b', '==', 2) + ); + const subQuery1 = queryWithAddedFilter(query('coll'), filter('a', '==', 1)); + const subQuery2 = queryWithAddedFilter(query('coll'), filter('b', '==', 2)); + await indexManager.createTargetIndexes(queryToTarget(subQuery1)); + await validateIsPartialIndex(query_); + await validateIsFullIndex(subQuery1); + await validateIsNoneIndex(subQuery2); + + await indexManager.createTargetIndexes(queryToTarget(query_)); + + await validateIsFullIndex(query_); + await validateIsFullIndex(subQuery1); + await validateIsNoneIndex(subQuery2); + }); + + it('createTargetIndexes() does nothing if a full index already exists', async () => { + const query_ = query('coll'); + await indexManager.createTargetIndexes(queryToTarget(query_)); + await validateIsFullIndex(query_); + + await indexManager.createTargetIndexes(queryToTarget(query_)); + + await validateIsFullIndex(query_); + }); + async function validateIsPartialIndex(query: Query): Promise { - const indexType = await indexManager.getIndexType(queryToTarget(query)); - expect(indexType).to.equal(IndexType.PARTIAL); + await validateIndexType(query, IndexType.PARTIAL); } async function validateIsFullIndex(query: Query): Promise { + await validateIndexType(query, IndexType.FULL); + } + + async function validateIsNoneIndex(query: Query): Promise { + await validateIndexType(query, IndexType.NONE); + } + + async function validateIndexType( + query: Query, + expectedIndexType: IndexType + ): Promise { const indexType = await indexManager.getIndexType(queryToTarget(query)); - expect(indexType).to.equal(IndexType.FULL); + expect( + indexType, + 'index type is ' + + displayNameForIndexType(indexType) + + ' but expected ' + + displayNameForIndexType(expectedIndexType) + ).to.equal(expectedIndexType); } async function setUpSingleValueFilter(): Promise { diff --git a/packages/firestore/test/unit/local/local_store.test.ts b/packages/firestore/test/unit/local/local_store.test.ts index 1c388c69776..66009fbe89e 100644 --- a/packages/firestore/test/unit/local/local_store.test.ts +++ b/packages/firestore/test/unit/local/local_store.test.ts @@ -31,6 +31,7 @@ import { import { SnapshotVersion } from '../../../src/core/snapshot_version'; import { Target } from '../../../src/core/target'; import { BatchId, TargetId } from '../../../src/core/types'; +import { IndexBackfiller } from '../../../src/local/index_backfiller'; import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence'; import { LocalStore } from '../../../src/local/local_store'; import { @@ -42,6 +43,7 @@ import { localStoreGetHighestUnacknowledgedBatchId, localStoreGetTargetData, localStoreGetNamedQuery, + localStoreSetIndexAutoCreationEnabled, localStoreHasNewerBundle, localStoreWriteLocally, LocalWriteResult, @@ -51,7 +53,8 @@ import { localStoreReleaseTarget, localStoreSaveBundle, localStoreSaveNamedQuery, - newLocalStore + newLocalStore, + TestingHooks as LocalStoreTestingHooks } from '../../../src/local/local_store_impl'; import { LocalViewChanges } from '../../../src/local/local_view_changes'; import { Persistence } from '../../../src/local/persistence'; @@ -134,6 +137,9 @@ class LocalStoreTester { private lastTargetId: TargetId | null = null; private batches: MutationBatch[] = []; private bundleConverter: BundleConverterImpl; + private indexBackfiller: IndexBackfiller; + + private queryExecutionCount = 0; constructor( public localStore: LocalStore, @@ -142,6 +148,7 @@ class LocalStoreTester { readonly gcIsEager: boolean ) { this.bundleConverter = new BundleConverterImpl(JSON_SERIALIZER); + this.indexBackfiller = new IndexBackfiller(localStore, persistence); } private prepareNextStep(): void { @@ -176,6 +183,10 @@ class LocalStoreTester { } } + afterMutation(mutation: Mutation): LocalStoreTester { + return this.afterMutations([mutation]); + } + afterMutations(mutations: Mutation[]): LocalStoreTester { this.prepareNextStep(); @@ -203,6 +214,11 @@ class LocalStoreTester { return this; } + afterRemoteEvents(remoteEvents: RemoteEvent[]): LocalStoreTester { + remoteEvents.forEach(remoteEvent => this.afterRemoteEvent(remoteEvent)); + return this; + } + afterBundleDocuments( documents: BundledDocuments, bundleName?: string @@ -321,12 +337,50 @@ class LocalStoreTester { query, /* usePreviousResults= */ true ).then(({ documents }) => { + this.queryExecutionCount++; this.lastChanges = documents; }) ); return this; } + afterIndexAutoCreationConfigure(config: { + isEnabled?: boolean; + indexAutoCreationMinCollectionSize?: number; + relativeIndexReadCostPerDocument?: number; + }): LocalStoreTester { + this.prepareNextStep(); + + this.promiseChain = this.promiseChain.then(() => { + if (config.isEnabled !== undefined) { + localStoreSetIndexAutoCreationEnabled( + this.localStore, + config.isEnabled + ); + } + LocalStoreTestingHooks.setIndexAutoCreationSettings( + this.localStore, + config + ); + }); + + return this; + } + + afterBackfillIndexes(options?: { + maxDocumentsToProcess?: number; + }): LocalStoreTester { + this.prepareNextStep(); + + this.promiseChain = this.promiseChain.then(() => + this.indexBackfiller + .backfill(options?.maxDocumentsToProcess) + .then(() => {}) + ); + + return this; + } + /** * Asserts the expected number of mutations and documents read by * the MutationQueue and the RemoteDocumentCache. @@ -348,36 +402,28 @@ class LocalStoreTester { overlayTypes?: { [k: string]: MutationType }; }): LocalStoreTester { this.promiseChain = this.promiseChain.then(() => { + const actualCount: typeof expectedCount = {}; if (expectedCount.overlaysByCollection !== undefined) { - expect(this.queryEngine.overlaysReadByCollection).to.be.eq( - expectedCount.overlaysByCollection, - 'Overlays read (by collection)' - ); + actualCount.overlaysByCollection = + this.queryEngine.overlaysReadByCollection; } if (expectedCount.overlaysByKey !== undefined) { - expect(this.queryEngine.overlaysReadByKey).to.be.eq( - expectedCount.overlaysByKey, - 'Overlays read (by key)' - ); + actualCount.overlaysByKey = this.queryEngine.overlaysReadByKey; } if (expectedCount.overlayTypes !== undefined) { - expect(this.queryEngine.overlayTypes).to.deep.equal( - expectedCount.overlayTypes, - 'Overlay types read' - ); + actualCount.overlayTypes = this.queryEngine.overlayTypes; } if (expectedCount.documentsByCollection !== undefined) { - expect(this.queryEngine.documentsReadByCollection).to.be.eq( - expectedCount.documentsByCollection, - 'Remote documents read (by collection)' - ); + actualCount.documentsByCollection = + this.queryEngine.documentsReadByCollection; } if (expectedCount.documentsByKey !== undefined) { - expect(this.queryEngine.documentsReadByKey).to.be.eq( - expectedCount.documentsByKey, - 'Remote documents read (by key)' - ); + actualCount.documentsByKey = this.queryEngine.documentsReadByKey; } + expect(actualCount).to.deep.eq( + expectedCount, + `query execution #${this.queryExecutionCount}` + ); }); return this; } @@ -414,7 +460,7 @@ class LocalStoreTester { } toReturnChangedInternal( - docs: Document[], + docsOrKeyStrs: Document[] | string[], isEqual?: (lhs: Document | null, rhs: Document | null) => boolean ): LocalStoreTester { this.promiseChain = this.promiseChain.then(() => { @@ -422,14 +468,21 @@ class LocalStoreTester { this.lastChanges !== null, 'Called toReturnChanged() without prior after()' ); - expect(this.lastChanges.size).to.equal(docs.length, 'number of changes'); - for (const doc of docs) { - const returned = this.lastChanges.get(doc.key); + expect(this.lastChanges.size).to.equal( + docsOrKeyStrs.length, + 'number of changes' + ); + for (const docOrKeyStr of docsOrKeyStrs) { + const docKey = + typeof docOrKeyStr === 'string' ? key(docOrKeyStr) : docOrKeyStr.key; + const returned = this.lastChanges.get(docKey); const message = `Expected '${returned}' to equal '${doc}'.`; - if (isEqual) { - expect(isEqual(doc, returned)).to.equal(true, message); + if (typeof docOrKeyStr === 'string') { + expect(returned?.isValidDocument()).to.equal(true, message); + } else if (isEqual) { + expect(isEqual(docOrKeyStr, returned)).to.equal(true, message); } else { - expectEqual(doc, returned, message); + expectEqual(docOrKeyStr, returned, message); } } this.lastChanges = null; @@ -437,8 +490,10 @@ class LocalStoreTester { return this; } - toReturnChanged(...docs: Document[]): LocalStoreTester { - return this.toReturnChangedInternal(docs); + toReturnChanged(...docs: Document[]): LocalStoreTester; + toReturnChanged(...docKeyStrs: string[]): LocalStoreTester; + toReturnChanged(...docsOrKeyStrs: Document[] | string[]): LocalStoreTester { + return this.toReturnChangedInternal(docsOrKeyStrs); } toReturnChangedWithDocComparator( @@ -631,7 +686,10 @@ describe('LocalStore w/ IndexedDB Persistence', () => { } addEqualityMatcher(); - genericLocalStoreTests(initialize, /* gcIsEager= */ false); + describe('genericLocalStoreTests', () => + genericLocalStoreTests(initialize, /* gcIsEager= */ false)); + describe('indexedDbLocalStoreTests', () => + indexedDbLocalStoreTests(initialize, /* gcIsEager= */ false)); }); function genericLocalStoreTests( @@ -663,6 +721,17 @@ function genericLocalStoreTests( ); } + it('localStoreSetIndexAutoCreationEnabled()', () => { + localStoreSetIndexAutoCreationEnabled(localStore, true); + expect(queryEngine.indexAutoCreationEnabled).to.be.true; + localStoreSetIndexAutoCreationEnabled(localStore, false); + expect(queryEngine.indexAutoCreationEnabled).to.be.false; + localStoreSetIndexAutoCreationEnabled(localStore, true); + expect(queryEngine.indexAutoCreationEnabled).to.be.true; + localStoreSetIndexAutoCreationEnabled(localStore, false); + expect(queryEngine.indexAutoCreationEnabled).to.be.false; + }); + it('handles SetMutation', () => { return expectLocalStore() .after(setMutation('foo/bar', { foo: 'bar' })) @@ -2621,3 +2690,301 @@ function genericLocalStoreTests( } ); } + +function indexedDbLocalStoreTests( + getComponents: () => Promise, + gcIsEager: boolean +): void { + let persistence: Persistence; + let localStore: LocalStore; + let queryEngine: CountingQueryEngine; + + beforeEach(async () => { + const components = await getComponents(); + persistence = components.persistence; + localStore = components.localStore; + queryEngine = components.queryEngine; + }); + + afterEach(async () => { + await persistence.shutdown(); + await persistenceHelpers.clearTestPersistence(); + }); + + function expectLocalStore(): LocalStoreTester { + return new LocalStoreTester( + localStore, + persistence, + queryEngine, + gcIsEager + ); + } + + it('can auto-create indexes', () => { + const query_ = query('coll', filter('matches', '==', true)); + return ( + expectLocalStore() + .afterAllocatingQuery(query_) + .toReturnTargetId(2) + .afterIndexAutoCreationConfigure({ + isEnabled: true, + indexAutoCreationMinCollectionSize: 0, + relativeIndexReadCostPerDocument: 2 + }) + .afterRemoteEvents([ + docAddedRemoteEvent(doc('coll/a', 10, { matches: true }), [2], []), + docAddedRemoteEvent(doc('coll/b', 10, { matches: false }), [2], []), + docAddedRemoteEvent(doc('coll/c', 10, { matches: false }), [2], []), + docAddedRemoteEvent(doc('coll/d', 10, { matches: false }), [2], []), + docAddedRemoteEvent(doc('coll/e', 10, { matches: true }), [2], []) + ]) + // First time query runs without indexes. + // Based on current heuristic, collection document counts (5) > + // 2 * resultSize (2). + // Full matched index should be created. + .afterExecutingQuery(query_) + .toHaveRead({ documentsByKey: 0, documentsByCollection: 2 }) + .toReturnChanged('coll/a', 'coll/e') + .afterBackfillIndexes() + .afterRemoteEvent( + docAddedRemoteEvent(doc('coll/f', 20, { matches: true }), [2], []) + ) + .afterExecutingQuery(query_) + .toHaveRead({ documentsByKey: 2, documentsByCollection: 1 }) + .toReturnChanged('coll/a', 'coll/e', 'coll/f') + .finish() + ); + }); + + it('does not auto-create indexes for small collections', () => { + const query_ = query('coll', filter('count', '>=', 3)); + return ( + expectLocalStore() + .afterAllocatingQuery(query_) + .toReturnTargetId(2) + .afterIndexAutoCreationConfigure({ + isEnabled: true, + relativeIndexReadCostPerDocument: 2 + }) + .afterRemoteEvents([ + docAddedRemoteEvent(doc('coll/a', 10, { count: 5 }), [2], []), + docAddedRemoteEvent(doc('coll/b', 10, { count: 1 }), [2], []), + docAddedRemoteEvent(doc('coll/c', 10, { count: 0 }), [2], []), + docAddedRemoteEvent(doc('coll/d', 10, { count: 1 }), [2], []), + docAddedRemoteEvent(doc('coll/e', 10, { count: 3 }), [2], []) + ]) + // SDK will not create indexes since collection size is too small. + .afterExecutingQuery(query_) + .toHaveRead({ documentsByKey: 0, documentsByCollection: 2 }) + .toReturnChanged('coll/a', 'coll/e') + .afterBackfillIndexes() + .afterRemoteEvent( + docAddedRemoteEvent(doc('coll/f', 20, { count: 4 }), [2], []) + ) + .afterExecutingQuery(query_) + .toHaveRead({ documentsByKey: 0, documentsByCollection: 3 }) + .toReturnChanged('coll/a', 'coll/e', 'coll/f') + .finish() + ); + }); + + it('does not auto create indexes when index lookup is expensive', () => { + const query_ = query('coll', filter('array', 'array-contains-any', [0, 7])); + return ( + expectLocalStore() + .afterAllocatingQuery(query_) + .toReturnTargetId(2) + .afterIndexAutoCreationConfigure({ + isEnabled: true, + indexAutoCreationMinCollectionSize: 0, + relativeIndexReadCostPerDocument: 5 + }) + .afterRemoteEvents([ + docAddedRemoteEvent(doc('coll/a', 10, { array: [2, 7] }), [2], []), + docAddedRemoteEvent(doc('coll/b', 10, { array: [] }), [2], []), + docAddedRemoteEvent(doc('coll/c', 10, { array: [3] }), [2], []), + docAddedRemoteEvent( + doc('coll/d', 10, { array: [2, 10, 20] }), + [2], + [] + ), + docAddedRemoteEvent(doc('coll/e', 10, { array: [2, 0, 8] }), [2], []) + ]) + // SDK will not create indexes since relative read cost is too large. + .afterExecutingQuery(query_) + .toHaveRead({ documentsByKey: 0, documentsByCollection: 2 }) + .toReturnChanged('coll/a', 'coll/e') + .afterBackfillIndexes() + .afterRemoteEvent( + docAddedRemoteEvent(doc('coll/f', 20, { array: [0] }), [2], []) + ) + .afterExecutingQuery(query_) + .toHaveRead({ documentsByKey: 0, documentsByCollection: 3 }) + .toReturnChanged('coll/a', 'coll/e', 'coll/f') + .finish() + ); + }); + + it('index auto creation works when backfiller runs halfway', () => { + const query_ = query('coll', filter('matches', '==', 'foo')); + return ( + expectLocalStore() + .afterAllocatingQuery(query_) + .toReturnTargetId(2) + .afterIndexAutoCreationConfigure({ + isEnabled: true, + indexAutoCreationMinCollectionSize: 0, + relativeIndexReadCostPerDocument: 2 + }) + .afterRemoteEvents([ + docAddedRemoteEvent(doc('coll/a', 10, { matches: 'foo' }), [2], []), + docAddedRemoteEvent(doc('coll/b', 10, { matches: '' }), [2], []), + docAddedRemoteEvent(doc('coll/c', 10, { matches: 'bar' }), [2], []), + docAddedRemoteEvent(doc('coll/d', 10, { matches: 7 }), [2], []), + docAddedRemoteEvent(doc('coll/e', 10, { matches: 'foo' }), [2], []) + ]) + // First time query runs without indexes. + // Based on current heuristic, collection document counts (5) > + // 2 * resultSize (2). + // Full matched index should be created. + .afterExecutingQuery(query_) + .toHaveRead({ documentsByKey: 0, documentsByCollection: 2 }) + .toReturnChanged('coll/a', 'coll/e') + .afterBackfillIndexes({ maxDocumentsToProcess: 2 }) + .afterRemoteEvent( + docAddedRemoteEvent(doc('coll/f', 20, { matches: 'foo' }), [2], []) + ) + .afterExecutingQuery(query_) + .toHaveRead({ documentsByKey: 1, documentsByCollection: 2 }) + .toReturnChanged('coll/a', 'coll/e', 'coll/f') + .finish() + ); + }); + + it('index created by index auto creation exists after turn off auto creation', () => { + const query_ = query('coll', filter('value', 'not-in', [3])); + return ( + expectLocalStore() + .afterAllocatingQuery(query_) + .toReturnTargetId(2) + .afterIndexAutoCreationConfigure({ + isEnabled: true, + indexAutoCreationMinCollectionSize: 0, + relativeIndexReadCostPerDocument: 2 + }) + .afterRemoteEvents([ + docAddedRemoteEvent(doc('coll/a', 10, { value: 5 }), [2], []), + docAddedRemoteEvent(doc('coll/b', 10, { value: 3 }), [2], []), + docAddedRemoteEvent(doc('coll/c', 10, { value: 3 }), [2], []), + docAddedRemoteEvent(doc('coll/d', 10, { value: 3 }), [2], []), + docAddedRemoteEvent(doc('coll/e', 10, { value: 2 }), [2], []) + ]) + // First time query runs without indexes. + // Based on current heuristic, collection document counts (5) > + // 2 * resultSize (2). + // Full matched index should be created. + .afterExecutingQuery(query_) + .toHaveRead({ documentsByKey: 0, documentsByCollection: 2 }) + .toReturnChanged('coll/a', 'coll/e') + .afterIndexAutoCreationConfigure({ isEnabled: false }) + .afterBackfillIndexes() + .afterRemoteEvent( + docAddedRemoteEvent(doc('coll/f', 20, { value: 7 }), [2], []) + ) + .afterExecutingQuery(query_) + .toHaveRead({ documentsByKey: 2, documentsByCollection: 1 }) + .toReturnChanged('coll/a', 'coll/e', 'coll/f') + .finish() + ); + }); + + it('disable index auto creation works', () => { + const query1 = query('coll', filter('value', 'in', [0, 1])); + const query2 = query('foo', filter('value', '!=', Number.NaN)); + return ( + expectLocalStore() + .afterAllocatingQuery(query1) + .toReturnTargetId(2) + .afterIndexAutoCreationConfigure({ + isEnabled: true, + indexAutoCreationMinCollectionSize: 0, + relativeIndexReadCostPerDocument: 2 + }) + .afterRemoteEvents([ + docAddedRemoteEvent(doc('coll/a', 10, { value: 1 }), [2], []), + docAddedRemoteEvent(doc('coll/b', 10, { value: 8 }), [2], []), + docAddedRemoteEvent(doc('coll/c', 10, { value: 'string' }), [2], []), + docAddedRemoteEvent(doc('coll/d', 10, { value: false }), [2], []), + docAddedRemoteEvent(doc('coll/e', 10, { value: 0 }), [2], []) + ]) + // First time query runs without indexes. + // Based on current heuristic, collection document counts (5) > + // 2 * resultSize (2). + // Full matched index should be created. + .afterExecutingQuery(query1) + .toHaveRead({ documentsByKey: 0, documentsByCollection: 2 }) + .toReturnChanged('coll/a', 'coll/e') + .afterIndexAutoCreationConfigure({ isEnabled: false }) + .afterBackfillIndexes() + .afterExecutingQuery(query1) + .toHaveRead({ documentsByKey: 2, documentsByCollection: 0 }) + .toReturnChanged('coll/a', 'coll/e') + .afterAllocatingQuery(query2) + .toReturnTargetId(4) + .afterRemoteEvents([ + docAddedRemoteEvent(doc('foo/a', 10, { value: 5 }), [2], []), + docAddedRemoteEvent(doc('foo/b', 10, { value: Number.NaN }), [2], []), + docAddedRemoteEvent(doc('foo/c', 10, { value: Number.NaN }), [2], []), + docAddedRemoteEvent(doc('foo/d', 10, { value: Number.NaN }), [2], []), + docAddedRemoteEvent(doc('foo/e', 10, { value: 'string' }), [2], []) + ]) + .afterExecutingQuery(query2) + .toHaveRead({ documentsByKey: 0, documentsByCollection: 2 }) + .afterBackfillIndexes() + .afterExecutingQuery(query2) + .toHaveRead({ documentsByKey: 0, documentsByCollection: 2 }) + .finish() + ); + }); + + it('index auto creation works with mutation', () => { + const query_ = query( + 'coll', + filter('value', 'array-contains-any', [8, 1, 'string']) + ); + return expectLocalStore() + .afterAllocatingQuery(query_) + .toReturnTargetId(2) + .afterIndexAutoCreationConfigure({ + isEnabled: true, + indexAutoCreationMinCollectionSize: 0, + relativeIndexReadCostPerDocument: 2 + }) + .afterRemoteEvents([ + docAddedRemoteEvent( + doc('coll/a', 10, { value: [8, 1, 'string'] }), + [2], + [] + ), + docAddedRemoteEvent(doc('coll/b', 10, { value: [] }), [2], []), + docAddedRemoteEvent(doc('coll/c', 10, { value: [3] }), [2], []), + docAddedRemoteEvent(doc('coll/d', 10, { value: [0, 5] }), [2], []), + docAddedRemoteEvent(doc('coll/e', 10, { value: ['string'] }), [2], []) + ]) + .afterExecutingQuery(query_) + .toHaveRead({ documentsByKey: 0, documentsByCollection: 2 }) + .toReturnChanged('coll/a', 'coll/e') + .afterMutation(deleteMutation('coll/e')) + .afterBackfillIndexes() + .afterMutation(setMutation('coll/f', { value: [1] })) + .afterExecutingQuery(query_) + .toHaveRead({ + documentsByKey: 1, + documentsByCollection: 0, + overlaysByKey: 1, + overlaysByCollection: 1 + }) + .toReturnChanged('coll/a', 'coll/f') + .finish(); + }); +} diff --git a/packages/firestore/test/unit/local/query_engine.test.ts b/packages/firestore/test/unit/local/query_engine.test.ts index 7bcf7743b43..bf8ada916ef 100644 --- a/packages/firestore/test/unit/local/query_engine.test.ts +++ b/packages/firestore/test/unit/local/query_engine.test.ts @@ -22,6 +22,7 @@ import { User } from '../../../src/auth/user'; import { LimitType, Query, + queryToTarget, queryWithAddedFilter, queryWithAddedOrderBy, queryWithLimit @@ -29,12 +30,17 @@ import { import { SnapshotVersion } from '../../../src/core/snapshot_version'; import { View } from '../../../src/core/view'; import { DocumentOverlayCache } from '../../../src/local/document_overlay_cache'; +import { + displayNameForIndexType, + IndexType +} from '../../../src/local/index_manager'; import { IndexedDbPersistence } from '../../../src/local/indexeddb_persistence'; import { LocalDocumentsView } from '../../../src/local/local_documents_view'; import { MutationQueue } from '../../../src/local/mutation_queue'; import { Persistence } from '../../../src/local/persistence'; import { PersistencePromise } from '../../../src/local/persistence_promise'; import { PersistenceTransaction } from '../../../src/local/persistence_transaction'; +import { QueryContext } from '../../../src/local/query_context'; import { QueryEngine } from '../../../src/local/query_engine'; import { RemoteDocumentCache } from '../../../src/local/remote_document_cache'; import { TargetCache } from '../../../src/local/target_cache'; @@ -94,7 +100,8 @@ class TestLocalDocumentsView extends LocalDocumentsView { getDocumentsMatchingQuery( transaction: PersistenceTransaction, query: Query, - offset: IndexOffset + offset: IndexOffset, + context?: QueryContext ): PersistencePromise { const skipsDocumentsBeforeSnapshot = indexOffsetComparator(IndexOffset.min(), offset) !== 0; @@ -104,49 +111,45 @@ class TestLocalDocumentsView extends LocalDocumentsView { 'Observed query execution mode did not match expectation' ); - return super.getDocumentsMatchingQuery(transaction, query, offset); + return super.getDocumentsMatchingQuery(transaction, query, offset, context); } } -describe('MemoryQueryEngine', async () => { - /* not durable and without client side indexing */ - genericQueryEngineTest( - false, - persistenceHelpers.testMemoryEagerPersistence, - false - ); -}); +describe('QueryEngine', async () => { + describe('MemoryEagerPersistence', async () => { + /* not durable and without client side indexing */ + genericQueryEngineTest( + persistenceHelpers.testMemoryEagerPersistence, + false + ); + }); -describe('IndexedDbQueryEngine', async () => { if (!IndexedDbPersistence.isAvailable()) { console.warn('No IndexedDB. Skipping IndexedDbQueryEngine tests.'); return; } - let persistencePromise: Promise; - beforeEach(async () => { - persistencePromise = persistenceHelpers.testIndexedDbPersistence(); + describe('IndexedDbPersistence configureCsi=false', async () => { + /* durable but without client side indexing */ + genericQueryEngineTest(persistenceHelpers.testIndexedDbPersistence, false); }); - /* durable but without client side indexing */ - genericQueryEngineTest(true, () => persistencePromise, false); - - /* durable and with client side indexing */ - genericQueryEngineTest(true, () => persistencePromise, true); + describe('IndexedDbQueryEngine configureCsi=true', async () => { + /* durable and with client side indexing */ + genericQueryEngineTest(persistenceHelpers.testIndexedDbPersistence, true); + }); }); /** * Defines the set of tests to run against the memory and IndexedDB-backed * query engine. * - * @param durable Whether the provided persistence is backed by IndexedDB * @param persistencePromise A factory function that returns an initialized * persistence layer. * @param configureCsi Whether tests should configure client side indexing * or use full table scans. */ function genericQueryEngineTest( - durable: boolean, persistencePromise: () => Promise, configureCsi: boolean ): void { @@ -232,7 +235,9 @@ function genericQueryEngineTest( 'expectOptimizedCollectionQuery()/expectFullCollectionQuery()' ); - return persistence.runTransaction('runQuery', 'readonly', txn => { + // NOTE: Use a `readwrite` transaction (instead of `readonly`) so that + // client-side indexes can be written to persistence. + return persistence.runTransaction('runQuery', 'readwrite', txn => { return targetCache .getMatchingKeysForTargetId(txn, TEST_TARGET_ID) .next(remoteKeys => { @@ -832,6 +837,128 @@ function genericQueryEngineTest( verifyResult(results, [doc1, doc2, doc4]); }); + + // A generic test for index auto-creation. + // This function can be called with explicit parameters from it() methods. + const testIndexAutoCreation = async (config: { + indexAutoCreationEnabled: boolean; + indexAutoCreationMinCollectionSize?: number; + relativeIndexReadCostPerDocument?: number; + matchingDocumentCount?: number; + nonmatchingDocumentCount?: number; + expectedPostQueryExecutionIndexType: IndexType; + }): Promise => { + debugAssert(configureCsi, 'Test requires durable persistence'); + + const matchingDocuments: MutableDocument[] = []; + for (let i = 0; i < (config.matchingDocumentCount ?? 3); i++) { + const matchingDocument = doc(`coll/A${i}`, 1, { 'foo': 'match' }); + matchingDocuments.push(matchingDocument); + } + await addDocument(...matchingDocuments); + + const nonmatchingDocuments: MutableDocument[] = []; + for (let i = 0; i < (config.nonmatchingDocumentCount ?? 3); i++) { + const nonmatchingDocument = doc(`coll/X${i}`, 1, { 'foo': 'nomatch' }); + nonmatchingDocuments.push(nonmatchingDocument); + } + await addDocument(...nonmatchingDocuments); + + queryEngine.indexAutoCreationEnabled = config.indexAutoCreationEnabled; + + if (config.indexAutoCreationMinCollectionSize !== undefined) { + queryEngine.indexAutoCreationMinCollectionSize = + config.indexAutoCreationMinCollectionSize; + } + if (config.relativeIndexReadCostPerDocument !== undefined) { + queryEngine.relativeIndexReadCostPerDocument = + config.relativeIndexReadCostPerDocument; + } + + const q = query('coll', filter('foo', '==', 'match')); + const target = queryToTarget(q); + const preQueryExecutionIndexType = await indexManager.getIndexType( + target + ); + expect( + preQueryExecutionIndexType, + 'index type for target _before_ running the query is ' + + displayNameForIndexType(preQueryExecutionIndexType) + + ', but expected ' + + displayNameForIndexType(IndexType.NONE) + ).to.equal(IndexType.NONE); + + const result = await expectFullCollectionQuery(() => + runQuery(q, SnapshotVersion.min()) + ); + verifyResult(result, matchingDocuments); + + const postQueryExecutionIndexType = await indexManager.getIndexType( + target + ); + expect( + postQueryExecutionIndexType, + 'index type for target _after_ running the query is ' + + displayNameForIndexType(postQueryExecutionIndexType) + + ', but expected ' + + displayNameForIndexType(config.expectedPostQueryExecutionIndexType) + ).to.equal(config.expectedPostQueryExecutionIndexType); + }; + + it('creates indexes when indexAutoCreationEnabled=true', () => + testIndexAutoCreation({ + indexAutoCreationEnabled: true, + indexAutoCreationMinCollectionSize: 0, + relativeIndexReadCostPerDocument: 0, + expectedPostQueryExecutionIndexType: IndexType.FULL + })); + + it('does not create indexes when indexAutoCreationEnabled=false', () => + testIndexAutoCreation({ + indexAutoCreationEnabled: false, + indexAutoCreationMinCollectionSize: 0, + relativeIndexReadCostPerDocument: 0, + expectedPostQueryExecutionIndexType: IndexType.NONE + })); + + it( + 'creates indexes when ' + + 'min collection size is met exactly ' + + 'and relative cost is ever-so-slightly better', + () => + testIndexAutoCreation({ + indexAutoCreationEnabled: true, + indexAutoCreationMinCollectionSize: 10, + relativeIndexReadCostPerDocument: 1.9999, + matchingDocumentCount: 5, + nonmatchingDocumentCount: 5, + expectedPostQueryExecutionIndexType: IndexType.FULL + }) + ); + + it( + 'does not create indexes when ' + + 'min collection size is not met by only 1 document', + () => + testIndexAutoCreation({ + indexAutoCreationEnabled: true, + indexAutoCreationMinCollectionSize: 10, + relativeIndexReadCostPerDocument: 0, + matchingDocumentCount: 5, + nonmatchingDocumentCount: 4, + expectedPostQueryExecutionIndexType: IndexType.NONE + }) + ); + + it('does not create indexes when relative cost is equal', () => + testIndexAutoCreation({ + indexAutoCreationEnabled: true, + indexAutoCreationMinCollectionSize: 0, + relativeIndexReadCostPerDocument: 2, + matchingDocumentCount: 5, + nonmatchingDocumentCount: 5, + expectedPostQueryExecutionIndexType: IndexType.NONE + })); } // Tests below this line execute with and without client side indexing diff --git a/packages/firestore/test/unit/local/test_index_manager.ts b/packages/firestore/test/unit/local/test_index_manager.ts index 8b31656cd02..94509073925 100644 --- a/packages/firestore/test/unit/local/test_index_manager.ts +++ b/packages/firestore/test/unit/local/test_index_manager.ts @@ -63,6 +63,14 @@ export class TestIndexManager { ); } + createTargetIndexes(target: Target): Promise { + return this.persistence.runTransaction( + 'createTargetIndexes', + 'readwrite', + txn => this.indexManager.createTargetIndexes(txn, target) + ); + } + getFieldIndexes(collectionGroup?: string): Promise { return this.persistence.runTransaction('getFieldIndexes', 'readonly', txn => collectionGroup diff --git a/packages/firestore/test/unit/model/target_index_matcher.test.ts b/packages/firestore/test/unit/model/target_index_matcher.test.ts index 2d48b12f1a1..8e295e17fef 100644 --- a/packages/firestore/test/unit/model/target_index_matcher.test.ts +++ b/packages/firestore/test/unit/model/target_index_matcher.test.ts @@ -24,6 +24,7 @@ import { Query, newQueryForCollectionGroup } from '../../../src/core/query'; +import { targetGetSegmentCount } from '../../../src/core/target'; import { IndexKind } from '../../../src/model/field_index'; import { TargetIndexMatcher } from '../../../src/model/target_index_matcher'; import { fieldIndex, filter, orderBy, query } from '../../util/helpers'; @@ -51,6 +52,23 @@ describe('Target Bounds', () => { ) ]; + const queriesWithOrderBy = [ + queryWithAddedOrderBy(query('collId'), orderBy('a')), + queryWithAddedOrderBy(query('collId'), orderBy('a', 'asc')), + queryWithAddedOrderBy(query('collId'), orderBy('a', 'desc')), + queryWithAddedOrderBy( + queryWithAddedOrderBy(query('collId'), orderBy('a', 'desc')), + orderBy('__name__') + ), + queryWithAddedOrderBy( + queryWithAddedFilter( + query('collId'), + filter('a', 'array-contains-any', ['a']) + ), + orderBy('b') + ) + ]; + it('can use merge join', () => { let q = queryWithAddedFilter( queryWithAddedFilter(query('collId'), filter('a', '==', 1)), @@ -713,6 +731,240 @@ describe('Target Bounds', () => { validateServesTarget(q, 'a', IndexKind.ASCENDING); }); + describe('buildTargetIndex()', () => { + it('queries with equalities', () => + queriesWithEqualities.forEach( + validateBuildTargetIndexCreateFullMatchIndex + )); + + it('queries with inequalities', () => + queriesWithInequalities.forEach( + validateBuildTargetIndexCreateFullMatchIndex + )); + + it('queries with array contains', () => + queriesWithArrayContains.forEach( + validateBuildTargetIndexCreateFullMatchIndex + )); + + it('queries with order by', () => + queriesWithOrderBy.forEach(validateBuildTargetIndexCreateFullMatchIndex)); + + it('queries with inequalities uses single field index', () => + validateBuildTargetIndexCreateFullMatchIndex( + queryWithAddedFilter( + queryWithAddedFilter(query('collId'), filter('a', '>', 1)), + filter('a', '<', 10) + ) + )); + + it('query of a collection', () => + validateBuildTargetIndexCreateFullMatchIndex(query('collId'))); + + it('query with array contains and order by', () => + validateBuildTargetIndexCreateFullMatchIndex( + queryWithAddedOrderBy( + queryWithAddedFilter( + queryWithAddedFilter( + query('collId'), + filter('a', 'array-contains', 'a') + ), + filter('a', '>', 'b') + ), + orderBy('a', 'asc') + ) + )); + + it('query with equality and descending order', () => + validateBuildTargetIndexCreateFullMatchIndex( + queryWithAddedOrderBy( + queryWithAddedFilter(query('collId'), filter('a', '==', 1)), + orderBy('__name__', 'desc') + ) + )); + + it('query with multiple equalities', () => + validateBuildTargetIndexCreateFullMatchIndex( + queryWithAddedFilter( + queryWithAddedFilter(query('collId'), filter('a1', '==', 'a')), + filter('a2', '==', 'b') + ) + )); + + describe('query with multiple equalities and inequality', () => { + it('inequality last', () => + validateBuildTargetIndexCreateFullMatchIndex( + queryWithAddedFilter( + queryWithAddedFilter( + queryWithAddedFilter( + query('collId'), + filter('equality1', '==', 'a') + ), + filter('equality2', '==', 'b') + ), + filter('inequality', '>=', 'c') + ) + )); + + it('inequality in middle', () => + validateBuildTargetIndexCreateFullMatchIndex( + queryWithAddedFilter( + queryWithAddedFilter( + queryWithAddedFilter( + query('collId'), + filter('equality1', '==', 'a') + ), + filter('inequality', '>=', 'c') + ), + filter('equality2', '==', 'b') + ) + )); + }); + + describe('query with multiple filters', () => { + it('== and > on different fields', () => + validateBuildTargetIndexCreateFullMatchIndex( + queryWithAddedOrderBy( + queryWithAddedFilter( + queryWithAddedFilter(query('collId'), filter('a1', '==', 'a')), + filter('a2', '>', 'b') + ), + orderBy('a2', 'asc') + ) + )); + + it('>=, ==, and <= filters on the same field', () => + validateBuildTargetIndexCreateFullMatchIndex( + queryWithAddedFilter( + queryWithAddedFilter( + queryWithAddedFilter(query('collId'), filter('a', '>=', 1)), + filter('a', '==', 5) + ), + filter('a', '<=', 10) + ) + )); + + it('not-in and >= on the same field', () => + validateBuildTargetIndexCreateFullMatchIndex( + queryWithAddedFilter( + queryWithAddedFilter( + query('collId'), + filter('a', 'not-in', [1, 2, 3]) + ), + filter('a', '>=', 2) + ) + )); + }); + + describe('query with multiple order-bys', () => { + it('order by fff, bar desc, __name__', () => + validateBuildTargetIndexCreateFullMatchIndex( + queryWithAddedOrderBy( + queryWithAddedOrderBy( + queryWithAddedOrderBy(query('collId'), orderBy('fff')), + orderBy('bar', 'desc') + ), + orderBy('__name__') + ) + )); + + it('order by foo, bar, __name__ desc', () => + validateBuildTargetIndexCreateFullMatchIndex( + queryWithAddedOrderBy( + queryWithAddedOrderBy( + queryWithAddedOrderBy(query('collId'), orderBy('foo')), + orderBy('bar') + ), + orderBy('__name__', 'desc') + ) + )); + }); + + it('query with in and not in filters', () => + validateBuildTargetIndexCreateFullMatchIndex( + queryWithAddedFilter( + queryWithAddedFilter( + query('collId'), + filter('a', 'not-in', [1, 2, 3]) + ), + filter('b', 'in', [1, 2, 3]) + ) + )); + + describe('query with equality and different order-by', () => { + it('filter on foo and bar, order by qux', () => + validateBuildTargetIndexCreateFullMatchIndex( + queryWithAddedOrderBy( + queryWithAddedFilter( + queryWithAddedFilter(query('collId'), filter('foo', '==', '')), + filter('bar', '==', '') + ), + orderBy('qux') + ) + )); + + it('filter on aaa, qqq, ccc, order by fff, bbb', () => + validateBuildTargetIndexCreateFullMatchIndex( + queryWithAddedOrderBy( + queryWithAddedOrderBy( + queryWithAddedFilter( + queryWithAddedFilter( + queryWithAddedFilter( + query('collId'), + filter('aaa', '==', '') + ), + filter('qqq', '==', '') + ), + filter('ccc', '==', '') + ), + orderBy('fff', 'desc') + ), + orderBy('bbb') + ) + )); + }); + + it('query with equals and not-in filters', () => + validateBuildTargetIndexCreateFullMatchIndex( + queryWithAddedFilter( + queryWithAddedFilter(query('collId'), filter('a', '==', '1')), + filter('b', 'not-in', [1, 2, 3]) + ) + )); + + describe('query with in and order-by', () => { + it('on different fields', () => + validateBuildTargetIndexCreateFullMatchIndex( + queryWithAddedOrderBy( + queryWithAddedOrderBy( + queryWithAddedFilter( + query('collId'), + filter('a', 'not-in', [1, 2, 3]) + ), + orderBy('a') + ), + orderBy('b') + ) + )); + + it('on the same field', () => + validateBuildTargetIndexCreateFullMatchIndex( + queryWithAddedOrderBy( + queryWithAddedFilter(query('collId'), filter('a', 'in', [1, 2, 3])), + orderBy('a') + ) + )); + }); + + function validateBuildTargetIndexCreateFullMatchIndex(q: Query): void { + const target = queryToTarget(q); + const targetIndexMatcher = new TargetIndexMatcher(target); + const expectedIndex = targetIndexMatcher.buildTargetIndex(); + expect(targetIndexMatcher.servedByIndex(expectedIndex)).is.true; + expect(expectedIndex.fields.length >= targetGetSegmentCount(target)); + } + }); + function validateServesTarget( query: Query, field: string,