Upgrade ES client to 9.0.0-alpha.4 (#213375)

## Summary

Upgrading the ES client to v9.0.0-alpha.4 to test the changes mentioned
in elastic/elasticsearch-js#2584

This new version introduces some type changes, most notably, the
`FieldValue` is `string | number | boolean | null | undefined` instead
of `any`, leading to some new type checks to be implemented (like on
aggregation results `bucket.key`, `search_after`, and `sort` options).

On top of that, it adds the new behavior where unknown properties are
placed in the `body` (when the request has a body). If they must be in
as a query parameter, they should be placed under the `querystring`
option.

cc @JoshMock 

TODO:
- [x] Stabilize the type errors
- [x] Address all the query parameters that are now placed in the body
(by wrapping them inside the option `querystring: {}`)

I will address `// @ts-expect-error [email protected]
https://github.com/elastic/elasticsearch-js/issues/2584` in a separate
PR to reduce noise.


Related #208776

Author: afharo

package.json

@@ -123,7 +123,7 @@
     "@elastic/datemath": "5.0.3",
     "@elastic/ebt": "^1.1.1",
     "@elastic/ecs": "^8.11.5",
-    "@elastic/elasticsearch": "9.0.0-alpha.3",
+    "@elastic/elasticsearch": "9.0.0-alpha.4",
     "@elastic/ems-client": "8.6.3",
     "@elastic/eui": "101.0.1",
     "@elastic/eui-theme-borealis": "0.1.0",

src/core/packages/elasticsearch/client-server-mocks/src/mocks.ts

@@ -19,6 +19,7 @@ const omittedProps = [
   'transport',
   'serializer',
   'helpers',
+  'acceptedParams',
 ] as Array<PublicKeys<Client>>;
 
 export type DeeplyMockedApi<T> = {

src/core/packages/saved-objects/api-server-internal/src/lib/apis/create.ts

@@ -159,7 +159,7 @@ export const performCreate = async <T>(
     refresh,
     document: raw._source,
     ...(overwrite && version ? decodeRequestVersion(version) : {}),
-    require_alias: true,
+    querystring: { require_alias: true },
   };
 
   const { body, statusCode, headers } =

src/core/packages/saved-objects/api-server-internal/src/lib/apis/update.ts

@@ -194,8 +194,7 @@ export const executeUpdate = async <T>(
       refresh,
       document: rawUpsert._source,
       ...(version ? decodeRequestVersion(version) : {}),
-      // @ts-expect-error
-      require_alias: true,
+      querystring: { require_alias: true },
     };
 
     const {

src/platform/packages/private/kbn-generate-csv/src/generate_csv.test.ts

@@ -1424,7 +1424,7 @@ describe('CsvGenerator', () => {
     expect(mockEsClient.asCurrentUser.openPointInTime).toHaveBeenCalledWith(
       {
         ignore_unavailable: true,
-        ignore_throttled: false,
+        querystring: { ignore_throttled: false },
         index: 'logstash-*',
         keep_alive: '30s',
       },

src/platform/packages/private/kbn-generate-csv/src/lib/search_cursor_pit.ts

@@ -47,13 +47,13 @@ export class SearchCursorPit extends SearchCursor {
           index: this.indexPatternTitle,
           keep_alive: scroll.duration(taskInstanceFields),
           ignore_unavailable: true,
-          // @ts-expect-error ignore_throttled is not in the type definition, but it is accepted by es
-          ignore_throttled: includeFrozen ? false : undefined, // "true" will cause deprecation warnings logged in ES
+          ...(includeFrozen ? { querystring: { ignore_throttled: false } } : {}), // "true" will cause deprecation warnings logged in ES
         },
         {
           signal: this.abortController.signal,
           requestTimeout: scroll.duration(taskInstanceFields),
           maxRetries: 0,
+          // @ts-expect-error not documented in the types. Is this still supported?
           maxConcurrentShardRequests,
         }
       );

src/platform/packages/shared/content-management/favorites/favorites_server/src/favorites_usage_collection.ts

@@ -115,11 +115,12 @@ export function registerFavoritesUsageCollection({
           []) as estypes.AggregationsStringTermsBucket[];
 
         typesBuckets.forEach((bucket) => {
-          favoritesUsage[bucket.key] = {
-            total: bucket.stats.sum,
-            total_users_spaces: bucket.stats.count,
-            avg_per_user_per_space: bucket.stats.avg,
-            max_per_user_per_space: bucket.stats.max,
+          const stats = bucket.stats as estypes.AggregationsStatsAggregate;
+          favoritesUsage[`${bucket.key}`] = {
+            total: stats.sum,
+            total_users_spaces: stats.count,
+            avg_per_user_per_space: stats.avg!,
+            max_per_user_per_space: stats.max!,
           };
         });
 

src/platform/packages/shared/kbn-rule-data-utils/src/alerts_as_data_rbac.ts

@@ -51,17 +51,19 @@ export const isValidFeatureId = (a: unknown): a is ValidFeatureId =>
  * @param sortIds estypes.SortResults | undefined
  * @returns SortResults
  */
-export const getSafeSortIds = (sortIds: estypes.SortResults | null | undefined) => {
+export const getSafeSortIds = (
+  sortIds: estypes.SortResults | null | undefined
+): Array<string | number> | undefined => {
   if (sortIds == null) {
-    return sortIds;
+    return sortIds as undefined;
   }
   return sortIds.map((sortId) => {
     // haven't determined when we would receive a null value for a sort id
     // but in case we do, default to sending the stringified Java max_int
-    if (sortId == null || sortId === '' || sortId >= Number.MAX_SAFE_INTEGER) {
+    if (sortId == null || sortId === '' || Number(sortId) >= Number.MAX_SAFE_INTEGER) {
       return '9223372036854775807';
     }
-    return sortId;
+    return sortId as string | number;
   });
 };
 

src/platform/plugins/private/kibana_usage_collection/server/collectors/saved_objects_counts/get_saved_object_counts/get_saved_object_counts.ts

@@ -80,16 +80,18 @@ export async function getSavedObjectsCounts(
   const nonExpectedTypes: string[] = [];
 
   const perType = buckets.map((perTypeEntry) => {
-    if (perTypeEntry.key !== MISSING_TYPE_KEY && !soTypes.includes(perTypeEntry.key)) {
+    const key = perTypeEntry.key as string;
+    if (key !== MISSING_TYPE_KEY && !soTypes.includes(key)) {
       // If the breakdown includes any SO types that are not expected, highlight them in the nonExpectedTypes list.
-      nonExpectedTypes.push(perTypeEntry.key);
+      nonExpectedTypes.push(key);
     }
 
     return { key: perTypeEntry.key, doc_count: perTypeEntry.doc_count };
   });
 
   return {
     total: body.total,
+    // @ts-expect-error `FieldValue` types now claim that bucket keys can be `null`
     per_type: perType,
     non_expected_types: nonExpectedTypes,
     others: body.aggregations?.types?.sum_other_doc_count ?? 0,

src/platform/plugins/shared/data/server/search/strategies/ese_search/ese_search_strategy.test.ts

@@ -314,7 +314,9 @@ describe('ES search strategy', () => {
         expect(mockRollupSearchCaller).toHaveBeenCalledWith(
           {
             index: 'foo-程',
-            ignore_unavailable: true,
+            querystring: {
+              ignore_unavailable: true,
+            },
             max_concurrent_shard_requests: undefined,
             timeout: '100ms',
             track_total_hits: true,

src/platform/plugins/shared/data/server/search/strategies/ese_search/ese_search_strategy.ts

@@ -166,12 +166,20 @@ export const enhancedEsSearchStrategyProvider = (
       throw new KbnSearchError(`"params.index" is required when performing a rollup search`, 400);
     }
 
+    // eslint-disable-next-line @typescript-eslint/naming-convention
+    const { ignore_unavailable, preference, ...params } = {
+      ...querystring,
+      ...request.params,
+      index: request.params.index,
+    };
+
     try {
       const esResponse = await client.rollup.rollupSearch(
         {
-          ...querystring,
-          ...request.params,
-          index: request.params.index,
+          ...params,
+          // Not defined in the spec, and the client places it in the body.
+          // This workaround allows us to force it as a query parameter.
+          querystring: { ...params.querystring, ignore_unavailable, preference },
         },
         {
           signal: options?.abortSignal,

src/platform/plugins/shared/saved_objects_management/server/lib/get_saved_objects_counts.ts

@@ -38,7 +38,7 @@ export const getSavedObjectCounts = async ({
     (body.aggregations?.types?.buckets as estypes.AggregationsStringTermsBucketKeys[]) || [];
 
   const counts = buckets.reduce((memo, bucket) => {
-    memo[bucket.key] = bucket.doc_count;
+    memo[`${bucket.key}`] = bucket.doc_count;
     return memo;
   }, {} as Record<string, number>);
 

x-pack/platform/plugins/private/index_lifecycle_management/__jest__/extend_index_management.test.tsx

@@ -162,7 +162,6 @@ const indexWithLifecyclePhaseDefinition: Index = {
     step_time_millis: 1544187776208,
     phase_execution: {
       policy: 'testy',
-      // @ts-expect-error ILM type is incorrect https://github.com/elastic/elasticsearch-specification/issues/2326
       phase_definition: { min_age: '0s', actions: { rollover: { max_size: '1gb' } } },
       version: 1,
       modified_date_in_millis: 1544031699844,

x-pack/platform/plugins/private/index_lifecycle_management/public/extend_index_management/components/index_lifecycle_summary.tsx

@@ -83,7 +83,7 @@ export const IndexLifecycleSummary: FunctionComponent<Props> = ({
           defaultMessage: 'Policy name',
         }
       ),
-      description: ilm.policy,
+      description: ilm.policy!,
     },
     {
       title: i18n.translate(
@@ -153,7 +153,7 @@ export const IndexLifecycleSummary: FunctionComponent<Props> = ({
                 <EuiLink
                   color="primary"
                   href={getUrlForApp('management', {
-                    path: `data/index_lifecycle_management/${getPolicyEditPath(ilm.policy)}`,
+                    path: `data/index_lifecycle_management/${getPolicyEditPath(ilm.policy!)}`,
                   })}
                   target="_blank"
                 >

x-pack/platform/plugins/private/upgrade_assistant/server/lib/es_deprecations_status/index.test.ts

@@ -7,7 +7,7 @@
 
 import _ from 'lodash';
 import { elasticsearchServiceMock } from '@kbn/core/server/mocks';
-import type * as estypes from '@elastic/elasticsearch/lib/api/types';
+import type { estypes } from '@elastic/elasticsearch';
 
 import fakeDeprecations from '../__fixtures__/fake_deprecations.json';
 import * as healthIndicatorsMock from '../__fixtures__/health_indicators';
@@ -126,7 +126,6 @@ describe('getESUpgradeStatus', () => {
         ],
       },
       data_streams: {},
-      // @ts-expect-error not in types yet
       ilm_policies: {},
       templates: {},
     });
@@ -148,7 +147,7 @@ describe('getESUpgradeStatus', () => {
       ...esMigrationsMock.getMockEsDeprecations(),
       ...esMigrationsMock.getMockMlSettingsDeprecations(),
     };
-    // @ts-ignore missing property definitions in ES resolve_during_rolling_upgrade and _meta
+    // @ts-expect-error missing property definitions in ES resolve_during_rolling_upgrade and _meta
     esClient.migration.deprecations.mockResponse(mockResponse);
 
     const enabledUpgradeStatus = await getESUpgradeStatus(esClient, {
@@ -217,7 +216,6 @@ describe('getESUpgradeStatus', () => {
           message: 'Index created before 7.0',
           url: 'https: //www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html',
           details: 'This index was created using version: 6.8.13',
-          // @ts-ignore
           resolve_during_rolling_upgrade: false,
           _meta: {
             reindex_required: true,
@@ -228,7 +226,6 @@ describe('getESUpgradeStatus', () => {
           message: 'Index created before 7.0',
           url: 'https: //www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html',
           details: 'This index was created using version: 6.8.13',
-          // @ts-ignore
           resolve_during_rolling_upgrade: false,
           _meta: {
             reindex_required: true,
@@ -238,7 +235,6 @@ describe('getESUpgradeStatus', () => {
       ml_settings: [],
       index_settings: {},
       data_streams: {},
-      // @ts-expect-error not in types yet
       ilm_policies: {},
       templates: {},
     });
@@ -313,7 +309,6 @@ describe('getESUpgradeStatus', () => {
           },
         ],
       },
-      // @ts-expect-error not in types yet
       ilm_policies: {},
       templates: {},
     });
@@ -369,7 +364,6 @@ describe('getESUpgradeStatus', () => {
         ],
       },
       data_streams: {},
-      // @ts-expect-error not in types yet
       ilm_policies: {},
       templates: {},
     });
@@ -398,7 +392,6 @@ describe('getESUpgradeStatus', () => {
           message: 'Index created before 7.0',
           url: 'https: //www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html',
           details: 'This index was created using version: 6.8.13',
-          // @ts-ignore
           resolve_during_rolling_upgrade: false,
           _meta: {
             reindex_required: true,
@@ -408,7 +401,6 @@ describe('getESUpgradeStatus', () => {
       ml_settings: [],
       index_settings: {},
       data_streams: {},
-      // @ts-expect-error not in types yet
       ilm_policies: {},
       templates: {},
     });
@@ -417,7 +409,7 @@ describe('getESUpgradeStatus', () => {
       cluster_name: 'mock',
       indicators: {
         disk: healthIndicatorsMock.diskIndicatorGreen,
-        // @ts-ignore
+        // @ts-expect-error
         shards_capacity: healthIndicatorsMock.shardCapacityIndicatorRed,
       },
     });

x-pack/platform/plugins/shared/actions/server/lib/get_execution_log_aggregation.ts

@@ -324,7 +324,7 @@ function formatExecutionLogAggBucket(bucket: IExecutionUuidAggBucket): IExecutio
   const connectorId = outcomeAndMessage?.kibana?.action?.id ?? '';
   const timedOut = (bucket?.timeoutMessage?.doc_count ?? 0) > 0;
   return {
-    id: bucket?.key ?? '',
+    id: bucket?.key ? `${bucket.key}` : '',
     timestamp: bucket?.actionExecution?.executeStartTime.value_as_string ?? '',
     duration_ms: durationUs / Millis2Nanos,
     status,

x-pack/platform/plugins/shared/alerting/server/lib/get_execution_log_aggregation.ts

@@ -530,7 +530,7 @@ function formatExecutionLogAggBucket(bucket: IExecutionUuidAggBucket): IExecutio
     ? outcomeMessageAndMaintenanceWindow.rule?.name ?? ''
     : '';
   return {
-    id: bucket?.key ?? '',
+    id: bucket?.key ? `${bucket.key}` : '', // `key` can be a number, this way we stringify it.
     timestamp: bucket?.ruleExecution?.executeStartTime.value_as_string ?? '',
     duration_ms: durationUs / Millis2Nanos,
     status,

x-pack/platform/plugins/shared/alerting/server/lib/rule_gaps/find_gaps.ts

@@ -101,7 +101,7 @@ export const findGapsSearchAfter = async ({
     return {
       total: gapsResponse.total,
       data: transformToGap(gapsResponse),
-      searchAfter: gapsResponse.search_after,
+      searchAfter: gapsResponse.search_after as SortResults[] | undefined,
       pitId: gapsResponse.pit_id,
     };
   } catch (err) {

x-pack/platform/plugins/shared/alerting/server/lib/wrap_scoped_cluster_client.ts

@@ -253,7 +253,7 @@ function getWrappedEqlSearchFn(opts: WrapEsClientOpts) {
             requestTimeout ? ` and ${requestTimeout}ms requestTimeout` : ''
           }`
       );
-      const result = (await originalEqlSearch.call(opts.esClient, params, {
+      const result = (await originalEqlSearch.call(opts.esClient.eql, params, {
         ...searchOptions,
         ...(requestTimeout
           ? {

x-pack/platform/plugins/shared/alerting/server/usage/lib/get_telemetry_from_event_log.ts

@@ -340,7 +340,7 @@ export function parseRuleTypeBucket(
     alertsPercentilesByType: { p50: {}, p90: {}, p99: {} },
   };
   for (const bucket of buckets ?? []) {
-    const ruleType: string = replaceDotSymbols(bucket?.key) ?? '';
+    const ruleType: string = replaceDotSymbols(`${bucket?.key ?? ''}`);
     const numExecutions: number = bucket?.doc_count ?? 0;
     const avgExecutionTimeNanos = bucket?.avg_execution_time?.value ?? 0;
     const avgEsSearchTimeMillis = bucket?.avg_es_search_duration?.value ?? 0;
@@ -391,7 +391,7 @@ export function parseExecutionFailureByRuleType(
   const executionFailuresWithRuleTypeBuckets: FlattenedExecutionFailureBucket[] = flatMap(
     buckets ?? [],
     (bucket) => {
-      const ruleType: string = replaceDotSymbols(bucket.key);
+      const ruleType: string = replaceDotSymbols(`${bucket.key}`);
 
       /**
        * Execution failure bucket format
@@ -409,7 +409,7 @@ export function parseExecutionFailureByRuleType(
 
       const executionFailuresBuckets = bucket?.execution_failures?.by_reason
         ?.buckets as AggregationsStringTermsBucketKeys[];
-      return (executionFailuresBuckets ?? []).map((b) => ({ ...b, ruleType }));
+      return (executionFailuresBuckets ?? []).map((b) => ({ ...b, key: `${b.key}`, ruleType }));
     }
   );
 
@@ -548,7 +548,7 @@ export function parseExecutionCountAggregationResults(results: {
     countTotalFailedExecutions: results?.execution_failures?.doc_count ?? 0,
     countFailedExecutionsByReason: executionFailuresByReasonBuckets.reduce<Record<string, number>>(
       (acc, bucket: AggregationsStringTermsBucketKeys) => {
-        const reason: string = bucket.key;
+        const reason: string = `${bucket.key}`;
         acc[reason] = bucket.doc_count ?? 0;
         return acc;
       },

x-pack/platform/plugins/shared/alerting/server/usage/lib/get_telemetry_from_task_manager.ts

@@ -163,12 +163,14 @@ export function parseBucket(
 > {
   return (buckets ?? []).reduce(
     (summary, bucket) => {
-      const status: string = bucket.key;
+      const status: string = `${bucket.key}`;
       const taskTypeBuckets = bucket?.by_task_type?.buckets as AggregationsStringTermsBucketKeys[];
 
       const byTaskType = (taskTypeBuckets ?? []).reduce<Record<string, number>>(
         (acc, taskTypeBucket: AggregationsStringTermsBucketKeys) => {
-          const taskType: string = replaceDotSymbols(taskTypeBucket.key.replace('alerting:', ''));
+          const taskType: string = replaceDotSymbols(
+            `${taskTypeBucket.key}`.replace('alerting:', '')
+          );
           acc[taskType] = taskTypeBucket.doc_count ?? 0;
           return acc;
         },

x-pack/platform/plugins/shared/alerting/server/usage/lib/parse_simple_rule_type_bucket.ts

@@ -16,7 +16,7 @@ export function parseSimpleRuleTypeBucket(
 ) {
   const buckets = ruleTypeBuckets as AggregationsStringTermsBucketKeys[];
   return (buckets ?? []).reduce((acc, bucket: AggregationsStringTermsBucketKeys) => {
-    const ruleType: string = replaceDotSymbols(bucket.key);
+    const ruleType: string = replaceDotSymbols(`${bucket.key}`);
     acc[ruleType] = bucket.doc_count ?? 0;
     return acc;
   }, {} as Record<string, number>);