Skip to content

Commit 54094bd

Browse files
KDKHDelasticmachinepatrykkopycinski
authored
[Security] [AI assistant] setup/cleanup indices for evaluations (#217078)
## Summary Summarize your PR. If it involves visual changes include a screenshot or gif. Setup indices and datastreams for evaluations. This will be used for ESQL evals and can be extended to setup other indices for other graphs. How to test: 1. Enable the evaluations feature flag in kibana.dev.yml ``` xpack.securitySolution.enableExperimental: ['assistantModelEvaluation'] ``` 2. Launch Kibana 4. Go to evaluations http://localhost:5601/app/management/kibana/securityAiAssistantManagement?tab=evaluation 5. Start evaluations for the default assistant graph <img width="1840" alt="image" src="https://github.com/user-attachments/assets/2974b34f-40a7-4300-8294-d25d4f72b27e" /> 6. Go to discover -> create a dataview 7. Search for `*evaluations*` and check there are datastreams and indices <img width="1840" alt="image" src="https://github.com/user-attachments/assets/b6e9e476-82de-4292-9757-487ac85d7fce" /> 8. These indices and datastreams are not cleaned up after the evaluation finishes. However, they are cleaned up when evaluations are re-run. To test this, run the evaluation again and see new datastreams and indices created. We can not do the cleanup after evaluations finish because evaluations happen asynchronously. ### Checklist Check the PR satisfies following conditions. Reviewers should verify this PR satisfies this list as well. - [X] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/src/platform/packages/shared/kbn-i18n/README.md) - [X] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [X] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [X] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [X] This was checked for breaking HTTP API changes, and any breaking changes have been approved by the breaking-change committee. The `release_note:breaking` label should be applied in these situations. - [X] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [X] The PR description includes the appropriate Release Notes section, and the correct `release_note:*` label is applied per the [guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) ### Identify risks Does this PR introduce any risks? For example, consider risks like hard to test bugs, performance regression, potential of data loss. Describe the risk, its severity, and mitigation for each identified risk. Invite stakeholders and evaluate how to proceed before merging. - [ ] [See some risk examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx) - [ ] ... --------- Co-authored-by: Elastic Machine <[email protected]> Co-authored-by: Patryk Kopyciński <[email protected]>
1 parent d359881 commit 54094bd

13 files changed

+475
-1
lines changed

x-pack/solutions/security/plugins/elastic_assistant/server/routes/evaluate/post_evaluate.ts

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ import { getLlmClass, getLlmType, isOpenSourceModel } from '../utils';
5353
import { getGraphsFromNames } from './get_graphs_from_names';
5454
import { DEFAULT_DATE_FORMAT_TZ } from '../../../common/constants';
5555
import { agentRunableFactory } from '../../lib/langchain/graphs/default_assistant_graph/agentRunnable';
56+
import { PrepareIndicesForAssistantGraphEvaluations } from './prepare_indices_for_evaluations/graph_type/assistant';
5657

5758
const DEFAULT_SIZE = 20;
5859
const ROUTE_HANDLER_TIMEOUT = 10 * 60 * 1000; // 10 * 60 seconds = 10 minutes
@@ -184,7 +185,18 @@ export const postEvaluateRoute = (
184185
// Fetch any tools registered to the security assistant
185186
const assistantTools = assistantContext.getRegisteredTools(DEFAULT_PLUGIN_NAME);
186187

187-
const { attackDiscoveryGraphs, defendInsightsGraphs } = getGraphsFromNames(graphNames);
188+
const { attackDiscoveryGraphs, defendInsightsGraphs, assistantGraphs } =
189+
getGraphsFromNames(graphNames);
190+
191+
const prepareIndicesForAssistantGraph = new PrepareIndicesForAssistantGraphEvaluations({
192+
esClient,
193+
logger,
194+
});
195+
196+
if (assistantGraphs?.length) {
197+
await prepareIndicesForAssistantGraph.cleanup();
198+
await prepareIndicesForAssistantGraph.setup();
199+
}
188200

189201
if (defendInsightsGraphs.length > 0) {
190202
const connectorsWithPrompts = await Promise.all(
@@ -299,6 +311,7 @@ export const postEvaluateRoute = (
299311
pluginId: 'security_ai_assistant',
300312
},
301313
});
314+
302315
const llm = createLlmInstance();
303316
const anonymizationFieldsRes =
304317
await dataClients?.anonymizationFieldsDataClient?.findDocuments<EsAnonymizationFieldsSchema>(
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { ElasticsearchClient } from '@kbn/core/server';
9+
import { Logger } from '@kbn/logging';
10+
import { IndexRequest, IndicesCreateRequest } from '@elastic/elasticsearch/lib/api/types';
11+
import { PrepareIndicesForEvaluations } from '../../prepare_indices_for_evalutations';
12+
import { indicesCreateRequests } from './indices_create_requests';
13+
import { indexRequests } from './index_requests';
14+
15+
const ENVIRONMENTS = ['production', 'staging', 'development'];
16+
export class PrepareIndicesForAssistantGraphEvaluations extends PrepareIndicesForEvaluations {
17+
constructor({ esClient, logger }: { esClient: ElasticsearchClient; logger: Logger }) {
18+
super({
19+
esClient,
20+
indicesCreateRequests: PrepareIndicesForAssistantGraphEvaluations.hydrateRequestTemplate(
21+
Object.values(indicesCreateRequests)
22+
),
23+
indexRequests: PrepareIndicesForAssistantGraphEvaluations.hydrateRequestTemplate(
24+
Object.values(indexRequests)
25+
),
26+
logger,
27+
});
28+
}
29+
30+
static hydrateRequestTemplate<T extends IndicesCreateRequest | IndexRequest>(requests: T[]): T[] {
31+
return requests
32+
.map((request) => {
33+
return ENVIRONMENTS.map((environment) => {
34+
return {
35+
...request,
36+
index: request.index
37+
.replace(/\[environment\]/g, environment)
38+
.replace(/\[date\]/g, this.getRandomDate()),
39+
} as T;
40+
});
41+
})
42+
.flat();
43+
}
44+
45+
async cleanup() {
46+
this.logger.debug('Deleting assistant indices for evaluations');
47+
48+
const requests = [...Object.values(indicesCreateRequests), ...Object.values(indexRequests)];
49+
const indexPatternsToDelete = Object.values(requests).map((index) =>
50+
index.index.replace(/\[environment\]/g, '*').replace(/\[date\]/g, '*')
51+
);
52+
53+
const indicesResolveIndexResponses = await Promise.all(
54+
indexPatternsToDelete.map(async (indexPattern) =>
55+
this.esClient.indices.resolveIndex({
56+
name: indexPattern,
57+
expand_wildcards: 'open',
58+
})
59+
)
60+
);
61+
62+
const indicesToDelete = indicesResolveIndexResponses
63+
.flatMap((response) => response.indices)
64+
.map((index) => index.name);
65+
66+
const dataStreamsToDelete = indicesResolveIndexResponses
67+
.flatMap((response) => response.data_streams)
68+
.map((dataStream) => dataStream.name);
69+
70+
if (indicesToDelete.length > 0) {
71+
this.logger.info('Deleting indices');
72+
await this.esClient.indices.delete({ index: indicesToDelete });
73+
}
74+
75+
if (dataStreamsToDelete.length > 0) {
76+
this.logger.info('Deleting data streams');
77+
await this.esClient.indices.deleteDataStream({ name: dataStreamsToDelete });
78+
}
79+
}
80+
81+
static getRandomDate() {
82+
const year = Math.floor(Math.random() * (2050 - 2000 + 1)) + 2000;
83+
const month = String(Math.floor(Math.random() * 12) + 1).padStart(2, '0');
84+
const day = String(Math.floor(Math.random() * 28) + 1).padStart(2, '0');
85+
return `${year}.${month}.${day}`;
86+
}
87+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { metricsApmIndexRequest } from './metrics-apm-[environment].evaluations.[date]';
9+
import { tracesApmIndexRequest } from './traces-apm-[environment].evaluations.[date]';
10+
11+
export const indexRequests = [metricsApmIndexRequest, tracesApmIndexRequest];
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { IndexRequest } from '@elastic/elasticsearch/lib/api/types';
9+
10+
export const metricsApmIndexRequest: IndexRequest = {
11+
index: 'metrics-apm-[environment].evaluations.[date]',
12+
document: {
13+
'@timestamp': '2024-04-02T12:00:00.000Z',
14+
metricset: {
15+
name: 'app',
16+
interval: '1m',
17+
},
18+
transaction: {
19+
duration: {
20+
us: 13980,
21+
},
22+
},
23+
},
24+
};
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { IndexRequest } from '@elastic/elasticsearch/lib/api/types';
9+
10+
export const tracesApmIndexRequest: IndexRequest = {
11+
index: 'traces-apm-[environment].evaluations.[date]',
12+
document: {
13+
'@timestamp': '2024-04-02T12:00:00.000Z',
14+
event: {
15+
outcome: 'success',
16+
},
17+
transaction: {
18+
duration: {
19+
us: 1000,
20+
},
21+
transaction: { id: '945254c567a5417e' },
22+
service: { name: 'my-service' },
23+
},
24+
},
25+
};
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { IndicesCreateRequest } from '@elastic/elasticsearch/lib/api/types';
9+
10+
export const employeesIndexCreateRequest: IndicesCreateRequest = {
11+
index: 'employees-[environment].evaluations.[date]',
12+
mappings: {
13+
properties: {
14+
emp_no: {
15+
type: 'keyword',
16+
},
17+
hire_date: {
18+
type: 'date',
19+
format: 'yyyy-MM-dd',
20+
},
21+
salary: {
22+
type: 'double',
23+
},
24+
},
25+
},
26+
};
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { nycTaxisIndexCreateRequest } from './nyc_taxis-[environment].evaluations.[date]';
9+
import { postgresLogsIndexCreateRequest } from './postgres-logs-[environment].evaluations.[date]';
10+
import { employeesIndexCreateRequest } from './employees-[environment].evaluations.[date]';
11+
import { metricbeatIndexCreateRequest } from './metricbeat-[environment].evaluations-[date]';
12+
import { packetbeatIndexCreateRequest } from './packetbeat-[environment].evaluations.[date]';
13+
import { logsIndexCreateRequest } from './logs-[environment].evaluations.[date]';
14+
15+
export const indicesCreateRequests = {
16+
nycTaxisIndexCreateRequest,
17+
postgresLogsIndexCreateRequest,
18+
employeesIndexCreateRequest,
19+
metricbeatIndexCreateRequest,
20+
packetbeatIndexCreateRequest,
21+
logsIndexCreateRequest,
22+
};
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { IndicesCreateRequest } from '@elastic/elasticsearch/lib/api/types';
9+
10+
export const logsIndexCreateRequest: IndicesCreateRequest = {
11+
index: 'logs-[environment].evaluations.[date]',
12+
mappings: {
13+
properties: {
14+
'@timestamp': { type: 'date' },
15+
bytes_transferred: { type: 'long' },
16+
command_line: { type: 'text' },
17+
destination: {
18+
properties: {
19+
ip: { type: 'ip' },
20+
port: { type: 'integer' },
21+
address: { type: 'keyword' },
22+
},
23+
},
24+
dns: {
25+
properties: {
26+
question: {
27+
properties: {
28+
name: { type: 'keyword' },
29+
registered_domain: { type: 'keyword' },
30+
},
31+
},
32+
},
33+
},
34+
error_code: { type: 'keyword' },
35+
event: {
36+
properties: {
37+
action: { type: 'keyword' },
38+
code: { type: 'long' },
39+
},
40+
},
41+
file: {
42+
properties: {
43+
name: { type: 'text' },
44+
},
45+
},
46+
group: {
47+
properties: {
48+
name: { type: 'keyword' },
49+
},
50+
},
51+
host: {
52+
properties: {
53+
name: { type: 'keyword' },
54+
},
55+
},
56+
ip: { type: 'ip' },
57+
log: {
58+
properties: {
59+
message: { type: 'text' },
60+
},
61+
},
62+
network: {
63+
properties: {
64+
bytes: { type: 'long' },
65+
},
66+
},
67+
process: {
68+
properties: {
69+
name: { type: 'keyword' },
70+
working_directory: { type: 'keyword' },
71+
},
72+
},
73+
source: {
74+
properties: {
75+
ip: { type: 'ip' },
76+
bytes: { type: 'long' },
77+
},
78+
},
79+
system: {
80+
properties: {
81+
cpu: {
82+
properties: {
83+
total: {
84+
properties: {
85+
norm: {
86+
properties: {
87+
pct: { type: 'float' },
88+
},
89+
},
90+
},
91+
},
92+
},
93+
},
94+
},
95+
},
96+
url: {
97+
properties: {
98+
domain: { type: 'keyword' },
99+
},
100+
},
101+
user: {
102+
properties: {
103+
name: { type: 'keyword' },
104+
},
105+
},
106+
user_agent: {
107+
properties: {
108+
original: { type: 'text' },
109+
},
110+
},
111+
},
112+
},
113+
};
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import { IndicesCreateRequest } from '@elastic/elasticsearch/lib/api/types';
9+
10+
export const metricbeatIndexCreateRequest: IndicesCreateRequest = {
11+
index: 'metricbeat-[environment].evaluations-[date]',
12+
mappings: {
13+
properties: {
14+
system: {
15+
properties: {
16+
cpu: {
17+
properties: {
18+
user: {
19+
properties: {
20+
pct: { type: 'float' },
21+
},
22+
},
23+
system: {
24+
properties: {
25+
pct: { type: 'float' },
26+
},
27+
},
28+
cores: { type: 'integer' },
29+
},
30+
},
31+
},
32+
},
33+
host: {
34+
properties: {
35+
name: { type: 'keyword' },
36+
},
37+
},
38+
},
39+
},
40+
};

0 commit comments

Comments
 (0)