Skip to content

Commit 75eb933

Browse files
MikeJamesWhiteMikeJamesWhite
authored
feat(logs): Add support for multiple parse and filter statements in QueryString (#24022)
Currently, `QueryString` is limited to only allow a single line/statement to be provided for each query command. For some commands this makes sense (e.g. `limit`), but for `parse` and `filter` this can be limiting. Adding multiple lines for these commands is possible in the AWS console, so it makes sense for it to be supported in CDK too. In this PR, I'm adding support for `filter` and `parse` to be provided as `string` or `string[]`, and adding/modifying various utility methods to handle this ambiguity. I left the existing tests the same to verify no breaking changes, and added a new test for the newly enabled behavior. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
1 parent 89802a9 commit 75eb933

11 files changed

+231
-43
lines changed

‎packages/@aws-cdk/aws-logs/README.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -330,6 +330,14 @@ new logs.QueryDefinition(this, 'QueryDefinition', {
330330
queryDefinitionName: 'MyQuery',
331331
queryString: new logs.QueryString({
332332
fields: ['@timestamp', '@message'],
333+
parseStatements: [
334+
'@message "[*] *" as loggingType, loggingMessage',
335+
'@message "<*>: *" as differentLoggingType, differentLoggingMessage',
336+
],
337+
filterStatements: [
338+
'loggingType = "ERROR"',
339+
'loggingMessage = "A very strange error occurred!"',
340+
],
333341
sort: '@timestamp desc',
334342
limit: 20,
335343
}),

‎packages/@aws-cdk/aws-logs/lib/query-definition.ts

Lines changed: 82 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -16,19 +16,43 @@ export interface QueryStringProps {
1616
readonly fields?: string[];
1717

1818
/**
19-
* Extracts data from a log field and creates one or more ephemeral fields that you can process further in the query.
19+
* A single statement for parsing data from a log field and creating ephemeral fields that can
20+
* be processed further in the query.
2021
*
22+
* @deprecated Use `parseStatements` instead
2123
* @default - no parse in QueryString
2224
*/
2325
readonly parse?: string;
2426

2527
/**
26-
* Filters the results of a query that's based on one or more conditions.
28+
* An array of one or more statements for parsing data from a log field and creating ephemeral
29+
* fields that can be processed further in the query. Each provided statement generates a separate
30+
* parse line in the query string.
2731
*
32+
* Note: If provided, this property overrides any value provided for the `parse` property.
33+
*
34+
* @default - no parse in QueryString
35+
*/
36+
readonly parseStatements?: string[];
37+
38+
/**
39+
* A single statement for filtering the results of a query based on a boolean expression.
40+
*
41+
* @deprecated Use `filterStatements` instead
2842
* @default - no filter in QueryString
2943
*/
3044
readonly filter?: string;
3145

46+
/**
47+
* An array of one or more statements for filtering the results of a query based on a boolean
48+
* expression. Each provided statement generates a separate filter line in the query string.
49+
*
50+
* Note: If provided, this property overrides any value provided for the `filter` property.
51+
*
52+
* @default - no filter in QueryString
53+
*/
54+
readonly filterStatements?: string[];
55+
3256
/**
3357
* Uses log field values to calculate aggregate statistics.
3458
*
@@ -58,62 +82,88 @@ export interface QueryStringProps {
5882
readonly display?: string;
5983
}
6084

61-
interface QueryStringMap {
62-
readonly fields?: string,
63-
readonly parse?: string,
64-
readonly filter?: string,
65-
readonly stats?: string,
66-
readonly sort?: string,
67-
readonly limit?: Number,
68-
readonly display?: string,
69-
}
70-
7185
/**
7286
* Define a QueryString
7387
*/
7488
export class QueryString {
7589
private readonly fields?: string[];
76-
private readonly parse?: string;
77-
private readonly filter?: string;
90+
private readonly parse: string[];
91+
private readonly filter: string[];
7892
private readonly stats?: string;
7993
private readonly sort?: string;
8094
private readonly limit?: Number;
8195
private readonly display?: string;
8296

8397
constructor(props: QueryStringProps = {}) {
8498
this.fields = props.fields;
85-
this.parse = props.parse;
86-
this.filter = props.filter;
8799
this.stats = props.stats;
88100
this.sort = props.sort;
89101
this.limit = props.limit;
90102
this.display = props.display;
103+
104+
// Determine parsing by either the parseStatements or parse properties, or default to empty array
105+
if (props.parseStatements) {
106+
this.parse = props.parseStatements;
107+
} else if (props.parse) {
108+
this.parse = [props.parse];
109+
} else {
110+
this.parse = [];
111+
}
112+
113+
// Determine filtering by either the filterStatements or filter properties, or default to empty array
114+
if (props.filterStatements) {
115+
this.filter = props.filterStatements;
116+
} else if (props.filter) {
117+
this.filter = [props.filter];
118+
} else {
119+
this.filter = [];
120+
}
91121
}
92122

93123
/**
94124
* String representation of this QueryString.
95125
*/
96126
public toString(): string {
97-
return noUndef({
98-
fields: this.fields !== undefined ? this.fields.join(', ') : this.fields,
99-
parse: this.parse,
100-
filter: this.filter,
101-
stats: this.stats,
102-
sort: this.sort,
103-
limit: this.limit,
104-
display: this.display,
105-
}).join('\n| ');
127+
return [
128+
this.buildQueryLine('fields', this.fields?.join(', ')),
129+
...this.buildQueryLines('parse', this.parse),
130+
...this.buildQueryLines('filter', this.filter),
131+
this.buildQueryLine('stats', this.stats),
132+
this.buildQueryLine('sort', this.sort),
133+
this.buildQueryLine('limit', this.limit?.toString()),
134+
this.buildQueryLine('display', this.display),
135+
].filter(
136+
(queryLine) => queryLine !== undefined && queryLine.length > 0,
137+
).join('\n| ');
106138
}
107-
}
108139

109-
function noUndef(x: QueryStringMap): string[] {
110-
const ret: string[] = [];
111-
for (const [key, value] of Object.entries(x)) {
112-
if (value !== undefined) {
113-
ret.push(`${key} ${value}`);
140+
/**
141+
* Build an array of query lines given a command and statement(s).
142+
*
143+
* @param command a query command
144+
* @param statements one or more query statements for the specified command, or undefined
145+
* @returns an array of the query string lines generated from the provided command and statements
146+
*/
147+
private buildQueryLines(command: string, statements?: string[]): string[] {
148+
if (statements === undefined) {
149+
return [];
114150
}
151+
152+
return statements.map(
153+
(statement: string): string => this.buildQueryLine(command, statement),
154+
);
155+
}
156+
157+
/**
158+
* Build a single query line given a command and statement.
159+
*
160+
* @param command a query command
161+
* @param statement a single query statement
162+
* @returns a single query string line generated from the provided command and statement
163+
*/
164+
private buildQueryLine(command: string, statement?: string): string {
165+
return statement ? `${command} ${statement}` : '';
115166
}
116-
return ret;
117167
}
118168

119169
/**

‎packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/LogsInsightsQueryDefinitionIntegTestDefaultTestDeployAssert902BAAD5.assets.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version": "21.0.0",
2+
"version": "29.0.0",
33
"files": {
44
"21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": {
55
"source": {

‎packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/aws-cdk-logs-insights-querydefinition-integ.assets.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
{
2-
"version": "21.0.0",
2+
"version": "29.0.0",
33
"files": {
4-
"e6a0a51961925fbc37d9b81431449c256ed453f98089eb70f83850f237b4d722": {
4+
"3546c78647ea20567832041c960a034787f9bfc0128226ea8fbc0894366a4dd0": {
55
"source": {
66
"path": "aws-cdk-logs-insights-querydefinition-integ.template.json",
77
"packaging": "file"
88
},
99
"destinations": {
1010
"current_account-current_region": {
1111
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
12-
"objectKey": "e6a0a51961925fbc37d9b81431449c256ed453f98089eb70f83850f237b4d722.json",
12+
"objectKey": "3546c78647ea20567832041c960a034787f9bfc0128226ea8fbc0894366a4dd0.json",
1313
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
1414
}
1515
}

‎packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/aws-cdk-logs-insights-querydefinition-integ.template.json

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,18 @@
1919
}
2020
]
2121
}
22+
},
23+
"QueryDefinitionWithMultipleStatements58A3EF74": {
24+
"Type": "AWS::Logs::QueryDefinition",
25+
"Properties": {
26+
"Name": "QueryDefinitionWithMultipleStatements",
27+
"QueryString": "fields @timestamp, @message\n| parse @message \"[*] *\" as loggingType, loggingMessage\n| parse @message \"<*>: *\" as differentLoggingType, differentLoggingMessage\n| filter loggingType = \"ERROR\"\n| filter loggingMessage = \"A very strange error occurred!\"\n| sort @timestamp desc\n| limit 20\n| display loggingMessage",
28+
"LogGroupNames": [
29+
{
30+
"Ref": "LogGroupF5B46931"
31+
}
32+
]
33+
}
2234
}
2335
},
2436
"Parameters": {

‎packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/cdk.out

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"version":"21.0.0"}
1+
{"version":"29.0.0"}

‎packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/integ.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version": "21.0.0",
2+
"version": "29.0.0",
33
"testCases": {
44
"LogsInsightsQueryDefinitionIntegTest/DefaultTest": {
55
"stacks": [

‎packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/manifest.json

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version": "21.0.0",
2+
"version": "29.0.0",
33
"artifacts": {
44
"aws-cdk-logs-insights-querydefinition-integ.assets": {
55
"type": "cdk:asset-manifest",
@@ -17,7 +17,7 @@
1717
"validateOnSynth": false,
1818
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}",
1919
"cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}",
20-
"stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/e6a0a51961925fbc37d9b81431449c256ed453f98089eb70f83850f237b4d722.json",
20+
"stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/3546c78647ea20567832041c960a034787f9bfc0128226ea8fbc0894366a4dd0.json",
2121
"requiresBootstrapStackVersion": 6,
2222
"bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version",
2323
"additionalDependencies": [
@@ -45,6 +45,12 @@
4545
"data": "QueryDefinition4190BC36"
4646
}
4747
],
48+
"/aws-cdk-logs-insights-querydefinition-integ/QueryDefinitionWithMultipleStatements/Resource": [
49+
{
50+
"type": "aws:cdk:logicalId",
51+
"data": "QueryDefinitionWithMultipleStatements58A3EF74"
52+
}
53+
],
4854
"/aws-cdk-logs-insights-querydefinition-integ/BootstrapVersion": [
4955
{
5056
"type": "aws:cdk:logicalId",

‎packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/tree.json

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,36 @@
6262
"version": "0.0.0"
6363
}
6464
},
65+
"QueryDefinitionWithMultipleStatements": {
66+
"id": "QueryDefinitionWithMultipleStatements",
67+
"path": "aws-cdk-logs-insights-querydefinition-integ/QueryDefinitionWithMultipleStatements",
68+
"children": {
69+
"Resource": {
70+
"id": "Resource",
71+
"path": "aws-cdk-logs-insights-querydefinition-integ/QueryDefinitionWithMultipleStatements/Resource",
72+
"attributes": {
73+
"aws:cdk:cloudformation:type": "AWS::Logs::QueryDefinition",
74+
"aws:cdk:cloudformation:props": {
75+
"name": "QueryDefinitionWithMultipleStatements",
76+
"queryString": "fields @timestamp, @message\n| parse @message \"[*] *\" as loggingType, loggingMessage\n| parse @message \"<*>: *\" as differentLoggingType, differentLoggingMessage\n| filter loggingType = \"ERROR\"\n| filter loggingMessage = \"A very strange error occurred!\"\n| sort @timestamp desc\n| limit 20\n| display loggingMessage",
77+
"logGroupNames": [
78+
{
79+
"Ref": "LogGroupF5B46931"
80+
}
81+
]
82+
}
83+
},
84+
"constructInfo": {
85+
"fqn": "@aws-cdk/aws-logs.CfnQueryDefinition",
86+
"version": "0.0.0"
87+
}
88+
}
89+
},
90+
"constructInfo": {
91+
"fqn": "@aws-cdk/aws-logs.QueryDefinition",
92+
"version": "0.0.0"
93+
}
94+
},
6595
"BootstrapVersion": {
6696
"id": "BootstrapVersion",
6797
"path": "aws-cdk-logs-insights-querydefinition-integ/BootstrapVersion",
@@ -97,7 +127,7 @@
97127
"path": "LogsInsightsQueryDefinitionIntegTest/DefaultTest/Default",
98128
"constructInfo": {
99129
"fqn": "constructs.Construct",
100-
"version": "10.1.161"
130+
"version": "10.1.237"
101131
}
102132
},
103133
"DeployAssert": {
@@ -143,7 +173,7 @@
143173
"path": "Tree",
144174
"constructInfo": {
145175
"fqn": "constructs.Construct",
146-
"version": "10.1.161"
176+
"version": "10.1.237"
147177
}
148178
}
149179
},

‎packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.ts

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ class LogsInsightsQueryDefinitionIntegStack extends Stack {
1111
removalPolicy: RemovalPolicy.DESTROY,
1212
});
1313

14+
// Test query creation with single parse and filter statements
1415
new QueryDefinition(this, 'QueryDefinition', {
1516
queryDefinitionName: 'QueryDefinition',
1617
queryString: new QueryString({
@@ -23,6 +24,26 @@ class LogsInsightsQueryDefinitionIntegStack extends Stack {
2324
}),
2425
logGroups: [logGroup],
2526
});
27+
28+
// Test query creation with multiple parse and filter statements
29+
new QueryDefinition(this, 'QueryDefinitionWithMultipleStatements', {
30+
queryDefinitionName: 'QueryDefinitionWithMultipleStatements',
31+
queryString: new QueryString({
32+
fields: ['@timestamp', '@message'],
33+
parseStatements: [
34+
'@message "[*] *" as loggingType, loggingMessage',
35+
'@message "<*>: *" as differentLoggingType, differentLoggingMessage',
36+
],
37+
filterStatements: [
38+
'loggingType = "ERROR"',
39+
'loggingMessage = "A very strange error occurred!"',
40+
],
41+
sort: '@timestamp desc',
42+
limit: 20,
43+
display: 'loggingMessage',
44+
}),
45+
logGroups: [logGroup],
46+
});
2647
}
2748
}
2849

0 commit comments

Comments
 (0)