Skip to content

Commit 02c6d24

Browse files
authored
Cleanup boto patching and include min/max (#3939)
* Cleanup boto patching and include min/max
1 parent 90af150 commit 02c6d24

File tree

1,462 files changed

+18680
-7629
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,462 files changed

+18680
-7629
lines changed

scripts/boto/_automated_patches.py

+15-25
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,17 @@
44
"""
55

66
import json
7+
import logging
78
from pathlib import Path
89
from typing import Any
910

11+
from _helpers import load_schema_file
1012
from _types import AllPatches, Patch, ResourcePatches
1113

1214
from cfnlint.schema.resolver import RefResolutionError, RefResolver
1315

16+
LOGGER = logging.getLogger("cfnlint")
17+
1418
skip = [
1519
"account",
1620
"chime",
@@ -33,8 +37,6 @@
3337

3438
skip_property_names = ["State"]
3539

36-
_fields = ["pattern", "enum"]
37-
3840
_visited_paths = []
3941

4042

@@ -125,9 +127,6 @@ def _nested_arrays(
125127
resolver, schema_data, boto_data, array_shap_data, path, source
126128
)
127129
else:
128-
# skip if we already have an enum or pattern
129-
if any([schema_data.get(field) for field in _fields]):
130-
return {}
131130
return {
132131
path: Patch(
133132
source=source,
@@ -167,10 +166,6 @@ def _nested_objects(
167166
except RefResolutionError:
168167
return results
169168

170-
# skip if we already have an enum or pattern
171-
if any([p_data.get(field) for field in _fields]):
172-
continue
173-
174169
member_shape_name = member_data.get("shape")
175170
member_shape = boto_data.get("shapes", {}).get(member_shape_name, {})
176171

@@ -189,9 +184,6 @@ def _nested_objects(
189184
)
190185
)
191186

192-
if not any([member_shape.get(field) for field in _fields]):
193-
continue
194-
195187
results[path] = Patch(
196188
source=source,
197189
shape=member_shape_name,
@@ -201,13 +193,13 @@ def _nested_objects(
201193

202194

203195
def _per_resource_patch(
204-
schema_data: dict[str, Any], boto_data: dict[str, Any], source: list[str]
196+
resolver: RefResolver, boto_data: dict[str, Any], source: list[str]
205197
) -> ResourcePatches:
206198
results: ResourcePatches = {}
199+
_, schema_data = resolver.resolve("/")
207200
create_operations = get_schema_create_operations(schema_data)
208201
shapes = {}
209202

210-
resolver = RefResolver.from_schema(schema_data)
211203
for create_operation in create_operations:
212204
shapes.update(get_shapes(boto_data, create_operation))
213205
create_shape = (
@@ -252,21 +244,19 @@ def get_resource_patches(
252244

253245
resources = list(schema_path.glob(f"aws-{service_name}-*.json"))
254246
if not resources:
255-
print(f"No resource files found for {service_name}")
247+
LOGGER.warning(f"No resource files found for {service_name}")
256248

257249
for resource in resources:
258-
with open(resource, "r") as f:
259-
schema_data = json.load(f)
250+
ref_resolver = load_schema_file(resource)
251+
_, schema_data = ref_resolver.resolve("/")
260252

261-
resource_type = schema_data.get("typeName", "")
262-
if resource_type not in results:
263-
results[resource_type] = {}
253+
resource_type = schema_data.get("typeName", "")
254+
if resource_type not in results:
255+
results[resource_type] = {}
264256

265-
results[resource_type].update(
266-
_per_resource_patch(
267-
schema_data, boto_data, [service_dir.name, last_date]
268-
)
269-
)
257+
results[resource_type].update(
258+
_per_resource_patch(ref_resolver, boto_data, [service_dir.name, last_date])
259+
)
270260

271261
return results
272262

scripts/boto/_helpers.py

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
"""
2+
Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
SPDX-License-Identifier: MIT-0
4+
"""
5+
6+
from __future__ import annotations
7+
8+
import json
9+
from pathlib import Path
10+
11+
from cfnlint.schema.resolver import RefResolver
12+
13+
14+
def load_schema_file(schema_path: Path) -> RefResolver:
15+
with open(schema_path, "r") as f:
16+
data = json.load(f)
17+
return RefResolver.from_schema(data)

scripts/boto/update_schemas_from_boto.py

+50-8
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import regex as re
1515
import requests
1616
from _automated_patches import build_automated_patches
17+
from _helpers import load_schema_file
1718
from _manual_patches import patches
1819
from _types import AllPatches, ResourcePatches
1920

@@ -39,7 +40,7 @@ def configure_logging():
3940
ch = logging.StreamHandler()
4041
ch.setLevel(logging.INFO)
4142

42-
LOGGER.setLevel(logging.INFO)
43+
LOGGER.setLevel(logging.WARNING)
4344
log_formatter = logging.Formatter(
4445
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
4546
)
@@ -52,10 +53,16 @@ def configure_logging():
5253

5354

5455
def build_resource_type_patches(
55-
dir: str, resource_name: str, resource_patches: ResourcePatches
56+
boto_path: Path,
57+
schema_path: Path,
58+
resource_name: str,
59+
resource_patches: ResourcePatches,
5660
):
5761
LOGGER.info(f"Applying patches for {resource_name}")
5862

63+
resolver = load_schema_file(
64+
schema_path / f"{resource_name.lower().replace('::', '-')}.json"
65+
)
5966
resource_name = resource_name.lower().replace("::", "_")
6067
output_path = Path("src/cfnlint/data/schemas/patches/extensions/all/")
6168
resource_path = output_path / resource_name
@@ -71,13 +78,38 @@ def build_resource_type_patches(
7178
service_path = (
7279
["botocore-master/botocore/data"] + patch.source + ["service-2.json"]
7380
)
74-
with open(os.path.join(dir, *service_path), "r") as f:
81+
_, schema_data = resolver.resolve(f"#{path}")
82+
with open(os.path.join(boto_path, *service_path), "r") as f:
7583
boto_d = json.load(f)
76-
77-
for field in ["enum", "pattern"]:
84+
shape_type = boto_d.get("shapes", {}).get(patch.shape, {}).get("type")
85+
for field in ["enum", "pattern", "max", "min"]:
7886
value = boto_d.get("shapes", {}).get(patch.shape, {}).get(field)
7987
if not value:
8088
continue
89+
if field in ["enum", "pattern"]:
90+
if any(f in schema_data for f in ["enum", "pattern"]):
91+
continue
92+
elif field == "max":
93+
if any(
94+
f in schema_data for f in ["maxLength", "maxItems", "maximum"]
95+
):
96+
continue
97+
if "pattern" in schema_data:
98+
if re.match(
99+
r"^.*\{[0-9]+,[0-9]+\}\$?$", schema_data["pattern"]
100+
):
101+
continue
102+
103+
elif field == "min":
104+
if any(
105+
f in schema_data for f in ["minLength", "minItems", "minimum"]
106+
):
107+
continue
108+
if "pattern" in schema_data:
109+
if re.match(
110+
r"^.*\{[0-9]+,[0-9]+\}\$?$", schema_data["pattern"]
111+
):
112+
continue
81113
if field == "pattern":
82114
if value in [".*", "^.*$"]:
83115
continue
@@ -92,6 +124,15 @@ def build_resource_type_patches(
92124
)
93125
)
94126
continue
127+
if field in ["max", "min"]:
128+
if shape_type == "string":
129+
field = "maxLength" if field == "max" else "minLength"
130+
elif shape_type == "list":
131+
field = "maxItems" if field == "max" else "minItems"
132+
elif shape_type in ["integer", "number"]:
133+
field = "maximum" if field == "max" else "minimum"
134+
else:
135+
continue
95136
if value:
96137
if patch.source[0] in exceptions:
97138
if path in exceptions[patch.source[0]]:
@@ -123,12 +164,13 @@ def build_resource_type_patches(
123164

124165

125166
def build_patches(
126-
dir: str,
167+
boto_path: Path,
168+
schema_path: Path,
127169
patches: AllPatches,
128170
):
129171
for resource_name, patch in patches.items():
130172
build_resource_type_patches(
131-
dir, resource_name=resource_name, resource_patches=patch
173+
boto_path, schema_path, resource_name=resource_name, resource_patches=patch
132174
)
133175

134176

@@ -159,7 +201,7 @@ def main():
159201
else:
160202
_patches[k][path] = patch
161203

162-
build_patches(boto_path, _patches)
204+
build_patches(boto_path, schema_path, _patches)
163205

164206

165207
if __name__ == "__main__":
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"b7df0a841e1d85eb2ca55fae9b63a199\"", "url": "https://schema.cloudformation.eu-south-1.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"3b5d6ce396c9a98b72b7fc1127bc1abc\"", "url": "https://schema.cloudformation.eu-south-1.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"75e66d980dea62bf67981ffe569bbb47\"", "url": "https://schema.cloudformation.us-gov-east-1.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"40c9314343c1df29f23241aca99ff5c0\"", "url": "https://schema.cloudformation.us-gov-east-1.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"b4ea48ef051d441d6fe68349a08772e5\"", "url": "https://schema.cloudformation.us-gov-west-1.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"4d01654834ffd168b5611579eb8746bc\"", "url": "https://schema.cloudformation.us-gov-west-1.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"6d426c7baa8bf4703793694e05b7699a\"", "url": "https://schema.cloudformation.me-central-1.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"c0cee52c664bec22aa9cad20452c8be4\"", "url": "https://schema.cloudformation.me-central-1.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"4eff125b6a8d9a53bfd01cd720058a77\"", "url": "https://schema.cloudformation.eu-west-2.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"704f1b32b8ed2357b4efeb93a9633e36\"", "url": "https://schema.cloudformation.eu-west-2.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"b44fc7ac1956ad50020dcc9bce2f7e65\"", "url": "https://schema.cloudformation.af-south-1.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"5d90eede497d94dc5acbe3947c384f0f\"", "url": "https://schema.cloudformation.af-south-1.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"cccc338e071c166a131d1cdc9d74fab7\"", "url": "https://schema.cloudformation.us-west-1.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"11cd6603e5e474388338daeea4ec18c5\"", "url": "https://schema.cloudformation.us-west-1.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"2a5f184af1076cf4909dd7dfd66c1091\"", "url": "https://schema.cloudformation.eu-central-1.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"f1245202dc7d03e32366cf183a60f5bf\"", "url": "https://schema.cloudformation.eu-central-1.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"81e2ca053c22e5c8e6f33990c3c1683a\"", "url": "https://schema.cloudformation.ap-south-1.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"be46d4992c4b8b9d9363ed0f29d67bfc\"", "url": "https://schema.cloudformation.ap-south-1.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"5b4a791385a5b8095875163b14a84e12\"", "url": "https://schema.cloudformation.ap-southeast-4.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"8b09581590dc797e7f8b9966df079bcd\"", "url": "https://schema.cloudformation.ap-southeast-4.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"735d724f69e6c702c966f1bd667266dc\"", "url": "https://schema.cloudformation.us-east-2.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"416452f27037f3b97f425284a73bcf9c\"", "url": "https://schema.cloudformation.us-east-2.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"d05bdba3e6c8b67d31218a116456c75b\"", "url": "https://schema.cloudformation.ap-southeast-1.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"b5126a7d839bd20951fcafe4da854d38\"", "url": "https://schema.cloudformation.ap-southeast-1.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"26a4a4a9660b28ecb31a40aa4db8b791\"", "url": "https://schema.cloudformation.ap-southeast-3.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"6edf94664df234608592965308b6f6b7\"", "url": "https://schema.cloudformation.ap-southeast-3.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"d37369e7d40a074400df68c04fb64d28\"", "url": "https://schema.cloudformation.ap-east-1.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"174ddf0e8bbf351c26704e4e2337b16a\"", "url": "https://schema.cloudformation.ap-east-1.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"1faad9cae8146a0781efc270a2cf2ed3\"", "url": "https://schema.cloudformation.sa-east-1.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"3e62904c961fae6ced725b12783ef45d\"", "url": "https://schema.cloudformation.sa-east-1.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"5da9230365aca3d094395f17b18cb452\"", "url": "https://schema.cloudformation.ap-southeast-2.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"83baf7b0a3b0ee8c5f5ba852405d693e\"", "url": "https://schema.cloudformation.ap-southeast-2.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"2b0a5723c1a5e739eea87299385382f9\"", "url": "https://schema.cloudformation.eu-north-1.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"e292cb731d4b0b927a8d86a6301127ae\"", "url": "https://schema.cloudformation.eu-north-1.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"260acbfe6a05bd388bccf119e9747069\"", "url": "https://schema.cloudformation.eu-south-2.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"c75fbe544296584ef4d56b592fb87ed3\"", "url": "https://schema.cloudformation.eu-south-2.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"10d837e83b0dee3b5bdd8025d2e030f2\"", "url": "https://schema.cloudformation.ca-central-1.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"485c9eb6f7e1efdb2af30a9800b05fab\"", "url": "https://schema.cloudformation.ca-central-1.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"316784e70cf0510fea4b81b31ecd8f15\"", "url": "https://schema.cloudformation.eu-west-3.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"8fc65b5326dd82fb678168c25661e666\"", "url": "https://schema.cloudformation.eu-west-3.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"b4ec2276c5d96d861a33de8d7ef645f4\"", "url": "https://schema.cloudformation.ap-northeast-3.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"d89138fadb813a9999c4f07c383b48e7\"", "url": "https://schema.cloudformation.ap-northeast-3.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"47806c7293decd2a8dc0b6e7ec30b75f\"", "url": "https://schema.cloudformation.us-west-2.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"dd30b5012826298b344563720602a94b\"", "url": "https://schema.cloudformation.us-west-2.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"aec4bdf0cc84ce7f3a8486d83c3b41de\"", "url": "https://schema.cloudformation.ap-south-2.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"707ec23d98096f232e1f68967ce937bd\"", "url": "https://schema.cloudformation.ap-south-2.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"6e1fd9ae967709a2a2069c40a2db3aa2\"", "url": "https://schema.cloudformation.us-east-1.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"91387a48563a194dd6a2d7b943bd8005\"", "url": "https://schema.cloudformation.us-east-1.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"854a88a3967badd9ca15fab91e861ad3\"", "url": "https://schema.cloudformation.il-central-1.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"0ff66b9ec501bfa3f5235ad1b46b6901\"", "url": "https://schema.cloudformation.il-central-1.amazonaws.com/CloudformationSchema.zip"}
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"etag": "\"cee4b474888719e8896f94090ac6e16d\"", "url": "https://schema.cloudformation.ap-northeast-1.amazonaws.com/CloudformationSchema.zip"}
1+
{"etag": "\"76471ec41ead11d47f95ac6393542b12\"", "url": "https://schema.cloudformation.ap-northeast-1.amazonaws.com/CloudformationSchema.zip"}

src/cfnlint/data/schemas/patches/extensions/all/aws_accessanalyzer_analyzer/boto.json

+10
Original file line numberDiff line numberDiff line change
@@ -18,5 +18,15 @@
1818
"op": "add",
1919
"path": "/definitions/ArchiveRule/properties/RuleName/pattern",
2020
"value": "[A-Za-z][A-Za-z0-9_.-]*"
21+
},
22+
{
23+
"op": "add",
24+
"path": "/definitions/ArchiveRule/properties/RuleName/maxLength",
25+
"value": 255
26+
},
27+
{
28+
"op": "add",
29+
"path": "/definitions/ArchiveRule/properties/RuleName/minLength",
30+
"value": 1
2131
}
2232
]

0 commit comments

Comments
 (0)