Skip to content

Commit 37aaa9e

Browse files
kddejongkddejong
authored
Fix integration tests (#3615)
* fix integration testing to make it actually work
1 parent fccb4d3 commit 37aaa9e

File tree

6 files changed

+74
-100
lines changed

6 files changed

+74
-100
lines changed

‎test/fixtures/results/integration/ref-no-value.json

+16-14
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@
6161
},
6262
{
6363
"Filename": "test/fixtures/templates/integration/ref-no-value.yaml",
64-
"Id": "2ed7922b-1cd5-30ae-0c5f-918f69c9b782",
64+
"Id": "93c41f0b-56b3-2349-8fb6-4be356befb25",
6565
"Level": "Error",
6666
"Location": {
6767
"End": {
@@ -71,25 +71,26 @@
7171
"Path": [
7272
"Resources",
7373
"IamRole2",
74-
"Properties"
74+
"Properties",
75+
"Ref"
7576
],
7677
"Start": {
7778
"ColumnNumber": 5,
7879
"LineNumber": 26
7980
}
8081
},
81-
"Message": "'AssumeRolePolicyDocument' is a required property",
82+
"Message": "{'Ref': 'AWS::NoValue'} is not of type object",
8283
"ParentId": null,
8384
"Rule": {
84-
"Description": "Make sure that Resources properties that are required exist",
85-
"Id": "E3003",
86-
"ShortDescription": "Required Resource properties are missing",
87-
"Source": "https://github.com/aws-cloudformation/cfn-lint/blob/main/docs/cfn-schema-specification.md#required"
85+
"Description": "Checks resource property values with Primitive Types for values that match those types.",
86+
"Id": "E3012",
87+
"ShortDescription": "Check resource properties values",
88+
"Source": "https://github.com/aws-cloudformation/cfn-lint/blob/main/docs/cfn-schema-specification.md#type"
8889
}
8990
},
9091
{
9192
"Filename": "test/fixtures/templates/integration/ref-no-value.yaml",
92-
"Id": "edfc8e64-ad37-e697-8fb7-5c89d2ff1735",
93+
"Id": "5714e7bb-7c4b-573d-88cb-cbda0df6276d",
9394
"Level": "Error",
9495
"Location": {
9596
"End": {
@@ -99,20 +100,21 @@
99100
"Path": [
100101
"Resources",
101102
"CloudFront1",
102-
"Properties"
103+
"Properties",
104+
"Ref"
103105
],
104106
"Start": {
105107
"ColumnNumber": 5,
106108
"LineNumber": 39
107109
}
108110
},
109-
"Message": "'DistributionConfig' is a required property",
111+
"Message": "{'Ref': 'AWS::NoValue'} is not of type object",
110112
"ParentId": null,
111113
"Rule": {
112-
"Description": "Make sure that Resources properties that are required exist",
113-
"Id": "E3003",
114-
"ShortDescription": "Required Resource properties are missing",
115-
"Source": "https://github.com/aws-cloudformation/cfn-lint/blob/main/docs/cfn-schema-specification.md#required"
114+
"Description": "Checks resource property values with Primitive Types for values that match those types.",
115+
"Id": "E3012",
116+
"ShortDescription": "Check resource properties values",
117+
"Source": "https://github.com/aws-cloudformation/cfn-lint/blob/main/docs/cfn-schema-specification.md#type"
116118
}
117119
},
118120
{

‎test/fixtures/results/quickstart/nist_iam.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@
5757
},
5858
{
5959
"Filename": "test/fixtures/templates/quickstart/nist_iam.yaml",
60-
"Id": "99273eea-4208-d5b5-e641-516b59429087",
60+
"Id": "2902b820-10fb-7008-c4f2-b74be7c678f6",
6161
"Level": "Warning",
6262
"Location": {
6363
"End": {
@@ -78,7 +78,7 @@
7878
"LineNumber": 165
7979
}
8080
},
81-
"Message": "'get*' is not one of ['associateappblockbuilderappblock', 'associateapplicationfleet', 'associateapplicationtoentitlement', 'associatefleet', 'batchassociateuserstack', 'batchdisassociateuserstack', 'copyimage', 'createappblock', 'createappblockbuilder', 'createappblockbuilderstreamingurl', 'createapplication', 'createdirectoryconfig', 'createentitlement', 'createfleet', 'createimagebuilder', 'createimagebuilderstreamingurl', 'createstack', 'createstreamingurl', 'createupdatedimage', 'createusagereportsubscription', 'createuser', 'deleteappblock', 'deleteappblockbuilder', 'deleteapplication', 'deletedirectoryconfig', 'deleteentitlement', 'deletefleet', 'deleteimage', 'deleteimagebuilder', 'deleteimagepermissions', 'deletestack', 'deleteusagereportsubscription', 'deleteuser', 'describeappblockbuilderappblockassociations', 'describeappblockbuilders', 'describeappblocks', 'describeapplicationfleetassociations', 'describeapplications', 'describedirectoryconfigs', 'describeentitlements', 'describefleets', 'describeimagebuilders', 'describeimagepermissions', 'describeimages', 'describesessions', 'describestacks', 'describeusagereportsubscriptions', 'describeuserstackassociations', 'describeusers', 'disableuser', 'disassociateappblockbuilderappblock', 'disassociateapplicationfleet', 'disassociateapplicationfromentitlement', 'disassociatefleet', 'enableuser', 'expiresession', 'listassociatedfleets', 'listassociatedstacks', 'listentitledapplications', 'listtagsforresource', 'startappblockbuilder', 'startfleet', 'startimagebuilder', 'stopappblockbuilder', 'stopfleet', 'stopimagebuilder', 'stream', 'tagresource', 'untagresource', 'updateappblockbuilder', 'updateapplication', 'updatedirectoryconfig', 'updateentitlement', 'updatefleet', 'updateimagepermissions', 'updatestack']",
81+
"Message": "'get*' is not one of ['associateappblockbuilderappblock', 'associateapplicationfleet', 'associateapplicationtoentitlement', 'associatefleet', 'batchassociateuserstack', 'batchdisassociateuserstack', 'copyimage', 'createappblock', 'createappblockbuilder', 'createappblockbuilderstreamingurl', 'createapplication', 'createdirectoryconfig', 'createentitlement', 'createfleet', 'createimagebuilder', 'createimagebuilderstreamingurl', 'createstack', 'createstreamingurl', 'createthemeforstack', 'createupdatedimage', 'createusagereportsubscription', 'createuser', 'deleteappblock', 'deleteappblockbuilder', 'deleteapplication', 'deletedirectoryconfig', 'deleteentitlement', 'deletefleet', 'deleteimage', 'deleteimagebuilder', 'deleteimagepermissions', 'deletestack', 'deletethemeforstack', 'deleteusagereportsubscription', 'deleteuser', 'describeappblockbuilderappblockassociations', 'describeappblockbuilders', 'describeappblocks', 'describeapplicationfleetassociations', 'describeapplications', 'describedirectoryconfigs', 'describeentitlements', 'describefleets', 'describeimagebuilders', 'describeimagepermissions', 'describeimages', 'describesessions', 'describestacks', 'describethemeforstack', 'describeusagereportsubscriptions', 'describeuserstackassociations', 'describeusers', 'disableuser', 'disassociateappblockbuilderappblock', 'disassociateapplicationfleet', 'disassociateapplicationfromentitlement', 'disassociatefleet', 'enableuser', 'expiresession', 'listassociatedfleets', 'listassociatedstacks', 'listentitledapplications', 'listtagsforresource', 'startappblockbuilder', 'startfleet', 'startimagebuilder', 'stopappblockbuilder', 'stopfleet', 'stopimagebuilder', 'stream', 'tagresource', 'untagresource', 'updateappblockbuilder', 'updateapplication', 'updatedirectoryconfig', 'updateentitlement', 'updatefleet', 'updateimagepermissions', 'updatestack', 'updatethemeforstack']",
8282
"ParentId": null,
8383
"Rule": {
8484
"Description": "Check for valid IAM Permissions",

‎test/integration/__init__.py

+23-13
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,15 @@
66
import json
77
import subprocess
88
import unittest
9+
from copy import deepcopy
10+
from io import StringIO
911
from pathlib import Path
1012
from typing import Any, Dict, List
13+
from unittest.mock import patch
1114

1215
from cfnlint.config import configure_logging
1316
from cfnlint.decode import cfn_yaml
17+
from cfnlint.formatters import JsonFormatter
1418
from cfnlint.runner import Runner
1519

1620

@@ -102,16 +106,22 @@ def run_module_integration_scenarios(self, config):
102106
for result in expected_results:
103107
result["Filename"] = str(Path(result.get("Filename")))
104108

105-
template = cfn_yaml.load(filename)
106-
107-
runner = Runner(config)
108-
matches = list(runner.validate_template(filename, template))
109-
110-
# Only check that the error count matches as the formats are different
111-
self.assertEqual(
112-
len(expected_results),
113-
len(matches),
114-
"Expected {} failures, got {} on {}".format(
115-
len(expected_results), matches, filename
116-
),
117-
)
109+
# template = cfn_yaml.load(filename)
110+
scenario_config = deepcopy(config)
111+
scenario_config.cli_args.template_alt = [filename]
112+
scenario_config.cli_args.format = "json"
113+
114+
runner = Runner(scenario_config)
115+
116+
print(f"Running test for {filename!r}")
117+
with patch("sys.exit") as exit:
118+
with patch("sys.stdout", new=StringIO()) as out:
119+
runner.cli()
120+
exit.assert_called_once_with(scenario.get("exit_code", 0))
121+
122+
output = json.loads(out.getvalue())
123+
self.assertEqual(
124+
expected_results,
125+
output,
126+
f"Test for {filename!r} got results: {output!r}",
127+
)

‎test/integration/test_good_templates.py

+23-61
Original file line numberDiff line numberDiff line change
@@ -40,96 +40,58 @@ class TestQuickStartTemplates(BaseCliTestCase):
4040
),
4141
"results": [
4242
{
43-
"Filename": str(
44-
Path(
45-
"test/fixtures/templates/bad/transform_serverless_template.yaml"
46-
)
47-
),
48-
"Id": "74181426-e865-10eb-96fd-908dfd30a358",
43+
"Filename": "test/fixtures/templates/bad/transform_serverless_template.yaml",
44+
"Id": "9e05773a-b0d0-f157-2955-596d9bd54749",
45+
"Level": "Error",
4946
"Location": {
50-
"Start": {"ColumnNumber": 1, "LineNumber": 1},
5147
"End": {"ColumnNumber": 2, "LineNumber": 1},
5248
"Path": None,
49+
"Start": {"ColumnNumber": 1, "LineNumber": 1},
5350
},
51+
"Message": "Error transforming template: Resource with id [myFunctionMyTimer] is invalid. Missing required property 'Schedule'.",
5452
"ParentId": None,
5553
"Rule": {
54+
"Description": "Errors found when performing transformation on the template",
5655
"Id": "E0001",
57-
"Description": (
58-
"Errors found when performing transformation on the"
59-
" template"
60-
),
61-
"Source": ("https://github.com/aws-cloudformation/cfn-lint"),
62-
"ShortDescription": (
63-
"Error found when transforming the template"
64-
),
56+
"ShortDescription": "Error found when transforming the template",
57+
"Source": "https://github.com/aws-cloudformation/cfn-lint",
6558
},
66-
"Level": "Error",
67-
"Message": (
68-
"Error transforming template: Resource with id [AppName] is"
69-
" invalid. Resource is missing the required [Location]"
70-
" property."
71-
),
7259
},
7360
{
74-
"Filename": str(
75-
Path(
76-
"test/fixtures/templates/bad/transform_serverless_template.yaml"
77-
)
78-
),
61+
"Filename": "test/fixtures/templates/bad/transform_serverless_template.yaml",
7962
"Id": "fd751fa3-7d1f-e194-7108-eb08352814c8",
63+
"Level": "Error",
8064
"Location": {
81-
"Start": {"ColumnNumber": 1, "LineNumber": 1},
8265
"End": {"ColumnNumber": 2, "LineNumber": 1},
8366
"Path": None,
67+
"Start": {"ColumnNumber": 1, "LineNumber": 1},
8468
},
69+
"Message": "Error transforming template: Resource with id [ExampleLayer] is invalid. Missing required property 'ContentUri'.",
8570
"ParentId": None,
8671
"Rule": {
72+
"Description": "Errors found when performing transformation on the template",
8773
"Id": "E0001",
88-
"Description": (
89-
"Errors found when performing transformation on the"
90-
" template"
91-
),
92-
"Source": ("https://github.com/aws-cloudformation/cfn-lint"),
93-
"ShortDescription": (
94-
"Error found when transforming the template"
95-
),
74+
"ShortDescription": "Error found when transforming the template",
75+
"Source": "https://github.com/aws-cloudformation/cfn-lint",
9676
},
97-
"Level": "Error",
98-
"Message": (
99-
"Error transforming template: Resource with id [ExampleLayer]"
100-
" is invalid. Missing required property 'ContentUri'."
101-
),
10277
},
10378
{
104-
"Filename": str(
105-
Path(
106-
"test/fixtures/templates/bad/transform_serverless_template.yaml"
107-
)
108-
),
109-
"Id": "9e05773a-b0d0-f157-2955-596d9bd54749",
79+
"Filename": "test/fixtures/templates/bad/transform_serverless_template.yaml",
80+
"Id": "74181426-e865-10eb-96fd-908dfd30a358",
81+
"Level": "Error",
11082
"Location": {
111-
"Start": {"ColumnNumber": 1, "LineNumber": 1},
11283
"End": {"ColumnNumber": 2, "LineNumber": 1},
11384
"Path": None,
85+
"Start": {"ColumnNumber": 1, "LineNumber": 1},
11486
},
87+
"Message": "Error transforming template: Resource with id [AppName] is invalid. Resource is missing the required [Location] property.",
11588
"ParentId": None,
11689
"Rule": {
90+
"Description": "Errors found when performing transformation on the template",
11791
"Id": "E0001",
118-
"Description": (
119-
"Errors found when performing transformation on the"
120-
" template"
121-
),
122-
"Source": ("https://github.com/aws-cloudformation/cfn-lint"),
123-
"ShortDescription": (
124-
"Error found when transforming the template"
125-
),
92+
"ShortDescription": "Error found when transforming the template",
93+
"Source": "https://github.com/aws-cloudformation/cfn-lint",
12694
},
127-
"Level": "Error",
128-
"Message": (
129-
"Error transforming template: Resource with id"
130-
" [myFunctionMyTimer] is invalid. Missing required property"
131-
" 'Schedule'."
132-
),
13395
},
13496
],
13597
"exit_code": 2,

‎test/integration/test_integration_templates.py

+5-5
Original file line numberDiff line numberDiff line change
@@ -24,21 +24,21 @@ class TestQuickStartTemplates(BaseCliTestCase):
2424
"exit_code": 0,
2525
},
2626
{
27-
"filename": ("test/fixtures/templates/integration/dynamic-references.yaml"),
27+
"filename": "test/fixtures/templates/integration/dynamic-references.yaml",
2828
"results_filename": (
29-
"test/fixtures/results/integration/dynamic-references.json"
29+
"test/fixtures/results/integration" "/dynamic-references.json"
3030
),
3131
"exit_code": 2,
3232
},
3333
{
34-
"filename": ("test/fixtures/templates/integration/ref-no-value.yaml"),
34+
"filename": "test/fixtures/templates/integration/ref-no-value.yaml",
3535
"results_filename": ("test/fixtures/results/integration/ref-no-value.json"),
3636
"exit_code": 2,
3737
},
3838
{
39-
"filename": ("test/fixtures/templates/integration/metdata.yaml"),
39+
"filename": "test/fixtures/templates/integration/metdata.yaml",
4040
"results_filename": ("test/fixtures/results/integration/metadata.json"),
41-
"exit_code": 2,
41+
"exit_code": 4,
4242
},
4343
]
4444

‎test/integration/test_quickstart_templates.py

+5-5
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,14 @@ class TestQuickStartTemplates(BaseCliTestCase):
3030
{
3131
"filename": "test/fixtures/templates/quickstart/nist_application.yaml",
3232
"results_filename": (
33-
"test/fixtures/results/quickstart/nist_application.json"
33+
"test/fixtures/results/quickstart" "/nist_application.json"
3434
),
3535
"exit_code": 14,
3636
},
3737
{
3838
"filename": "test/fixtures/templates/quickstart/nist_config_rules.yaml",
3939
"results_filename": (
40-
"test/fixtures/results/quickstart/nist_config_rules.json"
40+
"test/fixtures/results/quickstart/" "nist_config_rules.json"
4141
),
4242
"exit_code": 6,
4343
},
@@ -54,21 +54,21 @@ class TestQuickStartTemplates(BaseCliTestCase):
5454
{
5555
"filename": "test/fixtures/templates/quickstart/nist_vpc_management.yaml",
5656
"results_filename": (
57-
"test/fixtures/results/quickstart/nist_vpc_management.json"
57+
"test/fixtures/results/quickstart/" "nist_vpc_management.json"
5858
),
5959
"exit_code": 14,
6060
},
6161
{
6262
"filename": "test/fixtures/templates/quickstart/nist_vpc_production.yaml",
6363
"results_filename": (
64-
"test/fixtures/results/quickstart/nist_vpc_production.json"
64+
"test/fixtures/results/quickstart/" "nist_vpc_production.json"
6565
),
6666
"exit_code": 14,
6767
},
6868
{
6969
"filename": "test/fixtures/templates/quickstart/openshift_master.yaml",
7070
"results_filename": (
71-
"test/fixtures/results/quickstart/openshift_master.json"
71+
"test/fixtures/results/quickstart" "/openshift_master.json"
7272
),
7373
"exit_code": 8,
7474
},

0 commit comments

Comments
 (0)