New rule I3510 to validate action and resources match (#4019)

* New rule I3510 to validate IAM asterisk resources

Author: kddejong

src/cfnlint/data/AdditionalSpecs/Policies.json

@@ -239,6 +239,9 @@
      ]
     }
    ],
+   "cfnLint": [
+    "AWS::IAM::Policy/Properties/PolicyDocument/Statement"
+   ],
    "properties": {
     "Action": {
      "$ref": "#/definitions/Action"

src/cfnlint/data/schemas/other/iam/policy.json

@@ -23,6 +23,9 @@
      ]
     }
    ],
+   "cfnLint": [
+    "AWS::IAM::Policy/Properties/PolicyDocument/Statement"
+   ],
    "properties": {
     "Action": {
      "$ref": "policy#/definitions/Action"

src/cfnlint/data/schemas/other/iam/policy_identity.json

@@ -123,30 +123,72 @@ def update_documentation(rules):
 def update_iam_policies():
     """update iam policies file"""
 
-    url = "https://awspolicygen.s3.amazonaws.com/js/policies.js"
+    url = "https://servicereference.us-east-1.amazonaws.com"
 
     filename = os.path.join(
         os.path.dirname(cfnlint.data.AdditionalSpecs.__file__), "Policies.json"
     )
     LOGGER.debug("Downloading policies %s into %s", url, filename)
 
-    content = get_url_content(url)
+    services = json.loads(get_url_content(url))
 
-    content = content.split("app.PolicyEditorConfig=")[1]
-    content = json.loads(content)
+    def _clean_arn_formats(arn):
+        arn_parts = arn.split(":", 5)
 
-    actions = {
-        "Manage Amazon API Gateway": ["HEAD", "OPTIONS"],
-        "Amazon API Gateway Management": ["HEAD", "OPTIONS"],
-        "Amazon API Gateway Management V2": ["HEAD", "OPTIONS"],
-        "Amazon Kinesis Video Streams": ["StartStreamEncryption"],
-    }
-    for k, v in actions.items():
-        if content.get("serviceMap").get(k):
-            content["serviceMap"][k]["Actions"].extend(v)
+        resource = arn_parts[5]
+        delimiter = None
+        for d in [":", "/"]:
+            if d in resource:
+                delimiter = d
+                break
         else:
-            LOGGER.debug('"%s" was not found in the policies file', k)
+            delimiter = ":"
+
+        if delimiter:
+            resource_parts = []
+            for resource_part in resource.split(delimiter):
+                if "${" in resource_part:
+                    resource_parts.append(".*")
+                    break
+
+                resource_parts.append(resource_part)
+
+            arn_parts[5] = delimiter.join(resource_parts)
+
+        return ":".join(arn_parts)
+
+    def _processes_a_service(data):
+        results = {
+            "Actions": {},
+            "Resources": {},
+        }
+
+        for action in data.get("Actions", []):
+            results["Actions"][action.get("Name").lower()] = {}
+            if "Resources" in action:
+                results["Actions"][action.get("Name").lower()] = {
+                    "Resources": list([i["Name"] for i in action["Resources"]])
+                }
+
+        for resource in data.get("Resources", []):
+            results["Resources"][resource.get("Name").lower()] = {
+                "ARNFormats": [
+                    _clean_arn_formats(arn) for arn in resource.get("ARNFormats")
+                ],
+            }
+            if "ConditionKeys" in resource:
+                results["Resources"][resource.get("Name").lower()]["ConditionKeys"] = (
+                    resource.get("ConditionKeys")
+                )
+
+        return results
+
+    data = {}
+    for service in services:
+        name = service.get("service")
+        content = json.loads(get_url_content(service.get("url")))
+        data[name] = _processes_a_service(content)
 
     with open(filename, "w", encoding="utf-8") as f:
-        json.dump(content, f, indent=1, sort_keys=True, separators=(",", ": "))
+        json.dump(data, f, indent=1, sort_keys=True, separators=(",", ": "))
         f.write("\n")

src/cfnlint/maintenance.py

@@ -27,7 +27,7 @@ def __init__(self):
         super().__init__(
             ["AWS::IAM::Policy/Properties/PolicyDocument/Statement/Action"]
         )
-        self.service_map = self.load_service_map()
+        self.service_map = load_resource(AdditionalSpecs, "Policies.json")
 
     def validate(
         self, validator: Validator, _, instance: Any, schema: dict[str, Any]
@@ -56,7 +56,7 @@ def validate(
             permission = permission.lower()
 
             if service in self.service_map:
-                enums = self.service_map[service]
+                enums = list(self.service_map[service].get("Actions", []).keys())
                 if permission == "*":
                     pass
                 elif permission.endswith("*"):
@@ -69,15 +69,12 @@ def validate(
 
                 elif permission.startswith("*"):
                     wilcarded_permission = permission.split("*")[1]
-                    if not any(
-                        wilcarded_permission in action
-                        for action in self.service_map[service]
-                    ):
+                    if not any(wilcarded_permission in action for action in enums):
                         yield ValidationError(
                             f"{permission!r} is not one of {enums!r}",
                             rule=self,
                         )
-                elif permission not in self.service_map[service]:
+                elif permission not in enums:
                     yield ValidationError(
                         f"{permission!r} is not one of {enums!r}",
                         rule=self,
@@ -87,25 +84,3 @@ def validate(
                     f"{service!r} is not one of {list(self.service_map.keys())!r}",
                     rule=self,
                 )
-
-    def load_service_map(self):
-        """
-        Convert policies.json into a simpler version for more efficient key lookup.
-        """
-        service_map = load_resource(AdditionalSpecs, "Policies.json")["serviceMap"]
-
-        policy_service_map = {}
-
-        for _, properties in service_map.items():
-            # The services and actions are case insensitive
-            service = properties["StringPrefix"].lower()
-            actions = [x.lower() for x in properties["Actions"]]
-
-            # Some services have the same name for different
-            # generations; like elasticloadbalancing.
-            if service in policy_service_map:
-                policy_service_map[service] += actions
-            else:
-                policy_service_map[service] = actions
-
-        return policy_service_map

src/cfnlint/rules/resources/iam/Permissions.py

@@ -71,6 +71,4 @@ def validate(
             )
 
         for err in iam_validator.iter_errors(policy):
-            if not err.validator.startswith("fn_") and err.validator not in ["cfnLint"]:
-                err.rule = self
-            yield err
+            yield self._clean_error(err)

src/cfnlint/rules/resources/iam/Policy.py

@@ -0,0 +1,171 @@
+"""
+Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+SPDX-License-Identifier: MIT-0
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from typing import Any
+
+from cfnlint.data import AdditionalSpecs
+from cfnlint.helpers import ensure_list, is_function, load_resource
+from cfnlint.jsonschema import ValidationError, ValidationResult, Validator
+from cfnlint.rules.helpers import get_value_from_path
+from cfnlint.rules.jsonschema.CfnLintKeyword import CfnLintKeyword
+
+
+class _Arn:
+
+    def __init__(self, full_arn: str):
+        arn_parts = full_arn.split(":", 5)
+
+        self._parts: list[str] = ["*", "*", "*", "*", "*", "*"]
+
+        for i, arn_part in enumerate(arn_parts):
+            self._parts[i] = arn_part
+
+    def __repr__(self):
+        return ":".join(self._parts)
+
+    @property
+    def parts(self):
+        return self._parts
+
+    def __eq__(self, value: Any):
+        if not isinstance(value, _Arn):
+            return False
+
+        for i in range(0, 5):
+            if self.parts[i] == "*":
+                continue
+            if value.parts[i] in ["${Partition}", "${Region}", "${Account}"]:
+                continue
+            if self.parts[i] != value.parts[i]:
+                return False
+
+        if self.parts[5] == "*":
+            return True
+
+        delimiter = None
+        if ":" in value.parts[5]:
+            delimiter = ":"
+        elif "/" in value.parts[5]:
+            delimiter = "/"
+
+        if not delimiter:
+            return True
+
+        for x, y in zip(
+            self.parts[5].split(delimiter), value.parts[5].split(delimiter)
+        ):
+            if x == y:
+                continue
+            if x == "*" or y == "" or y == ".*":
+                return True
+            if x.startswith(y) and "*" in x:
+                return True
+
+            return False
+
+        return True
+
+
+class StatementResources(CfnLintKeyword):
+
+    id = "I3510"
+    shortdesc = "Validate statement resources match the actions"
+    description = (
+        "IAM policy statements have different constraints between actions and "
+        "resources. This rule validates that resource ARNs or asterisks match "
+        "the actions."
+    )
+    source_url = "https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/iam-access-control-overview-cwl.html"
+    tags = ["resources", "iam"]
+
+    def __init__(self):
+        super().__init__(
+            ["AWS::IAM::Policy/Properties/PolicyDocument/Statement"],
+        )
+        self.service_map = load_resource(AdditionalSpecs, "Policies.json")
+
+    def validate(
+        self, validator: Validator, keywords: Any, instance: Any, schema: dict[str, Any]
+    ) -> ValidationResult:
+
+        if not validator.is_type(instance, "object"):
+            return
+
+        using_fn_arns = False
+        all_resources: set[str] = set()
+        for resources, _ in get_value_from_path(
+            validator, instance, path=deque(["Resource"])
+        ):
+            resources = ensure_list(resources)
+
+            for resource in resources:
+                if not isinstance(resource, str):
+                    k, v = is_function(resource)
+                    if k is None:
+                        continue
+                    if k == "Ref" and validator.is_type(v, "string"):
+                        if v in validator.context.parameters:
+                            return
+                    using_fn_arns = True
+                    continue
+                if resource == "*":
+                    return
+                all_resources.add(resource)
+
+        all_resource_arns = [_Arn(a) for a in all_resources]
+        for actions, _ in get_value_from_path(
+            validator, instance, path=deque(["Action"])
+        ):
+            actions = ensure_list(actions)
+
+            for action in actions:
+
+                if not validator.is_type(action, "string"):
+                    continue
+                if "*" in action:
+                    continue
+                if ":" not in action:
+                    continue
+                service, permission = action.split(":", 1)
+                service = service.lower()
+                permission = permission.lower()
+
+                if service not in self.service_map:
+                    continue
+
+                if permission not in self.service_map[service]["Actions"]:
+                    continue
+
+                resources = self.service_map[service]["Actions"][permission].get(
+                    "Resources", None
+                )
+                if isinstance(resources, (list, str)):
+                    if using_fn_arns:
+                        continue
+                    resources = ensure_list(resources)
+                    for resource in resources:
+                        arn_formats = self.service_map[service]["Resources"][
+                            resource
+                        ].get("ARNFormats")
+                        for arn_format in arn_formats:
+                            arn = _Arn(arn_format)
+                            if arn not in all_resource_arns:
+                                yield ValidationError(
+                                    (
+                                        f"action {action!r} requires "
+                                        f"a resource of {arn_formats!r}"
+                                    ),
+                                    path=deque(["Resource"]),
+                                    rule=self,
+                                )
+                else:
+                    yield ValidationError(
+                        f"action {action!r} requires a resource of '*'",
+                        path=deque(["Resource"]),
+                        rule=self,
+                    )

src/cfnlint/rules/resources/iam/StatementResources.py

@@ -89,8 +89,8 @@
     },
     {
         "Filename": "test/fixtures/templates/quickstart/nist_application.yaml",
-        "Id": "81670f17-4f0b-a2b6-f94e-4385058cb73d",
-        "Level": "Error",
+        "Id": "e7287f1c-73b2-cb51-ec2b-8ab42c30ca27",
+        "Level": "Warning",
         "Location": {
             "End": {
                 "ColumnNumber": 12,
@@ -109,10 +109,10 @@
         "Message": "{'Ref': 'pSecurityAlarmTopic'} does not match '(^arn:(aws|aws-cn|aws-us-gov):[^:]+:[^:]*(:(?:\\\\d{12}|\\\\*|aws)?:.+|)|\\\\*)$' when 'Ref' is resolved",
         "ParentId": null,
         "Rule": {
-            "Description": "IAM identity polices are embedded JSON in CloudFormation. This rule validates those embedded policies.",
-            "Id": "E3510",
-            "ShortDescription": "Validate identity based IAM polices",
-            "Source": "https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_identity-vs-resource.html"
+            "Description": "Resolve the Ref and then validate the values against the schema",
+            "Id": "W1030",
+            "ShortDescription": "Validate the values that come from a Ref function",
+            "Source": "https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/intrinsic-function-reference-ref.html"
         }
     },
     {

test/fixtures/results/quickstart/nist_application.json

@@ -57,7 +57,7 @@
     },
     {
         "Filename": "test/fixtures/templates/quickstart/nist_iam.yaml",
-        "Id": "2902b820-10fb-7008-c4f2-b74be7c678f6",
+        "Id": "4c9089fd-fdc9-d2fc-b4ad-6c0aacb89a01",
         "Level": "Warning",
         "Location": {
             "End": {
@@ -78,7 +78,7 @@
                 "LineNumber": 165
             }
         },
-        "Message": "'get*' is not one of ['associateappblockbuilderappblock', 'associateapplicationfleet', 'associateapplicationtoentitlement', 'associatefleet', 'batchassociateuserstack', 'batchdisassociateuserstack', 'copyimage', 'createappblock', 'createappblockbuilder', 'createappblockbuilderstreamingurl', 'createapplication', 'createdirectoryconfig', 'createentitlement', 'createfleet', 'createimagebuilder', 'createimagebuilderstreamingurl', 'createstack', 'createstreamingurl', 'createthemeforstack', 'createupdatedimage', 'createusagereportsubscription', 'createuser', 'deleteappblock', 'deleteappblockbuilder', 'deleteapplication', 'deletedirectoryconfig', 'deleteentitlement', 'deletefleet', 'deleteimage', 'deleteimagebuilder', 'deleteimagepermissions', 'deletestack', 'deletethemeforstack', 'deleteusagereportsubscription', 'deleteuser', 'describeappblockbuilderappblockassociations', 'describeappblockbuilders', 'describeappblocks', 'describeapplicationfleetassociations', 'describeapplications', 'describedirectoryconfigs', 'describeentitlements', 'describefleets', 'describeimagebuilders', 'describeimagepermissions', 'describeimages', 'describesessions', 'describestacks', 'describethemeforstack', 'describeusagereportsubscriptions', 'describeuserstackassociations', 'describeusers', 'disableuser', 'disassociateappblockbuilderappblock', 'disassociateapplicationfleet', 'disassociateapplicationfromentitlement', 'disassociatefleet', 'enableuser', 'expiresession', 'listassociatedfleets', 'listassociatedstacks', 'listentitledapplications', 'listtagsforresource', 'startappblockbuilder', 'startfleet', 'startimagebuilder', 'stopappblockbuilder', 'stopfleet', 'stopimagebuilder', 'stream', 'tagresource', 'untagresource', 'updateappblockbuilder', 'updateapplication', 'updatedirectoryconfig', 'updateentitlement', 'updatefleet', 'updateimagepermissions', 'updatestack', 'updatethemeforstack']",
+        "Message": "'get*' is not one of ['associateappblockbuilderappblock', 'associateapplicationfleet', 'associateapplicationtoentitlement', 'associatefleet', 'batchassociateuserstack', 'batchdisassociateuserstack', 'copyimage', 'createappblock', 'createappblockbuilder', 'createappblockbuilderstreamingurl', 'createapplication', 'createdirectoryconfig', 'createentitlement', 'createfleet', 'createimagebuilder', 'createimagebuilderstreamingurl', 'createstack', 'createstreamingurl', 'createthemeforstack', 'createupdatedimage', 'createusagereportsubscription', 'createuser', 'deleteappblock', 'deleteappblockbuilder', 'deleteapplication', 'deletedirectoryconfig', 'deleteentitlement', 'deletefleet', 'deleteimage', 'deleteimagebuilder', 'deleteimagepermissions', 'deletestack', 'deletethemeforstack', 'deleteusagereportsubscription', 'deleteuser', 'describeappblockbuilderappblockassociations', 'describeappblockbuilders', 'describeappblocks', 'describeapplicationfleetassociations', 'describeapplications', 'describedirectoryconfigs', 'describeentitlements', 'describefleets', 'describeimagebuilders', 'describeimagepermissions', 'describeimages', 'describesessions', 'describestacks', 'describethemeforstack', 'describeusagereportsubscriptions', 'describeusers', 'describeuserstackassociations', 'disableuser', 'disassociateappblockbuilderappblock', 'disassociateapplicationfleet', 'disassociateapplicationfromentitlement', 'disassociatefleet', 'enableuser', 'expiresession', 'listassociatedfleets', 'listassociatedstacks', 'listentitledapplications', 'listtagsforresource', 'startappblockbuilder', 'startfleet', 'startimagebuilder', 'stopappblockbuilder', 'stopfleet', 'stopimagebuilder', 'stream', 'tagresource', 'untagresource', 'updateappblockbuilder', 'updateapplication', 'updatedirectoryconfig', 'updateentitlement', 'updatefleet', 'updateimagepermissions', 'updatestack', 'updatethemeforstack']",
         "ParentId": null,
         "Rule": {
             "Description": "Check for valid IAM Permissions",