fix(triggers): unable to trigger two lambda functions (#22124)

Closes #22110

This PR updates the custom resource provider in the Trigger construct to use `addToRolePolicy` to add new statements to the IAM policy, rather than the constructor, so it can be used to trigger more than one lambda function.

Also adds a one-minute retry in the custom resource provider lambda function as IAM policy changes take some time to propagate.

Also refactored tests to increase coverage.

----

### All Submissions:

* [X] Have you followed the guidelines in our [Contributing guide?](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md)

### Adding new Unconventional Dependencies:

* [ ] This PR adds new unconventional dependencies following the process described [here](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md/#adding-new-unconventional-dependencies)

### New Features

* [ ] Have you added the new feature to an [integration test](https://github.com/aws/aws-cdk/blob/main/INTEGRATION_TESTS.md)?
	* [ ] Did you use `yarn integ` to deploy the infrastructure and generate the snapshot (i.e. `yarn integ` without `--dry-run`)?

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

Author: cecheta

Diff for: packages/@aws-cdk/triggers/lib/lambda/index.ts

@@ -5,11 +5,35 @@ import * as AWS from 'aws-sdk';
 
 export type InvokeFunction = (functionName: string) => Promise<AWS.Lambda.InvocationResponse>;
 
-export const invoke: InvokeFunction = async functionName => {
+export const invoke: InvokeFunction = async (functionName) => {
   const lambda = new AWS.Lambda();
   const invokeRequest = { FunctionName: functionName };
   console.log({ invokeRequest });
-  const invokeResponse = await lambda.invoke(invokeRequest).promise();
+
+  // IAM policy changes can take some time to fully propagate
+  // Therefore, retry for up to one minute
+
+  let retryCount = 0;
+  const delay = 5000;
+
+  let invokeResponse;
+  while (true) {
+    try {
+      invokeResponse = await lambda.invoke(invokeRequest).promise();
+      break;
+    } catch (error) {
+      if (error instanceof Error && (error as AWS.AWSError).code === 'AccessDeniedException' && retryCount < 12) {
+        retryCount++;
+        await new Promise((resolve) => {
+          setTimeout(resolve, delay);
+        });
+        continue;
+      }
+
+      throw error;
+    }
+  }
+
   console.log({ invokeResponse });
   return invokeResponse;
 };
@@ -37,7 +61,7 @@ export async function handler(event: AWSLambda.CloudFormationCustomResourceEvent
   if (invokeResponse.FunctionError) {
     throw new Error(parseError(invokeResponse.Payload?.toString()));
   }
-};
+}
 
 /**
  * Parse the error message from the lambda function.

Diff for: packages/@aws-cdk/triggers/lib/trigger.ts

@@ -82,13 +82,12 @@ export class Trigger extends Construct implements ITrigger {
     const provider = CustomResourceProvider.getOrCreateProvider(this, 'AWSCDK.TriggerCustomResourceProvider', {
       runtime: CustomResourceProviderRuntime.NODEJS_14_X,
       codeDirectory: join(__dirname, 'lambda'),
-      policyStatements: [
-        {
-          Effect: 'Allow',
-          Action: ['lambda:InvokeFunction'],
-          Resource: [`${props.handler.functionArn}:*`],
-        },
-      ],
+    });
+
+    provider.addToRolePolicy({
+      Effect: 'Allow',
+      Action: ['lambda:InvokeFunction'],
+      Resource: [`${props.handler.functionArn}:*`],
     });
 
     new CustomResource(this, 'Default', {
@@ -133,4 +132,4 @@ export enum TriggerInvalidation {
    * of the AWS Lambda function, which gets recreated every time the handler changes.
    */
   HANDLER_CHANGE = 'WHEN_FUNCTION_CHANGES',
-}
+}

Diff for: packages/@aws-cdk/triggers/test/integ.triggers.js.snapshot/MyStack.assets.json

@@ -1,28 +1,28 @@
 {
-  "version": "20.0.0",
+  "version": "21.0.0",
   "files": {
-    "f942cf8dea09e8b74bc8da73a643a8b2639fe7f93c6eb60e338c56224decd68a": {
+    "ae344ba0df770ab1ea166e5b55e0ff5681c951c0a34d8e724e430b88957c50d4": {
       "source": {
-        "path": "asset.f942cf8dea09e8b74bc8da73a643a8b2639fe7f93c6eb60e338c56224decd68a",
+        "path": "asset.ae344ba0df770ab1ea166e5b55e0ff5681c951c0a34d8e724e430b88957c50d4",
         "packaging": "zip"
       },
       "destinations": {
         "current_account-current_region": {
           "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
-          "objectKey": "f942cf8dea09e8b74bc8da73a643a8b2639fe7f93c6eb60e338c56224decd68a.zip",
+          "objectKey": "ae344ba0df770ab1ea166e5b55e0ff5681c951c0a34d8e724e430b88957c50d4.zip",
           "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
         }
       }
     },
-    "e1b247eb3c41fa99451971925f9a24d2d7e5dec4d36ed03c43640d0b477a41ce": {
+    "4155391c7a2ef259603bc3a1d18d0bad9a2478b51ed773d2c06efab5a0a51c56": {
       "source": {
         "path": "MyStack.template.json",
         "packaging": "file"
       },
       "destinations": {
         "current_account-current_region": {
           "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
-          "objectKey": "e1b247eb3c41fa99451971925f9a24d2d7e5dec4d36ed03c43640d0b477a41ce.json",
+          "objectKey": "4155391c7a2ef259603bc3a1d18d0bad9a2478b51ed773d2c06efab5a0a51c56.json",
           "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
         }
       }

Diff for: packages/@aws-cdk/triggers/test/integ.triggers.js.snapshot/MyStack.template.json

@@ -133,6 +133,28 @@
            ]
           }
          ]
+        },
+        {
+         "Effect": "Allow",
+         "Action": [
+          "lambda:InvokeFunction"
+         ],
+         "Resource": [
+          {
+           "Fn::Join": [
+            "",
+            [
+             {
+              "Fn::GetAtt": [
+               "MySecondFunction0F0B51EB",
+               "Arn"
+              ]
+             },
+             ":*"
+            ]
+           ]
+          }
+         ]
         }
        ]
       }
@@ -147,7 +169,7 @@
      "S3Bucket": {
       "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
      },
-     "S3Key": "f942cf8dea09e8b74bc8da73a643a8b2639fe7f93c6eb60e338c56224decd68a.zip"
+     "S3Key": "ae344ba0df770ab1ea166e5b55e0ff5681c951c0a34d8e724e430b88957c50d4.zip"
     },
     "Timeout": 900,
     "MemorySize": 128,
@@ -163,6 +185,80 @@
    "DependsOn": [
     "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A"
    ]
+  },
+  "MySecondFunctionServiceRole5B930841": {
+   "Type": "AWS::IAM::Role",
+   "Properties": {
+    "AssumeRolePolicyDocument": {
+     "Statement": [
+      {
+       "Action": "sts:AssumeRole",
+       "Effect": "Allow",
+       "Principal": {
+        "Service": "lambda.amazonaws.com"
+       }
+      }
+     ],
+     "Version": "2012-10-17"
+    },
+    "ManagedPolicyArns": [
+     {
+      "Fn::Join": [
+       "",
+       [
+        "arn:",
+        {
+         "Ref": "AWS::Partition"
+        },
+        ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
+       ]
+      ]
+     }
+    ]
+   }
+  },
+  "MySecondFunction0F0B51EB": {
+   "Type": "AWS::Lambda::Function",
+   "Properties": {
+    "Code": {
+     "ZipFile": "exports.handler = function() { console.log(\"hello\"); };"
+    },
+    "Role": {
+     "Fn::GetAtt": [
+      "MySecondFunctionServiceRole5B930841",
+      "Arn"
+     ]
+    },
+    "Handler": "index.handler",
+    "Runtime": "nodejs16.x"
+   },
+   "DependsOn": [
+    "MySecondFunctionServiceRole5B930841"
+   ]
+  },
+  "MySecondFunctionTrigger8C61EC28": {
+   "Type": "Custom::Trigger",
+   "Properties": {
+    "ServiceToken": {
+     "Fn::GetAtt": [
+      "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91",
+      "Arn"
+     ]
+    },
+    "HandlerArn": {
+     "Ref": "MySecondFunctionCurrentVersion7D497B5D173a4bb1f758991022ea97d651403362"
+    }
+   },
+   "UpdateReplacePolicy": "Delete",
+   "DeletionPolicy": "Delete"
+  },
+  "MySecondFunctionCurrentVersion7D497B5D173a4bb1f758991022ea97d651403362": {
+   "Type": "AWS::Lambda::Version",
+   "Properties": {
+    "FunctionName": {
+     "Ref": "MySecondFunction0F0B51EB"
+    }
+   }
   }
  },
  "Parameters": {