chore: run e2e tests on demand

.github/workflows/on-merge-to-main.yml

@@ -7,7 +7,9 @@ on:
 jobs:
   publish:
     runs-on: ubuntu-latest
-
+    permissions:
+      id-token: write # needed to interact with GitHub's OIDC Token endpoint.
+      contents: read
     steps:
     - name: "Checkout"
       uses: actions/checkout@v2
@@ -32,6 +34,21 @@ jobs:
       run: npm run lerna-lint
     - name: Run tests
       run: npm run lerna-test
+    - name: "Version and publish"
+      env:
+        GH_TOKEN: ${{ secrets.GH_PUBLISH_TOKEN }}
+      run: |
+        git config --global user.name 'github-actions[bot]'
+        git config --global user.email 'github-actions[bot]@users.noreply.github.com'
+        git remote set-url origin https://x-access-token:${GH_TOKEN}@github.com/$GITHUB_REPOSITORY
+
+        # For merge to main we 
+        ## don't create github release, 
+        ## don't update changelog (--no-changelog) 
+        ## bump version as a pre-release (--conventional-prerelease)
+        ## add a custom preid (--preid dev): 0.2.0-dev.1 -> 0.2.0-dev.2
+        npx lerna version --conventional-commits --conventional-prerelease --preid dev --force-publish=* --yes --no-changelog
+        git push --delete origin $(git describe --abbrev=0)
     - name: update release draft
       uses: release-drafter/[email protected]
       env:

.github/workflows/run-e2e-tests.yml

@@ -0,0 +1,36 @@
+name: run-e2e-tests
+on:
+  workflow_dispatch: {}
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write # needed to interact with GitHub's OIDC Token endpoint.
+      contents: read
+    steps:
+    - name: "Checkout"
+      uses: actions/checkout@v2
+      with:
+        token: ${{ secrets.GH_PUBLISH_TOKEN }}
+        fetch-depth: 0
+    #########################    
+    # Release new version
+    #########################
+    - name: "Use NodeJS 14"
+      uses: actions/setup-node@v2
+      with:
+        node-version: '14'
+    - name: "Setup npm"
+      run: |
+        npm set "//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }}"
+    - name: Install packages
+      run: |
+        npm ci
+        npm run lerna-ci
+    - name: Configure AWS credentials
+      uses: aws-actions/configure-aws-credentials@master
+      with:
+        role-to-assume: ${{ secrets.AWS_ROLE_ARN_TO_ASSUME }}
+        aws-region: eu-west-1
+    - name: Run integration tests
+      run: npm run lerna-test:e2e

CONTRIBUTING.md

@@ -57,6 +57,75 @@ You can build and start a local docs website by running these two commands.
   - `npm run docs-buildDockerImage` OR `docker build -t squidfunk/mkdocs-material ./docs/`
   - `npm run docs-runLocalDocker` OR `docker run --rm -it -p 8000:8000 -v ${PWD}:/docs squidfunk/mkdocs-material`
 
+### Tests
+
+Tests are under `tests` folder of each modules and splitted into two categories: Unit tests and e2e tests.
+
+This split happen thanks to [jest-runner-groups](https://www.npmjs.com/package/jest-runner-groups).
+
+Unit tests, under `tests/unit` folder are standard jest tests.
+
+Integration tests, under `tests/e2e` folder, will test the module features by deploying lambdas into your AWS Account (thanks to CDK lib for typescript) and use aws sdk to invoke them and assert on expected behavior. All of it orchestrated using standard Jest framework. Since it's deploying infrastructure, it will need an AWS account.
+
+
+**Unit testing**
+
+**Write**
+
+As mentioned before, tests are splitted thanks to [jest-runner-groups](https://www.npmjs.com/package/jest-runner-groups) and therefore needs to be tagged properly by adding the following comments in your unit test file:
+
+```
+/**
+ * Tests metrics
+ *
+ * @group unit/<YOUR CATEGORY>/<YOUR SUB CATEGORY>
+ */
+```
+
+**Run**
+
+ `npm run test`
+
+You can run selective tests by restricting the group to the one you want. For instance `npx jest --group=unit/metrics/all`.
+
+**e2e tests**
+
+**Write**
+
+As mentioned before, unit and e2e tests are splitted thanks to [jest-runner-groups](https://www.npmjs.com/package/jest-runner-groups) and therefore needs to be tagged properly by adding the following comments in your unit test file:
+
+```
+/**
+ * Tests data lake catalog
+ *
+ * @group e2e/<YOUR CATEGORY>/<YOUR SUB CATEGORY>
+ */
+```
+
+and leverage `aws-cdk` package to programatically deploy and destroy stacks. See `metrics/tests/e2e/decorator.test.ts` as an example.
+
+
+**Run**
+
+To run unit tests you can either use projen task
+* `npm run test:e2e` which will only run jest integ tests
+* or jest directly `npx jest --group=e2e`
+
+You can run selective tests by restricting the group to the one you want. For instance `npx jest --group=e2e/other/example`.
+
+Two important env variable can be used:
+* `AWS_PROFILE` to use the right credentials
+* `DISABLE_TEARDOWN` if you don't want your stack to be destroyed at the end of the test (useful in dev mode when iterating over your code).
+
+Example: `DISABLE_TEARDOWN=true AWS_PROFILE=ara npx jest --group=integ/other/example`
+
+**Automate**
+
+1. Create AWS Role
+  As mention earlier we are leveraging CDK to deploy and clean resources on AWS. Therefore to run those tests through github actions you will need to grant specific permissions to your workflow. To do so you can leverage [@pahud/cdk-github-oidc](https://constructs.dev/packages/@pahud/cdk-github-oidc) construct which setup the right resources to leverage [Github OpenID Connect](https://github.blog/changelog/2021-10-27-github-actions-secure-cloud-deployments-with-openid-connect/) mechanism.
+1. Add your new role into your Github fork secrets under `AWS_ROLE_ARN_TO_ASSUME`.
+1. Run manually `run-e2e-tests` workflow.
+
 ### Conventions
 
 Category | Convention

package.json

@@ -14,6 +14,7 @@
     "package": "npm run package",
     "lerna-ci": "lerna exec -- npm ci",
     "lerna-test": "lerna exec -- npm run test",
+    "lerna-test:e2e": "lerna exec -- npm run test:e2e",
     "lerna-package": "lerna exec -- npm run package",
     "lerna-build": "lerna exec -- tsc",
     "lerna-lint": "lerna exec -- eslint \"./{src,tests}/**/*.ts ./src/*.ts\"",

packages/commons/package.json

@@ -12,6 +12,7 @@
   "scripts": {
     "commit": "commit",
     "test": "jest  --detectOpenHandles --coverage --verbose",
+    "test:e2e": "echo 'Not Applicable'",
     "watch": "jest --watch",
     "build": "tsc",
     "lint": "eslint --ext .ts --fix --no-error-on-unmatched-pattern src tests",

packages/logger/package.json

@@ -12,6 +12,7 @@
   "scripts": {
     "commit": "commit",
     "test": "jest  --detectOpenHandles --coverage --verbose",
+    "test:e2e": "jest --group=e2e",
     "watch": "jest --watch",
     "build": "tsc",
     "lint": "eslint --ext .ts --fix --no-error-on-unmatched-pattern src tests",

packages/metrics/tests/e2e/decorator.test.MyFunction.ts

@@ -14,7 +14,7 @@ const singleMetricName = process.env.EXPECTED_SINGLE_METRIC_NAME ?? 'MySingleMet
 const singleMetricUnit = (process.env.EXPECTED_SINGLE_METRIC_UNIT as MetricUnits) ?? MetricUnits.Percent;
 const singleMetricValue = process.env.EXPECTED_SINGLE_METRIC_VALUE ?? '2';
 
-const metrics = new Metrics({ namespace: namespace, service: serviceName });
+const metrics = new Metrics({ namespace: namespace, serviceName: serviceName });
 
 class Lambda implements LambdaInterface {
 

packages/metrics/tests/e2e/decorator.test.ts

@@ -23,6 +23,7 @@ const integTestApp = new App();
 const stack = new Stack(integTestApp, 'MetricsE2EDecoratorStack');
 
 // GIVEN
+const invocationCount = 2;
 const startTime = new Date();
 const expectedNamespace = randomUUID(); // to easily find metrics back at assert phase
 const expectedServiceName = 'decoratorService';
@@ -67,28 +68,29 @@ describe('happy cases', () => {
     // lambda function is deployed
     await cloudFormation.deployStack({
       stack: stackArtifact,
+      quiet: true,
     });
-  }, 200000);
 
-  it('capture ColdStart Metric', async () => {
-    // WHEN
-    // invoked
-    await lambdaClient
-      .invoke({
-        FunctionName: functionName,
-      })
-      .promise();
-    // twice
-    await lambdaClient
-      .invoke({
-        FunctionName: functionName,
-      })
-      .promise();
+    // and invoked
+    for (let i = 0; i < invocationCount; i++) {
+      await lambdaClient
+        .invoke({
+          FunctionName: functionName,
+        })
+        .promise();
+    }
 
     // THEN
     // sleep to allow metrics to be collected
-    await new Promise((resolve) => setTimeout(resolve, 10000));
+    await new Promise((resolve) => setTimeout(resolve, 15000));
+  }, 200000);
 
+  it('capture ColdStart Metric', async () => {
+    const expectedDimensions = [
+      { Name: 'service', Value: expectedServiceName },
+      { Name: 'function_name', Value: functionName },
+      { Name: Object.keys(expectedDefaultDimensions)[0], Value: expectedDefaultDimensions.MyDimension },
+    ];
     // Check coldstart metric dimensions
     const coldStartMetrics = await cloudwatchClient
       .listMetrics({
@@ -98,24 +100,19 @@ describe('happy cases', () => {
       .promise();
     expect(coldStartMetrics.Metrics?.length).toBe(1);
     const coldStartMetric = coldStartMetrics.Metrics?.[0];
-    expect(coldStartMetric?.Dimensions).toStrictEqual([
-      { Name: 'service', Value: expectedServiceName },
-      { Name: 'function_name', Value: functionName },
-      { Name: Object.keys(expectedDefaultDimensions)[0], Value: expectedDefaultDimensions.MyDimension },
-    ]);
+    expect(coldStartMetric?.Dimensions).toStrictEqual(expectedDimensions);
 
     // Check coldstart metric value
+    const adjustedStartTime = new Date(startTime.getTime() - 60 * 1000);
+    const endTime = new Date(new Date().getTime() + 60 * 1000);
+    console.log(`Manual command: aws cloudwatch get-metric-statistics --namespace ${expectedNamespace} --metric-name ColdStart --start-time ${Math.floor(adjustedStartTime.getTime()/1000)} --end-time ${Math.floor(endTime.getTime()/1000)} --statistics 'Sum' --period 60 --dimensions '${JSON.stringify(expectedDimensions)}'`);
     const coldStartMetricStat = await cloudwatchClient
       .getMetricStatistics(
         {
           Namespace: expectedNamespace,
-          StartTime: new Date(startTime.getTime() - 60 * 1000), // minus 1 minute,
-          Dimensions: [
-            { Name: 'service', Value: expectedServiceName },
-            { Name: 'function_name', Value: functionName },
-            { Name: Object.keys(expectedDefaultDimensions)[0], Value: expectedDefaultDimensions.MyDimension },
-          ],
-          EndTime: new Date(new Date().getTime() + 60 * 1000),
+          StartTime: adjustedStartTime, 
+          Dimensions: expectedDimensions,
+          EndTime: endTime,
           Period: 60,
           MetricName: 'ColdStart',
           Statistics: ['Sum'],
@@ -126,27 +123,10 @@ describe('happy cases', () => {
 
     // Despite lambda has been called twice, coldstart metric sum should only be 1
     const singleDataPoint = coldStartMetricStat.Datapoints ? coldStartMetricStat.Datapoints[0] : {};
-    expect(singleDataPoint.Sum).toBe(1);
+    expect(singleDataPoint?.Sum).toBe(1);
   }, 15000);
 
   it('produce added Metric with the default and extra one dimensions', async () => {
-    // GIVEN
-    const invocationCount = 2;
-
-    // WHEN
-    // invoked
-    for (let i = 0; i < invocationCount; i++) {
-      await lambdaClient
-        .invoke({
-          FunctionName: functionName,
-        })
-        .promise();
-    }
-
-    // THEN
-    // sleep to allow metrics to be collected
-    await new Promise((resolve) => setTimeout(resolve, 10000));
-
     // Check metric dimensions
     const metrics = await cloudwatchClient
       .listMetrics({
@@ -164,6 +144,9 @@ describe('happy cases', () => {
     expect(metric?.Dimensions).toStrictEqual(expectedDimensions);
 
     // Check coldstart metric value
+    const adjustedStartTime = new Date(startTime.getTime() - 60 * 1000);
+    const endTime = new Date(new Date().getTime() + 60 * 1000);
+    console.log(`Manual command: aws cloudwatch get-metric-statistics --namespace ${expectedNamespace} --metric-name ${expectedMetricName} --start-time ${Math.floor(adjustedStartTime.getTime()/1000)} --end-time ${Math.floor(endTime.getTime()/1000)} --statistics 'Sum' --period 60 --dimensions '${JSON.stringify(expectedDimensions)}'`);
     const metricStat = await cloudwatchClient
       .getMetricStatistics(
         {
@@ -181,7 +164,7 @@ describe('happy cases', () => {
 
     // Since lambda has been called twice in this test and potentially more in others, metric sum should be at least of expectedMetricValue * invocationCount
     const singleDataPoint = metricStat.Datapoints ? metricStat.Datapoints[0] : {};
-    expect(singleDataPoint.Sum).toBeGreaterThanOrEqual(parseInt(expectedMetricValue) * invocationCount);
+    expect(singleDataPoint?.Sum).toBeGreaterThanOrEqual(parseInt(expectedMetricValue) * invocationCount);
   }, 15000);
 
   afterAll(async () => {
@@ -195,6 +178,7 @@ describe('happy cases', () => {
 
       await cloudFormation.destroyStack({
         stack: stackArtifact,
+        quiet: true,
       });
     }
   }, 200000);