diff --git a/.github/scripts/setup_tmp_layer_files.sh b/.github/scripts/setup_tmp_layer_files.sh deleted file mode 100644 index 77c5875734..0000000000 --- a/.github/scripts/setup_tmp_layer_files.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -rm -rf tmp/nodejs -mkdir -p tmp/nodejs -cd tmp/nodejs -npm init -y -npm i \ - @aws-lambda-powertools/logger@$VERSION \ - @aws-lambda-powertools/metrics@$VERSION \ - @aws-lambda-powertools/tracer@$VERSION -rm -rf node_modules/@types \ - package.json \ - package-lock.json -cd ../.. \ No newline at end of file diff --git a/.github/workflows/publish_layer.yml b/.github/workflows/publish_layer.yml index d1059bee86..c67d0e58db 100644 --- a/.github/workflows/publish_layer.yml +++ b/.github/workflows/publish_layer.yml @@ -42,10 +42,6 @@ jobs: node-version: "18" - name: Setup dependencies uses: ./.github/actions/cached-node-modules - - name: Create layer files - run: | - export VERSION=${{ inputs.latest_published_version }} - bash .github/scripts/setup_tmp_layer_files.sh - name: CDK build run: npm run cdk -w layers -- synth --context PowertoolsPackageVersion=${{ inputs.latest_published_version }} -o cdk.out - name: Zip output diff --git a/.github/workflows/reusable-run-linting-check-and-unit-tests.yml b/.github/workflows/reusable-run-linting-check-and-unit-tests.yml index 407d2e25b3..dfd274e6bd 100644 --- a/.github/workflows/reusable-run-linting-check-and-unit-tests.yml +++ b/.github/workflows/reusable-run-linting-check-and-unit-tests.yml @@ -69,10 +69,6 @@ jobs: uses: ./.github/actions/cached-node-modules - name: Run linting run: npm run lint -w layers - - name: Create layer files - run: | - export VERSION=latest - bash .github/scripts/setup_tmp_layer_files.sh - name: Run tests run: npm run test:unit -w layers check-docs-snippets: diff --git a/.github/workflows/run-e2e-tests.yml b/.github/workflows/run-e2e-tests.yml index f77589063f..67a9c4678e 100644 --- a/.github/workflows/run-e2e-tests.yml +++ b/.github/workflows/run-e2e-tests.yml @@ -55,11 +55,6 @@ jobs: role-to-assume: ${{ secrets.AWS_ROLE_ARN_TO_ASSUME }} aws-region: eu-west-1 mask-aws-account-id: true - - name: Create layer files - if: ${{ matrix.package == 'layers' }} - run: | - export VERSION=latest - bash .github/scripts/setup_tmp_layer_files.sh - name: Run integration tests on utils env: RUNTIME: nodejs${{ matrix.version }}x diff --git a/layers/README.md b/layers/README.md index 19912bfa9d..128996a4d4 100644 --- a/layers/README.md +++ b/layers/README.md @@ -44,10 +44,6 @@ RUNTIME=node12.x VERSION=0.9.0 npm run test:e2e ```shell cdk bootstrap aws://AWS_ACCOUNT/NEW_REGION ``` -* Build the layer folder from the project root directory -```shell -bash ./.github/scripts/setup_tmp_layer_files.sh -``` * Deploy the first layer version to the new region, make sure to set the NEW_REGION in your AWS CLI configuration correctly, otherwise you will deploy to the wrong region ```shell npm run cdk -w layers -- deploy --app cdk.out --context region=NEW_REGION 'LayerPublisherStack' --require-approval never --verbose diff --git a/layers/bin/layers.ts b/layers/bin/layers.ts index e945a14aba..bc7390c4fd 100644 --- a/layers/bin/layers.ts +++ b/layers/bin/layers.ts @@ -10,6 +10,7 @@ const app = new App(); new LayerPublisherStack(app, 'LayerPublisherStack', { powertoolsPackageVersion: app.node.tryGetContext('PowertoolsPackageVersion'), + buildFromLocal: app.node.tryGetContext('BuildFromLocal') || false, layerName: 'AWSLambdaPowertoolsTypeScript', ssmParameterLayerArn: SSM_PARAM_LAYER_ARN, }); diff --git a/layers/package.json b/layers/package.json index 03d0fbcff9..b358e8367d 100644 --- a/layers/package.json +++ b/layers/package.json @@ -14,7 +14,8 @@ "lint": "eslint --ext .ts,.js --no-error-on-unmatched-pattern .", "lint-fix": "eslint --fix --ext .ts,.js --fix --no-error-on-unmatched-pattern .", "test:unit": "jest --group=unit", - "test:e2e": "jest --group=e2e" + "test:e2e": "jest --group=e2e", + "createLayerFolder": "cdk synth --context BuildFromLocal=true" }, "lint-staged": { "*.{js,ts}": "npm run lint-fix" diff --git a/layers/src/canary-stack.ts b/layers/src/canary-stack.ts index 476f83246f..7c710cd976 100644 --- a/layers/src/canary-stack.ts +++ b/layers/src/canary-stack.ts @@ -1,12 +1,12 @@ import { CustomResource, Duration, Stack, StackProps } from 'aws-cdk-lib'; import { Construct } from 'constructs'; -import { LayerVersion, Runtime } from 'aws-cdk-lib/aws-lambda'; +import { LayerVersion, Runtime, Tracing } from 'aws-cdk-lib/aws-lambda'; import { RetentionDays } from 'aws-cdk-lib/aws-logs'; import { randomUUID } from 'node:crypto'; import { Effect, PolicyStatement } from 'aws-cdk-lib/aws-iam'; import { Provider } from 'aws-cdk-lib/custom-resources'; import { StringParameter } from 'aws-cdk-lib/aws-ssm'; -import path from 'path'; +import path from 'node:path'; import { NodejsFunction } from 'aws-cdk-lib/aws-lambda-nodejs'; export interface CanaryStackProps extends StackProps { @@ -54,6 +54,7 @@ export class CanaryStack extends Stack { ], }, environment: { + LAYERS_PATH: '/opt/nodejs/node_modules', POWERTOOLS_SERVICE_NAME: 'canary', POWERTOOLS_PACKAGE_VERSION: powertoolsPackageVersion, POWERTOOLS_LAYER_NAME: layerName, @@ -61,6 +62,7 @@ export class CanaryStack extends Stack { }, layers: layer, logRetention: RetentionDays.ONE_DAY, + tracing: Tracing.ACTIVE, }); canaryFunction.addToRolePolicy( diff --git a/layers/src/layer-publisher-stack.ts b/layers/src/layer-publisher-stack.ts index 42bbd78e9a..ea75124e25 100644 --- a/layers/src/layer-publisher-stack.ts +++ b/layers/src/layer-publisher-stack.ts @@ -1,18 +1,21 @@ import { CfnOutput, RemovalPolicy, Stack, StackProps } from 'aws-cdk-lib'; -import { Construct } from 'constructs'; import { - LayerVersion, + CfnLayerVersionPermission, Code, + LayerVersion, Runtime, - CfnLayerVersionPermission, } from 'aws-cdk-lib/aws-lambda'; import { StringParameter } from 'aws-cdk-lib/aws-ssm'; -import { resolve } from 'node:path'; +import { Construct } from 'constructs'; +import { execSync } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { join, resolve, sep } from 'node:path'; export interface LayerPublisherStackProps extends StackProps { readonly layerName?: string; readonly powertoolsPackageVersion?: string; readonly ssmParameterLayerArn: string; + readonly buildFromLocal?: boolean; } export class LayerPublisherStack extends Stack { @@ -24,7 +27,7 @@ export class LayerPublisherStack extends Stack { ) { super(scope, id, props); - const { layerName, powertoolsPackageVersion } = props; + const { layerName, powertoolsPackageVersion, buildFromLocal } = props; console.log( `publishing layer ${layerName} version : ${powertoolsPackageVersion}` @@ -41,7 +44,112 @@ export class LayerPublisherStack extends Stack { license: 'MIT-0', // This is needed because the following regions do not support the compatibleArchitectures property #1400 // ...(![ 'eu-south-2', 'eu-central-2', 'ap-southeast-4' ].includes(Stack.of(this).region) ? { compatibleArchitectures: [Architecture.X86_64] } : {}), - code: Code.fromAsset(resolve(__dirname, '..', '..', 'tmp')), + code: Code.fromAsset(resolve(__dirname), { + bundling: { + // This is here only because is required by CDK, however it is not used since the bundling is done locally + image: Runtime.NODEJS_18_X.bundlingImage, + // We need to run a command to generate a random UUID to force the bundling to run every time + command: [`echo "${randomUUID()}"`], + local: { + tryBundle(outputDir: string) { + // This folder are relative to the layers folder + const tmpBuildPath = resolve(__dirname, '..', 'tmp'); + const tmpBuildDir = join(tmpBuildPath, 'nodejs'); + // This folder is the project root, relative to the current file + const projectRoot = resolve(__dirname, '..', '..'); + + // This is the list of packages that we need include in the Lambda Layer + // the name is the same as the npm workspace name + const utilities = ['commons', 'logger', 'metrics', 'tracer']; + + // These files are relative to the tmp folder + const filesToRemove = [ + 'node_modules/@types', + 'package.json', + 'package-lock.json', + 'node_modules/**/README.md', + 'node_modules/.bin/semver', + 'node_modules/async-hook-jl/test', + 'node_modules/shimmer/test', + 'node_modules/jmespath/artifacts', + // We remove the type definitions since they can't be used in the Lambda Layer + 'node_modules/@aws-lambda-powertools/*/lib/*.d.ts', + 'node_modules/@aws-lambda-powertools/*/lib/*.d.ts.map', + ]; + const buildCommands: string[] = []; + const modulesToInstall: string[] = []; + + if (buildFromLocal) { + for (const util of utilities) { + // Build latest version of the package + buildCommands.push(`npm run build -w packages/${util}`); + // Pack the package to a .tgz file + buildCommands.push(`npm pack -w packages/${util}`); + // Move the .tgz file to the tmp folder + buildCommands.push( + `mv aws-lambda-powertools-${util}-*.tgz ${tmpBuildDir}` + ); + } + modulesToInstall.push( + ...utilities.map((util) => + join(tmpBuildDir, `aws-lambda-powertools-${util}-*.tgz`) + ) + ); + filesToRemove.push( + ...utilities.map((util) => + join(`aws-lambda-powertools-${util}-*.tgz`) + ) + ); + } else { + // Dependencies to install in the Lambda Layer + modulesToInstall.push( + ...utilities.map( + (util) => + `@aws-lambda-powertools/${util}@${powertoolsPackageVersion}` + ) + ); + } + + // Phase 1: Cleanup & create tmp folder + execSync( + [ + // Clean up existing tmp folder from previous builds + `rm -rf ${tmpBuildDir}`, + // Create tmp folder again + `mkdir -p ${tmpBuildDir}`, + ].join(' && ') + ); + + // Phase 2: (Optional) Build packages & pack them + buildFromLocal && + execSync(buildCommands.join(' && '), { cwd: projectRoot }); + + // Phase 3: Install dependencies to tmp folder + execSync( + `npm i --prefix ${tmpBuildDir} ${modulesToInstall.join(' ')}` + ); + + // Phase 4: Remove unnecessary files + execSync( + `rm -rf ${filesToRemove + .map((filePath) => `${tmpBuildDir}/${filePath}`) + .join(' ')}` + ); + + // Phase 5: Copy files from tmp folder to cdk.out asset folder (the folder is created by CDK) + execSync(`cp -R ${tmpBuildPath}${sep}* ${outputDir}`); + + // Phase 6: (Optional) Restore changes to the project root made by the build + buildFromLocal && + execSync('git restore packages/*/package.json', { + cwd: projectRoot, + }); + + return true; + }, + }, + }, + }), }); const layerPermission = new CfnLayerVersionPermission( diff --git a/layers/tests/e2e/layerPublisher.class.test.functionCode.ts b/layers/tests/e2e/layerPublisher.class.test.functionCode.ts index 59fd1834c3..fd89f22a44 100644 --- a/layers/tests/e2e/layerPublisher.class.test.functionCode.ts +++ b/layers/tests/e2e/layerPublisher.class.test.functionCode.ts @@ -1,4 +1,5 @@ -import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { readFile } from 'node:fs/promises'; import { Logger } from '@aws-lambda-powertools/logger'; import { Metrics } from '@aws-lambda-powertools/metrics'; import { Tracer } from '@aws-lambda-powertools/tracer'; @@ -9,36 +10,63 @@ const logger = new Logger({ const metrics = new Metrics(); const tracer = new Tracer(); -export const handler = (): void => { - // Check that the packages version matches the expected one - const packageJSON = JSON.parse( - readFileSync( - '/opt/nodejs/node_modules/@aws-lambda-powertools/logger/package.json', - { - encoding: 'utf8', - flag: 'r', - } - ) +const layerPath = process.env.LAYERS_PATH || '/opt/nodejs/node_modules'; +const expectedVersion = process.env.POWERTOOLS_PACKAGE_VERSION || '0.0.0'; + +const getVersionFromModule = async (moduleName: string): Promise => { + const manifestPath = join( + layerPath, + '@aws-lambda-powertools', + moduleName, + 'package.json' ); - if (packageJSON.version != process.env.POWERTOOLS_PACKAGE_VERSION) { - throw new Error( - `Package version mismatch: ${packageJSON.version} != ${process.env.POWERTOOLS_PACKAGE_VERSION}` - ); + let manifest: string; + try { + manifest = await readFile(manifestPath, { encoding: 'utf8' }); + } catch (error) { + console.log(error); + throw new Error(`Unable to read/find package.json file at ${manifestPath}`); } - // Check that the logger is working - logger.debug('Hello World!'); + let moduleVersion: string; + try { + const { version } = JSON.parse(manifest); + moduleVersion = version; + } catch (error) { + console.log(error); + throw new Error(`Unable to parse package.json file at ${manifestPath}`); + } + + return moduleVersion; +}; + +export const handler = async (): Promise => { + // Check that the packages version matches the expected one + for (const moduleName of ['commons', 'logger', 'metrics', 'tracer']) { + const moduleVersion = await getVersionFromModule(moduleName); + if (moduleVersion != expectedVersion) { + throw new Error( + `Package version mismatch (${moduleName}): ${moduleVersion} != ${expectedVersion}` + ); + } + } // Check that the metrics is working metrics.captureColdStartMetric(); // Check that the tracer is working - const segment = tracer.getSegment(); - if (!segment) throw new Error('Segment not found'); - const handlerSegment = segment.addNewSubsegment('### index.handler'); - tracer.setSegment(handlerSegment); + const subsegment = tracer.getSegment()?.addNewSubsegment('### index.handler'); + if (!subsegment) { + throw new Error('Unable to create subsegment, check the Tracer'); + } + tracer.setSegment(subsegment); tracer.annotateColdStart(); - handlerSegment.close(); - tracer.setSegment(segment); + subsegment.close(); + tracer.setSegment(subsegment.parent); + + // Check that logger & tracer are both working + // the presence of a log will indicate that the logger is working + // while the content of the log will indicate that the tracer is working + logger.debug('subsegment', { subsegment: subsegment.format() }); }; diff --git a/layers/tests/e2e/layerPublisher.test.ts b/layers/tests/e2e/layerPublisher.test.ts index 7ec4d82aba..52eff56c61 100644 --- a/layers/tests/e2e/layerPublisher.test.ts +++ b/layers/tests/e2e/layerPublisher.test.ts @@ -22,6 +22,8 @@ import { import { join } from 'node:path'; import packageJson from '../../package.json'; +jest.spyOn(console, 'log').mockImplementation(); + /** * This test has two stacks: * 1. LayerPublisherStack - publishes a layer version using the LayerPublisher construct and containing the Powertools utilities from the repo @@ -60,6 +62,7 @@ describe(`Layers E2E tests, publisher stack`, () => { const layerStack = new LayerPublisherStack(layerApp, layerId, { layerName: layerId, powertoolsPackageVersion: powerToolsPackageVersion, + buildFromLocal: true, ssmParameterLayerArn: ssmParameterLayerName, }); const testLayerStack = new TestStack({ @@ -84,6 +87,7 @@ describe(`Layers E2E tests, publisher stack`, () => { { entry: lambdaFunctionCodeFilePath, environment: { + LAYERS_PATH: '/opt/nodejs/node_modules', POWERTOOLS_PACKAGE_VERSION: powerToolsPackageVersion, POWERTOOLS_SERVICE_NAME: 'LayerPublisherStack', }, @@ -111,7 +115,7 @@ describe(`Layers E2E tests, publisher stack`, () => { }); }, SETUP_TIMEOUT); - describe('LayerPublisherStack usage', () => { + describe('package version and path check', () => { it( 'should have no errors in the logs, which indicates the pacakges version matches the expected one', () => { @@ -121,7 +125,9 @@ describe(`Layers E2E tests, publisher stack`, () => { }, TEST_CASE_TIMEOUT ); + }); + describe('utilities usage', () => { it( 'should have one warning related to missing Metrics namespace', () => { @@ -145,12 +151,30 @@ describe(`Layers E2E tests, publisher stack`, () => { ); it( - 'should have one debug log that says Hello World!', + 'should have one debug log with tracer subsegment info', () => { const logs = invocationLogs.getFunctionLogs('DEBUG'); expect(logs.length).toBe(1); - expect(logs[0]).toContain('Hello World!'); + const logEntry = TestInvocationLogs.parseFunctionLog(logs[0]); + expect(logEntry.message).toContain('subsegment'); + expect(logEntry.subsegment).toBeDefined(); + const subsegment = JSON.parse(logEntry.subsegment as string); + const traceIdFromLog = subsegment.trace_id; + expect(subsegment).toEqual( + expect.objectContaining({ + id: expect.any(String), + name: '### index.handler', + start_time: expect.any(Number), + end_time: expect.any(Number), + type: 'subsegment', + annotations: { + ColdStart: true, + }, + parent_id: expect.any(String), + trace_id: traceIdFromLog, + }) + ); }, TEST_CASE_TIMEOUT ); diff --git a/layers/tests/unit/layer-publisher.test.ts b/layers/tests/unit/layer-publisher.test.ts index 151dc87b99..a365c34f3d 100644 --- a/layers/tests/unit/layer-publisher.test.ts +++ b/layers/tests/unit/layer-publisher.test.ts @@ -15,6 +15,7 @@ describe('Class: LayerPublisherStack', () => { const stack = new LayerPublisherStack(app, 'MyTestStack', { layerName: 'AWSLambdaPowertoolsTypeScript', powertoolsPackageVersion: '1.0.1', + buildFromLocal: true, ssmParameterLayerArn: '/layers/powertools-layer-arn', });