tests: add benchmark on AWS Lambda

Author: nmoutschen

benchmark/.gitignore

@@ -0,0 +1 @@
+.aws-sam

benchmark/benchmark.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+set -e
+trap cleanup EXIT
+
+export BENCHMARK_STACK_NAME=${BENCHMARK_STACK_NAME:-"powertools-benchmark"}
+
+function cleanup {
+    echo "Cleaning up stack..."
+    aws cloudformation delete-stack --stack-name $BENCHMARK_STACK_NAME
+}
+
+function run_function {
+    # Update function to force a cold start
+    aws lambda update-function-configuration --function-name $1 --memory-size 256 >/dev/null
+    aws lambda update-function-configuration --function-name $1 --memory-size 128 >/dev/null
+    # Cold-start invoke
+    aws lambda invoke --function-name $1 --payload '{}' /dev/null >/dev/null && echo -n . || echo -n e
+}
+
+# Retrieve statistics
+function get_stats {
+    # Gather results from CloudWatch Logs Insights
+    query_id=$(aws logs start-query --log-group-name $1 --query-string 'filter @type = "REPORT" | stats avg(@initDuration) as init_duration, avg(@duration) as duration' --start-time $(expr $(date +%s) - 86400) --end-time $(expr $(date +%s) + 0) --query 'queryId' --output text)
+    while true; do
+        result=$(aws logs get-query-results --query-id $query_id --query 'status' --output text)
+        if [ $result == "Complete" ]; then
+            break
+        fi
+        sleep 1
+    done
+
+    # Check if greater than threshold and print result
+    init_duration=$(aws logs get-query-results --query-id $query_id --query 'results[0][?field==`init_duration`].value' --output text)
+    duration=$(aws logs get-query-results --query-id $query_id --query 'results[0][?field==`duration`].value' --output text)
+    echo "$init_duration,$duration"
+}
+
+# Build and deploy the benchmark stack
+echo "Building and deploying..."
+sam build
+sam deploy --stack-name $BENCHMARK_STACK_NAME --guided
+
+# Retrieve output values
+echo "Retrieve values..."
+export INSTRUMENTED_FUNCTION=$(aws cloudformation describe-stacks --stack-name $BENCHMARK_STACK_NAME --query 'Stacks[0].Outputs[?OutputKey==`InstrumentedFunction`].OutputValue' --output text)
+export REFERENCE_FUNCTION=$(aws cloudformation describe-stacks --stack-name $BENCHMARK_STACK_NAME --query 'Stacks[0].Outputs[?OutputKey==`ReferenceFunction`].OutputValue' --output text)
+export INSTRUMENTED_LOG_GROUP=$(aws cloudformation describe-stacks --stack-name $BENCHMARK_STACK_NAME --query 'Stacks[0].Outputs[?OutputKey==`InstrumentedLogGroup`].OutputValue' --output text)
+export REFERENCE_LOG_GROUP=$(aws cloudformation describe-stacks --stack-name $BENCHMARK_STACK_NAME --query 'Stacks[0].Outputs[?OutputKey==`ReferenceLogGroup`].OutputValue' --output text)
+
+echo INSTRUMENTED_FUNCTION=$INSTRUMENTED_FUNCTION
+echo REFERENCE_FUNCTION=$REFERENCE_FUNCTION
+echo INSTRUMENTED_LOG_GROUP=$INSTRUMENTED_LOG_GROUP
+echo REFERENCE_LOG_GROUP=$REFERENCE_LOG_GROUP
+
+# Running cold starts
+echo "Running functions..."
+for i in {0..20}; do
+    run_function $INSTRUMENTED_FUNCTION
+done &
+process_id=$!
+for i in {0..20}; do
+    run_function $REFERENCE_FUNCTION
+done &
+wait $process_id
+wait $!
+echo
+
+# Gather statistics
+sleep 150
+return_code=0
+echo -n "INSTRUMENTED="
+get_stats $INSTRUMENTED_LOG_GROUP
+echo -n "REFERENCE="
+get_stats $REFERENCE_LOG_GROUP
+
+exit $return_code

benchmark/src/instrumented/main.py

@@ -0,0 +1,17 @@
+from aws_lambda_powertools import (Logger, Metrics, Tracer)
+
+
+# Initialize core utilities
+logger = Logger()
+metrics = Metrics()
+tracer = Tracer()
+
+
+# Instrument Lambda function
+@logger.inject_lambda_context
+@metrics.log_metrics
+@tracer.capture_lambda_handler
+def handler(event, context):
+    return {
+        "message": "success"
+    }

benchmark/src/instrumented/requirements.txt

@@ -0,0 +1 @@
+aws-lambda-powertools

benchmark/src/reference/main.py

@@ -0,0 +1,4 @@
+def handler(event, context):
+    return {
+        "message": "success"
+    }

benchmark/src/reference/requirements.txt

@@ -0,0 +1 @@
+aws-lambda-powertools

benchmark/template.yaml

@@ -0,0 +1,48 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Transform: AWS::Serverless-2016-10-31
+
+Globals:
+  Function:
+    Handler: main.handler
+    Runtime: python3.8
+    MemorySize: 128
+    Tracing: Active
+    Environment:
+      Variables:
+        POWERTOOLS_SERVICE_NAME: benchmark
+        POWERTOOLS_METRICS_NAMESPACE: LambdaPowertools
+        POWERTOOLS_LOGGER_LOG_EVENT: "true"
+        LOG_LEVEL: INFO
+
+Resources:
+  InstrumentedFunction:
+    Type: AWS::Serverless::Function
+    Properties:
+      CodeUri: ./src/instrumented/
+
+  ReferenceFunction:
+    Type: AWS::Serverless::Function
+    Properties:
+      CodeUri: ./src/reference/
+
+  InstrumentedLogGroup:
+    Type: AWS::Logs::LogGroup
+    Properties:
+      LogGroupName: !Sub "/aws/lambda/${InstrumentedFunction}"
+      RetentionInDays: 7
+
+  ReferenceLogGroup:
+    Type: AWS::Logs::LogGroup
+    Properties:
+      LogGroupName: !Sub "/aws/lambda/${ReferenceFunction}"
+      RetentionInDays: 7
+
+Outputs:
+  InstrumentedFunction:
+    Value: !Ref InstrumentedFunction
+  ReferenceFunction:
+    Value: !Ref ReferenceFunction
+  InstrumentedLogGroup:
+    Value: !Ref InstrumentedLogGroup
+  ReferenceLogGroup:
+    Value: !Ref ReferenceLogGroup