Fix integration tests (#165)

* Fix integration tests

Author: smirnoal

.github/workflows/test-on-push-and-pr.yml

@@ -11,46 +11,38 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Run 'pr' target
       run: make pr
 
   alpine:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Run alpine integration tests
       run: DISTRO=alpine make test-integ
 
   amazonlinux:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Run amazonlinux integration tests
       run: DISTRO=amazonlinux make test-integ
 
-  centos:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Run centos integration tests
-      run: DISTRO=centos make test-integ
-
   debian:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Run debian integration tests
       run: DISTRO=debian make test-integ
 
   ubuntu:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Run ubuntu integration tests
       run: DISTRO=ubuntu make test-integ

Makefile

@@ -41,9 +41,10 @@ dev: init test
 
 # Verifications to run before sending a pull request
 .PHONY: pr
-pr: init check-format check-security dev setup-codebuild-agent
-	CODEBUILD_IMAGE_TAG=codebuild-agent DISTRO="$(DISTRO)" tests/integration/codebuild-local/test_all.sh tests/integration/codebuild
+pr: init check-format check-security dev
 
+codebuild: setup-codebuild-agent
+	CODEBUILD_IMAGE_TAG=codebuild-agent DISTRO="$(DISTRO)" tests/integration/codebuild-local/test_all.sh tests/integration/codebuild
 
 .PHONY: clean
 clean:

README.md

@@ -19,7 +19,6 @@ First step is to choose the base image to be used. The supported Linux OS distri
 
  - Amazon Linux 2
  - Alpine
- - CentOS
  - Debian
  - Ubuntu
 

tests/integration/codebuild/buildspec.os.alpine.yml

@@ -15,11 +15,10 @@ batch:
       env:
         variables:
           DISTRO_VERSION:
-            - "3.13"
-            - "3.14"
-            - "3.15"
+            - "3.17"
+            - "3.18"
+            - "3.19"
           RUNTIME_VERSION:
-            - "3.8"
             - "3.9"
             - "3.10"
             - "3.11"
@@ -52,20 +51,16 @@ phases:
         echo "COPY ${SCRATCH_DIR}/${RIE} /usr/bin/${RIE}" >> \
           "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp"
       - >
-        if [[ -z "${DOCKERHUB_USERNAME}" && -z "${DOCKERHUB_PASSWORD}" ]];
-        then
-            echo "DockerHub credentials not set as CodeBuild environment variables. Continuing without docker login."
-        else
-            echo "Performing DockerHub login . . ."
-            docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
-        fi
+        echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' > /etc/docker/daemon.json
+        service docker restart
       - echo "Building image ${IMAGE_TAG}"
       - >
         docker build . \
           -f "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" \
           -t "${IMAGE_TAG}" \
           --build-arg RUNTIME_VERSION="${RUNTIME_VERSION}" \
-          --build-arg DISTRO_VERSION="${DISTRO_VERSION}"
+          --build-arg DISTRO_VERSION="${DISTRO_VERSION}" \
+          --load
   build:
     commands:
       - set -x

tests/integration/codebuild/buildspec.os.amazonlinux.1.yml

@@ -17,8 +17,6 @@ batch:
           DISTRO_VERSION:
             - "2"
           RUNTIME_VERSION:
-            - "3.7"
-            - "3.8"
             - "3.9"
             - "3.10"
             - "3.11"
@@ -48,21 +46,17 @@ phases:
         echo "COPY ${SCRATCH_DIR}/${RIE} /usr/bin/${RIE}" >> \
           "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp"
       - >
-        if [[ -z "${DOCKERHUB_USERNAME}" && -z "${DOCKERHUB_PASSWORD}" ]];
-        then
-            echo "DockerHub credentials not set as CodeBuild environment variables. Continuing without docker login."
-        else
-            echo "Performing DockerHub login . . ."
-            docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
-        fi
+        echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' > /etc/docker/daemon.json
+        service docker restart
       - echo "Building image ${IMAGE_TAG}"
       - >
         docker build . \
           -f "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" \
           -t "${IMAGE_TAG}" \
           --build-arg RUNTIME_VERSION="${RUNTIME_VERSION}" \
           --build-arg DISTRO_VERSION="${DISTRO_VERSION}" \
-          --build-arg ARCHITECTURE="${ARCHITECTURE}"
+          --build-arg ARCHITECTURE="${ARCHITECTURE}" \
+          --load
   build:
     commands:
       - set -x

tests/integration/codebuild/buildspec.os.amazonlinux.2.yml

@@ -15,10 +15,9 @@ batch:
       env:
         variables:
           DISTRO_VERSION:
-            - "buster"
+            - "bookworm"
             - "bullseye"
           RUNTIME_VERSION:
-            - "3.8"
             - "3.9"
             - "3.10"
             - "3.11"
@@ -51,20 +50,16 @@ phases:
         echo "RUN apt-get update && apt-get install -y curl" >> \
           "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp"
       - >
-        if [[ -z "${DOCKERHUB_USERNAME}" && -z "${DOCKERHUB_PASSWORD}" ]];
-        then
-            echo "DockerHub credentials not set as CodeBuild environment variables. Continuing without docker login."
-        else
-            echo "Performing DockerHub login . . ."
-            docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
-        fi
+        echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' > /etc/docker/daemon.json
+        service docker restart
       - echo "Building image ${IMAGE_TAG}"
       - >
         docker build . \
           -f "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" \
           -t "${IMAGE_TAG}" \
           --build-arg RUNTIME_VERSION="${RUNTIME_VERSION}" \
-          --build-arg DISTRO_VERSION="${DISTRO_VERSION}"
+          --build-arg DISTRO_VERSION="${DISTRO_VERSION}" \
+          --load
   build:
     commands:
       - set -x

tests/integration/codebuild/buildspec.os.centos.yml

@@ -15,10 +15,9 @@ batch:
       env:
         variables:
           DISTRO_VERSION:
-            - "20.04"
             - "22.04"
+            - "24.04"
           RUNTIME_VERSION:
-            - "3.8"
             - "3.9"
             - "3.10"
             - "3.11"
@@ -48,20 +47,16 @@ phases:
         echo "COPY ${SCRATCH_DIR}/${RIE} /usr/bin/${RIE}" >> \
           "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp"
       - >
-        if [[ -z "${DOCKERHUB_USERNAME}" && -z "${DOCKERHUB_PASSWORD}" ]];
-        then
-            echo "DockerHub credentials not set as CodeBuild environment variables. Continuing without docker login."
-        else
-            echo "Performing DockerHub login . . ."
-            docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
-        fi
+        echo '{"registry-mirrors": ["https://mirror.gcr.io"]}' > /etc/docker/daemon.json
+        service docker restart
       - echo "Building image ${IMAGE_TAG}"
       - >
         docker build . \
           -f "${SCRATCH_DIR}/Dockerfile.echo.${OS_DISTRIBUTION}.tmp" \
           -t "${IMAGE_TAG}" \
           --build-arg RUNTIME_VERSION="${RUNTIME_VERSION}" \
-          --build-arg DISTRO_VERSION="${DISTRO_VERSION}"
+          --build-arg DISTRO_VERSION="${DISTRO_VERSION}" \
+          --load
   build:
     commands:
       - set -x

tests/integration/codebuild/buildspec.os.debian.yml

@@ -7,7 +7,8 @@ ARG DISTRO_VERSION
 FROM public.ecr.aws/docker/library/python:${RUNTIME_VERSION}-alpine${DISTRO_VERSION} AS python-alpine
 # Install libstdc++
 RUN apk add --no-cache \
-    libstdc++
+    libstdc++ \
+    binutils
 
 
 # Stage 2 - build function and dependencies
@@ -39,15 +40,12 @@ ARG FUNCTION_DIR="/home/app/"
 RUN mkdir -p ${FUNCTION_DIR}
 # Copy function code
 COPY tests/integration/test-handlers/echo/* ${FUNCTION_DIR}
-# Copy Runtime Interface Client .tgz
-RUN cp ./dist/awslambdaric-test.tar.gz ${FUNCTION_DIR}/awslambdaric-test.tar.gz
 
 # Install the function's dependencies
 WORKDIR ${FUNCTION_DIR}
 RUN python${RUNTIME_VERSION} -m pip install \
-        awslambdaric-test.tar.gz \
-        --target ${FUNCTION_DIR} && \
-    rm awslambdaric-test.tar.gz
+        ${RIC_BUILD_DIR}/dist/awslambdaric-test.tar.gz \
+        --target ${FUNCTION_DIR}
 
 
 # Stage 3 - final runtime interface client image

tests/integration/codebuild/buildspec.os.ubuntu.yml

@@ -17,8 +17,10 @@ RUN yum install -y \
   freetype-devel \
   yum-utils \
   findutils \
-  openssl-devel \
   wget \
+  openssl11 \
+  openssl11-devel \
+  bzip2-devel \
   libffi-devel \
   sqlite-devel
 
@@ -78,6 +80,11 @@ RUN mkdir -p ${RIC_BUILD_DIR}
 # Copy function code and Runtime Interface Client .tgz
 WORKDIR ${RIC_BUILD_DIR}
 COPY . .
+
+# distutils no longer available in python3.12 and later
+#   https://docs.python.org/3/whatsnew/3.12.html
+#   https://peps.python.org/pep-0632/
+RUN if [ $(cut -d '.' -f 2 <<< ${RUNTIME_VERSION}) -ge 12 ]; then pip3 install setuptools; fi
 RUN make init build test && \
     mv ./dist/awslambdaric-*.tar.gz ./dist/awslambdaric-test.tar.gz
 

tests/integration/docker/Dockerfile.echo.alpine

@@ -9,40 +9,41 @@ ENV DEBIAN_FRONTEND=noninteractive
 ARG RUNTIME_VERSION
 
 # Install python and pip
-RUN apt-get update && \
-  apt-get install -y \
-    software-properties-common
+RUN apt-get update && apt-get install -y software-properties-common
 RUN add-apt-repository ppa:deadsnakes/ppa
 RUN apt-get update && \
-  apt-get install -y \
-    curl \
-    python${RUNTIME_VERSION} \
-    python${RUNTIME_VERSION}-distutils
+    apt-get install -y \
+        curl \
+        python${RUNTIME_VERSION} \
+        python3-pip \
+        python3-virtualenv
+
+# python3xx-distutils is needed for python < 3.12
+RUN if [ $(echo ${RUNTIME_VERSION} | cut -d '.' -f 2) -lt 12 ]; then \
+            apt-get install -y python${RUNTIME_VERSION}-distutils; \
+    fi
+RUN virtualenv --python /usr/bin/python${RUNTIME_VERSION} --no-setuptools /home/venv
+
 
-RUN ln -s /usr/bin/python${RUNTIME_VERSION} /usr/local/bin/python3
 
 # Stage 2 - build function and dependencies
 FROM python-image AS python-ubuntu-builder
 
 ARG RUNTIME_VERSION
 
-RUN curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py"
-RUN python${RUNTIME_VERSION} get-pip.py
-
 # Install aws-lambda-cpp build dependencies
-RUN apt-get update && \
-  apt-get install -y \
-    g++ \
-    gcc \
-    tar \
-    gzip \
-    make \
-    cmake \
-    autoconf \
-    automake \
-    libtool \
-    libcurl4-openssl-dev \
-    python${RUNTIME_VERSION}-dev
+RUN apt-get install -y \
+        g++ \
+        gcc \
+        tar \
+        gzip \
+        make \
+        cmake \
+        autoconf \
+        automake \
+        libtool \
+        libcurl4-openssl-dev \
+        python${RUNTIME_VERSION}-dev
 
 # Include global args in this stage of the build
 ARG RIC_BUILD_DIR="/home/build/"
@@ -51,27 +52,28 @@ RUN mkdir -p ${RIC_BUILD_DIR}
 # Copy function code and Runtime Interface Client .tgz
 WORKDIR ${RIC_BUILD_DIR}
 COPY . .
-RUN make init build test && \
+RUN . /home/venv/bin/activate && \
+    pip install setuptools && \
+    make init build test && \
     mv ./dist/awslambdaric-*.tar.gz ./dist/awslambdaric-test.tar.gz
 
+
+
 # Include global args in this stage of the build
 ARG FUNCTION_DIR="/home/app/"
 # Create function directory
 RUN mkdir -p ${FUNCTION_DIR}
 # Copy function code
 COPY tests/integration/test-handlers/echo/* ${FUNCTION_DIR}
-# Copy Runtime Interface Client .tgz
-RUN cp ./dist/awslambdaric-test.tar.gz ${FUNCTION_DIR}/awslambdaric-test.tar.gz
-
 # Install the function's dependencies
 WORKDIR ${FUNCTION_DIR}
-RUN python${RUNTIME_VERSION} -m pip install \
-        awslambdaric-test.tar.gz \
-        --target ${FUNCTION_DIR} && \
-    rm awslambdaric-test.tar.gz
+RUN . /home/venv/bin/activate && \
+    pip install ${RIC_BUILD_DIR}/dist/awslambdaric-test.tar.gz --target ${FUNCTION_DIR}
+
+
 
 
-# Stage 4 - final runtime interface client image
+# Stage 3 - final runtime interface client image
 # Grab a fresh copy of the Python image
 FROM python-image