refactored docker images to build mainnet and testnet version of clive (testnet includes embedded testnet)

Author: vogel76

.gitlab-ci.yml

@@ -34,12 +34,12 @@ variables:
   # uses registry.gitlab.syncad.com/hive/hive/ci-base-image:ubuntu24.04-1
   CI_BASE_IMAGE_TAG: "@sha256:813f18c42929193036fc4e0017981c16913766442ef8d146313c64785df29c2e"
   CI_BASE_IMAGE: "registry.gitlab.syncad.com/hive/hive/ci-base-image${CI_BASE_IMAGE_TAG}"
-  # uses registry.gitlab.syncad.com/hive/clive/clive-testnet-base-image:python-poetry-1.7-3.10
-  CLIVE_TESTNET_BASE_IMAGE_TAG: "@sha256:d38883407b5860a89e7491b1de4153c516dc5ae87ca9988fba0ed0400e97f98f"
-  CLIVE_TESTNET_BASE_IMAGE: "registry.gitlab.syncad.com/hive/clive/clive-testnet-base-image${CLIVE_TESTNET_BASE_IMAGE_TAG}"
-  # uses registry.gitlab.syncad.com/hive/common-ci-configuration/python:3.12.9-slim-bookworm
-  CLIVE_BASE_IMAGE_TAG: "@sha256:931bdcd3ecbaa230cde88cb3dabca72017be4dae45f9b8c3616fc9947782eca3"
-  CLIVE_BASE_IMAGE: "registry.gitlab.syncad.com/hive/common-ci-configuration/python${CLIVE_BASE_IMAGE_TAG}"
+  # uses registry.gitlab.syncad.com/hive/common-ci-configuration/python_development:3.12-u24.04@sha256:8ccd75ced8a1064b7eff8b4baef5ed66522c06df4a1c9df1759b65eb041c3cc0
+  CLIVE_TESTNET_BASE_IMAGE_TAG: "@sha256:8ccd75ced8a1064b7eff8b4baef5ed66522c06df4a1c9df1759b65eb041c3cc0"
+  CLIVE_TESTNET_BASE_IMAGE: "registry.gitlab.syncad.com/hive/common-ci-configuration/python_development:3.12-u24.04${CLIVE_TESTNET_BASE_IMAGE_TAG}"
+  # uses registry.gitlab.syncad.com/hive/common-ci-configuration/python_runtime:3.12-u24.04@sha256:aa235bdadebaa4764b923900e9fa1d17c82adddd431aaa86e1fece47cc478380
+  CLIVE_BASE_IMAGE_TAG: "@sha256:aa235bdadebaa4764b923900e9fa1d17c82adddd431aaa86e1fece47cc478380"
+  CLIVE_BASE_IMAGE: "registry.gitlab.syncad.com/hive/common-ci-configuration/python_runtime:3.12-u24.04${CLIVE_BASE_IMAGE_TAG}"
   # other:
   AFTER_SCRIPT_IGNORE_ERRORS: 'false'  # without this errors in after_script will be ignored and just "WARNING: after_script failed, but job will continue unaffected: exit code 1" will be shown
 

docker/Dockerfile

@@ -1,86 +1,90 @@
 # syntax=registry.gitlab.syncad.com/hive/common-ci-configuration/dockerfile:1.5
 ARG HIVED_IMAGE
-ARG BASE_IMAGE
+
+ARG BASE_IMAGE=registry.gitlab.syncad.com/hive/common-ci-configuration/python_runtime:3.12-u24.04
 
 ARG CI_REGISTRY_IMAGE=registry.gitlab.syncad.com/hive/clive/
 
 FROM $HIVED_IMAGE AS hived_image
 
-FROM registry.gitlab.syncad.com/hive/hive/ci-base-image:ubuntu24.04-1 AS stripped_beekeeper_source
-
-COPY --from=hived_image "/home/hived/bin/beekeeper" "/home/hived/bin/beekeeper"
-RUN strip -p /home/hived/bin/beekeeper -o /home/hived_admin/beekeeper.stripped
-
 FROM $BASE_IMAGE AS preconfigured_base_image
 
 ARG CLIVE_VERSION
 ENV CLIVE_VERSION=${CLIVE_VERSION}
-ENV CLIVE_UID=2001
+
+ARG CLIVE_UID=2001
+ENV CLIVE_UID=${CLIVE_UID}
 
 ENV APT_CACHE_DIR=/var/cache/buildkit/apt
 ENV PIP_CACHE_DIR=/var/cache/buildkit/pip
 
 # create and use python virtualenv (use directory outside clive user home to reduce usermod execution time)
-ENV PYTHON_VENV_PATH="/clive-venv"
+ENV PYTHON_VENV_PATH="/python_venv"
 ENV PATH="${PYTHON_VENV_PATH}/bin:$PATH" VIRTUAL_ENV=${PYTHON_VENV_PATH}
 
 RUN --mount=type=cache,mode=0777,sharing=locked,target=${APT_CACHE_DIR} \
-    apt-get update && \
-    apt-get install -y sudo && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/* && \
-    useradd -d /clive -ms /bin/bash -u ${CLIVE_UID} -g users -c "clive application account" "clive" && \
-    mkdir -p /clive && chown -R clive /clive && mkdir -p "${PYTHON_VENV_PATH}" && chown -R clive "${PYTHON_VENV_PATH}" && \
+    useradd -o -d /clive -ms /bin/bash -u ${CLIVE_UID} -g users -c "clive application account" "clive" && \
+    mkdir -p /clive && chown -R clive /clive && mkdir -vp "${PYTHON_VENV_PATH}" && chown -R clive "${PYTHON_VENV_PATH}" && \
     chown -R clive:users /var/cache/ && \
     chmod -R 777 /var/cache/
 
 USER clive
 
 SHELL ["/bin/bash", "-c"]
 
-ADD ./docker/entrypoint.sh .
+COPY ./docker/entrypoint.sh .
 RUN mkdir /clive/scripts
-ADD ./scripts/activate_beekeeper.sh /clive/scripts/
+COPY --chown=clive ./scripts/activate_beekeeper.sh /clive/scripts/
 
-ADD --chown=clive ./pyproject.toml /clive/
-ADD --chown=clive ./poetry.lock /clive/
-ADD --chown=clive ./README.md /clive/
+COPY --chown=clive ./pyproject.toml /clive/
+COPY --chown=clive ./poetry.lock /clive/
+COPY --chown=clive ./README.md /clive/
 
 WORKDIR /clive
 
-ENV BEEKEEPER_PATH_SOURCE="/home/hived_admin/beekeeper.stripped"
-ENV BEEKEEPER_PATH_DEST="/clive/beekeeper"
-COPY --chown=clive --from=stripped_beekeeper_source "${BEEKEEPER_PATH_SOURCE}" "${BEEKEEPER_PATH_DEST}"
-
 # crucial for proper display
 ENV COLORTERM=truecolor
 
-FROM preconfigured_base_image AS python_installer
+FROM registry.gitlab.syncad.com/hive/common-ci-configuration/python_development:3.12-u24.04 AS python_installer
+
+ARG CLIVE_VERSION
+ENV CLIVE_VERSION=${CLIVE_VERSION}
+
+ARG CLIVE_UID=2001
+ENV CLIVE_UID=${CLIVE_UID}
+
+ENV APT_CACHE_DIR=/var/cache/buildkit/apt
+ENV PIP_CACHE_DIR=/var/cache/buildkit/pip
+
+# create and use python virtualenv (use directory outside clive user home to reduce usermod execution time)
+ENV PYTHON_VENV_PATH="/python_venv"
+ENV PATH="${PYTHON_VENV_PATH}/bin:$PATH" VIRTUAL_ENV=${PYTHON_VENV_PATH}
 
-USER root
 RUN --mount=type=cache,mode=0777,sharing=locked,target=${APT_CACHE_DIR} \
-    apt-get update && \
-    apt-get install -y git && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+    useradd -o -d /clive -ms /bin/bash -u ${CLIVE_UID} -g users -c "clive application account" "clive" && \
+    mkdir -p /clive && chown -R clive /clive && mkdir -vp "${PYTHON_VENV_PATH}" && chown -R clive "${PYTHON_VENV_PATH}" && \
+    chown -R clive:users /var/cache/ && \
+    chmod -R 777 /var/cache/
 
 USER clive
 WORKDIR /clive
+
+SHELL ["/bin/bash", "-c"]
+
 # Project IDS:
 # - 362 -> schemas
 # - 392 -> clive
 # - 419 -> wax
 # - 434 -> beekeepy
 RUN --mount=type=cache,mode=0777,uid=${CLIVE_UID},target=${PIP_CACHE_DIR} \
-    python3 -m venv "${PYTHON_VENV_PATH}" && \
+    source /python_venv/bin/activate && \
     pip install clive=="${CLIVE_VERSION}" \
     --extra-index-url https://gitlab.syncad.com/api/v4/projects/362/packages/pypi/simple \
     --extra-index-url https://gitlab.syncad.com/api/v4/projects/393/packages/pypi/simple \
     --extra-index-url https://gitlab.syncad.com/api/v4/projects/419/packages/pypi/simple \
     --extra-index-url https://gitlab.syncad.com/api/v4/projects/434/packages/pypi/simple && \
     find "${PYTHON_VENV_PATH}" -name __pycache__ -exec rm -rf {} +
 
-
 FROM preconfigured_base_image AS instance
 
 ARG BUILD_TIME
@@ -112,15 +116,17 @@ LABEL io.hive.image.commit.date="$GIT_LAST_COMMIT_DATE"
 USER root
 ENTRYPOINT ["/entrypoint.sh"]
 
-
 # this target should be built using a testnet hived image as a base (to have embedded testnet)
 FROM instance AS embedded_testnet_instance
 
 ENV HIVED_BINARIES_DIR_SOURCE="/home/hived/bin"
-ENV HIVED_BINARIES_DIR_DEST=${HIVED_BINARIES_DIR_SOURCE}
+ENV HIVED_BINARIES_DIR_DEST="${HIVED_BINARIES_DIR_SOURCE}/"
 
 # Copy all the required executables from hived image
-COPY --from=hived_image "${HIVED_BINARIES_DIR_SOURCE}" "${HIVED_BINARIES_DIR_DEST}"
+COPY --chown=clive --from=hived_image "${HIVED_BINARIES_DIR_SOURCE}/hived" "${HIVED_BINARIES_DIR_SOURCE}/get_dev_key" \
+  "${HIVED_BINARIES_DIR_DEST}"
+
+RUN strip -s "${HIVED_BINARIES_DIR_DEST}"/*
 
 ARG CLIVE_SECRETS__DEFAULT_PRIVATE_KEY="5KTNAYSHVzhnVPrwHpKhc5QqNQt6aW8JsrMT7T4hyrKydzYvYik"
 ENV CLIVE_SECRETS__DEFAULT_PRIVATE_KEY=${CLIVE_SECRETS__DEFAULT_PRIVATE_KEY}
@@ -138,19 +144,21 @@ ENV GET_DEV_KEY_PATH="${HIVED_BINARIES_DIR_DEST}/get_dev_key"
 ENV COMPRESS_BLOCK_LOG_PATH="${HIVED_BINARIES_DIR_DEST}/compress_block_log"
 
 # Get all the files required for embedded testnet
-ADD ./testnet_node.py /clive
+COPY --chown=clive ./testnet_node.py /clive
 
 ENV TEST_TOOLS_PATH="hive/tests/python/hive-local-tools/test-tools/"
-ADD ${TEST_TOOLS_PATH} /clive/${TEST_TOOLS_PATH}
+COPY --chown=clive ${TEST_TOOLS_PATH} /clive/${TEST_TOOLS_PATH}
 
 ENV LOCAL_TOOLS_PATH="tests/clive-local-tools/"
-ADD ${LOCAL_TOOLS_PATH} /clive/${LOCAL_TOOLS_PATH}
+COPY --chown=clive ${LOCAL_TOOLS_PATH} /clive/${LOCAL_TOOLS_PATH}
+
+COPY --from=python_installer /poetry_venv /poetry_venv
 
-RUN curl -sSL https://install.python-poetry.org | python3 - --version 1.7.0
-ENV PATH="/home/hived_admin/.local/bin:$PATH"
+ENV PATH="/home/hived_admin/.local/bin:/poetry_venv/bin:$PATH"
 
 # Run dev installation to supplement with embeddedtestnet dependencies
-RUN poetry install --only embeddedtestnet --no-root
+RUN set -ex; source "${PYTHON_VENV_PATH}/bin/activate" && \
+    poetry install --only embeddedtestnet --no-root
 
 ENV TESTNET_MODE=1