From acc4bd44f0cc32a372c02e16d986b85faeef51c0 Mon Sep 17 00:00:00 2001 From: Kate Osborn Date: Mon, 30 Jan 2023 13:04:21 -0700 Subject: [PATCH 1/5] Add Dockefile for agent --- Makefile | 9 ++++++ build/agent/Dockerfile | 53 ++++++++++++++++++++++++++++++++++++ build/agent/agent.yaml | 32 ++++++++++++++++++++++ build/agent/entrypoint.sh | 47 ++++++++++++++++++++++++++++++++ build/agent/nginx-agent.conf | 42 ++++++++++++++++++++++++++++ build/agent/nginx.conf | 29 ++++++++++++++++++++ 6 files changed, 212 insertions(+) create mode 100644 build/agent/Dockerfile create mode 100644 build/agent/agent.yaml create mode 100644 build/agent/entrypoint.sh create mode 100644 build/agent/nginx-agent.conf create mode 100644 build/agent/nginx.conf diff --git a/Makefile b/Makefile index 307146c365..132d6b2468 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,10 @@ OUT_DIR=$(shell pwd)/build/.out .DEFAULT_GOAL := help +AGENT_VERSION ?= 2.22.1 +ALPINE_VERSION ?= 3.16 +AGENT_PREFIX ?= agent + .PHONY: help help: Makefile ## Display this help @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "; printf "Usage:\n\n make \033[36m\033[0m\n\nTargets:\n\n"}; {printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -21,6 +25,11 @@ container: build ## Build the container @docker -v || (code=$$?; printf "\033[0;31mError\033[0m: there was a problem with Docker\n"; exit $$code) docker build --build-arg VERSION=$(VERSION) --build-arg GIT_COMMIT=$(GIT_COMMIT) --build-arg DATE=$(DATE) --target $(TARGET) -f build/Dockerfile -t $(PREFIX):$(TAG) . +.PHONY: container +agent-container: # Build the agent container + @docker -v || (code=$$?; printf "\033[0;31mError\033[0m: there was a problem with Docker\n"; exit $$code) + docker build --build-arg AGENT_VERSION=$(AGENT_VERSION) --build-arg ALPINE_VERSION=$(ALPINE_VERSION) -f build/agent/Dockerfile -t $(PREFIX)/$(AGENT_PREFIX):$(TAG) . + .PHONY: build build: ## Build the binary ifeq (${TARGET},local) diff --git a/build/agent/Dockerfile b/build/agent/Dockerfile new file mode 100644 index 0000000000..9378eddcd0 --- /dev/null +++ b/build/agent/Dockerfile @@ -0,0 +1,53 @@ +FROM nginx:1.22.1-alpine +ARG AGENT_VERSION +ARG ALPINE_VERSION + +WORKDIR /agent + +RUN apk add --no-cache libcap + +# For now, get the agent apk package from github release. Eventually, we will pull the pre-build package from nginx.org. +RUN wget -nv -O agent.apk https://github.com/nginx/agent/releases/download/v$AGENT_VERSION/nginx-agent-$AGENT_VERSION-v$ALPINE_VERSION-x86_64.apk \ + && apk add --allow-untrusted agent.apk + +# Copy nginx-agent config file and entrypont script. +# We could also mount this to the Pod. +COPY ./build/agent/nginx-agent.conf /etc/nginx-agent/nginx-agent.conf +COPY ./build/agent/entrypoint.sh /agent/entrypoint.sh + +# Copy nginx config file and httpmatches njs module. +# We could also mount this to the Pod. +COPY ./internal/nginx/modules/src/httpmatches.js /usr/lib/nginx/modules/njs/httpmatches.js +COPY ./build/agent/nginx.conf /etc/nginx/nginx.conf + +# Create nginx directories. +RUN mkdir -p /etc/nginx/secrets /var/lib/nginx /var/log/nginx + +# Clear /conf.d directory. +RUN rm -f /etc/nginx/conf.d/* + +# Change owner of nginx directories to nginx user 101. +RUN chown -R 101:101 /etc/nginx /var/lib/nginx /var/log/nginx /var/cache/nginx + +# Change owner of agent directories to nginx user 101. +RUN chown -R 101:101 /var/log/nginx-agent /etc/nginx-agent + +# Make the entrypoint script executable. +RUN chmod +x /agent/entrypoint.sh + +# The following instructions allow nginx and nginx-debug binaries to bind to privileged ports. +# However, adding this capability prevents the agent from reading nginx's /proc//exe symlink which is required by +# the agent to determine the path to the nginx binary. While we wait for a more permanent fix, we will work around this +# by having nginx bind to non-privileged ports. See this write-up for more details: https://dxuuu.xyz/filecaps.html + +#RUN setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx 'cap_net_bind_service=+ep' /usr/sbin/nginx-debug +#RUN setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx 'cap_net_bind_service=+ep' /usr/sbin/nginx-debug + +# Set user to 101 (nginx) +USER 101:101 + +STOPSIGNAL SIGTERM + +EXPOSE 8080 8443 + +ENTRYPOINT ["/agent/entrypoint.sh"] diff --git a/build/agent/agent.yaml b/build/agent/agent.yaml new file mode 100644 index 0000000000..16b611c121 --- /dev/null +++ b/build/agent/agent.yaml @@ -0,0 +1,32 @@ +# This manifest is for testing purposes and is not the final manifest for the agent. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-agent +spec: + replicas: 1 + selector: + matchLabels: + app: nginx-agent + template: + metadata: + labels: + app: nginx-agent + spec: + serviceAccountName: default + containers: + - image: docker.io/nginx-kubernetes-gateway/agent:edge + imagePullPolicy: IfNotPresent + name: nginx-agent + securityContext: + allowPrivilegeEscalation: true + runAsNonRoot: true + runAsUser: 101 #nginx + capabilities: + drop: + - ALL + ports: + - name: http + containerPort: 8080 + - name: https + containerPort: 8443 diff --git a/build/agent/entrypoint.sh b/build/agent/entrypoint.sh new file mode 100644 index 0000000000..549f2ddc97 --- /dev/null +++ b/build/agent/entrypoint.sh @@ -0,0 +1,47 @@ +#!/bin/sh + +set -e +set -x +set -euxo pipefail + +handle_term() +{ + echo "received TERM signal" + echo "stopping nginx-agent ..." + kill -TERM "${agent_pid}" 2>/dev/null + echo "stopping nginx ..." + kill -TERM "${nginx_pid}" 2>/dev/null +} + +trap 'handle_term' TERM + +# Launch nginx +echo "starting nginx ..." +nginx -g "daemon off;" & + +nginx_pid=$! + +cat /etc/nginx-agent/nginx-agent.conf +# start nginx-agent, pass args +echo "starting nginx-agent ..." +nginx-agent "$@" & + +agent_pid=$! + +if [ $? != 0 ]; then + echo "couldn't start the agent, please check the log file" + exit 1 +fi + +wait_term() +{ + wait ${agent_pid} + trap - TERM + kill -QUIT "${nginx_pid}" 2>/dev/null + echo "waiting for nginx to stop..." + wait ${nginx_pid} +} + +wait_term + +echo "nginx-agent process has stopped, exiting." diff --git a/build/agent/nginx-agent.conf b/build/agent/nginx-agent.conf new file mode 100644 index 0000000000..8afe6ba148 --- /dev/null +++ b/build/agent/nginx-agent.conf @@ -0,0 +1,42 @@ +# +# /etc/nginx-agent/nginx-agent.conf +# +# Configuration file for NGINX Agent. +# +# This file is to track agent configuration values that are meant to be statically set. There +# are additional agent configuration values that are set via the API and agent install script +# which can be found in /etc/nginx-agent/agent-dynamic.conf. + +log: + # set log level (panic, fatal, error, info, debug, trace; default "info") + level: info + # set log path. if empty, don't log to file. + path: /var/log/nginx-agent/ + +nginx: + # path of NGINX logs to exclude + exclude_logs: "" + socket: "" + +dataplane: + status: + # poll interval for data plane status - the frequency the agent will query the dataplane for changes + poll_interval: 30s + # report interval for data plane status - the maximum duration to wait before syncing dataplane information if no updates have being observed + report_interval: 24h + +metrics: + # specify the size of a buffer to build before sending metrics + bulk_size: 20 + # specify metrics poll interval + report_interval: 1m + collection_interval: 15s + mode: aggregated + +# OSS NGINX default config path +# path to aux file dirs can also be added +config_dirs: "/etc/nginx" + +api: + # default port for Agent API, this is for the server configuration of the REST API + port: 8081 diff --git a/build/agent/nginx.conf b/build/agent/nginx.conf new file mode 100644 index 0000000000..5c1c9ef898 --- /dev/null +++ b/build/agent/nginx.conf @@ -0,0 +1,29 @@ +load_module /usr/lib/nginx/modules/ngx_http_js_module.so; + +events {} + +pid /etc/nginx/nginx.pid; + +error_log /var/log/nginx/error.log debug; + +http { + include /etc/nginx/conf.d/*.conf; + js_import /usr/lib/nginx/modules/njs/httpmatches.js; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for" '; + + access_log /var/log/nginx/access.log main; + + # stub status API + # needed by the agent in order to collect metrics + server { + listen 127.0.0.1:8082; + location /api { + stub_status; + allow 127.0.0.1; + deny all; + } + } +} From d823852e41c8d82ab83d89d8a4873c5ff6c3b886 Mon Sep 17 00:00:00 2001 From: Kate Osborn Date: Tue, 31 Jan 2023 11:22:45 -0700 Subject: [PATCH 2/5] Collapse RUN statements --- build/agent/Dockerfile | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/build/agent/Dockerfile b/build/agent/Dockerfile index 9378eddcd0..0b3d820158 100644 --- a/build/agent/Dockerfile +++ b/build/agent/Dockerfile @@ -20,20 +20,14 @@ COPY ./build/agent/entrypoint.sh /agent/entrypoint.sh COPY ./internal/nginx/modules/src/httpmatches.js /usr/lib/nginx/modules/njs/httpmatches.js COPY ./build/agent/nginx.conf /etc/nginx/nginx.conf -# Create nginx directories. -RUN mkdir -p /etc/nginx/secrets /var/lib/nginx /var/log/nginx +# Create nginx directories, clear /conf.d directory, change owner of nginx anf agent directories to nginx user 101, +# and make the entrypoint script executable. +RUN mkdir -p /etc/nginx/secrets /var/lib/nginx /var/log/nginx \ + && rm -f /etc/nginx/conf.d/* \ + && chown -R 101:101 /etc/nginx /var/lib/nginx /var/log/nginx /var/cache/nginx \ + && chown -R 101:101 /var/log/nginx-agent /etc/nginx-agent /var/log/nginx-agent /etc/nginx-agent\ + && chmod +x /agent/entrypoint.sh -# Clear /conf.d directory. -RUN rm -f /etc/nginx/conf.d/* - -# Change owner of nginx directories to nginx user 101. -RUN chown -R 101:101 /etc/nginx /var/lib/nginx /var/log/nginx /var/cache/nginx - -# Change owner of agent directories to nginx user 101. -RUN chown -R 101:101 /var/log/nginx-agent /etc/nginx-agent - -# Make the entrypoint script executable. -RUN chmod +x /agent/entrypoint.sh # The following instructions allow nginx and nginx-debug binaries to bind to privileged ports. # However, adding this capability prevents the agent from reading nginx's /proc//exe symlink which is required by From 12c4575198ad608e0549ae73c9786c0454fd69f6 Mon Sep 17 00:00:00 2001 From: Kate Osborn Date: Tue, 31 Jan 2023 11:23:07 -0700 Subject: [PATCH 3/5] Fix Makefile errors --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 132d6b2468..e8006e8d97 100644 --- a/Makefile +++ b/Makefile @@ -25,8 +25,8 @@ container: build ## Build the container @docker -v || (code=$$?; printf "\033[0;31mError\033[0m: there was a problem with Docker\n"; exit $$code) docker build --build-arg VERSION=$(VERSION) --build-arg GIT_COMMIT=$(GIT_COMMIT) --build-arg DATE=$(DATE) --target $(TARGET) -f build/Dockerfile -t $(PREFIX):$(TAG) . -.PHONY: container -agent-container: # Build the agent container +.PHONY: agent-container +agent-container: ## Build the agent container @docker -v || (code=$$?; printf "\033[0;31mError\033[0m: there was a problem with Docker\n"; exit $$code) docker build --build-arg AGENT_VERSION=$(AGENT_VERSION) --build-arg ALPINE_VERSION=$(ALPINE_VERSION) -f build/agent/Dockerfile -t $(PREFIX)/$(AGENT_PREFIX):$(TAG) . From 3d1a19d47b7b54d06f1ee40837004a96fb7c01a3 Mon Sep 17 00:00:00 2001 From: Kate Osborn Date: Tue, 31 Jan 2023 13:43:06 -0700 Subject: [PATCH 4/5] agent -> nginx-with-agent --- Makefile | 8 ++++---- build/{agent => nginx-with-agent}/Dockerfile | 14 +++++++------- build/{agent => nginx-with-agent}/entrypoint.sh | 0 build/{agent => nginx-with-agent}/nginx-agent.conf | 0 .../nginx-with-agent.yaml} | 12 ++++++------ build/{agent => nginx-with-agent}/nginx.conf | 0 6 files changed, 17 insertions(+), 17 deletions(-) rename build/{agent => nginx-with-agent}/Dockerfile (82%) rename build/{agent => nginx-with-agent}/entrypoint.sh (100%) rename build/{agent => nginx-with-agent}/nginx-agent.conf (100%) rename build/{agent/agent.yaml => nginx-with-agent/nginx-with-agent.yaml} (73%) rename build/{agent => nginx-with-agent}/nginx.conf (100%) diff --git a/Makefile b/Makefile index e8006e8d97..e6b1c12a7b 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ OUT_DIR=$(shell pwd)/build/.out AGENT_VERSION ?= 2.22.1 ALPINE_VERSION ?= 3.16 -AGENT_PREFIX ?= agent +NGINX_WITH_AGENT_PREFIX ?= nginx-with-agent .PHONY: help help: Makefile ## Display this help @@ -25,10 +25,10 @@ container: build ## Build the container @docker -v || (code=$$?; printf "\033[0;31mError\033[0m: there was a problem with Docker\n"; exit $$code) docker build --build-arg VERSION=$(VERSION) --build-arg GIT_COMMIT=$(GIT_COMMIT) --build-arg DATE=$(DATE) --target $(TARGET) -f build/Dockerfile -t $(PREFIX):$(TAG) . -.PHONY: agent-container -agent-container: ## Build the agent container +.PHONY: nginx-with-agent-container +nginx-with-agent-container: ## Build the nginx-with-agent container @docker -v || (code=$$?; printf "\033[0;31mError\033[0m: there was a problem with Docker\n"; exit $$code) - docker build --build-arg AGENT_VERSION=$(AGENT_VERSION) --build-arg ALPINE_VERSION=$(ALPINE_VERSION) -f build/agent/Dockerfile -t $(PREFIX)/$(AGENT_PREFIX):$(TAG) . + docker build --build-arg AGENT_VERSION=$(AGENT_VERSION) --build-arg ALPINE_VERSION=$(ALPINE_VERSION) -f build/nginx-with-agent/Dockerfile -t $(PREFIX)/$(NGINX_WITH_AGENT_PREFIX):$(TAG) . .PHONY: build build: ## Build the binary diff --git a/build/agent/Dockerfile b/build/nginx-with-agent/Dockerfile similarity index 82% rename from build/agent/Dockerfile rename to build/nginx-with-agent/Dockerfile index 0b3d820158..d3ab17ac38 100644 --- a/build/agent/Dockerfile +++ b/build/nginx-with-agent/Dockerfile @@ -2,7 +2,7 @@ FROM nginx:1.22.1-alpine ARG AGENT_VERSION ARG ALPINE_VERSION -WORKDIR /agent +WORKDIR /nginx-with-agent RUN apk add --no-cache libcap @@ -12,21 +12,21 @@ RUN wget -nv -O agent.apk https://github.com/nginx/agent/releases/download/v$AGE # Copy nginx-agent config file and entrypont script. # We could also mount this to the Pod. -COPY ./build/agent/nginx-agent.conf /etc/nginx-agent/nginx-agent.conf -COPY ./build/agent/entrypoint.sh /agent/entrypoint.sh +COPY ./build/nginx-with-agent/nginx-agent.conf /etc/nginx-agent/nginx-agent.conf +COPY ./build/nginx-with-agent/entrypoint.sh /nginx-with-agent/entrypoint.sh # Copy nginx config file and httpmatches njs module. # We could also mount this to the Pod. COPY ./internal/nginx/modules/src/httpmatches.js /usr/lib/nginx/modules/njs/httpmatches.js -COPY ./build/agent/nginx.conf /etc/nginx/nginx.conf +COPY ./build/nginx-with-agent/nginx.conf /etc/nginx/nginx.conf -# Create nginx directories, clear /conf.d directory, change owner of nginx anf agent directories to nginx user 101, +# Create nginx directories, clear /conf.d directory, change owner of nginx and agent directories to nginx user 101, # and make the entrypoint script executable. RUN mkdir -p /etc/nginx/secrets /var/lib/nginx /var/log/nginx \ && rm -f /etc/nginx/conf.d/* \ && chown -R 101:101 /etc/nginx /var/lib/nginx /var/log/nginx /var/cache/nginx \ && chown -R 101:101 /var/log/nginx-agent /etc/nginx-agent /var/log/nginx-agent /etc/nginx-agent\ - && chmod +x /agent/entrypoint.sh + && chmod +x /nginx-with-agent/entrypoint.sh # The following instructions allow nginx and nginx-debug binaries to bind to privileged ports. @@ -44,4 +44,4 @@ STOPSIGNAL SIGTERM EXPOSE 8080 8443 -ENTRYPOINT ["/agent/entrypoint.sh"] +ENTRYPOINT ["/nginx-with-agent/entrypoint.sh"] diff --git a/build/agent/entrypoint.sh b/build/nginx-with-agent/entrypoint.sh similarity index 100% rename from build/agent/entrypoint.sh rename to build/nginx-with-agent/entrypoint.sh diff --git a/build/agent/nginx-agent.conf b/build/nginx-with-agent/nginx-agent.conf similarity index 100% rename from build/agent/nginx-agent.conf rename to build/nginx-with-agent/nginx-agent.conf diff --git a/build/agent/agent.yaml b/build/nginx-with-agent/nginx-with-agent.yaml similarity index 73% rename from build/agent/agent.yaml rename to build/nginx-with-agent/nginx-with-agent.yaml index 16b611c121..56773115f4 100644 --- a/build/agent/agent.yaml +++ b/build/nginx-with-agent/nginx-with-agent.yaml @@ -1,23 +1,23 @@ -# This manifest is for testing purposes and is not the final manifest for the agent. +# This manifest is for testing purposes and is not the final manifest for the nginx-with-agent. apiVersion: apps/v1 kind: Deployment metadata: - name: nginx-agent + name: nginx-with-agent spec: replicas: 1 selector: matchLabels: - app: nginx-agent + app: nginx-with-agent template: metadata: labels: - app: nginx-agent + app: nginx-with-agent spec: serviceAccountName: default containers: - - image: docker.io/nginx-kubernetes-gateway/agent:edge + - image: docker.io/nginx-kubernetes-gateway/nginx-with-agent:edge imagePullPolicy: IfNotPresent - name: nginx-agent + name: nginx-with-agent securityContext: allowPrivilegeEscalation: true runAsNonRoot: true diff --git a/build/agent/nginx.conf b/build/nginx-with-agent/nginx.conf similarity index 100% rename from build/agent/nginx.conf rename to build/nginx-with-agent/nginx.conf From b099b6fcf547587b0cb25665e7d6910743bc314d Mon Sep 17 00:00:00 2001 From: Kate Osborn Date: Tue, 31 Jan 2023 13:43:24 -0700 Subject: [PATCH 5/5] Prevent automount of service account token --- build/nginx-with-agent/nginx-with-agent.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/build/nginx-with-agent/nginx-with-agent.yaml b/build/nginx-with-agent/nginx-with-agent.yaml index 56773115f4..ace3670874 100644 --- a/build/nginx-with-agent/nginx-with-agent.yaml +++ b/build/nginx-with-agent/nginx-with-agent.yaml @@ -14,6 +14,7 @@ spec: app: nginx-with-agent spec: serviceAccountName: default + automountServiceAccountToken: false containers: - image: docker.io/nginx-kubernetes-gateway/nginx-with-agent:edge imagePullPolicy: IfNotPresent