Add Dockerfile for agent (#399)

Author: kate-osborn

Makefile

@@ -12,6 +12,10 @@ OUT_DIR=$(shell pwd)/build/.out
 
 .DEFAULT_GOAL := help
 
+AGENT_VERSION ?= 2.22.1
+ALPINE_VERSION ?= 3.16
+NGINX_WITH_AGENT_PREFIX ?= nginx-with-agent
+
 .PHONY: help
 help: Makefile ## Display this help
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "; printf "Usage:\n\n    make \033[36m<target>\033[0m\n\nTargets:\n\n"}; {printf "    \033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -21,6 +25,11 @@ container: build ## Build the container
 	@docker -v || (code=$$?; printf "\033[0;31mError\033[0m: there was a problem with Docker\n"; exit $$code)
 	docker build --build-arg VERSION=$(VERSION) --build-arg GIT_COMMIT=$(GIT_COMMIT) --build-arg DATE=$(DATE) --target $(TARGET) -f build/Dockerfile -t $(PREFIX):$(TAG) .
 
+.PHONY: nginx-with-agent-container
+nginx-with-agent-container: ## Build the nginx-with-agent container
+	@docker -v || (code=$$?; printf "\033[0;31mError\033[0m: there was a problem with Docker\n"; exit $$code)
+	docker build --build-arg AGENT_VERSION=$(AGENT_VERSION) --build-arg ALPINE_VERSION=$(ALPINE_VERSION) -f build/nginx-with-agent/Dockerfile -t $(PREFIX)/$(NGINX_WITH_AGENT_PREFIX):$(TAG) .
+
 .PHONY: build
 build: ## Build the binary
 ifeq (${TARGET},local)

build/nginx-with-agent/Dockerfile

@@ -0,0 +1,47 @@
+FROM nginx:1.22.1-alpine
+ARG AGENT_VERSION
+ARG ALPINE_VERSION
+
+WORKDIR /nginx-with-agent
+
+RUN apk add --no-cache libcap
+
+# For now, get the agent apk package from github release. Eventually, we will pull the pre-build package from nginx.org.
+RUN wget -nv -O agent.apk https://github.com/nginx/agent/releases/download/v$AGENT_VERSION/nginx-agent-$AGENT_VERSION-v$ALPINE_VERSION-x86_64.apk \
+    && apk add --allow-untrusted agent.apk
+
+# Copy nginx-agent config file and entrypont script.
+# We could also mount this to the Pod.
+COPY ./build/nginx-with-agent/nginx-agent.conf /etc/nginx-agent/nginx-agent.conf
+COPY ./build/nginx-with-agent/entrypoint.sh /nginx-with-agent/entrypoint.sh
+
+# Copy nginx config file and httpmatches njs module.
+# We could also mount this to the Pod.
+COPY ./internal/nginx/modules/src/httpmatches.js /usr/lib/nginx/modules/njs/httpmatches.js
+COPY ./build/nginx-with-agent/nginx.conf /etc/nginx/nginx.conf
+
+# Create nginx directories, clear /conf.d directory, change owner of nginx and agent directories to nginx user 101,
+# and make the entrypoint script executable.
+RUN mkdir -p /etc/nginx/secrets /var/lib/nginx /var/log/nginx \
+    && rm -f /etc/nginx/conf.d/* \
+    && chown -R 101:101 /etc/nginx /var/lib/nginx /var/log/nginx /var/cache/nginx \
+    && chown -R 101:101 /var/log/nginx-agent /etc/nginx-agent /var/log/nginx-agent /etc/nginx-agent\
+    && chmod +x /nginx-with-agent/entrypoint.sh
+
+
+# The following instructions allow nginx and nginx-debug binaries to bind to privileged ports.
+# However, adding this capability prevents the agent from reading nginx's /proc/<pid>/exe symlink which is required by
+# the agent to determine the path to the nginx binary. While we wait for a more permanent fix, we will work around this
+# by having nginx bind to non-privileged ports. See this write-up for more details: https://dxuuu.xyz/filecaps.html
+
+#RUN setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx 'cap_net_bind_service=+ep' /usr/sbin/nginx-debug
+#RUN setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx 'cap_net_bind_service=+ep' /usr/sbin/nginx-debug
+
+# Set user to 101 (nginx)
+USER 101:101
+
+STOPSIGNAL SIGTERM
+
+EXPOSE 8080 8443
+
+ENTRYPOINT ["/nginx-with-agent/entrypoint.sh"]

build/nginx-with-agent/entrypoint.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+
+set -e
+set -x
+set -euxo pipefail
+
+handle_term()
+{
+    echo "received TERM signal"
+    echo "stopping nginx-agent ..."
+    kill -TERM "${agent_pid}" 2>/dev/null
+    echo "stopping nginx ..."
+    kill -TERM "${nginx_pid}" 2>/dev/null
+}
+
+trap 'handle_term' TERM
+
+# Launch nginx
+echo "starting nginx ..."
+nginx -g "daemon off;" &
+
+nginx_pid=$!
+
+cat /etc/nginx-agent/nginx-agent.conf
+# start nginx-agent, pass args
+echo "starting nginx-agent ..."
+nginx-agent "$@" &
+
+agent_pid=$!
+
+if [ $? != 0 ]; then
+    echo "couldn't start the agent, please check the log file"
+    exit 1
+fi
+
+wait_term()
+{
+    wait ${agent_pid}
+    trap - TERM
+    kill -QUIT "${nginx_pid}" 2>/dev/null
+    echo "waiting for nginx to stop..."
+    wait ${nginx_pid}
+}
+
+wait_term
+
+echo "nginx-agent process has stopped, exiting."

build/nginx-with-agent/nginx-agent.conf

@@ -0,0 +1,42 @@
+#
+# /etc/nginx-agent/nginx-agent.conf
+#
+# Configuration file for NGINX Agent.
+#
+# This file is to track agent configuration values that are meant to be statically set. There  
+# are additional agent configuration values that are set via the API and agent install script
+# which can be found in /etc/nginx-agent/agent-dynamic.conf. 
+
+log:
+  # set log level (panic, fatal, error, info, debug, trace; default "info")
+  level: info
+  # set log path. if empty, don't log to file.
+  path: /var/log/nginx-agent/
+
+nginx:
+  # path of NGINX logs to exclude
+  exclude_logs: ""
+  socket: ""
+
+dataplane:
+  status:
+    # poll interval for data plane status - the frequency the agent will query the dataplane for changes
+    poll_interval: 30s
+    # report interval for data plane status - the maximum duration to wait before syncing dataplane information if no updates have being observed
+    report_interval: 24h
+
+metrics:
+  # specify the size of a buffer to build before sending metrics
+  bulk_size: 20
+  # specify metrics poll interval
+  report_interval: 1m
+  collection_interval: 15s
+  mode: aggregated
+
+# OSS NGINX default config path
+# path to aux file dirs can also be added
+config_dirs: "/etc/nginx"
+
+api:
+  # default port for Agent API, this is for the server configuration of the REST API
+  port: 8081

build/nginx-with-agent/nginx-with-agent.yaml

@@ -0,0 +1,33 @@
+# This manifest is for testing purposes and is not the final manifest for the nginx-with-agent.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-with-agent
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: nginx-with-agent
+  template:
+    metadata:
+      labels:
+        app: nginx-with-agent
+    spec:
+      serviceAccountName: default
+      automountServiceAccountToken: false
+      containers:
+      - image: docker.io/nginx-kubernetes-gateway/nginx-with-agent:edge
+        imagePullPolicy: IfNotPresent
+        name: nginx-with-agent
+        securityContext:
+          allowPrivilegeEscalation: true
+          runAsNonRoot: true
+          runAsUser: 101 #nginx
+          capabilities:
+            drop:
+            - ALL
+        ports:
+        - name: http
+          containerPort: 8080
+        - name: https
+          containerPort: 8443

build/nginx-with-agent/nginx.conf

@@ -0,0 +1,29 @@
+load_module /usr/lib/nginx/modules/ngx_http_js_module.so;
+
+events {}
+
+pid /etc/nginx/nginx.pid;
+
+error_log /var/log/nginx/error.log debug;
+
+http {
+    include /etc/nginx/conf.d/*.conf;
+    js_import /usr/lib/nginx/modules/njs/httpmatches.js;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for" ';
+
+    access_log /var/log/nginx/access.log  main;
+
+    # stub status API
+    # needed by the agent in order to collect metrics
+    server {
+        listen 127.0.0.1:8082;
+        location /api {
+            stub_status;
+            allow 127.0.0.1;
+            deny all;
+        }
+    }
+}