diff --git a/Makefile b/Makefile index 307146c365..e6b1c12a7b 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,10 @@ OUT_DIR=$(shell pwd)/build/.out .DEFAULT_GOAL := help +AGENT_VERSION ?= 2.22.1 +ALPINE_VERSION ?= 3.16 +NGINX_WITH_AGENT_PREFIX ?= nginx-with-agent + .PHONY: help help: Makefile ## Display this help @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "; printf "Usage:\n\n make \033[36m\033[0m\n\nTargets:\n\n"}; {printf " \033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -21,6 +25,11 @@ container: build ## Build the container @docker -v || (code=$$?; printf "\033[0;31mError\033[0m: there was a problem with Docker\n"; exit $$code) docker build --build-arg VERSION=$(VERSION) --build-arg GIT_COMMIT=$(GIT_COMMIT) --build-arg DATE=$(DATE) --target $(TARGET) -f build/Dockerfile -t $(PREFIX):$(TAG) . +.PHONY: nginx-with-agent-container +nginx-with-agent-container: ## Build the nginx-with-agent container + @docker -v || (code=$$?; printf "\033[0;31mError\033[0m: there was a problem with Docker\n"; exit $$code) + docker build --build-arg AGENT_VERSION=$(AGENT_VERSION) --build-arg ALPINE_VERSION=$(ALPINE_VERSION) -f build/nginx-with-agent/Dockerfile -t $(PREFIX)/$(NGINX_WITH_AGENT_PREFIX):$(TAG) . + .PHONY: build build: ## Build the binary ifeq (${TARGET},local) diff --git a/build/nginx-with-agent/Dockerfile b/build/nginx-with-agent/Dockerfile new file mode 100644 index 0000000000..d3ab17ac38 --- /dev/null +++ b/build/nginx-with-agent/Dockerfile @@ -0,0 +1,47 @@ +FROM nginx:1.22.1-alpine +ARG AGENT_VERSION +ARG ALPINE_VERSION + +WORKDIR /nginx-with-agent + +RUN apk add --no-cache libcap + +# For now, get the agent apk package from github release. Eventually, we will pull the pre-build package from nginx.org. +RUN wget -nv -O agent.apk https://github.com/nginx/agent/releases/download/v$AGENT_VERSION/nginx-agent-$AGENT_VERSION-v$ALPINE_VERSION-x86_64.apk \ + && apk add --allow-untrusted agent.apk + +# Copy nginx-agent config file and entrypont script. +# We could also mount this to the Pod. +COPY ./build/nginx-with-agent/nginx-agent.conf /etc/nginx-agent/nginx-agent.conf +COPY ./build/nginx-with-agent/entrypoint.sh /nginx-with-agent/entrypoint.sh + +# Copy nginx config file and httpmatches njs module. +# We could also mount this to the Pod. +COPY ./internal/nginx/modules/src/httpmatches.js /usr/lib/nginx/modules/njs/httpmatches.js +COPY ./build/nginx-with-agent/nginx.conf /etc/nginx/nginx.conf + +# Create nginx directories, clear /conf.d directory, change owner of nginx and agent directories to nginx user 101, +# and make the entrypoint script executable. +RUN mkdir -p /etc/nginx/secrets /var/lib/nginx /var/log/nginx \ + && rm -f /etc/nginx/conf.d/* \ + && chown -R 101:101 /etc/nginx /var/lib/nginx /var/log/nginx /var/cache/nginx \ + && chown -R 101:101 /var/log/nginx-agent /etc/nginx-agent /var/log/nginx-agent /etc/nginx-agent\ + && chmod +x /nginx-with-agent/entrypoint.sh + + +# The following instructions allow nginx and nginx-debug binaries to bind to privileged ports. +# However, adding this capability prevents the agent from reading nginx's /proc//exe symlink which is required by +# the agent to determine the path to the nginx binary. While we wait for a more permanent fix, we will work around this +# by having nginx bind to non-privileged ports. See this write-up for more details: https://dxuuu.xyz/filecaps.html + +#RUN setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx 'cap_net_bind_service=+ep' /usr/sbin/nginx-debug +#RUN setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx 'cap_net_bind_service=+ep' /usr/sbin/nginx-debug + +# Set user to 101 (nginx) +USER 101:101 + +STOPSIGNAL SIGTERM + +EXPOSE 8080 8443 + +ENTRYPOINT ["/nginx-with-agent/entrypoint.sh"] diff --git a/build/nginx-with-agent/entrypoint.sh b/build/nginx-with-agent/entrypoint.sh new file mode 100644 index 0000000000..549f2ddc97 --- /dev/null +++ b/build/nginx-with-agent/entrypoint.sh @@ -0,0 +1,47 @@ +#!/bin/sh + +set -e +set -x +set -euxo pipefail + +handle_term() +{ + echo "received TERM signal" + echo "stopping nginx-agent ..." + kill -TERM "${agent_pid}" 2>/dev/null + echo "stopping nginx ..." + kill -TERM "${nginx_pid}" 2>/dev/null +} + +trap 'handle_term' TERM + +# Launch nginx +echo "starting nginx ..." +nginx -g "daemon off;" & + +nginx_pid=$! + +cat /etc/nginx-agent/nginx-agent.conf +# start nginx-agent, pass args +echo "starting nginx-agent ..." +nginx-agent "$@" & + +agent_pid=$! + +if [ $? != 0 ]; then + echo "couldn't start the agent, please check the log file" + exit 1 +fi + +wait_term() +{ + wait ${agent_pid} + trap - TERM + kill -QUIT "${nginx_pid}" 2>/dev/null + echo "waiting for nginx to stop..." + wait ${nginx_pid} +} + +wait_term + +echo "nginx-agent process has stopped, exiting." diff --git a/build/nginx-with-agent/nginx-agent.conf b/build/nginx-with-agent/nginx-agent.conf new file mode 100644 index 0000000000..8afe6ba148 --- /dev/null +++ b/build/nginx-with-agent/nginx-agent.conf @@ -0,0 +1,42 @@ +# +# /etc/nginx-agent/nginx-agent.conf +# +# Configuration file for NGINX Agent. +# +# This file is to track agent configuration values that are meant to be statically set. There +# are additional agent configuration values that are set via the API and agent install script +# which can be found in /etc/nginx-agent/agent-dynamic.conf. + +log: + # set log level (panic, fatal, error, info, debug, trace; default "info") + level: info + # set log path. if empty, don't log to file. + path: /var/log/nginx-agent/ + +nginx: + # path of NGINX logs to exclude + exclude_logs: "" + socket: "" + +dataplane: + status: + # poll interval for data plane status - the frequency the agent will query the dataplane for changes + poll_interval: 30s + # report interval for data plane status - the maximum duration to wait before syncing dataplane information if no updates have being observed + report_interval: 24h + +metrics: + # specify the size of a buffer to build before sending metrics + bulk_size: 20 + # specify metrics poll interval + report_interval: 1m + collection_interval: 15s + mode: aggregated + +# OSS NGINX default config path +# path to aux file dirs can also be added +config_dirs: "/etc/nginx" + +api: + # default port for Agent API, this is for the server configuration of the REST API + port: 8081 diff --git a/build/nginx-with-agent/nginx-with-agent.yaml b/build/nginx-with-agent/nginx-with-agent.yaml new file mode 100644 index 0000000000..ace3670874 --- /dev/null +++ b/build/nginx-with-agent/nginx-with-agent.yaml @@ -0,0 +1,33 @@ +# This manifest is for testing purposes and is not the final manifest for the nginx-with-agent. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-with-agent +spec: + replicas: 1 + selector: + matchLabels: + app: nginx-with-agent + template: + metadata: + labels: + app: nginx-with-agent + spec: + serviceAccountName: default + automountServiceAccountToken: false + containers: + - image: docker.io/nginx-kubernetes-gateway/nginx-with-agent:edge + imagePullPolicy: IfNotPresent + name: nginx-with-agent + securityContext: + allowPrivilegeEscalation: true + runAsNonRoot: true + runAsUser: 101 #nginx + capabilities: + drop: + - ALL + ports: + - name: http + containerPort: 8080 + - name: https + containerPort: 8443 diff --git a/build/nginx-with-agent/nginx.conf b/build/nginx-with-agent/nginx.conf new file mode 100644 index 0000000000..5c1c9ef898 --- /dev/null +++ b/build/nginx-with-agent/nginx.conf @@ -0,0 +1,29 @@ +load_module /usr/lib/nginx/modules/ngx_http_js_module.so; + +events {} + +pid /etc/nginx/nginx.pid; + +error_log /var/log/nginx/error.log debug; + +http { + include /etc/nginx/conf.d/*.conf; + js_import /usr/lib/nginx/modules/njs/httpmatches.js; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for" '; + + access_log /var/log/nginx/access.log main; + + # stub status API + # needed by the agent in order to collect metrics + server { + listen 127.0.0.1:8082; + location /api { + stub_status; + allow 127.0.0.1; + deny all; + } + } +}