bugfix: buffer bloat and CPU 100% when download large file was filtered by body_filter_by_lua.

Author: zhuizhuhaomeng

src/ngx_http_lua_bodyfilterby.c

@@ -235,15 +235,12 @@ ngx_http_lua_body_filter(ngx_http_request_t *r, ngx_chain_t *in)
     uint16_t                     old_context;
     ngx_http_cleanup_t          *cln;
     ngx_chain_t                 *out;
+    ngx_chain_t                 *cl, *ln, **ll;
     ngx_http_lua_main_conf_t    *lmcf;
 
     ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
                    "lua body filter for user lua code, uri \"%V\"", &r->uri);
 
-    if (in == NULL) {
-        return ngx_http_next_body_filter(r, in);
-    }
-
     llcf = ngx_http_get_module_loc_conf(r, ngx_http_lua_module);
 
     if (llcf->body_filter_handler == NULL) {
@@ -272,6 +269,50 @@ ngx_http_lua_body_filter(ngx_http_request_t *r, ngx_chain_t *in)
         return NGX_OK;
     }
 
+    if (in != NULL) {
+        ll = &ctx->in_bufs;
+        while (*ll != NULL) {
+            ll = &(*ll)->next;
+        }
+
+        for (ln = in; ln != NULL; ln = ln->next) {
+            cl = ngx_alloc_chain_link(r->pool);
+            if (cl == NULL) {
+                return NGX_ERROR;
+            }
+
+            cl->buf = ln->buf;
+            cl->next = NULL;
+            *ll = cl;
+            ll = &cl->next;
+        }
+    }
+
+    if (ctx->last_body_filter_rc == NGX_AGAIN && ctx->busy_bufs != NULL) {
+        /* Socket write buffer was full on last write.
+         * Try to write the remain data, if still can not write
+         * do not execute body_filter_by_lua otherwise the `in` chain will be
+         * replaced by new content from lua and buf of `in` mark as consumed.
+         * And then ngx_output_chain will call the filter chain again which
+         * make all the data cached in the memory and long ngx_chain_t link
+         * cause CPU 100%.
+         */
+        rc = ngx_http_next_body_filter(r, NULL);
+        ctx->last_body_filter_rc = rc;
+
+        if (rc == NGX_ERROR) {
+            return rc;
+        }
+
+        out = NULL;
+        ngx_chain_update_chains(r->pool,
+                                &ctx->free_bufs, &ctx->busy_bufs, &out,
+                                (ngx_buf_tag_t) &ngx_http_lua_module);
+        if (rc != NGX_OK) {
+            return rc;
+        }
+    }
+
     if (ctx->cleanup == NULL) {
         cln = ngx_http_cleanup_add(r, 0);
         if (cln == NULL) {
@@ -286,6 +327,9 @@ ngx_http_lua_body_filter(ngx_http_request_t *r, ngx_chain_t *in)
     old_context = ctx->context;
     ctx->context = NGX_HTTP_LUA_CONTEXT_BODY_FILTER;
 
+    in = ctx->in_bufs;
+    ctx->in_bufs = NULL;
+
     dd("calling body filter handler");
     rc = llcf->body_filter_handler(r, in);
 
@@ -300,8 +344,21 @@ ngx_http_lua_body_filter(ngx_http_request_t *r, ngx_chain_t *in)
     lmcf = ngx_http_get_module_main_conf(r, ngx_http_lua_module);
     out = lmcf->body_filter_chain;
 
-    if (in == out) {
-        return ngx_http_next_body_filter(r, in);
+    if (in != out) {
+        /* content of body was replaced in ngx_http_lua_body_filter_param_set
+         * and the buffer was marked as consumed.
+         */
+        for (cl = in; cl != NULL; cl = ln) {
+            ln = cl->next;
+            if (ngx_buf_size(cl->buf) == 0) {
+                ngx_free_chain(r->pool, cl);
+
+            } else {
+                ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
+                              "chain link was not marked as consumed");
+                return NGX_ERROR;
+            }
+        }
     }
 
     if (out == NULL) {
@@ -316,6 +373,8 @@ ngx_http_lua_body_filter(ngx_http_request_t *r, ngx_chain_t *in)
         return NGX_ERROR;
     }
 
+    ctx->last_body_filter_rc = rc;
+
     ngx_chain_update_chains(r->pool,
                             &ctx->free_bufs, &ctx->busy_bufs, &out,
                             (ngx_buf_tag_t) &ngx_http_lua_module);
@@ -622,3 +681,5 @@ ngx_http_lua_body_filter_param_set(lua_State *L, ngx_http_request_t *r,
 }
 
 /* vi:set ft=c ts=4 sw=4 et fdm=marker: */
+
+

src/ngx_http_lua_common.h

@@ -545,6 +545,7 @@ typedef struct ngx_http_lua_ctx_s {
     ngx_chain_t             *free_bufs;
     ngx_chain_t             *busy_bufs;
     ngx_chain_t             *free_recv_bufs;
+    ngx_chain_t             *in_bufs;  /* for the body filter */
 
     ngx_http_cleanup_pt     *cleanup;
 
@@ -560,6 +561,8 @@ typedef struct ngx_http_lua_ctx_s {
 
     ngx_int_t                exit_code;
 
+    ngx_int_t                last_body_filter_rc;
+
     void                    *downstream;  /* can be either
                                              ngx_http_lua_socket_tcp_upstream_t
                                              or ngx_http_lua_co_ctx_t */