bugfix: ensured we cleanup up aborted queued connect operations.

Signed-off-by: Thibault Charbonnier <[email protected]>

Author: spacewander

src/ngx_http_lua_socket_tcp.c

@@ -140,6 +140,7 @@ static ngx_int_t ngx_http_lua_socket_tcp_read_resume(ngx_http_request_t *r);
 static ngx_int_t ngx_http_lua_socket_tcp_write_resume(ngx_http_request_t *r);
 static ngx_int_t ngx_http_lua_socket_tcp_resume_helper(ngx_http_request_t *r,
     int socket_op);
+static void ngx_http_lua_tcp_queue_conn_op_cleanup(void *data);
 static void ngx_http_lua_tcp_resolve_cleanup(void *data);
 static void ngx_http_lua_coctx_cleanup(void *data);
 static void ngx_http_lua_socket_free_pool(ngx_log_t *log,
@@ -630,6 +631,10 @@ ngx_http_lua_socket_tcp_connect_helper(lua_State *L,
                 u->write_co_ctx = ctx->cur_co_ctx;
 
                 conn_op_ctx->u = u;
+                ctx->cur_co_ctx->cleanup =
+                    ngx_http_lua_tcp_queue_conn_op_cleanup;
+                ctx->cur_co_ctx->data = conn_op_ctx;
+
                 ngx_memzero(&conn_op_ctx->event, sizeof(ngx_event_t));
                 conn_op_ctx->event.handler =
                     ngx_http_lua_socket_tcp_conn_op_timeout_handler;
@@ -3977,25 +3982,10 @@ ngx_http_lua_socket_tcp_resume_conn_op(ngx_http_lua_socket_pool_t *spool)
      * is exiting.
      */
     if (ngx_queue_empty(&spool->wait_connect_op)) {
-#if (NGX_DEBUG)
-        ngx_http_lua_assert(!(spool->backlog >= 0
-                              && spool->connections > spool->size));
-
-#else
-        if (spool->backlog >= 0 && spool->connections > spool->size) {
-            ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0,
-                          "lua tcp socket connections count mismatched for "
-                          "connection pool \"%s\", connections: %i, size: %i",
-                          spool->key, spool->connections, spool->size);
-            spool->connections = spool->size;
-        }
-#endif
-
         return;
     }
 
     q = ngx_queue_head(&spool->wait_connect_op);
-    ngx_queue_remove(q);
     conn_op_ctx = ngx_queue_data(q, ngx_http_lua_socket_tcp_conn_op_ctx_t,
                                  queue);
     ngx_log_debug4(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
@@ -4034,27 +4024,52 @@ ngx_http_lua_socket_tcp_conn_op_ctx_cleanup(void *data)
 static void
 ngx_http_lua_socket_tcp_conn_op_resume_handler(ngx_event_t *ev)
 {
-    ngx_http_lua_socket_tcp_upstream_t      *u;
-    ngx_http_lua_ctx_t                      *ctx;
+    ngx_queue_t                             *q;
     ngx_connection_t                        *c;
+    ngx_http_lua_ctx_t                      *ctx;
     ngx_http_request_t                      *r;
     ngx_http_cleanup_t                      *cln;
     ngx_http_lua_co_ctx_t                   *coctx;
+    ngx_http_lua_socket_pool_t              *spool;
+    ngx_http_lua_socket_tcp_upstream_t      *u;
     ngx_http_lua_socket_tcp_conn_op_ctx_t   *conn_op_ctx;
 
     conn_op_ctx = ev->data;
     u = conn_op_ctx->u;
     r = u->request;
+    spool = u->socket_pool;
+
+    if (ngx_queue_empty(&spool->wait_connect_op)) {
+#if (NGX_DEBUG)
+        ngx_http_lua_assert(!(spool->backlog >= 0
+                              && spool->connections > spool->size));
+
+#else
+        if (spool->backlog >= 0 && spool->connections > spool->size) {
+            ngx_log_error(NGX_LOG_ERR, ngx_cycle->log, 0,
+                          "lua tcp socket connections count mismatched for "
+                          "connection pool \"%s\", connections: %i, size: %i",
+                          spool->key, spool->connections, spool->size);
+            spool->connections = spool->size;
+        }
+#endif
+
+        return;
+    }
+
+    q = ngx_queue_head(&spool->wait_connect_op);
+    ngx_queue_remove(q);
 
     coctx = u->write_co_ctx;
     coctx->cleanup = NULL;
     /* note that we store conn_op_ctx in coctx->data instead of u */
     coctx->data = conn_op_ctx;
+    /* clear ngx_http_lua_tcp_queue_conn_op_cleanup */
     u->write_co_ctx = NULL;
 
     ctx = ngx_http_get_module_ctx(r, ngx_http_lua_module);
     if (ctx == NULL) {
-        ngx_queue_insert_head(&u->socket_pool->cache_connect_op,
+        ngx_queue_insert_head(&spool->cache_connect_op,
                               &conn_op_ctx->queue);
         return;
     }
@@ -6021,6 +6036,36 @@ ngx_http_lua_socket_tcp_resume_helper(ngx_http_request_t *r, int socket_op)
 }
 
 
+static void
+ngx_http_lua_tcp_queue_conn_op_cleanup(void *data)
+{
+    ngx_http_lua_co_ctx_t                  *coctx = data;
+    ngx_http_lua_socket_tcp_upstream_t     *u;
+    ngx_http_lua_socket_tcp_conn_op_ctx_t  *conn_op_ctx;
+
+    conn_op_ctx = coctx->data;
+    u = conn_op_ctx->u;
+
+    ngx_log_debug2(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0,
+                   "lua tcp socket abort queueing, conn_op_ctx: %p, u: %p",
+                   conn_op_ctx, u);
+
+    if (conn_op_ctx->event.posted) {
+        ngx_delete_posted_event(&conn_op_ctx->event);
+
+    } else if (conn_op_ctx->event.timer_set) {
+        ngx_del_timer(&conn_op_ctx->event);
+    }
+
+    ngx_queue_remove(&conn_op_ctx->queue);
+    ngx_queue_insert_head(&u->socket_pool->cache_connect_op,
+                          &conn_op_ctx->queue);
+
+    u->socket_pool->connections--;
+    ngx_http_lua_socket_tcp_resume_conn_op(u->socket_pool);
+}
+
+
 static void
 ngx_http_lua_tcp_resolve_cleanup(void *data)
 {

t/068-socket-keepalive.t

@@ -2909,3 +2909,50 @@ GET /t
 [error]
 --- response_body
 ok
+
+
+
+=== TEST 52: conn queuing: clean up pending connect operations which are in queue
+--- config
+    set $port $TEST_NGINX_MEMCACHED_PORT;
+
+    location /sub {
+        content_by_lua_block {
+            local opts = {pool = "test", pool_size = 1, backlog = 1}
+            local sock, err = ngx.socket.connect("127.0.0.1", ngx.var.port, opts)
+            if not sock then
+                ngx.say("connect failed: " .. err)
+                return
+            end
+
+            local function f()
+                assert(ngx.socket.connect("127.0.0.1", ngx.var.port, opts))
+            end
+
+            local th = ngx.thread.spawn(f)
+            local ok, err = ngx.thread.kill(th)
+            if not ok then
+                ngx.log(ngx.ERR, "kill thread failed: ", err)
+                return
+            end
+
+            sock:close()
+        }
+    }
+
+    location /t {
+        content_by_lua_block {
+            ngx.location.capture("/sub")
+            -- let pending connect operation resumes first
+            ngx.sleep(0)
+            ngx.say("ok")
+        }
+    }
+--- request
+GET /t
+--- no_error_log
+[error]
+--- error_log
+lua tcp socket abort queueing
+--- response_body
+ok