Skip to content

Instantly share code, notes, and snippets.

@gregvish
Created September 10, 2013 16:18
Show Gist options
  • Save gregvish/6511822 to your computer and use it in GitHub Desktop.
Save gregvish/6511822 to your computer and use it in GitHub Desktop.
diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c
index 5e62caa..9d41d71 100644
--- a/src/http/modules/ngx_http_proxy_module.c
+++ b/src/http/modules/ngx_http_proxy_module.c
@@ -513,6 +513,27 @@ static ngx_command_t ngx_http_proxy_commands[] = {
#endif
+ { ngx_string("proxy_upstream_default_keepalive"),
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
+ ngx_conf_set_flag_slot,
+ NGX_HTTP_LOC_CONF_OFFSET,
+ offsetof(ngx_http_proxy_loc_conf_t, upstream.default_keepalive),
+ NULL },
+
+ { ngx_string("proxy_upstream_default_keepalive_max_hosts"),
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
+ ngx_conf_set_size_slot,
+ NGX_HTTP_LOC_CONF_OFFSET,
+ offsetof(ngx_http_proxy_loc_conf_t, upstream.default_keepalive_max_hosts),
+ NULL },
+
+ { ngx_string("proxy_upstream_default_keepalive_max_connections"),
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
+ ngx_conf_set_size_slot,
+ NGX_HTTP_LOC_CONF_OFFSET,
+ offsetof(ngx_http_proxy_loc_conf_t, upstream.default_keepalive_max_connections),
+ NULL },
+
ngx_null_command
};
@@ -2385,6 +2406,10 @@ ngx_http_proxy_create_loc_conf(ngx_conf_t *cf)
* conf->redirects = NULL;
*/
+ conf->upstream.default_keepalive = NGX_CONF_UNSET;
+ conf->upstream.default_keepalive_max_hosts = NGX_CONF_UNSET_SIZE;
+ conf->upstream.default_keepalive_max_connections = NGX_CONF_UNSET_SIZE;
+
conf->upstream.store = NGX_CONF_UNSET;
conf->upstream.store_access = NGX_CONF_UNSET_UINT;
conf->upstream.buffering = NGX_CONF_UNSET;
@@ -2467,6 +2492,21 @@ ngx_http_proxy_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child)
}
}
+ ngx_conf_merge_uint_value(conf->upstream.default_keepalive,
+ prev->upstream.default_keepalive, 0);
+
+ ngx_conf_merge_size_value(conf->upstream.default_keepalive_max_connections,
+ prev->upstream.default_keepalive_max_connections, 10);
+
+ ngx_conf_merge_size_value(conf->upstream.default_keepalive_max_hosts,
+ prev->upstream.default_keepalive_max_hosts, 10);
+
+ if (conf->upstream.default_keepalive) {
+ if (ngx_http_upstream_default_keepalive_init(cf, &(conf->upstream)) != NGX_CONF_OK) {
+ return NGX_CONF_ERROR;
+ }
+ }
+
ngx_conf_merge_uint_value(conf->upstream.store_access,
prev->upstream.store_access, 0600);
diff --git a/src/http/modules/ngx_http_upstream_keepalive_module.c b/src/http/modules/ngx_http_upstream_keepalive_module.c
index eed1174..9f4d624 100644
--- a/src/http/modules/ngx_http_upstream_keepalive_module.c
+++ b/src/http/modules/ngx_http_upstream_keepalive_module.c
@@ -51,6 +51,19 @@ typedef struct {
} ngx_http_upstream_keepalive_cache_t;
+typedef struct {
+ ngx_queue_t queue;
+ char host[NGX_MAXHOSTNAMELEN];
+ uint16_t host_len;
+ uint16_t port;
+ ngx_http_upstream_keepalive_srv_conf_t kcf;
+} kcf_cache_entry_t;
+
+struct ngx_http_upstream_default_keepalive_cache_s {
+ ngx_queue_t kcf_cache;
+ ngx_queue_t kcf_free_list;
+};
+
static ngx_int_t ngx_http_upstream_init_keepalive_peer(ngx_http_request_t *r,
ngx_http_upstream_srv_conf_t *us);
@@ -537,3 +550,214 @@ invalid:
return NGX_CONF_ERROR;
}
+
+static ngx_int_t
+kcf_host_cache_add(ngx_http_upstream_default_keepalive_cache_t *cache,
+ ngx_str_t *host,
+ uint16_t port)
+{
+ kcf_cache_entry_t *kcf_cache_entry;
+ ngx_queue_t *q;
+
+ if (ngx_queue_empty(&cache->kcf_free_list)) {
+ return NGX_ERROR;
+ }
+
+ q = ngx_queue_head(&cache->kcf_free_list);
+ ngx_queue_remove(q);
+ kcf_cache_entry = ngx_queue_data(q, kcf_cache_entry_t, queue);
+
+ ngx_memcpy(&kcf_cache_entry->host, host->data, ngx_min(host->len, NGX_MAXHOSTNAMELEN));
+ kcf_cache_entry->host_len = host->len;
+ kcf_cache_entry->port = port;
+
+ ngx_queue_insert_head(&cache->kcf_cache, q);
+
+ return NGX_OK;
+}
+
+static ngx_http_upstream_keepalive_srv_conf_t *
+kcf_host_cache_get(ngx_http_upstream_default_keepalive_cache_t *cache,
+ ngx_str_t *host,
+ uint16_t port)
+{
+ kcf_cache_entry_t *kcf_cache_entry;
+ ngx_queue_t *q;
+
+ if (ngx_queue_empty(&cache->kcf_cache)) {
+ return NULL;
+ }
+
+ for (q = ngx_queue_head(&cache->kcf_cache);
+ q != ngx_queue_sentinel(&cache->kcf_cache);
+ q = ngx_queue_next(q))
+ {
+ kcf_cache_entry = ngx_queue_data(q, kcf_cache_entry_t, queue);
+
+ if ((kcf_cache_entry->host_len == host->len) &&
+ (ngx_memcmp(kcf_cache_entry->host, host->data, host->len) == 0) &&
+ (kcf_cache_entry->port == port)) {
+ return &kcf_cache_entry->kcf;
+ }
+ }
+
+ return NULL;
+}
+
+static void
+kcf_host_cache_collect_garbage(ngx_http_upstream_default_keepalive_cache_t *cache)
+{
+ kcf_cache_entry_t *kcf_cache_entry;
+ ngx_queue_t *q;
+ ngx_queue_t *next_q;
+
+ for (q = ngx_queue_head(&cache->kcf_cache);
+ q != ngx_queue_sentinel(&cache->kcf_cache);
+ q = next_q)
+ {
+ kcf_cache_entry = ngx_queue_data(q, kcf_cache_entry_t, queue);
+ next_q = ngx_queue_next(q);
+
+ if (!ngx_queue_empty(&kcf_cache_entry->kcf.cache)) {
+ continue;
+ }
+
+ ngx_memzero(kcf_cache_entry->host, NGX_MAXHOSTNAMELEN);
+ kcf_cache_entry->host_len = 0;
+ kcf_cache_entry->port = 0;
+
+ ngx_queue_remove(q);
+ ngx_queue_insert_head(&cache->kcf_free_list, q);
+ }
+}
+
+static ngx_http_upstream_keepalive_srv_conf_t *
+get_cached_keepalive_srv_conf_for_host(ngx_http_request_t *r,
+ ngx_str_t *host,
+ uint16_t port)
+{
+ ngx_http_upstream_keepalive_srv_conf_t *kcf;
+ ngx_http_upstream_default_keepalive_cache_t *cache;
+
+ cache = r->upstream->conf->default_keepalive_cache;
+
+ kcf = kcf_host_cache_get(cache, host, port);
+ if (kcf != NULL) {
+ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "default keepalive srv conf for host: cache hit");
+ return kcf;
+ }
+
+ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "default keepalive srv conf for host: cache miss");
+
+ if (ngx_queue_empty(&cache->kcf_free_list)) {
+ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "default keepalive srv conf for host: collecting garbage from cache");
+ kcf_host_cache_collect_garbage(cache);
+ }
+
+ if (kcf_host_cache_add(cache, host, port) != NGX_OK) {
+ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "default keepalive srv conf for host: adding host failed");
+ return NULL;
+ }
+
+ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "default keepalive srv conf for host: new host added");
+
+ kcf = kcf_host_cache_get(cache, host, port);
+
+ return kcf;
+}
+
+char *
+ngx_http_upstream_default_keepalive_init(ngx_conf_t *cf,
+ ngx_http_upstream_conf_t *umcf)
+{
+ ngx_http_upstream_keepalive_srv_conf_t *kcf;
+ ngx_http_upstream_keepalive_cache_t *cached;
+ kcf_cache_entry_t *kcf_cached;
+ ngx_uint_t i;
+ ngx_uint_t j;
+ ngx_http_upstream_default_keepalive_cache_t *host_cache;
+
+ host_cache = ngx_pcalloc(cf->pool,
+ sizeof(ngx_http_upstream_default_keepalive_cache_t));
+ if (host_cache == NULL) {
+ return NGX_CONF_ERROR;
+ }
+
+ ngx_queue_init(&host_cache->kcf_cache);
+ ngx_queue_init(&host_cache->kcf_free_list);
+
+ kcf_cached = ngx_pcalloc(cf->pool,
+ sizeof(kcf_cache_entry_t) * umcf->default_keepalive_max_hosts);
+
+ for (i = 0; i < umcf->default_keepalive_max_hosts; i += 1) {
+ kcf = &kcf_cached[i].kcf;
+ kcf->max_cached = umcf->default_keepalive_max_connections;
+
+ cached = ngx_pcalloc(cf->pool,
+ sizeof(ngx_http_upstream_keepalive_cache_t) * kcf->max_cached);
+ if (cached == NULL) {
+ return NGX_CONF_ERROR;
+ }
+
+ ngx_queue_init(&kcf->cache);
+ ngx_queue_init(&kcf->free);
+
+ for (j = 0; j < kcf->max_cached; j += 1) {
+ ngx_queue_insert_head(&kcf->free, &cached[j].queue);
+ cached[j].conf = kcf;
+ }
+
+ ngx_queue_insert_head(&(host_cache->kcf_free_list), &(kcf_cached[i].queue));
+ }
+
+ umcf->default_keepalive_cache = host_cache;
+
+ return NGX_CONF_OK;
+}
+
+ngx_int_t
+ngx_http_upstream_default_keepalive_adapt_peer(ngx_http_request_t *r,
+ ngx_http_upstream_resolved_t *ur)
+{
+ ngx_http_upstream_keepalive_peer_data_t *kp;
+ ngx_http_upstream_keepalive_srv_conf_t *kcf;
+
+ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "default keepalive adapt peer");
+
+ kcf = get_cached_keepalive_srv_conf_for_host(r, &ur->host, ur->port);
+ if (kcf == NULL) {
+ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "default keepalive adapt peer: failed to keepalive host");
+ return NGX_ERROR;
+ }
+
+ kp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_keepalive_peer_data_t));
+ if (kp == NULL) {
+ return NGX_ERROR;
+ }
+
+ kp->conf = kcf;
+ kp->upstream = r->upstream;
+ kp->data = r->upstream->peer.data;
+ kp->original_get_peer = r->upstream->peer.get;
+ kp->original_free_peer = r->upstream->peer.free;
+
+ r->upstream->peer.data = kp;
+ r->upstream->peer.get = ngx_http_upstream_get_keepalive_peer;
+ r->upstream->peer.free = ngx_http_upstream_free_keepalive_peer;
+
+#if (NGX_HTTP_SSL)
+ kp->original_set_session = r->upstream->peer.set_session;
+ kp->original_save_session = r->upstream->peer.save_session;
+ r->upstream->peer.set_session = ngx_http_upstream_keepalive_set_session;
+ r->upstream->peer.save_session = ngx_http_upstream_keepalive_save_session;
+#endif
+
+ return NGX_OK;
+}
diff --git a/src/http/modules/ngx_http_upstream_keepalive_module.h b/src/http/modules/ngx_http_upstream_keepalive_module.h
new file mode 100644
index 0000000..727fe5b
--- /dev/null
+++ b/src/http/modules/ngx_http_upstream_keepalive_module.h
@@ -0,0 +1,23 @@
+
+/*
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#ifndef _NGX_HTTP_UPSTREAM_KEEPALIVE_H_INCLUDED_
+#define _NGX_HTTP_UPSTREAM_KEEPALIVE_H_INCLUDED_
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_http.h>
+
+char *
+ngx_http_upstream_default_keepalive_init(ngx_conf_t *cf,
+ ngx_http_upstream_conf_t *umcf);
+
+ngx_int_t
+ngx_http_upstream_default_keepalive_adapt_peer(ngx_http_request_t *r,
+ ngx_http_upstream_resolved_t *ur);
+
+#endif /* _NGX_HTTP_UPSTREAM_KEEPALIVE_H_INCLUDED_ */
diff --git a/src/http/ngx_http.h b/src/http/ngx_http.h
index d4dc1bd..a5c5f34 100644
--- a/src/http/ngx_http.h
+++ b/src/http/ngx_http.h
@@ -19,6 +19,7 @@ typedef struct ngx_http_cache_s ngx_http_cache_t;
typedef struct ngx_http_file_cache_s ngx_http_file_cache_t;
typedef struct ngx_http_log_ctx_s ngx_http_log_ctx_t;
typedef struct ngx_http_chunked_s ngx_http_chunked_t;
+typedef struct ngx_http_upstream_default_keepalive_cache_s ngx_http_upstream_default_keepalive_cache_t;
#if (NGX_HTTP_SPDY)
typedef struct ngx_http_spdy_stream_s ngx_http_spdy_stream_t;
@@ -36,6 +37,7 @@ typedef u_char *(*ngx_http_log_handler_pt)(ngx_http_request_t *r,
#include <ngx_http_script.h>
#include <ngx_http_upstream.h>
#include <ngx_http_upstream_round_robin.h>
+#include <ngx_http_upstream_keepalive_module.h>
#include <ngx_http_busy_lock.h>
#include <ngx_http_core_module.h>
diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c
index 45e2eb7..39ad475 100644
--- a/src/http/ngx_http_upstream.c
+++ b/src/http/ngx_http_upstream.c
@@ -931,6 +931,13 @@ ngx_http_upstream_resolve_handler(ngx_resolver_ctx_t *ctx)
goto failed;
}
+ if (r->upstream->conf->default_keepalive) {
+ if (ngx_http_upstream_default_keepalive_adapt_peer(r, ur) != NGX_OK) {
+ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "http upstream default keepalive: can't keepalive peer");
+ }
+ }
+
ngx_resolve_name_done(ctx);
ur->ctx = NULL;
diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h
index 29ebf9b..fe0cdea 100644
--- a/src/http/ngx_http_upstream.h
+++ b/src/http/ngx_http_upstream.h
@@ -194,6 +194,11 @@ typedef struct {
#endif
ngx_str_t module;
+
+ ngx_uint_t default_keepalive;
+ ngx_uint_t default_keepalive_max_hosts;
+ ngx_uint_t default_keepalive_max_connections;
+ ngx_http_upstream_default_keepalive_cache_t *default_keepalive_cache;
} ngx_http_upstream_conf_t;
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment