NGINX Dynamic Proxy Example (Kik)

nginx.conf

# Credit: https://www.kik.com/blog/using-lua-to-extend-nginx-configuration/

worker_processes auto;  # process per cpu
worker_rlimit_nofile 8192;
events {
  worker_connections 65536;
  }
 
http {
  # https://t37.net/nginx-optimization-understanding-sendfile-tcp_nodelay-and-tcp_nopush.html
  sendfile        on;
  tcp_nopush      on;
  tcp_nodelay     on;
 
  keepalive_timeout   65;
  keepalive_requests  100000;
  types_hash_max_size 2048;
 
  # dynamic upstream
  upstream edge {
      server 0.0.0.1;   # just an invalid address as a place holder
 
      balancer_by_lua_block {
          local balancer = require “ngx.balancer”
          — init dynamic upstream with resolved target IP
          local ok, err = balancer.set_current_peer(ngx.var.target_ip, 443)
          if not ok then
              ngx.log(ngx.ERR, “failed to set the current peer: “, err)
              ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
          end
      }
 
      keepalive 1000;    # your connection pool
  }
 
  lua_shared_dict routes 1m;  # route cache
 
  server {
      listen 8080 default_server;
      location / {
          # target server variables to be inited by Lua
          set $target_name “”;
          set $target_ip “”;
 
          rewrite_by_lua_block {
              — get the target server name
              local m = ngx.re.match(ngx.var.uri, “^/([^/]+)/”, “o”)
              if not m then
                  ngx.exit(ngx.HTTP_BAD_REQUEST)
              end
              local target = m[1]
 
              — try cached route, resolve IP if missing
              local routes = ngx.shared.routes
              local route = routes:get(target)
              if route == nil then
                  local resolver = require “resty.dns.resolver”
                  local r, err = resolver:new{
                      — we use google, you are free to use your own resolver
                      nameservers = {“8.8.8.8”, {“8.8.4.4”, 53} },
                      retrans = 3,     — 3 retransmissions on receive timeout
                      timeout = 2000,  — 2 sec
                  }
 
                  if not r then
                      ngx.log(ngx.ERR, “failed to instantiate the resolver: “, err)
                      ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
                  end
 
                  local answers, err = r:query(target)
                  if not answers then
                      ngx.log(ngx.ERR, “failed to query the DNS server: “, err)
                      ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
                  end
 
                  if answers.errcode then
                      ngx.log(ngx.ERR, “resolver returned error code: “,
                          answers.errcode, “: “, answers.errstr)
                      ngx.status = ngx.HTTP_INTERNAL_SERVER_ERROR
                      ngx.say(“host “, target ” not found”)
                      ngx.exit(ngx.status)
                  end
 
                  — use the first entry for now
                  route = answers[1].address
 
                  — update global route cache
                  routes:set(ngx.var.target_server, route)
              end
 
              ngx.var.target_name = target
              ngx.var.target_ip = route
 
              — strip target prefix from uri
              local target_uri = ngx.re.sub(ngx.var.uri, “^/[^/]+/(.*)”, “/$1”)
              ngx.req.set_uri(target_uri)
          }
 
          # use our dynamic upstream
          proxy_pass https://edge;
 
          # proxy config
          proxy_redirect          off;
          proxy_buffering         off;
          proxy_set_header        Host $target_name;
          proxy_set_header        X-Real-IP $remote_addr;
          proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
          proxy_connect_timeout   5;
          proxy_send_timeout      10;
          proxy_read_timeout      10;
 
          # make sure http keepalives are enabled for upstream connections
          proxy_http_version      1.1;
          proxy_set_header        Connection “”;
          proxy_ssl_session_reuse on;  # not required, should be a default
      }
  }
  }