nginx balancer example

nginx_for_production.conf

worker_processes  1;
daemon off;
error_log /dev/stdout;

events {
    worker_connections  1024;
}


http {
    resolver 127.0.0.1 valid=1s;

    include       mime.types;
    default_type  application/octet-stream;

    log_format ltsv "time:$time_local"
                    "\tstatus:$status"
                    "\treqtime:$request_time"
                    "\tappaddr:$upstream_addr"
                    "\tappcache:$upstream_cache_status"
                    "\tapptime:$upstream_response_time"
                    ;

    access_log  /dev/stdout global_ltsv;

    server {
        listen 80;
        
        set $proxy_upstream_host 'oreore-upstream.com';
        set $proxy_upstream_port '80';

        # upstream
        upstream backend_lua {
            server 0.0.0.1;

            balancer_by_lua_block {
                local balancer = require "ngx.balancer"
                if not ngx.ctx.tries then
                    ngx.ctx.tries = 0
                end

                if ngx.ctx.tries < #ngx.ctx.upstream_servers then
                    b.set_more_tries(1)
                end
                ngx.ctx.tries = ngx.ctx.tries + 1

                -- get upstream server
                local host = ngx.ctx.upstream_servers[ngx.ctx.tries]["address"]
                local port = ngx.var.proxy_upstream_port
                local ok, err = balancer.set_current_peer(host, port)
                if not ok then
                    ngx.log(ngx.ERR, "failed to set the peer: ", err)
                end
            }

            keepalive 10;  # connection pool
        }

        # location
        location / {
            rewrite_by_lua_block {
                local resolver = require "resty.dns.resolver"
                local r, err = resolver:new {
                  nameservers = {{"127.0.0.1", 53}},
                  retrans = 3,    -- 3 retransmissions on receive timeout
                  timeout = 1000, -- 1 sec
                }
                if not r then
                  ngx.log(ngx.ERR, "failed to instantiate the resolver: ", err)
                  ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
                end

                -- dns query
                local answers, err = r:query(ngx.var.proxy_upstream_port)
                if not answers then
                  ngx.log(ngx.ERR, "failed to query the DNS server: ", err)
                  ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
                end
                if answers.errcode then
                  ngx.log(ngx.ERR, "dns server returned error code: ", answers.errcode, ": ", answers.errstr)
                  ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
                end

                -- set ctx
                ngx.ctx.upstream_servers = answers
            }

            proxy_next_upstream_tries 10;
            proxy_pass http://external_backend_lua;
        }
    }

}

nginx_for_verification.conf

worker_processes  2;
daemon off;
error_log /dev/stdout;

events {
    worker_connections  1024;
}


http {
    resolver 127.0.0.1 valid=1s;

    include       mime.types;
    default_type  application/octet-stream;

    log_format ltsv "time:$time_local"
                           "\thost:$remote_addr"
                           "\tuser:$remote_user"
                           "\tprotocol:$server_protocol"
                           "\tmethod:$request_method"
                           "\tforwardedfor:$http_x_forwarded_for"
                           "\turi:$uri"
                           "\tquery_string:$query_string"
                           "\tstatus:$status"
                           "\tsize:$body_bytes_sent"
                           "\treqsize:$request_length"
                           "\treferer:$http_referer"
                           "\tua:$http_user_agent"
                           "\tvhost:$host"
                           "\treqtime:$request_time"
                           "\tappaddr:$upstream_addr"
                           "\tappcache:$upstream_cache_status"
                           "\tapptime:$upstream_response_time"
                           ;

    access_log  logs/access.log  ltsv;

    sendfile    on;
    tcp_nopush     on;
    keepalive_timeout  65;

    init_by_lua_block {
        require 'resty.core';
    }

    # local fakeserver
    server {
        listen 127.0.0.2:8080;

        location = /fake {
            echo "this is the fake backend peer...";
        }
    }
    
    # 80: nginx: to internal
    #################################################
    server {
        listen 80;

        upstream backend {
            server 127.0.0.2:8080;
            keepalive 10;
        }

        location / {
            proxy_pass http://backend/fake;
            proxy_http_version 1.1;
            proxy_set_header Connection "";
        }
    }

    # 81: lua: to internal
    #################################################
    upstream backend_lua {
        server 0.0.0.1;
    
        balancer_by_lua_block {
            local balancer = require "ngx.balancer"
            local host = "127.0.0.2"
            local port = 8080
            local ok, err = balancer.set_current_peer(host, port)
            if not ok then
                ngx.log(ngx.ERR, "failed to set the current peer: ", err)
                return ngx.exit(500)
            end
        }
    
        keepalive 10;  # connection pool
    }
    server {
        listen 81;

        location / {
            proxy_pass http://backend_lua/fake;
            proxy_http_version 1.1;
            proxy_set_header Connection "";
        }
    }

    # 82: nginx: to external
    #################################################
    upstream external_backend {
        server oreore-upstream.com:80 max_fails=0 fail_timeout=0;
        keepalive 10;
    }
    server {
        listen 82;

        location / {
            proxy_pass http://external_backend;
            proxy_http_version 1.1;
            proxy_set_header Connection "";
        }
    }

    # 83: lua: to external(and dns resolver)
    #################################################
    upstream external_backend_lua {
        server 0.0.0.1;

        balancer_by_lua_block {
            local balancer = require "ngx.balancer"
            if not ngx.ctx.tries then
                ngx.ctx.tries = 0
            end

            if ngx.ctx.tries < #ngx.ctx.upstream_servers then
                balancer.set_more_tries(1)
            end
            ngx.ctx.tries = ngx.ctx.tries + 1

            -- get upstream server
            local host = ngx.ctx.upstream_servers[ngx.ctx.tries]["address"]
            local port = ngx.var.proxy_upstream_port
            local ok, err = balancer.set_current_peer(host, port)
            if not ok then
                ngx.log(ngx.ERR, "failed to set the peer: ", err)
            end
        }

        keepalive 10;  # connection pool
    }
    server {
        listen 83;

        set $proxy_upstream_host 'oreore-upstream.com';
        set $proxy_upstream_port '80';
        
        location / {
            rewrite_by_lua_block {
                local resolver = require "resty.dns.resolver"
                local r, err = resolver:new {
                  nameservers = {{"127.0.0.1", 53}},
                  retrans = 3,    -- 3 retransmissions on receive timeout
                  timeout = 1000, -- 1 sec
                }
                if not r then
                  ngx.log(ngx.ERR, "failed to instantiate the resolver: ", err)
                  ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
                end

                -- dns query
                local answers, err = r:query(ngx.var.proxy_upstream_host)
                if not answers then
                  ngx.log(ngx.ERR, "failed to query the DNS server: ", err)
                  ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
                end
                if answers.errcode then
                  ngx.log(ngx.ERR, "dns server returned error code: ", answers.errcode, ": ", answers.errstr)
                  ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
                end

                -- set ctx
                ngx.ctx.upstream_servers = answers
            }

            # proxy_next_upstream_tries 3;
            proxy_pass http://external_backend_lua;
            proxy_http_version 1.1;
            proxy_set_header Connection "";
        }
    }
    
    # 84: lua: to external(and dns resolver and dns cache)
    #######################################################
    upstream external_backend_lua_2 {
        server 0.0.0.1;

        balancer_by_lua_block {
            local balancer = require "ngx.balancer"
            -- split
            function split(str, del)
              local p, nrep = str:gsub("%s*"..del.."%s*", "")
              return { str:match((("%s*(.-)%s*"..del.."%s*"):rep(nrep).."(.*)")) }
            end

            -- hosts random choice
            local hosts = split(ngx.ctx.upstream_hosts, ',')
            local host = hosts[ math.random(1, table.maxn(hosts)) ]
            local port = 80
            -- set current peer
            local ok, err = balancer.set_current_peer(host, port)
            if not ok then
              ngx.log(ngx.ERR, "failed to set the peer: ", err)
            end
        }

        keepalive 10;  # connection pool
    }
    server {
        listen 84;

        set $proxy_upstream_host 'oreore-upstream.com';
        set $proxy_upstream_port '80';

        location / {
            rewrite_by_lua_block {
                local resolver = require "resty.dns.resolver"
                local table_insert = table.insert
                local smem = ngx.shared.dns

                local key   = "hosts::" .. ngx.var.proxy_upstream_host
                local hosts = smem:get(key) or "" -- csv format
                if hosts == "" then
                  local r, err = resolver:new {
                    nameservers = {{"127.0.0.1", 53}},
                    retrans = 3,    -- 3 retransmissions on receive timeout
                    timeout = 1000, -- 1 sec
                  }
                  if not r then
                    ngx.log(ngx.ERR, "failed to instantiate the resolver: ", err)
                    ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
                  end

                  -- dns query
                  local answers, err = r:query(ngx.var.proxy_upstream_host)
                  if not answers then
                    ngx.log(ngx.ERR, "failed to query the DNS server: ", err)
                    ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
                  end
                  if answers.errcode then
                    ngx.log(ngx.ERR, "dns server returned error code: ", answers.errcode, ": ", answers.errstr)
                    ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
                  end

                  local i, answer
                  for i, answer in ipairs(answers) do
                    if answer["address"] then
                      if hosts ~= "" then hosts = hosts .. ","; end
                      hosts = hosts .. answer["address"]
                    end
                  end
				  
                  -- set shared memory (expire: 5sec)
                  smem:set(key, hosts, 5)
                end

                -- set ctx
                ngx.ctx.upstream_hosts = hosts
            }

            # proxy_next_upstream_tries 3;
            proxy_pass http://external_backend_lua_2;
            proxy_http_version 1.1;
            proxy_set_header Connection "";
        }
    }
}