Broken SSL Passthrough

failure

 [FAILED] in [It] - /go/src/k8s.io/ingress-nginx/test/e2e/framework/ssl.go:149 @ 09/05/23 22:20:44.704
  < Exit [It] should pass unknown traffic to default backend and handle known traffic

Hypothesis

For the default backend, apparently it is using proxy_protocol. Proxy is not implemented yet, as NGINX does not support setting proxy using a variable.

Probably need to create 2 segregated internal listeners, one with proxy and one without it.

Maybe, if I can already pass inside what backend should be used (a dynamic variable with the script loaded globally? otherwise will have to query the shared map twice)

nginx.conf

# Configuration checksum: 7779263455934520812

  # setup custom paths that do not require root access
  pid /tmp/nginx/nginx.pid;

  load_module /etc/nginx/modules/ngx_stream_js_module.so;

  daemon off;

  worker_processes 1;

  worker_rlimit_nofile 1073740792;

  worker_shutdown_timeout 240s ;

  events {
     multi_accept        on;
     worker_connections  16384;
     use                 epoll;

  }

  http {

     lua_package_path "/etc/nginx/lua/?.lua;;";

     lua_shared_dict balancer_ewma 10M;
     lua_shared_dict balancer_ewma_last_touched_at 10M;
     lua_shared_dict balancer_ewma_locks 1M;
     lua_shared_dict certificate_data 20M;
     lua_shared_dict certificate_servers 5M;
     lua_shared_dict configuration_data 20M;
     lua_shared_dict global_throttle_cache 10M;
     lua_shared_dict ocsp_response_cache 5M;

     init_by_lua_block {
        collectgarbage("collect")

        -- init modules
        local ok, res

        ok, res = pcall(require, "lua_ingress")
        if not ok then
        error("require failed: " .. tostring(res))
        else
        lua_ingress = res
        lua_ingress.set_config({
         use_forwarded_headers = false,
         use_proxy_protocol = false,
         is_ssl_passthrough_enabled = true,
         http_redirect_code = 308,
         listen_ports = { ssl_proxy = "442", https = "443" },

         hsts = true,
         hsts_max_age = 15724800,
         hsts_include_subdomains = true,
         hsts_preload = false,

         global_throttle = {
           memcached = {
            host = "", port = 11211, connect_timeout = 50, max_idle_timeout = 10000, pool_size = 50,
           },
           status_code = 429,
         }
        })
        end

        ok, res = pcall(require, "configuration")
        if not ok then
        error("require failed: " .. tostring(res))
        else
        configuration = res
        configuration.prohibited_localhost_port = '10246'
        end

        ok, res = pcall(require, "balancer")
        if not ok then
        error("require failed: " .. tostring(res))
        else
        balancer = res
        end

        ok, res = pcall(require, "monitor")
        if not ok then
        error("require failed: " .. tostring(res))
        else
        monitor = res
        end

        ok, res = pcall(require, "certificate")
        if not ok then
        error("require failed: " .. tostring(res))
        else
        certificate = res
        certificate.is_ocsp_stapling_enabled = false
        end

        ok, res = pcall(require, "plugins")
        if not ok then
        error("require failed: " .. tostring(res))
        else
        plugins = res
        end
        -- load all plugins that'll be used here
        plugins.init({  })
     }

     init_worker_by_lua_block {
        lua_ingress.init_worker()
        balancer.init_worker()

        monitor.init_worker(10000)

        plugins.run()
     }

     geoip_country       /etc/nginx/geoip/GeoIP.dat;
     geoip_city          /etc/nginx/geoip/GeoLiteCity.dat;
     geoip_org           /etc/nginx/geoip/GeoIPASNum.dat;
     geoip_proxy_recursive on;

     aio                 threads;
     aio_write           on;

     tcp_nopush          on;
     tcp_nodelay         on;

     log_subrequest      on;

     reset_timedout_connection on;

     keepalive_timeout  75s;
     keepalive_requests 1000;

     client_body_temp_path           /tmp/nginx/client-body;
     fastcgi_temp_path               /tmp/nginx/fastcgi-temp;
     proxy_temp_path                 /tmp/nginx/proxy-temp;

     client_header_buffer_size       1k;
     client_header_timeout           60s;
     large_client_header_buffers     4 8k;
     client_body_buffer_size         8k;
     client_body_timeout             60s;

     http2_max_concurrent_streams    128;

     types_hash_max_size             2048;
     server_names_hash_max_size      1024;
     server_names_hash_bucket_size   64;
     map_hash_bucket_size            64;

     proxy_headers_hash_max_size     512;
     proxy_headers_hash_bucket_size  64;

     variables_hash_bucket_size      256;
     variables_hash_max_size         2048;

     underscores_in_headers          off;
     ignore_invalid_headers          on;

     limit_req_status                503;
     limit_conn_status               503;

     include /etc/nginx/mime.types;
     default_type text/html;

     # Custom headers for response

     server_tokens off;

     more_clear_headers Server;

     # disable warnings
     uninitialized_variable_warn off;

     # Additional available variables:
     # $namespace
     # $ingress_name
     # $service_name
     # $service_port
     log_format upstreaminfo '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_length $request_time [$proxy_upstream_name] [$proxy_alternative_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status $req_id';

     map $request_uri $loggable {

        default 1;
     }

     access_log /var/log/nginx/access.log upstreaminfo  if=$loggable;

     error_log  /var/log/nginx/error.log notice;

     resolver 10.96.0.10 valid=30s;

     # See https://www.nginx.com/blog/websocket-nginx
     map $http_upgrade $connection_upgrade {
        default          upgrade;

        # See http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive
        ''               '';

     }

     # Reverse proxies can detect if a client provides a X-Request-ID header, and pass it on to the backend server.
     # If no such header is provided, it can provide a random value.
     map $http_x_request_id $req_id {
        default   $http_x_request_id;

        ""        $request_id;

     }

     # Create a variable that contains the literal $ character.
     # This works because the geo module will not resolve variables.
     geo $literal_dollar {
        default "$";
     }

     server_name_in_redirect off;
     port_in_redirect        off;

     ssl_protocols TLSv1.2 TLSv1.3;

     ssl_early_data off;

     # turn on session caching to drastically improve performance

     ssl_session_cache shared:SSL:10m;
     ssl_session_timeout 10m;

     # allow configuring ssl session tickets
     ssl_session_tickets off;

     # slightly reduce the time-to-first-byte
     ssl_buffer_size 4k;

     # allow configuring custom ssl ciphers
     ssl_ciphers 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384';
     ssl_prefer_server_ciphers on;

     ssl_ecdh_curve auto;

     # PEM sha: 752cc9cf9ffb27b0ed35bb3c4c221356aa774b00
     ssl_certificate     /etc/ingress-controller/ssl/default-fake-certificate.pem;
     ssl_certificate_key /etc/ingress-controller/ssl/default-fake-certificate.pem;

     proxy_ssl_session_reuse on;

     upstream upstream_balancer {
        server 0.0.0.1; # placeholder

        balancer_by_lua_block {
         balancer.balance()
        }

        keepalive 320;
        keepalive_time 1h;
        keepalive_timeout  60s;
        keepalive_requests 10000;

     }

     # Cache for internal auth checks
     proxy_cache_path /tmp/nginx/nginx-cache-auth levels=1:2 keys_zone=auth_cache:10m max_size=128m inactive=30m use_temp_path=off;

     # Global filters

     ## start server _
     server {
        server_name _ ;

        listen 80 default_server reuseport backlog=4096 ;
        listen [::]:80 default_server reuseport backlog=4096 ;
        listen 442 proxy_protocol default_server reuseport backlog=4096 ssl http2 ;
        listen [::]:442 proxy_protocol default_server reuseport backlog=4096 ssl http2 ;

        set $proxy_upstream_name "-";

        ssl_reject_handshake off;

        ssl_certificate_by_lua_block {
         certificate.call()
        }

        location / {

         set $namespace      "";
         set $ingress_name   "";
         set $service_name   "";
         set $service_port   "";
         set $location_path  "";
         set $global_rate_limit_exceeding n;

         rewrite_by_lua_block {
           lua_ingress.rewrite({
            force_ssl_redirect = false,
            ssl_redirect = false,
            force_no_ssl_redirect = false,
            preserve_trailing_slash = false,
            use_port_in_redirects = false,
            global_throttle = { namespace = "", limit = 0, window_size = 0, key = { }, ignored_cidrs = { } },
           })
           balancer.rewrite()
           plugins.run()
         }

         # be careful with `access_by_lua_block` and `satisfy any` directives as satisfy any
         # will always succeed when there's `access_by_lua_block` that does not have any lua code doing `ngx.exit(ngx.DECLINED)`
         # other authentication method such as basic auth or external auth useless - all requests will be allowed.
         #access_by_lua_block {
         #}

         header_filter_by_lua_block {
           lua_ingress.header()
           plugins.run()
         }

         body_filter_by_lua_block {
           plugins.run()
         }

         log_by_lua_block {
           balancer.log()

           monitor.call()

           plugins.run()
         }

         access_log off;

         port_in_redirect off;

         set $balancer_ewma_score -1;
         set $proxy_upstream_name "upstream-default-backend";
         set $proxy_host          $proxy_upstream_name;
         set $pass_access_scheme  $scheme;

         set $pass_server_port    $server_port;

         set $best_http_host      $http_host;
         set $pass_port           $pass_server_port;

         set $proxy_alternative_upstream_name "";

         client_max_body_size                    1m;

         proxy_set_header Host                   $best_http_host;

         # Pass the extracted client certificate to the backend

         # Allow websocket connections
         proxy_set_header                        Upgrade           $http_upgrade;

         proxy_set_header                        Connection        $connection_upgrade;

         proxy_set_header X-Request-ID           $req_id;
         proxy_set_header X-Real-IP              $remote_addr;

         proxy_set_header X-Forwarded-For        $remote_addr;

         proxy_set_header X-Forwarded-Host       $best_http_host;
         proxy_set_header X-Forwarded-Port       $pass_port;
         proxy_set_header X-Forwarded-Proto      $pass_access_scheme;
         proxy_set_header X-Forwarded-Scheme     $pass_access_scheme;

         proxy_set_header X-Scheme               $pass_access_scheme;

         # Pass the original X-Forwarded-For
         proxy_set_header X-Original-Forwarded-For $http_x_forwarded_for;

         # mitigate HTTPoxy Vulnerability
         # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/
         proxy_set_header Proxy                  "";

         # Custom headers to proxied server

         proxy_connect_timeout                   5s;
         proxy_send_timeout                      60s;
         proxy_read_timeout                      60s;

         proxy_buffering                         off;
         proxy_buffer_size                       4k;
         proxy_buffers                           4 4k;

         proxy_max_temp_file_size                1024m;

         proxy_request_buffering                 on;
         proxy_http_version                      1.1;

         proxy_cookie_domain                     off;
         proxy_cookie_path                       off;

         # In case of errors try the next upstream server before returning an error
         proxy_next_upstream                     error timeout;
         proxy_next_upstream_timeout             0;
         proxy_next_upstream_tries               3;

         proxy_pass http://upstream_balancer;

         proxy_redirect                          off;

        }

        # health checks in cloud providers require the use of port 80
        location /healthz {

         access_log off;
         return 200;
        }

        # this is required to avoid error if nginx is being monitored
        # with an external software (like sysdig)
        location /nginx_status {

         allow 127.0.0.1;

         allow ::1;

         deny all;

         access_log off;
         stub_status on;
        }

     }
     ## end server _

     ## start server noannotationnopassthrough.com
     server {
        server_name noannotationnopassthrough.com ;

        listen 80  ;
        listen [::]:80  ;
        listen 442 proxy_protocol  ssl http2 ;
        listen [::]:442 proxy_protocol  ssl http2 ;

        set $proxy_upstream_name "-";

        ssl_certificate_by_lua_block {
         certificate.call()
        }

        location / {

         set $namespace      "e2e-tests-ssl-passthrough-1693952310890885285-hn2td";
         set $ingress_name   "noannotationnopassthrough.com";
         set $service_name   "echopass";
         set $service_port   "80";
         set $location_path  "/";
         set $global_rate_limit_exceeding n;

         rewrite_by_lua_block {
           lua_ingress.rewrite({
            force_ssl_redirect = false,
            ssl_redirect = true,
            force_no_ssl_redirect = false,
            preserve_trailing_slash = false,
            use_port_in_redirects = false,
            global_throttle = { namespace = "", limit = 0, window_size = 0, key = { }, ignored_cidrs = { } },
           })
           balancer.rewrite()
           plugins.run()
         }

         # be careful with `access_by_lua_block` and `satisfy any` directives as satisfy any
         # will always succeed when there's `access_by_lua_block` that does not have any lua code doing `ngx.exit(ngx.DECLINED)`
         # other authentication method such as basic auth or external auth useless - all requests will be allowed.
         #access_by_lua_block {
         #}

         header_filter_by_lua_block {
           lua_ingress.header()
           plugins.run()
         }

         body_filter_by_lua_block {
           plugins.run()
         }

         log_by_lua_block {
           balancer.log()

           monitor.call()

           plugins.run()
         }

         port_in_redirect off;

         set $balancer_ewma_score -1;
         set $proxy_upstream_name "e2e-tests-ssl-passthrough-1693952310890885285-hn2td-echopass-80";
         set $proxy_host          $proxy_upstream_name;
         set $pass_access_scheme  $scheme;

         set $pass_server_port    $server_port;

         set $best_http_host      $http_host;
         set $pass_port           $pass_server_port;

         set $proxy_alternative_upstream_name "";

         client_max_body_size                    1m;

         proxy_set_header Host                   $best_http_host;

         # Pass the extracted client certificate to the backend

         # Allow websocket connections
         proxy_set_header                        Upgrade           $http_upgrade;

         proxy_set_header                        Connection        $connection_upgrade;

         proxy_set_header X-Request-ID           $req_id;
         proxy_set_header X-Real-IP              $remote_addr;

         proxy_set_header X-Forwarded-For        $remote_addr;

         proxy_set_header X-Forwarded-Host       $best_http_host;
         proxy_set_header X-Forwarded-Port       $pass_port;
         proxy_set_header X-Forwarded-Proto      $pass_access_scheme;
         proxy_set_header X-Forwarded-Scheme     $pass_access_scheme;

         proxy_set_header X-Scheme               $pass_access_scheme;

         # Pass the original X-Forwarded-For
         proxy_set_header X-Original-Forwarded-For $http_x_forwarded_for;

         # mitigate HTTPoxy Vulnerability
         # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/
         proxy_set_header Proxy                  "";

         # Custom headers to proxied server

         proxy_connect_timeout                   5s;
         proxy_send_timeout                      60s;
         proxy_read_timeout                      60s;

         proxy_buffering                         off;
         proxy_buffer_size                       4k;
         proxy_buffers                           4 4k;

         proxy_max_temp_file_size                1024m;

         proxy_request_buffering                 on;
         proxy_http_version                      1.1;

         proxy_cookie_domain                     off;
         proxy_cookie_path                       off;

         # In case of errors try the next upstream server before returning an error
         proxy_next_upstream                     error timeout;
         proxy_next_upstream_timeout             0;
         proxy_next_upstream_tries               3;

         proxy_pass http://upstream_balancer;

         proxy_redirect                          off;

        }

     }
     ## end server noannotationnopassthrough.com

     ## start server testpassthrough.com
     server {
        server_name testpassthrough.com ;

        listen 80  ;
        listen [::]:80  ;
        listen 442 proxy_protocol  ssl http2 ;
        listen [::]:442 proxy_protocol  ssl http2 ;

        set $proxy_upstream_name "-";

        ssl_certificate_by_lua_block {
         certificate.call()
        }

        location / {

         set $namespace      "e2e-tests-ssl-passthrough-1693952310890885285-hn2td";
         set $ingress_name   "testpassthrough.com";
         set $service_name   "echopass";
         set $service_port   "80";
         set $location_path  "/";
         set $global_rate_limit_exceeding n;

         rewrite_by_lua_block {
           lua_ingress.rewrite({
            force_ssl_redirect = false,
            ssl_redirect = true,
            force_no_ssl_redirect = false,
            preserve_trailing_slash = false,
            use_port_in_redirects = false,
            global_throttle = { namespace = "", limit = 0, window_size = 0, key = { }, ignored_cidrs = { } },
           })
           balancer.rewrite()
           plugins.run()
         }

         # be careful with `access_by_lua_block` and `satisfy any` directives as satisfy any
         # will always succeed when there's `access_by_lua_block` that does not have any lua code doing `ngx.exit(ngx.DECLINED)`
         # other authentication method such as basic auth or external auth useless - all requests will be allowed.
         #access_by_lua_block {
         #}

         header_filter_by_lua_block {
           lua_ingress.header()
           plugins.run()
         }

         body_filter_by_lua_block {
           plugins.run()
         }

         log_by_lua_block {
           balancer.log()

           monitor.call()

           plugins.run()
         }

         port_in_redirect off;

         set $balancer_ewma_score -1;
         set $proxy_upstream_name "e2e-tests-ssl-passthrough-1693952310890885285-hn2td-echopass-80";
         set $proxy_host          $proxy_upstream_name;
         set $pass_access_scheme  $scheme;

         set $pass_server_port    $server_port;

         set $best_http_host      $http_host;
         set $pass_port           $pass_server_port;

         set $proxy_alternative_upstream_name "";

         client_max_body_size                    1m;

         proxy_set_header Host                   $best_http_host;

         # Pass the extracted client certificate to the backend

         # Allow websocket connections
         proxy_set_header                        Upgrade           $http_upgrade;

         proxy_set_header                        Connection        $connection_upgrade;

         proxy_set_header X-Request-ID           $req_id;
         proxy_set_header X-Real-IP              $remote_addr;

         proxy_set_header X-Forwarded-For        $remote_addr;

         proxy_set_header X-Forwarded-Host       $best_http_host;
         proxy_set_header X-Forwarded-Port       $pass_port;
         proxy_set_header X-Forwarded-Proto      $pass_access_scheme;
         proxy_set_header X-Forwarded-Scheme     $pass_access_scheme;

         proxy_set_header X-Scheme               $pass_access_scheme;

         # Pass the original X-Forwarded-For
         proxy_set_header X-Original-Forwarded-For $http_x_forwarded_for;

         # mitigate HTTPoxy Vulnerability
         # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/
         proxy_set_header Proxy                  "";

         # Custom headers to proxied server

         proxy_connect_timeout                   5s;
         proxy_send_timeout                      60s;
         proxy_read_timeout                      60s;

         proxy_buffering                         off;
         proxy_buffer_size                       4k;
         proxy_buffers                           4 4k;

         proxy_max_temp_file_size                1024m;

         proxy_request_buffering                 on;
         proxy_http_version                      1.1;

         proxy_cookie_domain                     off;
         proxy_cookie_path                       off;

         # In case of errors try the next upstream server before returning an error
         proxy_next_upstream                     error timeout;
         proxy_next_upstream_timeout             0;
         proxy_next_upstream_tries               3;

         proxy_pass http://upstream_balancer;

         proxy_redirect                          off;

        }

     }
     ## end server testpassthrough.com

     # backend for when default-backend-service is not configured or it does not have endpoints
     server {
        listen 8181 default_server reuseport backlog=4096;
        listen [::]:8181 default_server reuseport backlog=4096;
        set $proxy_upstream_name "internal";

        access_log off;

        location / {
         return 404;
        }
     }

     # default server, used for NGINX healthcheck and access to nginx stats
     server {
        # Ensure that modsecurity will not run on an internal location as this is not accessible from outside

        listen 127.0.0.1:10246;
        set $proxy_upstream_name "internal";

        keepalive_timeout 0;
        gzip off;

        access_log off;

        location /healthz {
         return 200;
        }

        location /is-dynamic-lb-initialized {
         content_by_lua_block {
           local configuration = require("configuration")
           local backend_data = configuration.get_backends_data()
           if not backend_data then
           ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
           return
           end

           ngx.say("OK")
           ngx.exit(ngx.HTTP_OK)
         }
        }

        location /nginx_status {
         stub_status on;
        }

        location /configuration {
         client_max_body_size                    21M;
         client_body_buffer_size                 21M;
         proxy_buffering                         off;

         content_by_lua_block {
           configuration.call()
         }
        }

        location / {
         content_by_lua_block {
           ngx.exit(ngx.HTTP_NOT_FOUND)
         }
        }
     }
  }

  stream {
     lua_package_path "/etc/nginx/lua/?.lua;/etc/nginx/lua/vendor/?.lua;;";

     lua_shared_dict tcp_udp_configuration_data 5M;

     resolver 10.96.0.10 valid=30s;

     init_by_lua_block {
        collectgarbage("collect")

        -- init modules
        local ok, res

        ok, res = pcall(require, "configuration")
        if not ok then
        error("require failed: " .. tostring(res))
        else
        configuration = res
        end

        ok, res = pcall(require, "tcp_udp_configuration")
        if not ok then
        error("require failed: " .. tostring(res))
        else
        tcp_udp_configuration = res
        tcp_udp_configuration.prohibited_localhost_port = '10246'

        end

        ok, res = pcall(require, "tcp_udp_balancer")
        if not ok then
        error("require failed: " .. tostring(res))
        else
        tcp_udp_balancer = res
        end
     }

     init_worker_by_lua_block {
        tcp_udp_balancer.init_worker()
     }

     lua_add_variable $proxy_upstream_name;

     log_format log_stream '[$remote_addr] [$time_local] $protocol $status $bytes_sent $bytes_received $session_time';

     access_log /var/log/nginx/access.log log_stream ;

     error_log  /var/log/nginx/error.log notice;

     upstream upstream_balancer {
        server 0.0.0.1:1234; # placeholder

        balancer_by_lua_block {
         tcp_udp_balancer.balance()
        }
     }

     js_import /etc/nginx/njs/passthrough.js;
     js_shared_dict_zone zone=ptbackends:32m type=string;

     server {
        listen 19090;
        js_preread passthrough.configBackends;
        js_set $cfgreturn passthrough.getConfigStatus;
        return $cfgreturn;
     }

     server {
        # TODO: Remove Hardcode
        listen 443;
        ssl_preread on;
        js_set $ptupstream passthrough.getBackend;
        proxy_pass $ptupstream;
     }

     server {
        listen 127.0.0.1:10247;

        access_log off;

        content_by_lua_block {
         tcp_udp_configuration.call()
        }
     }

     # TCP services

     # UDP services

     # Stream Snippets

  }