diff --git a/mariadb/Chart.yaml b/mariadb/Chart.yaml index 7feb54f33..7637bc4d9 100644 --- a/mariadb/Chart.yaml +++ b/mariadb/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v10.2.31 description: OpenStack-Helm MariaDB name: mariadb -version: 0.1.8 +version: 0.1.9 home: https://mariadb.com/kb/en/ icon: http://badges.mariadb.org/mariadb-badge-180x60.png sources: diff --git a/mariadb/files/nginx.tmpl b/mariadb/files/nginx.tmpl index 356fad502..fc069b2fc 100644 --- a/mariadb/files/nginx.tmpl +++ b/mariadb/files/nginx.tmpl @@ -7,38 +7,132 @@ {{ $proxyHeaders := .ProxySetHeaders }} {{ $addHeaders := .AddHeaders }} -{{ if $cfg.EnableModsecurity }} +# Configuration checksum: {{ $all.Cfg.Checksum }} + +# setup custom paths that do not require root access +pid {{ .PID }}; + +{{ if $cfg.UseGeoIP2 }} +load_module /etc/nginx/modules/ngx_http_geoip2_module.so; +{{ end }} + +{{ if $cfg.EnableBrotli }} +load_module /etc/nginx/modules/ngx_http_brotli_filter_module.so; +load_module /etc/nginx/modules/ngx_http_brotli_static_module.so; +{{ end }} + +{{ if (shouldLoadInfluxDBModule $servers) }} +load_module /etc/nginx/modules/ngx_http_influxdb_module.so; +{{ end }} + +{{ if (shouldLoadAuthDigestModule $servers) }} +load_module /etc/nginx/modules/ngx_http_auth_digest_module.so; +{{ end }} + +{{ if (shouldLoadModSecurityModule $cfg $servers) }} load_module /etc/nginx/modules/ngx_http_modsecurity_module.so; {{ end }} -{{ if $cfg.EnableOpentracing }} +{{ if (shouldLoadOpentracingModule $cfg $servers) }} load_module /etc/nginx/modules/ngx_http_opentracing_module.so; {{ end }} -{{ if (and $cfg.EnableOpentracing (ne $cfg.ZipkinCollectorHost "")) }} -load_module /etc/nginx/modules/ngx_http_zipkin_module.so; -{{ end }} - daemon off; worker_processes {{ $cfg.WorkerProcesses }}; -pid /run/nginx.pid; -{{ if ne .MaxOpenFiles 0 }} -worker_rlimit_nofile {{ .MaxOpenFiles }}; +{{ if gt (len $cfg.WorkerCPUAffinity) 0 }} +worker_cpu_affinity {{ $cfg.WorkerCPUAffinity }}; {{ end }} +worker_rlimit_nofile {{ $cfg.MaxWorkerOpenFiles }}; + {{/* http://nginx.org/en/docs/ngx_core_module.html#worker_shutdown_timeout */}} {{/* avoid waiting too long during a reload */}} worker_shutdown_timeout {{ $cfg.WorkerShutdownTimeout }} ; +{{ if not (empty $cfg.MainSnippet) }} +{{ $cfg.MainSnippet }} +{{ end }} + events { - multi_accept on; + multi_accept {{ if $cfg.EnableMultiAccept }}on{{ else }}off{{ end }}; worker_connections {{ $cfg.MaxWorkerConnections }}; use epoll; } http { - {{/* we use the value of the header X-Forwarded-For to be able to use the geo_ip module */}} + lua_package_path "/etc/nginx/lua/?.lua;;"; + + {{ buildLuaSharedDictionaries $cfg $servers }} + + init_by_lua_block { + collectgarbage("collect") + + -- init modules + local ok, res + + ok, res = pcall(require, "lua_ingress") + if not ok then + error("require failed: " .. tostring(res)) + else + lua_ingress = res + lua_ingress.set_config({{ configForLua $all }}) + end + + ok, res = pcall(require, "configuration") + if not ok then + error("require failed: " .. tostring(res)) + else + configuration = res + end + + ok, res = pcall(require, "balancer") + if not ok then + error("require failed: " .. tostring(res)) + else + balancer = res + end + + {{ if $all.EnableMetrics }} + ok, res = pcall(require, "monitor") + if not ok then + error("require failed: " .. tostring(res)) + else + monitor = res + end + {{ end }} + + ok, res = pcall(require, "certificate") + if not ok then + error("require failed: " .. tostring(res)) + else + certificate = res + certificate.is_ocsp_stapling_enabled = {{ $cfg.EnableOCSP }} + end + + ok, res = pcall(require, "plugins") + if not ok then + error("require failed: " .. tostring(res)) + else + plugins = res + end + -- load all plugins that'll be used here + plugins.init({ {{ range $idx, $plugin := $cfg.Plugins }}{{ if $idx }},{{ end }}{{ $plugin | quote }}{{ end }} }) + } + + init_worker_by_lua_block { + lua_ingress.init_worker() + balancer.init_worker() + {{ if $all.EnableMetrics }} + monitor.init_worker({{ $all.MonitorMaxBatchSize }}) + {{ end }} + + plugins.run() + } + + {{/* Enable the real_ip module only if we use either X-Forwarded headers or Proxy Protocol. */}} + {{/* we use the value of the real IP for the geo_ip module */}} + {{ if or (or $cfg.UseForwardedHeaders $cfg.UseProxyProtocol) $cfg.EnableRealIp }} {{ if $cfg.UseProxyProtocol }} real_ip_header proxy_protocol; {{ else }} @@ -49,20 +143,121 @@ http { {{ range $trusted_ip := $cfg.ProxyRealIPCIDR }} set_real_ip_from {{ $trusted_ip }}; {{ end }} + {{ end }} + {{ if $all.Cfg.EnableModsecurity }} + modsecurity on; + + modsecurity_rules_file /etc/nginx/modsecurity/modsecurity.conf; + + {{ if $all.Cfg.EnableOWASPCoreRules }} + modsecurity_rules_file /etc/nginx/owasp-modsecurity-crs/nginx-modsecurity.conf; + {{ else if (not (empty $all.Cfg.ModsecuritySnippet)) }} + modsecurity_rules ' + {{ $all.Cfg.ModsecuritySnippet }} + '; + {{ end }} + + {{ end }} + + {{ if $cfg.UseGeoIP }} {{/* databases used to determine the country depending on the client IP address */}} {{/* http://nginx.org/en/docs/http/ngx_http_geoip_module.html */}} {{/* this is require to calculate traffic for individual country using GeoIP in the status page */}} - geoip_country /etc/nginx/GeoIP.dat; - geoip_city /etc/nginx/GeoLiteCity.dat; + geoip_country /etc/nginx/geoip/GeoIP.dat; + geoip_city /etc/nginx/geoip/GeoLiteCity.dat; + geoip_org /etc/nginx/geoip/GeoIPASNum.dat; geoip_proxy_recursive on; - - {{ if $cfg.EnableVtsStatus }} - vhost_traffic_status_zone shared:vhost_traffic_status:{{ $cfg.VtsStatusZoneSize }}; - vhost_traffic_status_filter_by_set_key {{ $cfg.VtsDefaultFilterKey }}; {{ end }} - sendfile on; + {{ if $cfg.UseGeoIP2 }} + # https://github.com/leev/ngx_http_geoip2_module#example-usage + + {{ range $index, $file := $all.MaxmindEditionFiles }} + {{ if eq $file "GeoLite2-Country.mmdb" }} + geoip2 /etc/nginx/geoip/GeoLite2-Country.mmdb { + $geoip2_country_code source=$remote_addr country iso_code; + $geoip2_country_name source=$remote_addr country names en; + $geoip2_continent_name source=$remote_addr continent names en; + } + {{ end }} + + {{ if eq $file "GeoIP2-Country.mmdb" }} + geoip2 /etc/nginx/geoip/GeoIP2-Country.mmdb { + $geoip2_country_code source=$remote_addr country iso_code; + $geoip2_country_name source=$remote_addr country names en; + $geoip2_continent_name source=$remote_addr continent names en; + } + {{ end }} + + {{ if eq $file "GeoLite2-City.mmdb" }} + geoip2 /etc/nginx/geoip/GeoLite2-City.mmdb { + $geoip2_city_country_code source=$remote_addr country iso_code; + $geoip2_city_country_name source=$remote_addr country names en; + $geoip2_city source=$remote_addr city names en; + $geoip2_postal_code source=$remote_addr postal code; + $geoip2_dma_code source=$remote_addr location metro_code; + $geoip2_latitude source=$remote_addr location latitude; + $geoip2_longitude source=$remote_addr location longitude; + $geoip2_time_zone source=$remote_addr location time_zone; + $geoip2_region_code source=$remote_addr subdivisions 0 iso_code; + $geoip2_region_name source=$remote_addr subdivisions 0 names en; + } + {{ end }} + + {{ if eq $file "GeoIP2-City.mmdb" }} + geoip2 /etc/nginx/geoip/GeoIP2-City.mmdb { + $geoip2_city_country_code source=$remote_addr country iso_code; + $geoip2_city_country_name source=$remote_addr country names en; + $geoip2_city source=$remote_addr city names en; + $geoip2_postal_code source=$remote_addr postal code; + $geoip2_dma_code source=$remote_addr location metro_code; + $geoip2_latitude source=$remote_addr location latitude; + $geoip2_longitude source=$remote_addr location longitude; + $geoip2_time_zone source=$remote_addr location time_zone; + $geoip2_region_code source=$remote_addr subdivisions 0 iso_code; + $geoip2_region_name source=$remote_addr subdivisions 0 names en; + } + {{ end }} + + {{ if eq $file "GeoLite2-ASN.mmdb" }} + geoip2 /etc/nginx/geoip/GeoLite2-ASN.mmdb { + $geoip2_asn source=$remote_addr autonomous_system_number; + $geoip2_org source=$remote_addr autonomous_system_organization; + } + {{ end }} + + {{ if eq $file "GeoIP2-ASN.mmdb" }} + geoip2 /etc/nginx/geoip/GeoIP2-ASN.mmdb { + $geoip2_asn source=$remote_addr autonomous_system_number; + $geoip2_org source=$remote_addr autonomous_system_organization; + } + {{ end }} + + {{ if eq $file "GeoIP2-ISP.mmdb" }} + geoip2 /etc/nginx/geoip/GeoIP2-ISP.mmdb { + $geoip2_isp isp; + $geoip2_isp_org organization; + } + {{ end }} + + {{ if eq $file "GeoIP2-Connection-Type.mmdb" }} + geoip2 /etc/nginx/geoip/GeoIP2-Connection-Type.mmdb { + $geoip2_connection_type connection_type; + } + {{ end }} + + {{ if eq $file "GeoIP2-Anonymous-IP.mmdb" }} + geoip2 /etc/nginx/geoip/GeoIP2-Anonymous-IP.mmdb { + $geoip2_is_anon source=$remote_addr is_anonymous; + $geoip2_is_hosting_provider source=$remote_addr is_hosting_provider; + $geoip2_is_public_proxy source=$remote_addr is_public_proxy; + } + {{ end }} + + {{ end }} + + {{ end }} aio threads; aio_write on; @@ -77,6 +272,11 @@ http { keepalive_timeout {{ $cfg.KeepAlive }}s; keepalive_requests {{ $cfg.KeepAliveRequests }}; + client_body_temp_path /tmp/client-body; + fastcgi_temp_path /tmp/fastcgi-temp; + proxy_temp_path /tmp/proxy-temp; + ajp_temp_path /tmp/ajp-temp; + client_header_buffer_size {{ $cfg.ClientHeaderBufferSize }}; client_header_timeout {{ $cfg.ClientHeaderTimeout }}s; large_client_header_buffers {{ $cfg.LargeClientHeaderBuffers }}; @@ -85,6 +285,8 @@ http { http2_max_field_size {{ $cfg.HTTP2MaxFieldSize }}; http2_max_header_size {{ $cfg.HTTP2MaxHeaderSize }}; + http2_max_requests {{ $cfg.HTTP2MaxRequests }}; + http2_max_concurrent_streams {{ $cfg.HTTP2MaxConcurrentStreams }}; types_hash_max_size 2048; server_names_hash_max_size {{ $cfg.ServerNameHashMaxSize }}; @@ -100,18 +302,13 @@ http { underscores_in_headers {{ if $cfg.EnableUnderscoresInHeaders }}on{{ else }}off{{ end }}; ignore_invalid_headers {{ if $cfg.IgnoreInvalidHeaders }}on{{ else }}off{{ end }}; - {{ if $cfg.EnableOpentracing }} - opentracing on; - {{ end }} + limit_req_status {{ $cfg.LimitReqStatusCode }}; + limit_conn_status {{ $cfg.LimitConnStatusCode }}; - {{ if (and $cfg.EnableOpentracing (ne $cfg.ZipkinCollectorHost "")) }} - zipkin_collector_host {{ $cfg.ZipkinCollectorHost }}; - zipkin_collector_port {{ $cfg.ZipkinCollectorPort }}; - zipkin_service_name {{ $cfg.ZipkinServiceName }}; - {{ end }} + {{ buildOpentracing $cfg $servers }} include /etc/nginx/mime.types; - default_type text/html; + default_type {{ $cfg.DefaultType }}; {{ if $cfg.EnableBrotli }} brotli on; @@ -121,9 +318,9 @@ http { {{ if $cfg.UseGzip }} gzip on; - gzip_comp_level 5; + gzip_comp_level {{ $cfg.GzipLevel }}; gzip_http_version 1.1; - gzip_min_length 256; + gzip_min_length {{ $cfg.GzipMinLength}}; gzip_types {{ $cfg.GzipTypes }}; gzip_proxied any; gzip_vary on; @@ -131,10 +328,13 @@ http { # Custom headers for response {{ range $k, $v := $addHeaders }} - add_header {{ $k }} "{{ $v }}"; + more_set_headers {{ printf "%s: %s" $k $v | quote }}; {{ end }} server_tokens {{ if $cfg.ShowServerTokens }}on{{ else }}off{{ end }}; + {{ if not $cfg.ShowServerTokens }} + more_clear_headers Server; + {{ end }} # disable warnings uninitialized_variable_warn off; @@ -143,7 +343,8 @@ http { # $namespace # $ingress_name # $service_name - log_format upstreaminfo {{ if $cfg.LogFormatEscapeJSON }}escape=json {{ end }}'{{ buildLogFormatUpstream $cfg }}'; + # $service_port + log_format upstreaminfo {{ if $cfg.LogFormatEscapeJSON }}escape=json {{ end }}'{{ $cfg.LogFormatUpstream }}'; {{/* map urls that should not appear in access.log */}} {{/* http://nginx.org/en/docs/http/ngx_http_log_module.html#access_log */}} @@ -153,72 +354,45 @@ http { default 1; } - {{ if $cfg.DisableAccessLog }} + {{ if or $cfg.DisableAccessLog $cfg.DisableHTTPAccessLog }} access_log off; {{ else }} - access_log {{ $cfg.AccessLogPath }} upstreaminfo if=$loggable; + {{ if $cfg.EnableSyslog }} + access_log syslog:server={{ $cfg.SyslogHost }}:{{ $cfg.SyslogPort }} upstreaminfo if=$loggable; + {{ else }} + access_log {{ or $cfg.HttpAccessLogPath $cfg.AccessLogPath }} upstreaminfo {{ $cfg.AccessLogParams }} if=$loggable; {{ end }} + {{ end }} + + {{ if $cfg.EnableSyslog }} + error_log syslog:server={{ $cfg.SyslogHost }}:{{ $cfg.SyslogPort }} {{ $cfg.ErrorLogLevel }}; + {{ else }} error_log {{ $cfg.ErrorLogPath }} {{ $cfg.ErrorLogLevel }}; + {{ end }} - {{ buildResolvers $cfg.Resolver }} + {{ buildResolvers $cfg.Resolver $cfg.DisableIpv6DNS }} - {{/* Whenever nginx proxies a request without a "Connection" header, the "Connection" header is set to "close" */}} - {{/* when making the target request. This means that you cannot simply use */}} - {{/* "proxy_set_header Connection $http_connection" for WebSocket support because in this case, the */}} - {{/* "Connection" header would be set to "" whenever the original request did not have a "Connection" header, */}} - {{/* which would mean no "Connection" header would be in the target request. Since this would deviate from */}} - {{/* normal nginx behavior we have to use this approach. */}} - # Retain the default nginx handling of requests without a "Connection" header + # See https://www.nginx.com/blog/websocket-nginx map $http_upgrade $connection_upgrade { default upgrade; + {{ if (gt $cfg.UpstreamKeepaliveConnections 0) }} + # See http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive + '' ''; + {{ else }} '' close; + {{ end }} } - map {{ buildForwardedFor $cfg.ForwardedForHeader }} $the_real_ip { - {{ if $cfg.UseProxyProtocol }} - # Get IP address from Proxy Protocol - default $proxy_protocol_addr; - {{ else }} - default $remote_addr; - {{ end }} + # Reverse proxies can detect if a client provides a X-Request-ID header, and pass it on to the backend server. + # If no such header is provided, it can provide a random value. + map $http_x_request_id $req_id { + default $http_x_request_id; + {{ if $cfg.GenerateRequestID }} + "" $request_id; + {{ end }} } - # trust http_x_forwarded_proto headers correctly indicate ssl offloading - map $http_x_forwarded_proto $pass_access_scheme { - default $http_x_forwarded_proto; - '' $scheme; - } - - map $http_x_forwarded_port $pass_server_port { - default $http_x_forwarded_port; - '' $server_port; - } - - map $http_x_forwarded_host $best_http_host { - default $http_x_forwarded_host; - '' $this_host; - } - - {{ if $all.IsSSLPassthroughEnabled }} - # map port {{ $all.ListenPorts.SSLProxy }} to 443 for header X-Forwarded-Port - map $pass_server_port $pass_port { - {{ $all.ListenPorts.SSLProxy }} 443; - default $pass_server_port; - } - {{ else }} - map $pass_server_port $pass_port { - 443 443; - default $pass_server_port; - } - {{ end }} - - # Obtain best http host - map $http_host $this_host { - default $http_host; - '' $host; - } - - {{ if $cfg.ComputeFullForwardedFor }} + {{ if and $cfg.UseForwardedHeaders $cfg.ComputeFullForwardedFor }} # We can't use $proxy_add_x_forwarded_for because the realip module # replaces the remote_addr too soon map $http_x_forwarded_for $full_x_forwarded_for { @@ -228,15 +402,24 @@ http { {{ else }} default "$http_x_forwarded_for, $realip_remote_addr"; '' "$realip_remote_addr"; - {{ end }} + {{ end}} } + {{ end }} + # Create a variable that contains the literal $ character. + # This works because the geo module will not resolve variables. + geo $literal_dollar { + default "$"; + } + server_name_in_redirect off; port_in_redirect off; ssl_protocols {{ $cfg.SSLProtocols }}; + ssl_early_data {{ if $cfg.SSLEarlyData }}on{{ else }}off{{ end }}; + # turn on session caching to drastically improve performance {{ if $cfg.SSLSessionCache }} ssl_session_cache builtin:1000 shared:SSL:{{ $cfg.SSLSessionCacheSize }}; @@ -264,19 +447,18 @@ http { ssl_dhparam {{ $cfg.SSLDHParam }}; {{ end }} - {{ if not $cfg.EnableDynamicTLSRecords }} - ssl_dyn_rec_size_lo 0; - {{ end }} - ssl_ecdh_curve {{ $cfg.SSLECDHCurve }}; - {{ if .CustomErrors }} - # Custom error pages + # PEM sha: {{ $cfg.DefaultSSLCertificate.PemSHA }} + ssl_certificate {{ $cfg.DefaultSSLCertificate.PemFileName }}; + ssl_certificate_key {{ $cfg.DefaultSSLCertificate.PemFileName }}; + + {{ if gt (len $cfg.CustomHTTPErrors) 0 }} proxy_intercept_errors on; {{ end }} {{ range $errCode := $cfg.CustomHTTPErrors }} - error_page {{ $errCode }} = @custom_{{ $errCode }};{{ end }} + error_page {{ $errCode }} = @custom_upstream-default-backend_{{ $errCode }};{{ end }} proxy_ssl_session_reuse on; @@ -284,71 +466,43 @@ http { proxy_pass_header Server; {{ end }} + {{ range $header := $cfg.HideHeaders }}proxy_hide_header {{ $header }}; + {{ end }} + {{ if not (empty $cfg.HTTPSnippet) }} # Custom code snippet configured in the configuration configmap {{ $cfg.HTTPSnippet }} {{ end }} - {{ range $name, $upstream := $backends }} - {{ if eq $upstream.SessionAffinity.AffinityType "cookie" }} - upstream sticky-{{ $upstream.Name }} { - sticky hash={{ $upstream.SessionAffinity.CookieSessionAffinity.Hash }} name={{ $upstream.SessionAffinity.CookieSessionAffinity.Name }} httponly; + upstream upstream_balancer { + ### Attention!!! + # + # We no longer create "upstream" section for every backend. + # Backends are handled dynamically using Lua. If you would like to debug + # and see what backends ingress-nginx has in its memory you can + # install our kubectl plugin https://kubernetes.github.io/ingress-nginx/kubectl-plugin. + # Once you have the plugin you can use "kubectl ingress-nginx backends" command to + # inspect current backends. + # + ### + + server 0.0.0.1; # placeholder + + balancer_by_lua_block { + balancer.balance() + } {{ if (gt $cfg.UpstreamKeepaliveConnections 0) }} keepalive {{ $cfg.UpstreamKeepaliveConnections }}; - {{ end }} - {{ range $server := $upstream.Endpoints }}server {{ $server.Address | formatIP }}:{{ $server.Port }} max_fails={{ $server.MaxFails }} fail_timeout={{ $server.FailTimeout }}; - {{ end }} - - } - - {{ end }} - - - upstream {{ $upstream.Name }} { - # Load balance algorithm; empty for round robin, which is the default - {{ if ne $cfg.LoadBalanceAlgorithm "round_robin" }} - {{ $cfg.LoadBalanceAlgorithm }}; - {{ end }} - - {{ if $upstream.UpstreamHashBy }} - hash {{ $upstream.UpstreamHashBy }} consistent; - {{ end }} - - {{ if (gt $cfg.UpstreamKeepaliveConnections 0) }} - keepalive {{ $cfg.UpstreamKeepaliveConnections }}; - {{ end }} - - {{ range $server := $upstream.Endpoints }}server {{ $server.Address | formatIP }}:{{ $server.Port }} max_fails={{ $server.MaxFails }} fail_timeout={{ $server.FailTimeout }}; + keepalive_timeout {{ $cfg.UpstreamKeepaliveTimeout }}s; + keepalive_requests {{ $cfg.UpstreamKeepaliveRequests }}; {{ end }} } - {{ end }} - - {{/* build the maps that will be use to validate the Whitelist */}} - {{ range $index, $server := $servers }} - {{ range $location := $server.Locations }} - {{ $path := buildLocation $location }} - - {{ if isLocationAllowed $location }} - {{ if gt (len $location.Whitelist.CIDR) 0 }} - - # Deny for {{ print $server.Hostname $path }} - geo $the_real_ip {{ buildDenyVariable (print $server.Hostname "_" $path) }} { - default 1; - - {{ range $ip := $location.Whitelist.CIDR }} - {{ $ip }} 0;{{ end }} - } - {{ end }} - {{ end }} - {{ end }} - {{ end }} - {{ range $rl := (filterRateLimits $servers ) }} # Ratelimit {{ $rl.Name }} - geo $the_real_ip $whitelist_{{ $rl.ID }} { + geo $remote_addr $whitelist_{{ $rl.ID }} { default 0; {{ range $ip := $rl.Whitelist }} {{ $ip }} 1;{{ end }} @@ -367,41 +521,91 @@ http { {{ $zone }} {{ end }} - {{/* Build server redirects (from/to www) */}} - {{ range $hostname, $to := .RedirectServers }} - server { - {{ range $address := $all.Cfg.BindAddressIpv4 }} - listen {{ $address }}:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}; - listen {{ $address }}:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }} ssl; - {{ else }} - listen {{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}; - listen {{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }} ssl; - {{ end }} - {{ if $IsIPV6Enabled }} - {{ range $address := $all.Cfg.BindAddressIpv6 }} - listen {{ $address }}:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}; - listen {{ $address }}:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }}; - {{ else }} - listen [::]:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}; - listen [::]:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }}; - {{ end }} - {{ end }} - server_name {{ $hostname }}; + # Cache for internal auth checks + proxy_cache_path /tmp/nginx-cache-auth levels=1:2 keys_zone=auth_cache:10m max_size=128m inactive=30m use_temp_path=off; - {{ if ne $all.ListenPorts.HTTPS 443 }} - {{ $redirect_port := (printf ":%v" $all.ListenPorts.HTTPS) }} - return {{ $all.Cfg.HTTPRedirectCode }} $scheme://{{ $to }}{{ $redirect_port }}$request_uri; - {{ else }} - return {{ $all.Cfg.HTTPRedirectCode }} $scheme://{{ $to }}$request_uri; + # Global filters + {{ range $ip := $cfg.BlockCIDRs }}deny {{ trimSpace $ip }}; + {{ end }} + + {{ if gt (len $cfg.BlockUserAgents) 0 }} + map $http_user_agent $block_ua { + default 0; + + {{ range $ua := $cfg.BlockUserAgents }}{{ trimSpace $ua }} 1; {{ end }} } {{ end }} - {{ range $index, $server := $servers }} + {{ if gt (len $cfg.BlockReferers) 0 }} + map $http_referer $block_ref { + default 0; + + {{ range $ref := $cfg.BlockReferers }}{{ trimSpace $ref }} 1; + {{ end }} + } + {{ end }} + + {{/* Build server redirects (from/to www) */}} + {{ range $redirect := .RedirectServers }} + ## start server {{ $redirect.From }} + server { + server_name {{ $redirect.From }}; + + {{ buildHTTPListener $all $redirect.From }} + {{ buildHTTPSListener $all $redirect.From }} + + ssl_certificate_by_lua_block { + certificate.call() + } + + {{ if gt (len $cfg.BlockUserAgents) 0 }} + if ($block_ua) { + return 403; + } + {{ end }} + {{ if gt (len $cfg.BlockReferers) 0 }} + if ($block_ref) { + return 403; + } + {{ end }} + + set_by_lua_block $redirect_to { + local request_uri = ngx.var.request_uri + if string.sub(request_uri, -1) == "/" then + request_uri = string.sub(request_uri, 1, -2) + end + + {{ if ne $all.ListenPorts.HTTPS 443 }} + {{ $redirect_port := (printf ":%v" $all.ListenPorts.HTTPS) }} + return string.format("%s://%s%s%s", ngx.var.scheme, "{{ $redirect.To }}", "{{ $redirect_port }}", request_uri) + {{ else }} + return string.format("%s://%s%s", ngx.var.scheme, "{{ $redirect.To }}", request_uri) + {{ end }} + } + + return {{ $all.Cfg.HTTPRedirectCode }} $redirect_to; + } + ## end server {{ $redirect.From }} + {{ end }} + + {{ range $server := $servers }} ## start server {{ $server.Hostname }} server { - server_name {{ $server.Hostname }} {{ $server.Alias }}; + server_name {{ buildServerName $server.Hostname }} {{range $server.Aliases }}{{ . }} {{ end }}; + + {{ if gt (len $cfg.BlockUserAgents) 0 }} + if ($block_ua) { + return 403; + } + {{ end }} + {{ if gt (len $cfg.BlockReferers) 0 }} + if ($block_ref) { + return 403; + } + {{ end }} + {{ template "SERVER" serverConfig $all $server }} {{ if not (empty $cfg.ServerSnippet) }} @@ -409,79 +613,175 @@ http { {{ $cfg.ServerSnippet }} {{ end }} - {{ template "CUSTOM_ERRORS" $all }} + {{ template "CUSTOM_ERRORS" (buildCustomErrorDeps "upstream-default-backend" $cfg.CustomHTTPErrors $all.EnableMetrics) }} } ## end server {{ $server.Hostname }} {{ end }} + # backend for when default-backend-service is not configured or it does not have endpoints + server { + listen {{ $all.ListenPorts.Default }} default_server {{ if $all.Cfg.ReusePort }}reuseport{{ end }} backlog={{ $all.BacklogSize }}; + {{ if $IsIPV6Enabled }}listen [::]:{{ $all.ListenPorts.Default }} default_server {{ if $all.Cfg.ReusePort }}reuseport{{ end }} backlog={{ $all.BacklogSize }};{{ end }} + set $proxy_upstream_name "internal"; + + access_log off; + + location / { + return 404; + } + } + # default server, used for NGINX healthcheck and access to nginx stats server { - # Use the port {{ $all.ListenPorts.Status }} (random value just to avoid known ports) as default port for nginx. - # Changing this value requires a change in: - # https://github.com/kubernetes/ingress-nginx/blob/master/controllers/nginx/pkg/cmd/controller/nginx.go - listen {{ $all.ListenPorts.Status }} default_server reuseport backlog={{ $all.BacklogSize }}; - {{ if $IsIPV6Enabled }}listen [::]:{{ $all.ListenPorts.Status }} default_server reuseport backlog={{ $all.BacklogSize }};{{ end }} - set $proxy_upstream_name "-"; + listen 127.0.0.1:{{ .StatusPort }}; + set $proxy_upstream_name "internal"; + + keepalive_timeout 0; + gzip off; + + access_log off; + + {{ if $cfg.EnableOpentracing }} + opentracing off; + {{ end }} location {{ $healthzURI }} { - access_log off; return 200; } - location /nginx_status { - set $proxy_upstream_name "internal"; + location /is-dynamic-lb-initialized { + content_by_lua_block { + local configuration = require("configuration") + local backend_data = configuration.get_backends_data() + if not backend_data then + ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) + return + end - {{ if $cfg.EnableVtsStatus }} - vhost_traffic_status_display; - vhost_traffic_status_display_format html; - {{ else }} - access_log off; + ngx.say("OK") + ngx.exit(ngx.HTTP_OK) + } + } + + location {{ .StatusPath }} { stub_status on; - {{ end }} + } + + location /configuration { + client_max_body_size {{ luaConfigurationRequestBodySize $cfg }}m; + client_body_buffer_size {{ luaConfigurationRequestBodySize $cfg }}m; + proxy_buffering off; + + content_by_lua_block { + configuration.call() + } } location / { - {{ if .CustomErrors }} - proxy_set_header X-Code 404; - {{ end }} - set $proxy_upstream_name "upstream-default-backend"; - proxy_pass http://upstream-default-backend; + content_by_lua_block { + ngx.exit(ngx.HTTP_NOT_FOUND) + } } - - {{ template "CUSTOM_ERRORS" $all }} } } stream { - log_format log_stream {{ $cfg.LogFormatStream }}; + lua_package_path "/etc/nginx/lua/?.lua;/etc/nginx/lua/vendor/?.lua;;"; - {{ if $cfg.DisableAccessLog }} + lua_shared_dict tcp_udp_configuration_data 5M; + + init_by_lua_block { + collectgarbage("collect") + + -- init modules + local ok, res + + ok, res = pcall(require, "configuration") + if not ok then + error("require failed: " .. tostring(res)) + else + configuration = res + end + + ok, res = pcall(require, "tcp_udp_configuration") + if not ok then + error("require failed: " .. tostring(res)) + else + tcp_udp_configuration = res + end + + ok, res = pcall(require, "tcp_udp_balancer") + if not ok then + error("require failed: " .. tostring(res)) + else + tcp_udp_balancer = res + end + } + + init_worker_by_lua_block { + tcp_udp_balancer.init_worker() + } + + lua_add_variable $proxy_upstream_name; + + log_format log_stream '{{ $cfg.LogFormatStream }}'; + + {{ if or $cfg.DisableAccessLog $cfg.DisableStreamAccessLog }} access_log off; {{ else }} - access_log {{ $cfg.AccessLogPath }} log_stream; + access_log {{ or $cfg.StreamAccessLogPath $cfg.AccessLogPath }} log_stream {{ $cfg.AccessLogParams }}; {{ end }} - error_log {{ $cfg.ErrorLogPath }}; + error_log {{ $cfg.ErrorLogPath }} {{ $cfg.ErrorLogLevel }}; + + {{ if $cfg.EnableRealIp }} + {{ range $trusted_ip := $cfg.ProxyRealIPCIDR }} + set_real_ip_from {{ $trusted_ip }}; + {{ end }} + {{ end }} + + upstream upstream_balancer { + server 0.0.0.1:1234; # placeholder + + balancer_by_lua_block { + local cjson = require("cjson.safe") + local b = require "ngx.balancer" + local ngx = ngx + local ngx_log = ngx.log + local backends_data = tcp_udp_configuration.get_backends_data() + local new_backends, err = cjson.decode(backends_data) + if not new_backends then + ngx.log(ngx.ERR, "could not parse backends data: ", err) + return + end + for _, new_backend in pairs(new_backends) do + for _, addr in pairs(new_backend.endpoints) do + local address = addr["address"] + local port = addr["port"] + local ok, err = b.set_current_peer(address, port) + end + end + } + } + + server { + listen 127.0.0.1:{{ .StreamPort }}; + + access_log off; + + content_by_lua_block { + tcp_udp_configuration.call() + } + } # TCP services - {{ range $i, $tcpServer := .TCPBackends }} - upstream tcp-{{ $tcpServer.Port }}-{{ $tcpServer.Backend.Namespace }}-{{ $tcpServer.Backend.Name }}-{{ $tcpServer.Backend.Port }} { - # NOTE(portdirect): mark the 1st server as up, the 2nd as backup, and all others as down. - # The ingress controller will manage this list, based on the health checks in the backend pods, - # which approximates the pattern commonly used by Haproxy's httpchk. - {{ range $j, $endpoint := $tcpServer.Endpoints }} - {{ if eq $j 0 }} - # NOTE(portdirect): see https://docs.nginx.com/nginx/admin-guide/load-balancer/tcp-health-check/#passive-tcp-health-checks to tune passive healthchecks - server {{ formatIP $endpoint.Address }}:{{ $endpoint.Port }}; - {{ else if eq $j 1 }} - server {{ formatIP $endpoint.Address }}:{{ $endpoint.Port }} backup; - {{ else }} - server {{ formatIP $endpoint.Address }}:{{ $endpoint.Port }} down; - {{ end }} - {{ end }} - } + {{ range $tcpServer := .TCPBackends }} server { + preread_by_lua_block { + ngx.var.proxy_upstream_name="tcp-{{ $tcpServer.Backend.Namespace }}-{{ $tcpServer.Backend.Name }}-{{ $tcpServer.Backend.Port }}"; + } + {{ range $address := $all.Cfg.BindAddressIpv4 }} listen {{ $address }}:{{ $tcpServer.Port }}{{ if $tcpServer.Backend.ProxyProtocol.Decode }} proxy_protocol{{ end }}; {{ else }} @@ -495,23 +795,24 @@ stream { {{ end }} {{ end }} proxy_timeout {{ $cfg.ProxyStreamTimeout }}; - proxy_pass tcp-{{ $tcpServer.Port }}-{{ $tcpServer.Backend.Namespace }}-{{ $tcpServer.Backend.Name }}-{{ $tcpServer.Backend.Port }}; + proxy_next_upstream {{ if $cfg.ProxyStreamNextUpstream }}on{{ else }}off{{ end }}; + proxy_next_upstream_timeout {{ $cfg.ProxyStreamNextUpstreamTimeout }}; + proxy_next_upstream_tries {{ $cfg.ProxyStreamNextUpstreamTries }}; + + proxy_pass upstream_balancer; {{ if $tcpServer.Backend.ProxyProtocol.Encode }} proxy_protocol on; {{ end }} } - {{ end }} # UDP services - {{ range $i, $udpServer := .UDPBackends }} - upstream udp-{{ $udpServer.Port }}-{{ $udpServer.Backend.Namespace }}-{{ $udpServer.Backend.Name }}-{{ $udpServer.Backend.Port }} { - {{ range $j, $endpoint := $udpServer.Endpoints }} - server {{ $endpoint.Address }}:{{ $endpoint.Port }}; - {{ end }} - } - + {{ range $udpServer := .UDPBackends }} server { + preread_by_lua_block { + ngx.var.proxy_upstream_name="udp-{{ $udpServer.Backend.Namespace }}-{{ $udpServer.Backend.Name }}-{{ $udpServer.Backend.Port }}"; + } + {{ range $address := $all.Cfg.BindAddressIpv4 }} listen {{ $address }}:{{ $udpServer.Port }} udp; {{ else }} @@ -526,17 +827,20 @@ stream { {{ end }} proxy_responses {{ $cfg.ProxyStreamResponses }}; proxy_timeout {{ $cfg.ProxyStreamTimeout }}; - proxy_pass udp-{{ $udpServer.Port }}-{{ $udpServer.Backend.Namespace }}-{{ $udpServer.Backend.Name }}-{{ $udpServer.Backend.Port }}; + proxy_next_upstream {{ if $cfg.ProxyStreamNextUpstream }}on{{ else }}off{{ end }}; + proxy_next_upstream_timeout {{ $cfg.ProxyStreamNextUpstreamTimeout }}; + proxy_next_upstream_tries {{ $cfg.ProxyStreamNextUpstreamTries }}; + proxy_pass upstream_balancer; } - {{ end }} } {{/* definition of templates to avoid repetitions */}} {{ define "CUSTOM_ERRORS" }} - {{ $proxySetHeaders := .ProxySetHeaders }} - {{ range $errCode := .Cfg.CustomHTTPErrors }} - location @custom_{{ $errCode }} { + {{ $enableMetrics := .EnableMetrics }} + {{ $upstreamName := .UpstreamName }} + {{ range $errCode := .ErrorCodes }} + location @custom_{{ $upstreamName }}_{{ $errCode }} { internal; proxy_intercept_errors off; @@ -547,9 +851,20 @@ stream { proxy_set_header X-Namespace $namespace; proxy_set_header X-Ingress-Name $ingress_name; proxy_set_header X-Service-Name $service_name; + proxy_set_header X-Service-Port $service_port; + proxy_set_header X-Request-ID $req_id; + proxy_set_header Host $best_http_host; + + set $proxy_upstream_name {{ $upstreamName | quote }}; rewrite (.*) / break; - proxy_pass http://upstream-default-backend; + + proxy_pass http://upstream_balancer; + log_by_lua_block { + {{ if $enableMetrics }} + monitor.call() + {{ end }} + } } {{ end }} {{ end }} @@ -559,20 +874,20 @@ stream { {{ $cors := .CorsConfig }} # Cors Preflight methods needs additional options and different Return Code if ($request_method = 'OPTIONS') { - add_header 'Access-Control-Allow-Origin' '{{ $cors.CorsAllowOrigin }}' always; - {{ if $cors.CorsAllowCredentials }} add_header 'Access-Control-Allow-Credentials' '{{ $cors.CorsAllowCredentials }}' always; {{ end }} - add_header 'Access-Control-Allow-Methods' '{{ $cors.CorsAllowMethods }}' always; - add_header 'Access-Control-Allow-Headers' '{{ $cors.CorsAllowHeaders }}' always; - add_header 'Access-Control-Max-Age' 1728000; - add_header 'Content-Type' 'text/plain charset=UTF-8'; - add_header 'Content-Length' 0; + more_set_headers 'Access-Control-Allow-Origin: {{ $cors.CorsAllowOrigin }}'; + {{ if $cors.CorsAllowCredentials }} more_set_headers 'Access-Control-Allow-Credentials: {{ $cors.CorsAllowCredentials }}'; {{ end }} + more_set_headers 'Access-Control-Allow-Methods: {{ $cors.CorsAllowMethods }}'; + more_set_headers 'Access-Control-Allow-Headers: {{ $cors.CorsAllowHeaders }}'; + {{ if not (empty $cors.CorsExposeHeaders) }} more_set_headers 'Access-Control-Expose-Headers: {{ $cors.CorsExposeHeaders }}'; {{ end }} + more_set_headers 'Access-Control-Max-Age: {{ $cors.CorsMaxAge }}'; + more_set_headers 'Content-Type: text/plain charset=UTF-8'; + more_set_headers 'Content-Length: 0'; return 204; } - add_header 'Access-Control-Allow-Origin' '{{ $cors.CorsAllowOrigin }}' always; - {{ if $cors.CorsAllowCredentials }} add_header 'Access-Control-Allow-Credentials' '{{ $cors.CorsAllowCredentials }}' always; {{ end }} - add_header 'Access-Control-Allow-Methods' '{{ $cors.CorsAllowMethods }}' always; - add_header 'Access-Control-Allow-Headers' '{{ $cors.CorsAllowHeaders }}' always; + more_set_headers 'Access-Control-Allow-Origin: {{ $cors.CorsAllowOrigin }}'; + {{ if $cors.CorsAllowCredentials }} more_set_headers 'Access-Control-Allow-Credentials: {{ $cors.CorsAllowCredentials }}'; {{ end }} + {{ if not (empty $cors.CorsExposeHeaders) }} more_set_headers 'Access-Control-Expose-Headers: {{ $cors.CorsExposeHeaders }}'; {{ end }} {{ end }} @@ -580,161 +895,314 @@ stream { {{ define "SERVER" }} {{ $all := .First }} {{ $server := .Second }} - {{ range $address := $all.Cfg.BindAddressIpv4 }} - listen {{ $address }}:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_" }} default_server reuseport backlog={{ $all.BacklogSize }}{{ end }}; - {{ else }} - listen {{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_" }} default_server reuseport backlog={{ $all.BacklogSize }}{{ end }}; - {{ end }} - {{ if $all.IsIPV6Enabled }} - {{ range $address := $all.Cfg.BindAddressIpv6 }} - listen {{ $address }}:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_" }} default_server reuseport backlog={{ $all.BacklogSize }}{{ end }}; - {{ else }} - listen [::]:{{ $all.ListenPorts.HTTP }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ if eq $server.Hostname "_" }} default_server reuseport backlog={{ $all.BacklogSize }}{{ end }}; - {{ end }} - {{ end }} + + {{ buildHTTPListener $all $server.Hostname }} + {{ buildHTTPSListener $all $server.Hostname }} + set $proxy_upstream_name "-"; - {{/* Listen on {{ $all.ListenPorts.SSLProxy }} because port {{ $all.ListenPorts.HTTPS }} is used in the TLS sni server */}} - {{/* This listener must always have proxy_protocol enabled, because the SNI listener forwards on source IP info in it. */}} - {{ if not (empty $server.SSLCertificate) }} - {{ range $address := $all.Cfg.BindAddressIpv4 }} - listen {{ $address }}:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol {{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }} {{ if eq $server.Hostname "_" }} default_server reuseport backlog={{ $all.BacklogSize }}{{ end }} ssl {{ if $all.Cfg.UseHTTP2 }}http2{{ end }}; - {{ else }} - listen {{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol {{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }} {{ if eq $server.Hostname "_" }} default_server reuseport backlog={{ $all.BacklogSize }}{{ end }} ssl {{ if $all.Cfg.UseHTTP2 }}http2{{ end }}; - {{ end }} - {{ if $all.IsIPV6Enabled }} - {{ range $address := $all.Cfg.BindAddressIpv6 }} - {{ if not (empty $server.SSLCertificate) }}listen {{ $address }}:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }}{{ end }} {{ if eq $server.Hostname "_" }} default_server reuseport backlog={{ $all.BacklogSize }}{{ end }} ssl {{ if $all.Cfg.UseHTTP2 }}http2{{ end }}; - {{ else }} - {{ if not (empty $server.SSLCertificate) }}listen [::]:{{ if $all.IsSSLPassthroughEnabled }}{{ $all.ListenPorts.SSLProxy }} proxy_protocol{{ else }}{{ $all.ListenPorts.HTTPS }}{{ if $all.Cfg.UseProxyProtocol }} proxy_protocol{{ end }}{{ end }}{{ end }} {{ if eq $server.Hostname "_" }} default_server reuseport backlog={{ $all.BacklogSize }}{{ end }} ssl {{ if $all.Cfg.UseHTTP2 }}http2{{ end }}; - {{ end }} - {{ end }} - {{/* comment PEM sha is required to detect changes in the generated configuration and force a reload */}} - # PEM sha: {{ $server.SSLPemChecksum }} - ssl_certificate {{ $server.SSLCertificate }}; - ssl_certificate_key {{ $server.SSLCertificate }}; - {{ if not (empty $server.SSLFullChainCertificate) }} - ssl_trusted_certificate {{ $server.SSLFullChainCertificate }}; - ssl_stapling on; - ssl_stapling_verify on; - {{ end }} - {{ end }} - - {{ if (and (not (empty $server.SSLCertificate)) $all.Cfg.HSTS) }} - more_set_headers "Strict-Transport-Security: max-age={{ $all.Cfg.HSTSMaxAge }}{{ if $all.Cfg.HSTSIncludeSubdomains }}; includeSubDomains{{ end }};{{ if $all.Cfg.HSTSPreload }} preload{{ end }}"; - {{ end }} + ssl_certificate_by_lua_block { + certificate.call() + } + {{ if not (empty $server.AuthTLSError) }} + # {{ $server.AuthTLSError }} + return 403; + {{ else }} {{ if not (empty $server.CertificateAuth.CAFileName) }} - # PEM sha: {{ $server.CertificateAuth.PemSHA }} + # PEM sha: {{ $server.CertificateAuth.CASHA }} ssl_client_certificate {{ $server.CertificateAuth.CAFileName }}; ssl_verify_client {{ $server.CertificateAuth.VerifyClient }}; ssl_verify_depth {{ $server.CertificateAuth.ValidationDepth }}; - {{ if not (empty $server.CertificateAuth.ErrorPage) }} + + {{ if not (empty $server.CertificateAuth.CRLFileName) }} + # PEM sha: {{ $server.CertificateAuth.CRLSHA }} + ssl_crl {{ $server.CertificateAuth.CRLFileName }}; + {{ end }} + + {{ if not (empty $server.CertificateAuth.ErrorPage)}} error_page 495 496 = {{ $server.CertificateAuth.ErrorPage }}; {{ end }} {{ end }} + {{ if not (empty $server.ProxySSL.CAFileName) }} + # PEM sha: {{ $server.ProxySSL.CASHA }} + proxy_ssl_trusted_certificate {{ $server.ProxySSL.CAFileName }}; + proxy_ssl_ciphers {{ $server.ProxySSL.Ciphers }}; + proxy_ssl_protocols {{ $server.ProxySSL.Protocols }}; + proxy_ssl_verify {{ $server.ProxySSL.Verify }}; + proxy_ssl_verify_depth {{ $server.ProxySSL.VerifyDepth }}; + {{ if not (empty $server.ProxySSL.ProxySSLName) }} + proxy_ssl_name {{ $server.ProxySSL.ProxySSLName }}; + proxy_ssl_server_name {{ $server.ProxySSL.ProxySSLServerName }}; + {{ end }} + {{ end }} + + {{ if not (empty $server.ProxySSL.PemFileName) }} + proxy_ssl_certificate {{ $server.ProxySSL.PemFileName }}; + proxy_ssl_certificate_key {{ $server.ProxySSL.PemFileName }}; + {{ end }} + + {{ if not (empty $server.SSLCiphers) }} + ssl_ciphers {{ $server.SSLCiphers }}; + {{ end }} + + {{ if not (empty $server.SSLPreferServerCiphers) }} + ssl_prefer_server_ciphers {{ $server.SSLPreferServerCiphers }}; + {{ end }} + {{ if not (empty $server.ServerSnippet) }} + # Custom code snippet configured for host {{ $server.Hostname }} {{ $server.ServerSnippet }} {{ end }} + {{ range $errorLocation := (buildCustomErrorLocationsPerServer $server) }} + {{ template "CUSTOM_ERRORS" (buildCustomErrorDeps $errorLocation.UpstreamName $errorLocation.Codes $all.EnableMetrics) }} + {{ end }} + + {{ buildMirrorLocations $server.Locations }} + + {{ $enforceRegex := enforceRegexModifier $server.Locations }} {{ range $location := $server.Locations }} - {{ $path := buildLocation $location }} - {{ $authPath := buildAuthLocation $location }} + {{ $path := buildLocation $location $enforceRegex }} + {{ $proxySetHeader := proxySetHeader $location }} + {{ $authPath := buildAuthLocation $location $all.Cfg.GlobalExternalAuth.URL }} + {{ $applyGlobalAuth := shouldApplyGlobalAuth $location $all.Cfg.GlobalExternalAuth.URL }} + + {{ $externalAuth := $location.ExternalAuth }} + {{ if eq $applyGlobalAuth true }} + {{ $externalAuth = $all.Cfg.GlobalExternalAuth }} + {{ end }} {{ if not (empty $location.Rewrite.AppRoot) }} if ($uri = /) { - return 302 {{ $location.Rewrite.AppRoot }}; + return 302 $scheme://$http_host{{ $location.Rewrite.AppRoot }}; } {{ end }} - {{ if not (empty $authPath) }} + {{ if $authPath }} location = {{ $authPath }} { internal; - set $proxy_upstream_name "external-authentication"; + + {{ if (or $all.Cfg.EnableOpentracing $location.Opentracing.Enabled) }} + opentracing on; + opentracing_propagate_context; + {{ end }} + + {{ if $externalAuth.AuthCacheKey }} + set $tmp_cache_key '{{ $server.Hostname }}{{ $authPath }}{{ $externalAuth.AuthCacheKey }}'; + set $cache_key ''; + + rewrite_by_lua_block { + ngx.var.cache_key = ngx.encode_base64(ngx.sha1_bin(ngx.var.tmp_cache_key)) + } + + proxy_cache auth_cache; + + {{- range $dur := $externalAuth.AuthCacheDuration }} + proxy_cache_valid {{ $dur }}; + {{- end }} + + proxy_cache_key "$cache_key"; + {{ end }} + + # ngx_auth_request module overrides variables in the parent request, + # therefore we have to explicitly set this variable again so that when the parent request + # resumes it has the correct value set for this variable so that Lua can pick backend correctly + set $proxy_upstream_name {{ buildUpstreamName $location | quote }}; proxy_pass_request_body off; - proxy_set_header Content-Length ""; + proxy_set_header Content-Length ""; + proxy_set_header X-Forwarded-Proto ""; + proxy_set_header X-Request-ID $req_id; - {{ if not (empty $location.ExternalAuth.Method) }} - proxy_method {{ $location.ExternalAuth.Method }}; + {{ if $externalAuth.Method }} + proxy_method {{ $externalAuth.Method }}; proxy_set_header X-Original-URI $request_uri; proxy_set_header X-Scheme $pass_access_scheme; {{ end }} - proxy_set_header Host {{ $location.ExternalAuth.Host }}; + proxy_set_header Host {{ $externalAuth.Host }}; proxy_set_header X-Original-URL $scheme://$http_host$request_uri; proxy_set_header X-Original-Method $request_method; - proxy_set_header X-Auth-Request-Redirect $request_uri; proxy_set_header X-Sent-From "nginx-ingress-controller"; + proxy_set_header X-Real-IP $remote_addr; + {{ if and $all.Cfg.UseForwardedHeaders $all.Cfg.ComputeFullForwardedFor }} + proxy_set_header X-Forwarded-For $full_x_forwarded_for; + {{ else }} + proxy_set_header X-Forwarded-For $remote_addr; + {{ end }} + + {{ if $externalAuth.RequestRedirect }} + proxy_set_header X-Auth-Request-Redirect {{ $externalAuth.RequestRedirect }}; + {{ else }} + proxy_set_header X-Auth-Request-Redirect $request_uri; + {{ end }} + + {{ if $externalAuth.AuthCacheKey }} + proxy_buffering "on"; + {{ else }} + proxy_buffering {{ $location.Proxy.ProxyBuffering }}; + {{ end }} + proxy_buffer_size {{ $location.Proxy.BufferSize }}; + proxy_buffers {{ $location.Proxy.BuffersNumber }} {{ $location.Proxy.BufferSize }}; + proxy_request_buffering {{ $location.Proxy.RequestBuffering }}; + proxy_http_version {{ $location.Proxy.ProxyHTTPVersion }}; - proxy_http_version 1.1; proxy_ssl_server_name on; proxy_pass_request_headers on; - client_max_body_size "{{ $location.Proxy.BodySize }}"; - {{ if isValidClientBodyBufferSize $location.ClientBodyBufferSize }} + {{ if isValidByteSize $location.Proxy.BodySize true }} + client_max_body_size {{ $location.Proxy.BodySize }}; + {{ end }} + {{ if isValidByteSize $location.ClientBodyBufferSize false }} client_body_buffer_size {{ $location.ClientBodyBufferSize }}; {{ end }} - set $target {{ $location.ExternalAuth.URL }}; + # Pass the extracted client certificate to the auth provider + {{ if not (empty $server.CertificateAuth.CAFileName) }} + {{ if $server.CertificateAuth.PassCertToUpstream }} + proxy_set_header ssl-client-cert $ssl_client_escaped_cert; + {{ end }} + proxy_set_header ssl-client-verify $ssl_client_verify; + proxy_set_header ssl-client-subject-dn $ssl_client_s_dn; + proxy_set_header ssl-client-issuer-dn $ssl_client_i_dn; + {{ end }} + + {{- range $line := buildAuthProxySetHeaders $externalAuth.ProxySetHeaders}} + {{ $line }} + {{- end }} + + {{ if not (empty $externalAuth.AuthSnippet) }} + {{ $externalAuth.AuthSnippet }} + {{ end }} + + set $target {{ $externalAuth.URL }}; proxy_pass $target; } {{ end }} + {{ if isLocationAllowed $location }} + {{ if $externalAuth.SigninURL }} + location {{ buildAuthSignURLLocation $location.Path $externalAuth.SigninURL }} { + internal; + + add_header Set-Cookie $auth_cookie; + + return 302 {{ buildAuthSignURL $externalAuth.SigninURL $externalAuth.SigninURLRedirectParam }}; + } + {{ end }} + {{ end }} + location {{ $path }} { - {{ if $all.Cfg.EnableVtsStatus }}{{ if $location.VtsFilterKey }} vhost_traffic_status_filter_by_set_key {{ $location.VtsFilterKey }};{{ end }}{{ end }} + {{ $ing := (getIngressInformation $location.Ingress $server.Hostname $location.IngressPath) }} + set $namespace {{ $ing.Namespace | quote}}; + set $ingress_name {{ $ing.Rule | quote }}; + set $service_name {{ $ing.Service | quote }}; + set $service_port {{ $ing.ServicePort | quote }}; + set $location_path {{ $ing.Path | escapeLiteralDollar | quote }}; - set $proxy_upstream_name "{{ buildUpstreamName $server.Hostname $all.Backends $location }}"; + {{ buildOpentracingForLocation $all.Cfg.EnableOpentracing $location }} - {{ $ing := (getIngressInformation $location.Ingress $path) }} - {{/* $ing.Metadata contains the Ingress metadata */}} - set $namespace "{{ $ing.Namespace }}"; - set $ingress_name "{{ $ing.Rule }}"; - set $service_name "{{ $ing.Service }}"; + {{ if $location.Mirror.Source }} + mirror {{ $location.Mirror.Source }}; + mirror_request_body {{ $location.Mirror.RequestBody }}; + {{ end }} - {{ if (or $location.Rewrite.ForceSSLRedirect (and (not (empty $server.SSLCertificate)) $location.Rewrite.SSLRedirect)) }} - # enforce ssl on server side - if ($pass_access_scheme = http) { - {{ if ne $all.ListenPorts.HTTPS 443 }} - {{ $redirect_port := (printf ":%v" $all.ListenPorts.HTTPS) }} - return {{ $all.Cfg.HTTPRedirectCode }} https://$best_http_host{{ $redirect_port }}$request_uri; - {{ else }} - return {{ $all.Cfg.HTTPRedirectCode }} https://$best_http_host$request_uri; + rewrite_by_lua_block { + lua_ingress.rewrite({{ locationConfigForLua $location $all }}) + balancer.rewrite() + plugins.run() + } + + # be careful with `access_by_lua_block` and `satisfy any` directives as satisfy any + # will always succeed when there's `access_by_lua_block` that does not have any lua code doing `ngx.exit(ngx.DECLINED)` + # other authentication method such as basic auth or external auth useless - all requests will be allowed. + #access_by_lua_block { + #} + + header_filter_by_lua_block { + lua_ingress.header() + plugins.run() + } + + body_filter_by_lua_block { + } + + log_by_lua_block { + balancer.log() + {{ if $all.EnableMetrics }} + monitor.call() {{ end }} + + plugins.run() } + + {{ if not $location.Logs.Access }} + access_log off; {{ end }} - {{ if $all.Cfg.EnableModsecurity }} - modsecurity on; - - modsecurity_rules_file /etc/nginx/modsecurity/modsecurity.conf; - {{ if $all.Cfg.EnableOWASPCoreRules }} - modsecurity_rules_file /etc/nginx/owasp-modsecurity-crs/nginx-modsecurity.conf; - {{ end }} + {{ if $location.Logs.Rewrite }} + rewrite_log on; {{ end }} - {{ if isLocationAllowed $location }} - {{ if gt (len $location.Whitelist.CIDR) 0 }} - if ({{ buildDenyVariable (print $server.Hostname "_" $path) }}) { - return 403; - } + {{ if $location.HTTP2PushPreload }} + http2_push_preload on; {{ end }} port_in_redirect {{ if $location.UsePortInRedirects }}on{{ else }}off{{ end }}; - {{ if not (empty $authPath) }} + set $balancer_ewma_score -1; + set $proxy_upstream_name {{ buildUpstreamName $location | quote }}; + set $proxy_host $proxy_upstream_name; + set $pass_access_scheme $scheme; + + {{ if $all.Cfg.UseProxyProtocol }} + set $pass_server_port $proxy_protocol_server_port; + {{ else }} + set $pass_server_port $server_port; + {{ end }} + + set $best_http_host $http_host; + set $pass_port $pass_server_port; + + set $proxy_alternative_upstream_name ""; + + {{ buildModSecurityForLocation $all.Cfg $location }} + + {{ if isLocationAllowed $location }} + {{ if gt (len $location.Whitelist.CIDR) 0 }} + {{ range $ip := $location.Whitelist.CIDR }} + allow {{ $ip }};{{ end }} + deny all; + {{ end }} + + {{ if not (isLocationInLocationList $location $all.Cfg.NoAuthLocations) }} + {{ if $authPath }} # this location requires authentication auth_request {{ $authPath }}; auth_request_set $auth_cookie $upstream_http_set_cookie; add_header Set-Cookie $auth_cookie; - {{- range $idx, $line := buildAuthResponseHeaders $location }} + {{- range $line := buildAuthResponseHeaders $externalAuth.ResponseHeaders }} {{ $line }} {{- end }} {{ end }} - {{ if not (empty $location.ExternalAuth.SigninURL) }} - error_page 401 = {{ buildAuthSignURL $location.ExternalAuth.SigninURL }}; + {{ if $externalAuth.SigninURL }} + set_escape_uri $escaped_request_uri $request_uri; + error_page 401 = {{ buildAuthSignURLLocation $location.Path $externalAuth.SigninURL }}; + {{ end }} + + {{ if $location.BasicDigestAuth.Secured }} + {{ if eq $location.BasicDigestAuth.Type "basic" }} + auth_basic {{ $location.BasicDigestAuth.Realm | quote }}; + auth_basic_user_file {{ $location.BasicDigestAuth.File }}; + {{ else }} + auth_digest {{ $location.BasicDigestAuth.Realm | quote }}; + auth_digest_user_file {{ $location.BasicDigestAuth.File }}; + {{ end }} + proxy_set_header Authorization ""; + {{ end }} {{ end }} {{/* if the location contains a rate limit annotation, create one */}} @@ -742,109 +1210,93 @@ stream { {{ range $limit := $limits }} {{ $limit }}{{ end }} - {{ if $location.BasicDigestAuth.Secured }} - {{ if eq $location.BasicDigestAuth.Type "basic" }} - auth_basic "{{ $location.BasicDigestAuth.Realm }}"; - auth_basic_user_file {{ $location.BasicDigestAuth.File }}; - {{ else }} - auth_digest "{{ $location.BasicDigestAuth.Realm }}"; - auth_digest_user_file {{ $location.BasicDigestAuth.File }}; - {{ end }} - proxy_set_header Authorization ""; - {{ end }} - {{ if $location.CorsConfig.CorsEnabled }} {{ template "CORS" $location }} {{ end }} - {{ if not (empty $location.Redirect.URL) }} - if ($uri ~* {{ $path }}) { - return {{ $location.Redirect.Code }} {{ $location.Redirect.URL }}; - } - {{ end }} + {{ buildInfluxDB $location.InfluxDB }} - client_max_body_size "{{ $location.Proxy.BodySize }}"; - {{ if isValidClientBodyBufferSize $location.ClientBodyBufferSize }} + {{ if isValidByteSize $location.Proxy.BodySize true }} + client_max_body_size {{ $location.Proxy.BodySize }}; + {{ end }} + {{ if isValidByteSize $location.ClientBodyBufferSize false }} client_body_buffer_size {{ $location.ClientBodyBufferSize }}; {{ end }} {{/* By default use vhost as Host to upstream, but allow overrides */}} + {{ if not (eq $proxySetHeader "grpc_set_header") }} {{ if not (empty $location.UpstreamVhost) }} - proxy_set_header Host "{{ $location.UpstreamVhost }}"; + {{ $proxySetHeader }} Host {{ $location.UpstreamVhost | quote }}; {{ else }} - proxy_set_header Host $best_http_host; + {{ $proxySetHeader }} Host $best_http_host; + {{ end }} {{ end }} - # Pass the extracted client certificate to the backend {{ if not (empty $server.CertificateAuth.CAFileName) }} {{ if $server.CertificateAuth.PassCertToUpstream }} - proxy_set_header ssl-client-cert $ssl_client_escaped_cert; - {{ else }} - proxy_set_header ssl-client-cert ""; + {{ $proxySetHeader }} ssl-client-cert $ssl_client_escaped_cert; {{ end }} - proxy_set_header ssl-client-verify $ssl_client_verify; - proxy_set_header ssl-client-dn $ssl_client_s_dn; - {{ else }} - proxy_set_header ssl-client-cert ""; - proxy_set_header ssl-client-verify ""; - proxy_set_header ssl-client-dn ""; + {{ $proxySetHeader }} ssl-client-verify $ssl_client_verify; + {{ $proxySetHeader }} ssl-client-subject-dn $ssl_client_s_dn; + {{ $proxySetHeader }} ssl-client-issuer-dn $ssl_client_i_dn; {{ end }} # Allow websocket connections - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - - proxy_set_header X-Real-IP $the_real_ip; - {{ if $all.Cfg.ComputeFullForwardedFor }} - proxy_set_header X-Forwarded-For $full_x_forwarded_for; + {{ $proxySetHeader }} Upgrade $http_upgrade; + {{ if $location.Connection.Enabled}} + {{ $proxySetHeader }} Connection {{ $location.Connection.Header }}; {{ else }} - proxy_set_header X-Forwarded-For $the_real_ip; + {{ $proxySetHeader }} Connection $connection_upgrade; {{ end }} - proxy_set_header X-Forwarded-Host $best_http_host; - proxy_set_header X-Forwarded-Port $pass_port; - proxy_set_header X-Forwarded-Proto $pass_access_scheme; - proxy_set_header X-Original-URI $request_uri; - proxy_set_header X-Scheme $pass_access_scheme; + + {{ $proxySetHeader }} X-Request-ID $req_id; + {{ $proxySetHeader }} X-Real-IP $remote_addr; + {{ if and $all.Cfg.UseForwardedHeaders $all.Cfg.ComputeFullForwardedFor }} + {{ $proxySetHeader }} X-Forwarded-For $full_x_forwarded_for; + {{ else }} + {{ $proxySetHeader }} X-Forwarded-For $remote_addr; + {{ end }} + {{ $proxySetHeader }} X-Forwarded-Host $best_http_host; + {{ $proxySetHeader }} X-Forwarded-Port $pass_port; + {{ $proxySetHeader }} X-Forwarded-Proto $pass_access_scheme; + {{ if $all.Cfg.ProxyAddOriginalURIHeader }} + {{ $proxySetHeader }} X-Original-URI $request_uri; + {{ end }} + {{ $proxySetHeader }} X-Scheme $pass_access_scheme; # Pass the original X-Forwarded-For - proxy_set_header X-Original-Forwarded-For {{ buildForwardedFor $all.Cfg.ForwardedForHeader }}; + {{ $proxySetHeader }} X-Original-Forwarded-For {{ buildForwardedFor $all.Cfg.ForwardedForHeader }}; # mitigate HTTPoxy Vulnerability # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/ - proxy_set_header Proxy ""; + {{ $proxySetHeader }} Proxy ""; # Custom headers to proxied server {{ range $k, $v := $all.ProxySetHeaders }} - proxy_set_header {{ $k }} "{{ $v }}"; + {{ $proxySetHeader }} {{ $k }} {{ $v | quote }}; {{ end }} proxy_connect_timeout {{ $location.Proxy.ConnectTimeout }}s; proxy_send_timeout {{ $location.Proxy.SendTimeout }}s; proxy_read_timeout {{ $location.Proxy.ReadTimeout }}s; - {{ if (or (eq $location.Proxy.ProxyRedirectFrom "default") (eq $location.Proxy.ProxyRedirectFrom "off")) }} - proxy_redirect {{ $location.Proxy.ProxyRedirectFrom }}; - {{ else }} - proxy_redirect {{ $location.Proxy.ProxyRedirectFrom }} {{ $location.Proxy.ProxyRedirectTo }}; + proxy_buffering {{ $location.Proxy.ProxyBuffering }}; + proxy_buffer_size {{ $location.Proxy.BufferSize }}; + proxy_buffers {{ $location.Proxy.BuffersNumber }} {{ $location.Proxy.BufferSize }}; + {{ if isValidByteSize $location.Proxy.ProxyMaxTempFileSize true }} + proxy_max_temp_file_size {{ $location.Proxy.ProxyMaxTempFileSize }}; {{ end }} - proxy_buffering off; - proxy_buffer_size "{{ $location.Proxy.BufferSize }}"; - proxy_buffers 4 "{{ $location.Proxy.BufferSize }}"; - proxy_request_buffering "{{ $location.Proxy.RequestBuffering }}"; - - proxy_http_version 1.1; + proxy_request_buffering {{ $location.Proxy.RequestBuffering }}; + proxy_http_version {{ $location.Proxy.ProxyHTTPVersion }}; proxy_cookie_domain {{ $location.Proxy.CookieDomain }}; proxy_cookie_path {{ $location.Proxy.CookiePath }}; # In case of errors try the next upstream server before returning an error proxy_next_upstream {{ buildNextUpstream $location.Proxy.NextUpstream $all.Cfg.RetryNonIdempotent }}; - - {{/* rewrite only works if the content is not compressed */}} - {{ if $location.Rewrite.AddBaseURL }} - proxy_set_header Accept-Encoding ""; - {{ end }} + proxy_next_upstream_timeout {{ $location.Proxy.NextUpstreamTimeout }}; + proxy_next_upstream_tries {{ $location.Proxy.NextUpstreamTries }}; {{/* Add any additional configuration defined */}} {{ $location.ConfigurationSnippet }} @@ -861,26 +1313,78 @@ stream { proxy_set_header X-Namespace $namespace; proxy_set_header X-Ingress-Name $ingress_name; proxy_set_header X-Service-Name $service_name; + proxy_set_header X-Service-Port $service_port; + proxy_set_header X-Request-ID $req_id; {{ end }} + {{ if $location.Satisfy }} + satisfy {{ $location.Satisfy }}; + {{ end }} + + {{/* if a location-specific error override is set, add the proxy_intercept here */}} + {{ if $location.CustomHTTPErrors }} + # Custom error pages per ingress + proxy_intercept_errors on; + {{ end }} + + {{ range $errCode := $location.CustomHTTPErrors }} + error_page {{ $errCode }} = @custom_{{ $location.DefaultBackendUpstreamName }}_{{ $errCode }};{{ end }} + + {{ if (eq $location.BackendProtocol "FCGI") }} + include /etc/nginx/fastcgi_params; + {{ end }} + {{- if $location.FastCGI.Index -}} + fastcgi_index {{ $location.FastCGI.Index | quote }}; + {{- end -}} + {{ range $k, $v := $location.FastCGI.Params }} + fastcgi_param {{ $k }} {{ $v | quote }}; + {{ end }} + + {{ if not (empty $location.Redirect.URL) }} + return {{ $location.Redirect.Code }} {{ $location.Redirect.URL }}; + {{ end }} - {{ if not (empty $location.Backend) }} {{ buildProxyPass $server.Hostname $all.Backends $location }} - {{ else }} - # No endpoints available for the request - return 503; + {{ if (or (eq $location.Proxy.ProxyRedirectFrom "default") (eq $location.Proxy.ProxyRedirectFrom "off")) }} + proxy_redirect {{ $location.Proxy.ProxyRedirectFrom }}; + {{ else if not (eq $location.Proxy.ProxyRedirectTo "off") }} + proxy_redirect {{ $location.Proxy.ProxyRedirectFrom }} {{ $location.Proxy.ProxyRedirectTo }}; {{ end }} {{ else }} - # Location denied. Reason: {{ $location.Denied }} + # Location denied. Reason: {{ $location.Denied | quote }} return 503; {{ end }} + {{ if not (empty $location.ProxySSL.CAFileName) }} + # PEM sha: {{ $location.ProxySSL.CASHA }} + proxy_ssl_trusted_certificate {{ $location.ProxySSL.CAFileName }}; + proxy_ssl_ciphers {{ $location.ProxySSL.Ciphers }}; + proxy_ssl_protocols {{ $location.ProxySSL.Protocols }}; + proxy_ssl_verify {{ $location.ProxySSL.Verify }}; + proxy_ssl_verify_depth {{ $location.ProxySSL.VerifyDepth }}; + {{ end }} + + {{ if not (empty $location.ProxySSL.ProxySSLName) }} + proxy_ssl_name {{ $location.ProxySSL.ProxySSLName }}; + {{ end }} + {{ if not (empty $location.ProxySSL.ProxySSLServerName) }} + proxy_ssl_server_name {{ $location.ProxySSL.ProxySSLServerName }}; + {{ end }} + + {{ if not (empty $location.ProxySSL.PemFileName) }} + proxy_ssl_certificate {{ $location.ProxySSL.PemFileName }}; + proxy_ssl_certificate_key {{ $location.ProxySSL.PemFileName }}; + {{ end }} } - + {{ end }} {{ end }} {{ if eq $server.Hostname "_" }} # health checks in cloud providers require the use of port {{ $all.ListenPorts.HTTP }} location {{ $all.HealthzURI }} { + {{ if $all.Cfg.EnableOpentracing }} + opentracing off; + {{ end }} + access_log off; return 200; } @@ -888,8 +1392,18 @@ stream { # this is required to avoid error if nginx is being monitored # with an external software (like sysdig) location /nginx_status { - allow 127.0.0.1; - {{ if $all.IsIPV6Enabled }}allow ::1;{{ end }} + {{ if $all.Cfg.EnableOpentracing }} + opentracing off; + {{ end }} + + {{ range $v := $all.NginxStatusIpv4Whitelist }} + allow {{ $v }}; + {{ end }} + {{ if $all.IsIPV6Enabled -}} + {{ range $v := $all.NginxStatusIpv6Whitelist }} + allow {{ $v }}; + {{ end }} + {{ end -}} deny all; access_log off; diff --git a/mariadb/templates/bin/_mariadb-ingress-controller.sh.tpl b/mariadb/templates/bin/_mariadb-ingress-controller.sh.tpl index 116c34eb1..bc057809f 100644 --- a/mariadb/templates/bin/_mariadb-ingress-controller.sh.tpl +++ b/mariadb/templates/bin/_mariadb-ingress-controller.sh.tpl @@ -18,10 +18,9 @@ set -ex COMMAND="${@:-start}" function start () { - find /tmp/ -maxdepth 1 -writable | grep -v "^/tmp/$" | xargs -L1 -r rm -rfv + find /tmp -maxdepth 1 \! -path /tmp -perm /222 -exec rm -rfv {} \; exec /usr/bin/dumb-init \ /nginx-ingress-controller \ - --force-namespace-isolation \ --watch-namespace ${POD_NAMESPACE} \ --election-id=${RELEASE_NAME} \ --ingress-class=${INGRESS_CLASS} \ diff --git a/mariadb/templates/deployment-ingress.yaml b/mariadb/templates/deployment-ingress.yaml index 72bea94af..add8501c2 100644 --- a/mariadb/templates/deployment-ingress.yaml +++ b/mariadb/templates/deployment-ingress.yaml @@ -21,6 +21,82 @@ limitations under the License. {{ tuple $envAll "ingress" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - "networking.k8s.io" + resources: + - ingressclasses + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: {{ $serviceAccountName }} @@ -123,6 +199,13 @@ metadata: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} labels: {{ tuple $envAll "mariadb" "ingress" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + app.kubernetes.io/instance: {{ $serviceAccountName }} + app.kubernetes.io/name: "mariadb" + app.kubernetes.io/component: "ingress" + app.kubernetes.io/managed-by: {{ $envAll.Release.Service }} +{{- if $envAll.Chart.AppVersion }} + app.kubernetes.io/version: {{ $envAll.Chart.AppVersion | quote }} +{{- end }} spec: replicas: {{ .Values.pod.replicas.ingress }} selector: @@ -133,6 +216,13 @@ spec: metadata: labels: {{ tuple $envAll "mariadb" "ingress" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + app.kubernetes.io/instance: {{ $serviceAccountName }} + app.kubernetes.io/name: "mariadb" + app.kubernetes.io/component: "ingress" + app.kubernetes.io/managed-by: {{ $envAll.Release.Service }} +{{- if $envAll.Chart.AppVersion }} + app.kubernetes.io/version: {{ $envAll.Chart.AppVersion | quote }} +{{- end }} annotations: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} diff --git a/mariadb/values.yaml b/mariadb/values.yaml index 9a46357e9..ffe556229 100644 --- a/mariadb/values.yaml +++ b/mariadb/values.yaml @@ -21,7 +21,7 @@ release_group: null images: tags: mariadb: docker.io/openstackhelm/mariadb:latest-ubuntu_xenial - ingress: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.9.0 + ingress: k8s.gcr.io/ingress-nginx/controller:v0.42.0 error_pages: k8s.gcr.io/defaultbackend:1.4 prometheus_create_mysql_user: docker.io/mariadb:10.2.31 prometheus_mysql_exporter: docker.io/prom/mysqld-exporter:v0.11.0 diff --git a/releasenotes/notes/mariadb.yaml b/releasenotes/notes/mariadb.yaml index 96ddc94f6..9284e00d1 100644 --- a/releasenotes/notes/mariadb.yaml +++ b/releasenotes/notes/mariadb.yaml @@ -9,4 +9,5 @@ mariadb: - 0.1.6 Change Issuer to ClusterIssuer - 0.1.7 Revert - Change Issuer to ClusterIssuer - 0.1.8 Change Issuer to ClusterIssuer with logic in place to support cert-manager versioning + - 0.1.9 Uplift Mariadb-ingress to 0.42.0 ...