From 7e8a32376366b675875a1e17097296306027991e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20=C4=8Cajka?= Date: Sun, 20 Jan 2019 20:30:48 +0100 Subject: [PATCH] Initial commit --- Dockerfile | 44 +++ conf/default_pub_keys.pem | 34 ++ conf/error-page-503.http | 140 +++++++++ conf/haproxy-config.template | 582 +++++++++++++++++++++++++++++++++++ reload-haproxy | 140 +++++++++ 5 files changed, 940 insertions(+) create mode 100644 Dockerfile create mode 100644 conf/default_pub_keys.pem create mode 100644 conf/error-page-503.http create mode 100644 conf/haproxy-config.template create mode 100755 reload-haproxy diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6284654 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,44 @@ +# +# This is the HAProxy router for OpenShift Origin. +# +# The standard name for this image is openshift/origin-haproxy-router +# +FROM registry.fedoraproject.org/f29/origin-cli:latest + +ENV NAME=origin-haproxy-router \ + VERSION=3.11 \ + ARCH=x86_64 + +RUN INSTALL_PKGS="haproxy" && \ + dnf install -y $INSTALL_PKGS && \ + rpm -V $INSTALL_PKGS && \ + dnf clean all && \ + mkdir -p /var/lib/haproxy/router/{certs,cacerts} && \ + mkdir -p /var/lib/haproxy/{conf,run,bin,log} && \ + touch /var/lib/haproxy/conf/{{os_http_be,os_edge_reencrypt_be,os_tcp_be,os_sni_passthrough,os_route_http_redirect,cert_config,os_wildcard_domain}.map,haproxy.config} && \ + setcap 'cap_net_bind_service=ep' /usr/sbin/haproxy && \ + chown -R :0 /var/lib/haproxy && \ + chmod -R g+w /var/lib/haproxy + +COPY . /var/lib/haproxy/ + +LABEL io.k8s.display-name="OpenShift Origin HAProxy Router" \ + io.k8s.description="This is a component of OpenShift Origin and contains an HAProxy instance that automatically exposes services within the cluster through routes, and offers TLS termination, reencryption, or SNI-passthrough on ports 80 and 443." \ + io.openshift.tags="openshift,router,haproxy" \ + summary="This is a component of OpenShift Origin and contains an HAProxy instance that automatically exposes services within the cluster through routes, and offers TLS termination, reencryption, or SNI-passthrough on ports 80 and 443." \ + maintainer="Jakub Cajka " \ + License="GPLv2+" \ + name="$FGC/$NAME" \ + com.redhat.component="$NAME" \ + version="$VERSION" \ + architecture="$ARCH" \ + usage="This is a component of OpenShift Origin and contains an HAProxy instance that automatically exposes services within the cluster through routes, and offers TLS termination, reencryption, or SNI-passthrough on ports 80 and 443." + +COPY README.md README.md + +USER 1001 +EXPOSE 80 443 +WORKDIR /var/lib/haproxy/conf +ENV TEMPLATE_FILE=/var/lib/haproxy/conf/haproxy-config.template \ + RELOAD_SCRIPT=/var/lib/haproxy/reload-haproxy +ENTRYPOINT ["/usr/bin/openshift-router"] diff --git a/conf/default_pub_keys.pem b/conf/default_pub_keys.pem new file mode 100644 index 0000000..fa3783d --- /dev/null +++ b/conf/default_pub_keys.pem @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIDIjCCAgqgAwIBAgIBBjANBgkqhkiG9w0BAQUFADCBoTELMAkGA1UEBhMCVVMx +CzAJBgNVBAgMAlNDMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0Rl +ZmF1bHQgQ29tcGFueSBMdGQxEDAOBgNVBAsMB1Rlc3QgQ0ExGjAYBgNVBAMMEXd3 +dy5leGFtcGxlY2EuY29tMSIwIAYJKoZIhvcNAQkBFhNleGFtcGxlQGV4YW1wbGUu +Y29tMB4XDTE2MDExMzE5NDA1N1oXDTI2MDExMDE5NDA1N1owfDEYMBYGA1UEAxMP +d3d3LmV4YW1wbGUuY29tMQswCQYDVQQIEwJTQzELMAkGA1UEBhMCVVMxIjAgBgkq +hkiG9w0BCQEWE2V4YW1wbGVAZXhhbXBsZS5jb20xEDAOBgNVBAoTB0V4YW1wbGUx +EDAOBgNVBAsTB0V4YW1wbGUwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAM0B +u++oHV1wcphWRbMLUft8fD7nPG95xs7UeLPphFZuShIhhdAQMpvcsFeg+Bg9PWCu +v3jZljmk06MLvuWLfwjYfo9q/V+qOZVfTVHHbaIO5RTXJMC2Nn+ACF0kHBmNcbth +OOgF8L854a/P8tjm1iPR++vHnkex0NH7lyosVc/vAgMBAAGjDTALMAkGA1UdEwQC +MAAwDQYJKoZIhvcNAQEFBQADggEBADjFm5AlNH3DNT1Uzx3m66fFjqqrHEs25geT +yA3rvBuynflEHQO95M/8wCxYVyuAx4Z1i4YDC7tx0vmOn/2GXZHY9MAj1I8KCnwt +Jik7E2r1/yY0MrkawljOAxisXs821kJ+Z/51Ud2t5uhGxS6hJypbGspMS7OtBbw7 +8oThK7cWtCXOldNF6ruqY1agWnhRdAq5qSMnuBXuicOP0Kbtx51a1ugE3SnvQenJ +nZxdtYUXvEsHZC/6bAtTfNh+/SwgxQJuL2ZM+VG3X2JIKY8xTDui+il7uTh422lq +wED8uwKl+bOj6xFDyw4gWoBxRobsbFaME8pkykP1+GnKDberyAM= +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIICWwIBAAKBgQDNAbvvqB1dcHKYVkWzC1H7fHw+5zxvecbO1Hiz6YRWbkoSIYXQ +EDKb3LBXoPgYPT1grr942ZY5pNOjC77li38I2H6Pav1fqjmVX01Rx22iDuUU1yTA +tjZ/gAhdJBwZjXG7YTjoBfC/OeGvz/LY5tYj0fvrx55HsdDR+5cqLFXP7wIDAQAB +AoGAfE7P4Zsj6zOzGPI/Izj7Bi5OvGnEeKfzyBiH9Dflue74VRQkqqwXs/DWsNv3 +c+M2Y3iyu5ncgKmUduo5X8D9To2ymPRLGuCdfZTxnBMpIDKSJ0FTwVPkr6cYyyBk +5VCbc470pQPxTAAtl2eaO1sIrzR4PcgwqrSOjwBQQocsGAECQQD8QOra/mZmxPbt +bRh8U5lhgZmirImk5RY3QMPI/1/f4k+fyjkU5FRq/yqSyin75aSAXg8IupAFRgyZ +W7BT6zwBAkEA0A0ugAGorpCbuTa25SsIOMxkEzCiKYvh0O+GfGkzWG4lkSeJqGME +keuJGlXrZNKNoCYLluAKLPmnd72X2yTL7wJARM0kAXUP0wn324w8+HQIyqqBj/gF +Vt9Q7uMQQ3s72CGu3ANZDFS2nbRZFU5koxrggk6lRRk1fOq9NvrmHg10AQJABOea +pgfj+yGLmkUw8JwgGH6xCUbHO+WBUFSlPf+Y50fJeO+OrjqPXAVKeSV3ZCwWjKT4 +9viXJNJJ4WfF0bO/XwJAOMB1wQnEOSZ4v+laMwNtMq6hre5K8woqteXICoGcIWe8 +u3YLAbyW/lHhOCiZu2iAI8AbmXem9lW6Tr7p/97s0w== +-----END RSA PRIVATE KEY----- diff --git a/conf/error-page-503.http b/conf/error-page-503.http new file mode 100644 index 0000000..cb826df --- /dev/null +++ b/conf/error-page-503.http @@ -0,0 +1,140 @@ +HTTP/1.0 503 Service Unavailable +Pragma: no-cache +Cache-Control: private, max-age=0, no-cache, no-store +Connection: close +Content-Type: text/html + + + + + + + + +
+

Application is not available

+

The application is currently not serving requests at this endpoint. It may not have been started or is still starting.

+ +
+

+ Possible reasons you are seeing this page: +

+
    +
  • + The host doesn't exist. + Make sure the hostname was typed correctly and that a route matching this hostname exists. +
    • +
  • + The host exists, but doesn't have a matching path. + Check if the URL path was typed correctly and that the route was created using the desired path. +
    • +
  • + Route and path matches, but all pods are down. + Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running. +
    • +
+
+
+ + diff --git a/conf/haproxy-config.template b/conf/haproxy-config.template new file mode 100644 index 0000000..8563d59 --- /dev/null +++ b/conf/haproxy-config.template @@ -0,0 +1,582 @@ +{{/* + haproxy-config.cfg: contains the main config with helper backends that are used to terminate + encryption before finally sending to a host_be which is the backend that is the final + backend for a route and contains all the endpoints for the service +*/}} +{{- define "/var/lib/haproxy/conf/haproxy.config" }} +{{- $workingDir := .WorkingDir }} +{{- $defaultDestinationCA := .DefaultDestinationCA }} +{{- $router_ip_v4_v6_mode := env "ROUTER_IP_V4_V6_MODE" "v4" }} + + +{{- /* A bunch of regular expressions. Each should be wrapped in (?:) so that it is safe to include bare */}} +{{- /* quadPattern: Match a quad in an IP address; e.g. 123 */}} +{{- $quadPattern := `(?:[0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])` -}} + +{{- /* ipPattern: Match an IPv4 address; e.g. 192.168.21.23 */}} +{{- $ipPattern := printf `(?:%s\.%s\.%s\.%s)` $quadPattern $quadPattern $quadPattern $quadPattern -}} + +{{- /* cidrPattern: Match an IP and network size in CIDR form; e.g. 192.168.21.23/24 */}} +{{- $cidrPattern := printf `(?:%s(?:/(?:[0-9]|[1-2][0-9]|3[0-2]))?)` $ipPattern -}} + +{{- /* cidrListPattern: Match a space separated list of CIDRs; e.g. 192.168.21.23/24 192.10.2.12 */}} +{{- $cidrListPattern := printf `(?:%s(?: +%s)*)` $cidrPattern $cidrPattern -}} + +{{- /* cookie name pattern: */}} +{{- $cookieNamePattern := `[a-zA-Z0-9_-]+` -}} + +{{- $timeSpecPattern := `[1-9][0-9]*(us|ms|s|m|h|d)?` }} + +{{- /* hsts header in response: */}} +{{- $hstsOptionalTokenPattern := `(?:includeSubDomains|preload)` }} +{{- $hstsPattern := printf `(?:%[1]s[;])*max-age=(?:\d+|"\d+")(?:[;]%[1]s)*` $hstsOptionalTokenPattern -}} + +global + maxconn {{env "ROUTER_MAX_CONNECTIONS" "20000"}} + + daemon +{{- with (env "ROUTER_SYSLOG_ADDRESS") }} + log {{.}} {{env "ROUTER_LOG_FACILITY" "local1"}} {{env "ROUTER_LOG_LEVEL" "warning"}} +{{- end}} + ca-base /etc/ssl + crt-base /etc/ssl + stats socket /var/lib/haproxy/run/haproxy.sock mode 600 level admin expose-fd listeners + stats timeout 2m + + # Increase the default request size to be comparable to modern cloud load balancers (ALB: 64kb), affects + # total memory use when large numbers of connections are open. + tune.maxrewrite 8192 + tune.bufsize 32768 + + # Prevent vulnerability to POODLE attacks + ssl-default-bind-options no-sslv3 + +# The default cipher suite can be selected from the three sets recommended by https://wiki.mozilla.org/Security/Server_Side_TLS, +# or the user can provide one using the ROUTER_CIPHERS environment variable. +# By default when a cipher set is not provided, intermediate is used. +{{- if eq (env "ROUTER_CIPHERS" "intermediate") "modern" }} + # Modern cipher suite (no legacy browser support) from https://wiki.mozilla.org/Security/Server_Side_TLS + tune.ssl.default-dh-param 2048 + ssl-default-bind-ciphers ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256 +{{ else }} + + {{- if eq (env "ROUTER_CIPHERS" "intermediate") "intermediate" }} + # Intermediate cipher suite (default) from https://wiki.mozilla.org/Security/Server_Side_TLS + tune.ssl.default-dh-param 2048 + ssl-default-bind-ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS + {{ else }} + + {{- if eq (env "ROUTER_CIPHERS" "intermediate") "old" }} + + # Old cipher suite (maximum compatibility but insecure) from https://wiki.mozilla.org/Security/Server_Side_TLS + tune.ssl.default-dh-param 1024 + ssl-default-bind-ciphers ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:HIGH:SEED:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!RSAPSK:!aDH:!aECDH:!EDH-DSS-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA:!SRP + + {{- else }} + # user provided list of ciphers (Colon separated list as seen above) + # the env default is not used here since we can't get here with empty ROUTER_CIPHERS + tune.ssl.default-dh-param 2048 + ssl-default-bind-ciphers {{env "ROUTER_CIPHERS" "ECDHE-ECDSA-CHACHA20-POLY1305"}} + {{- end }} + {{- end }} +{{- end }} + +defaults + maxconn {{env "ROUTER_MAX_CONNECTIONS" "20000"}} + + # Add x-forwarded-for header. +{{- if ne (env "ROUTER_SYSLOG_ADDRESS") "" }} + {{- if ne (env "ROUTER_SYSLOG_FORMAT") "" }} + log-format {{env "ROUTER_SYSLOG_FORMAT"}} + {{- else }} + option httplog + {{- end }} + log global +{{- end }} + + # To configure custom default errors, you can either uncomment the + # line below (server ... 127.0.0.1:8080) and point it to your custom + # backend service or alternatively, you can send a custom 503 error. + # + # server openshift_backend 127.0.0.1:8080 + errorfile 503 /var/lib/haproxy/conf/error-page-503.http + + timeout connect {{firstMatch $timeSpecPattern (env "ROUTER_DEFAULT_CONNECT_TIMEOUT") "5s"}} + timeout client {{firstMatch $timeSpecPattern (env "ROUTER_DEFAULT_CLIENT_TIMEOUT") "30s"}} + timeout client-fin {{firstMatch $timeSpecPattern (env "ROUTER_CLIENT_FIN_TIMEOUT") "1s"}} + timeout server {{firstMatch $timeSpecPattern (env "ROUTER_DEFAULT_SERVER_TIMEOUT") "30s"}} + timeout server-fin {{firstMatch $timeSpecPattern (env "ROUTER_DEFAULT_SERVER_FIN_TIMEOUT") "1s"}} + timeout http-request {{firstMatch $timeSpecPattern (env "ROUTER_SLOWLORIS_TIMEOUT") "10s" }} + timeout http-keep-alive {{firstMatch $timeSpecPattern (env "ROUTER_SLOWLORIS_HTTP_KEEPALIVE") "300s" }} + + # Long timeout for WebSocket connections. + timeout tunnel {{firstMatch $timeSpecPattern (env "ROUTER_DEFAULT_TUNNEL_TIMEOUT") "1h" }} + +{{- if isTrue (env "ROUTER_ENABLE_COMPRESSION") }} + compression algo gzip + compression type {{env "ROUTER_COMPRESSION_MIME" "text/html text/plain text/css"}} +{{- end }} + +{{ if (gt .StatsPort -1) }} +listen stats + bind :{{if (gt .StatsPort 0)}}{{.StatsPort}}{{else}}1936{{end}} + mode http + # Health check monitoring uri. + monitor-uri /healthz + +{{- if and (and (ne .StatsUser "") (ne .StatsPassword "")) (gt .StatsPort 0) }} + # Add your custom health check monitoring failure condition here. + # monitor fail if + stats enable + stats hide-version + stats realm Haproxy\ Statistics + stats uri / + stats auth {{.StatsUser}}:{{.StatsPassword}} +{{- end }} +{{- end }} + +{{ if .BindPorts -}} +frontend public + {{ if eq "v4v6" $router_ip_v4_v6_mode }} + bind :::{{env "ROUTER_SERVICE_HTTP_PORT" "80"}} v4v6 + {{- else if eq "v6" $router_ip_v4_v6_mode }} + bind :::{{env "ROUTER_SERVICE_HTTP_PORT" "80"}} v6only + {{- else }} + bind :{{env "ROUTER_SERVICE_HTTP_PORT" "80"}} + {{- end }} + {{- if isTrue (env "ROUTER_USE_PROXY_PROTOCOL") }} accept-proxy{{ end }} + mode http + tcp-request inspect-delay 5s + tcp-request content accept if HTTP + + {{- if (eq .StatsPort -1) }} + monitor-uri /_______internal_router_healthz + {{- end }} + + # Strip off Proxy headers to prevent HTTpoxy (https://httpoxy.org/) + http-request del-header Proxy + + # DNS labels are case insensitive (RFC 4343), we need to convert the hostname into lowercase + # before matching, or any requests containing uppercase characters will never match. + http-request set-header Host %[req.hdr(Host),lower] + + # check if we need to redirect/force using https. + acl secure_redirect base,map_reg(/var/lib/haproxy/conf/os_route_http_redirect.map) -m found + redirect scheme https if secure_redirect + + use_backend %[base,map_reg(/var/lib/haproxy/conf/os_http_be.map)] + + default_backend openshift_default + +# public ssl accepts all connections and isn't checking certificates yet certificates to use will be +# determined by the next backend in the chain which may be an app backend (passthrough termination) or a backend +# that terminates encryption in this router (edge) +frontend public_ssl + {{ if eq "v4v6" $router_ip_v4_v6_mode }} + bind :::{{env "ROUTER_SERVICE_HTTPS_PORT" "443"}} v4v6 + {{- else if eq "v6" $router_ip_v4_v6_mode }} + bind :::{{env "ROUTER_SERVICE_HTTPS_PORT" "443"}} v6only + {{- else }} + bind :{{env "ROUTER_SERVICE_HTTPS_PORT" "443"}} + {{- end }} + {{- if isTrue (env "ROUTER_USE_PROXY_PROTOCOL") }} accept-proxy{{ end }} + tcp-request inspect-delay 5s + tcp-request content accept if { req_ssl_hello_type 1 } + + # if the connection is SNI and the route is a passthrough don't use the termination backend, just use the tcp backend + # for the SNI case, we also need to compare it in case-insensitive mode (by converting it to lowercase) as RFC 4343 says + acl sni req.ssl_sni -m found + acl sni_passthrough req.ssl_sni,lower,map_reg(/var/lib/haproxy/conf/os_sni_passthrough.map) -m found + use_backend be_tcp:%[req.ssl_sni,lower,map_reg(/var/lib/haproxy/conf/os_tcp_be.map)] if sni sni_passthrough + + # if the route is SNI and NOT passthrough enter the termination flow + use_backend be_sni if sni + + # non SNI requests should enter a default termination backend rather than the custom cert SNI backend since it + # will not be able to match a cert to an SNI host + default_backend be_no_sni + +########################################################################## +# TLS SNI +# +# When using SNI we can terminate encryption with custom certificates. +# Certs will be stored in a directory and will be matched with the SNI host header +# which must exist in the CN of the certificate. Certificates must be concatenated +# as a single file (handled by the plugin writer) per the haproxy documentation. +# +# Finally, check re-encryption settings and re-encrypt or just pass along the unencrypted +# traffic +########################################################################## +backend be_sni + server fe_sni 127.0.0.1:{{env "ROUTER_SERVICE_SNI_PORT" "10444"}} weight 1 send-proxy + +frontend fe_sni + # terminate ssl on edge + bind 127.0.0.1:{{env "ROUTER_SERVICE_SNI_PORT" "10444"}} ssl no-sslv3 + {{- if isTrue (env "ROUTER_STRICT_SNI") }} strict-sni {{ end }} + {{- ""}} crt {{firstMatch ".+" .DefaultCertificate "/var/lib/haproxy/conf/default_pub_keys.pem"}} + {{- ""}} crt-list /var/lib/haproxy/conf/cert_config.map accept-proxy + mode http + + # Strip off Proxy headers to prevent HTTpoxy (https://httpoxy.org/) + http-request del-header Proxy + + # DNS labels are case insensitive (RFC 4343), we need to convert the hostname into lowercase + # before matching, or any requests containing uppercase characters will never match. + http-request set-header Host %[req.hdr(Host),lower] + + # map to backend + # Search from most specific to general path (host case). + # Note: If no match, haproxy uses the default_backend, no other + # use_backend directives below this will be processed. + use_backend %[base,map_reg(/var/lib/haproxy/conf/os_edge_reencrypt_be.map)] + + default_backend openshift_default + +########################################################################## +# END TLS SNI +########################################################################## + +########################################################################## +# TLS NO SNI +# +# When we don't have SNI the only thing we can try to do is terminate the encryption +# using our wild card certificate. Once that is complete we can either re-encrypt +# the traffic or pass it on to the backends +########################################################################## +# backend for when sni does not exist, or ssl term needs to happen on the edge +backend be_no_sni + server fe_no_sni 127.0.0.1:{{env "ROUTER_SERVICE_NO_SNI_PORT" "10443"}} weight 1 send-proxy + +frontend fe_no_sni + # terminate ssl on edge + bind 127.0.0.1:{{env "ROUTER_SERVICE_NO_SNI_PORT" "10443"}} ssl no-sslv3 crt {{firstMatch ".+" .DefaultCertificate "/var/lib/haproxy/conf/default_pub_keys.pem"}} accept-proxy + mode http + + # Strip off Proxy headers to prevent HTTpoxy (https://httpoxy.org/) + http-request del-header Proxy + + # DNS labels are case insensitive (RFC 4343), we need to convert the hostname into lowercase + # before matching, or any requests containing uppercase characters will never match. + http-request set-header Host %[req.hdr(Host),lower] + + + # map to backend + # Search from most specific to general path (host case). + # Note: If no match, haproxy uses the default_backend, no other + # use_backend directives below this will be processed. + use_backend %[base,map_reg(/var/lib/haproxy/conf/os_edge_reencrypt_be.map)] + + default_backend openshift_default + +########################################################################## +# END TLS NO SNI +########################################################################## + +backend openshift_default + mode http + option forwardfor + #option http-keep-alive + option http-pretend-keepalive + +##-------------- app level backends ---------------- +{{/* + 1. If termination is not set: This is plain http -> http. Create a be_http: backend. + Incoming http traffic is terminated and sent as http to the pods. + + 2. If termination is type 'edge': This is https -> http. Create a be_edge_http: backend. + Incoming https traffic is terminated and sent as http to the pods. + + 3. If termination is type 'reencrypt': This is https -> https. Create a be_secure: backend. + Incoming https traffic is terminated and then sent as https to the pods. + + 4. If termination is type 'passthrough': This is https (or any SNI TLS connection) passthrough. + Create a be_tcp: backend. + Incoming traffic is inspected to get the hostname from the SNI header, but then all traffic is + passed through to the backend pod by just looking at the TCP headers. +*/}} +{{- range $cfgIdx, $cfg := .State }} + {{- if matchValues (print $cfg.TLSTermination) "" "edge" "reencrypt" }} + {{- if (eq $cfg.TLSTermination "") }} + +# Plain http backend +backend be_http:{{$cfgIdx}} + {{- else if (eq $cfg.TLSTermination "edge") }} + +# Plain http backend but request is TLS, terminated at edge +backend be_edge_http:{{$cfgIdx}} + {{ else if (eq $cfg.TLSTermination "reencrypt") }} + +# Secure backend which requires re-encryption +backend be_secure:{{$cfgIdx}} + {{- end }}{{/* end chceck for router type */}} + mode http + option redispatch + option forwardfor + + {{- with $balanceAlgo := firstMatch "roundrobin|leastconn|source" (index $cfg.Annotations "haproxy.router.openshift.io/balance") (env "ROUTER_LOAD_BALANCE_ALGORITHM") }} + balance {{ $balanceAlgo }} + {{- else }} + balance {{ if gt $cfg.ActiveServiceUnits 1 }}roundrobin{{ else }}leastconn{{ end }} + {{- end }} + {{- with $ip_whiteList := firstMatch $cidrListPattern (index $cfg.Annotations "haproxy.router.openshift.io/ip_whitelist") }} + acl whitelist src {{ $ip_whiteList }} + tcp-request content reject if !whitelist + {{- end }} + {{- with $value := firstMatch $timeSpecPattern (index $cfg.Annotations "haproxy.router.openshift.io/timeout")}} + timeout server {{$value}} + {{- end }} + + {{- if isTrue (index $cfg.Annotations "haproxy.router.openshift.io/rate-limit-connections") }} + stick-table type ip size 100k expire 30s store conn_cur,conn_rate(3s),http_req_rate(10s) + tcp-request content track-sc2 src + {{- if (isInteger (index $cfg.Annotations "haproxy.router.openshift.io/rate-limit-connections.concurrent-tcp")) }} + tcp-request content reject if { src_conn_cur ge {{ index $cfg.Annotations "haproxy.router.openshift.io/rate-limit-connections.concurrent-tcp" }} } + {{- else }} + # concurrent TCP connections not restricted + {{- end }} + + {{- if (isInteger (index $cfg.Annotations "haproxy.router.openshift.io/rate-limit-connections.rate-tcp")) }} + tcp-request content reject if { src_conn_rate ge {{ index $cfg.Annotations "haproxy.router.openshift.io/rate-limit-connections.rate-tcp" }} } + {{- else }} + #TCP connection rate not restricted + {{- end }} + + {{- if (isInteger (index $cfg.Annotations "haproxy.router.openshift.io/rate-limit-connections.rate-http")) }} + tcp-request content reject if { src_http_req_rate ge {{ index $cfg.Annotations "haproxy.router.openshift.io/rate-limit-connections.rate-http" }} } + {{- else }} + #HTTP request rate not restricted + {{- end }} + {{- end }} + + timeout check 5000ms + http-request set-header X-Forwarded-Host %[req.hdr(host)] + http-request set-header X-Forwarded-Port %[dst_port] + http-request set-header X-Forwarded-Proto http if !{ ssl_fc } + http-request set-header X-Forwarded-Proto https if { ssl_fc } + {{- if matchPattern "(v4)?v6" $router_ip_v4_v6_mode }} + # See the quoting rules in https://tools.ietf.org/html/rfc7239 for IPv6 addresses (v4 addresses get translated to v6 when in hybrid mode) + http-request set-header Forwarded for="[%[src]]";host=%[req.hdr(host)];proto=%[req.hdr(X-Forwarded-Proto)] + {{- else }} + http-request set-header Forwarded for=%[src];host=%[req.hdr(host)];proto=%[req.hdr(X-Forwarded-Proto)] + {{- end }} + + {{- if not (isTrue (index $cfg.Annotations "haproxy.router.openshift.io/disable_cookies")) }} + cookie {{firstMatch $cookieNamePattern (index $cfg.Annotations "router.openshift.io/cookie_name") (env "ROUTER_COOKIE_NAME" "") $cfg.RoutingKeyName}} insert indirect nocache httponly + {{- if and (matchValues (print $cfg.TLSTermination) "edge" "reencrypt") (ne $cfg.InsecureEdgeTerminationPolicy "Allow") }} secure + {{- end }} + {{- end }}{{/* end disable cookies check */}} + + {{- if matchValues (print $cfg.TLSTermination) "edge" "reencrypt" }} + {{- with $hsts := firstMatch $hstsPattern (index $cfg.Annotations "haproxy.router.openshift.io/hsts_header") }} + http-response set-header Strict-Transport-Security {{$hsts}} + {{- end }}{{/* hsts header */}} + {{- end }}{{/* is "edge" or "reencrypt" */}} + + {{- range $serviceUnitName, $weight := $cfg.ServiceUnitNames }} + {{- if ne $weight 0 }} + {{- with $serviceUnit := index $.ServiceUnits $serviceUnitName }} + {{- range $idx, $endpoint := processEndpointsForAlias $cfg $serviceUnit (env "ROUTER_BACKEND_PROCESS_ENDPOINTS" "") }} + server {{$endpoint.ID}} {{$endpoint.IP}}:{{$endpoint.Port}} cookie {{$endpoint.IdHash}} weight {{$weight}} + {{- if (eq $cfg.TLSTermination "reencrypt") }} ssl + {{- if $cfg.VerifyServiceHostname }} verifyhost {{ $serviceUnit.Hostname }} + {{- end }} + {{- if gt (len (index $cfg.Certificates (printf "%s_pod" $cfg.Host)).Contents) 0 }} verify required ca-file {{ $workingDir }}/cacerts/{{$cfgIdx}}.pem + {{- else }} + {{- if gt (len $defaultDestinationCA) 0 }} verify required ca-file {{ $defaultDestinationCA }} + {{- else }} verify none + {{- end }} + {{- end }} + + {{- else if or (eq $cfg.TLSTermination "") (eq $cfg.TLSTermination "edge") }} + {{- end }}{{/* end type specific options*/}} + + {{- if and (not $endpoint.NoHealthCheck) (gt $cfg.ActiveEndpoints 1) }} check inter {{firstMatch $timeSpecPattern (index $cfg.Annotations "router.openshift.io/haproxy.health.check.interval") (env "ROUTER_BACKEND_CHECK_INTERVAL") "5000ms"}} + {{- end }}{{/* end else no health check */}} + {{- with $podMaxConn := index $cfg.Annotations "haproxy.router.openshift.io/pod-concurrent-connections" }} + {{- if (isInteger (index $cfg.Annotations "haproxy.router.openshift.io/pod-concurrent-connections")) }} maxconn {{$podMaxConn}} {{- end }} + {{- end}}{{/* end pod-concurrent-connections annotation */}} + + {{- end }}{{/* end if cg.TLSTermination */}} + {{- end }}{{/* end range processEndpointsForAlias */}} + {{- end }}{{/* end get serviceUnit from its name */}} + {{- end }}{{/* end range over serviceUnitNames */}} + + {{- end }}{{/* end if tls==edge/none/reencrypt */}} + + {{- if eq $cfg.TLSTermination "passthrough" }} + +# Secure backend, pass through +backend be_tcp:{{$cfgIdx}} +{{- if ne (env "ROUTER_SYSLOG_ADDRESS") ""}} + option tcplog +{{- end }} + {{- with $balanceAlgo := firstMatch "roundrobin|leastconn|source" (index $cfg.Annotations "haproxy.router.openshift.io/balance") (env "ROUTER_LOAD_BALANCE_ALGORITHM") }} + balance {{ $balanceAlgo }} + {{- else }} + balance {{ if gt $cfg.ActiveServiceUnits 1 }}roundrobin{{ else }}source{{ end }} + {{- end }} + {{- with $ip_whiteList := firstMatch $cidrListPattern (index $cfg.Annotations "haproxy.router.openshift.io/ip_whitelist") }} + acl whitelist src {{$ip_whiteList}} + tcp-request content reject if !whitelist + {{- end }} + {{- with $value := firstMatch $timeSpecPattern (index $cfg.Annotations "haproxy.router.openshift.io/timeout")}} + timeout tunnel {{$value}} + {{- end }} + +{{- if isTrue (index $cfg.Annotations "haproxy.router.openshift.io/rate-limit-connections") }} + stick-table type ip size 100k expire 30s store conn_cur,conn_rate(3s),http_req_rate(10s) + tcp-request content track-sc2 src + {{- if (isInteger (index $cfg.Annotations "haproxy.router.openshift.io/rate-limit-connections.concurrent-tcp")) }} + tcp-request content reject if { src_conn_cur ge {{ index $cfg.Annotations "haproxy.router.openshift.io/rate-limit-connections.concurrent-tcp" }} } + {{- else }} + # concurrent TCP connections not restricted + {{- end }} + + {{- if (isInteger (index $cfg.Annotations "haproxy.router.openshift.io/rate-limit-connections.rate-tcp")) }} + tcp-request content reject if { src_conn_rate ge {{ index $cfg.Annotations "haproxy.router.openshift.io/rate-limit-connections.rate-tcp" }} } + {{- else }} + #TCP connection rate not restricted + {{- end }} +{{- end }} + + hash-type consistent + timeout check 5000ms + {{- range $serviceUnitName, $weight := $cfg.ServiceUnitNames }} + {{- if ne $weight 0 }} + {{- with $serviceUnit := index $.ServiceUnits $serviceUnitName }} + {{- range $idx, $endpoint := processEndpointsForAlias $cfg $serviceUnit (env "ROUTER_BACKEND_PROCESS_ENDPOINTS" "") }} + server {{$endpoint.ID}} {{$endpoint.IP}}:{{$endpoint.Port}} weight {{$weight}} + {{- if and (not $endpoint.NoHealthCheck) (gt $cfg.ActiveEndpoints 1) }} check inter {{firstMatch $timeSpecPattern (index $cfg.Annotations "router.openshift.io/haproxy.health.check.interval") (env "ROUTER_BACKEND_CHECK_INTERVAL") "5000ms"}} + {{- end }}{{/* end else no health check */}} + {{- with $podMaxConn := index $cfg.Annotations "haproxy.router.openshift.io/pod-concurrent-connections" }} + {{- if (isInteger (index $cfg.Annotations "haproxy.router.openshift.io/pod-concurrent-connections")) }} maxconn {{$podMaxConn}} {{- end }} + {{- end}}{{/* end pod-concurrent-connections annotation */}} + + {{- end }}{{/* end range processEndpointsForAlias */}} + {{- end }}{{/* end get ServiceUnit from serviceUnitName */}} + {{- end }}{{/* end if weight != 0 */}} + {{- end }}{{/* end iterate over services*/}} + {{- end }}{{/*end tls==passthrough*/}} + +{{- end }}{{/* end loop over routes */}} +{{- else }} +# Avoiding binding ports until routing configuration has been synchronized. +{{- end }}{{/* end bind ports after sync */}} +{{ end }}{{/* end haproxy config template */}} + +{{/*--------------------------------- END OF HAPROXY CONFIG, BELOW ARE MAPPING FILES ------------------------*/}} +{{/* + os_wildcard_domain.map: contains a mapping of wildcard hosts for a + [sub]domain regexps. This map is used to check if + a host matches a [sub]domain with has wildcard support. +*/}} +{{ define "/var/lib/haproxy/conf/os_wildcard_domain.map" -}} +{{ if isTrue (env "ROUTER_ALLOW_WILDCARD_ROUTES") -}} +{{ range $idx, $cfg := .State -}} +{{ if ne $cfg.Host "" -}} +{{ if $cfg.IsWildcard -}} +{{generateRouteRegexp $cfg.Host "" true}} 1 +{{ end -}} +{{ end -}} +{{ end -}} +{{ end -}}{{/* end if router allows wildcard routes */}} +{{ end -}}{{/* end wildcard domain map template */}} + + + +{{/* + os_http_be.map : contains a mapping of www.example.com -> . This map is used to discover the correct backend + by attaching a prefix: be_http for http routes + be_edge_http for edge routes with InsecureEdgeTerminationPolicy Allow + be_secure for reencrypt routes with InsecureEdgeTerminationPolicy Allow +*/}} +{{ define "/var/lib/haproxy/conf/os_http_be.map" -}} +{{ range $idx, $cfg := .State -}} +{{ if and (ne $cfg.Host "") (eq $cfg.TLSTermination "") -}} +{{generateRouteRegexp $cfg.Host $cfg.Path $cfg.IsWildcard}} be_http:{{$idx}} +{{ end -}} +{{ if and (ne $cfg.Host "") (and (matchValues (print $cfg.TLSTermination) "edge" "reencrypt") (eq $cfg.InsecureEdgeTerminationPolicy "Allow")) -}} +{{ if (eq $cfg.TLSTermination "edge") -}} +{{generateRouteRegexp $cfg.Host $cfg.Path $cfg.IsWildcard}} be_edge_http:{{$idx}} +{{ else -}} +{{generateRouteRegexp $cfg.Host $cfg.Path $cfg.IsWildcard}} be_secure:{{$idx}} +{{ end -}} +{{ end -}} +{{ end -}} +{{ end -}} + +{{/* + os_edge_reencrypt_be.map : contains a mapping of www.example.com -> . This map is similar to os_http_be.map but for tls routes. + by attaching prefix: be_edge_http for edge terminated routes + be_secure for reencrypt routes +*/}} +{{ define "/var/lib/haproxy/conf/os_edge_reencrypt_be.map" -}} +{{ range $idx, $cfg := .State -}} +{{ if and (ne $cfg.Host "") (eq $cfg.TLSTermination "edge") -}} +{{generateRouteRegexp $cfg.Host $cfg.Path $cfg.IsWildcard}} be_edge_http:{{$idx}} +{{ end -}} +{{ if and (ne $cfg.Host "") (eq $cfg.TLSTermination "reencrypt") -}} +{{generateRouteRegexp $cfg.Host $cfg.Path $cfg.IsWildcard}} be_secure:{{$idx}} +{{ end -}} +{{ end -}} +{{ end -}}{{/* end edge http host map template */}} + + +{{/* + os_route_http_redirect.map: contains a mapping of www.example.com -> . + Map is used to redirect insecure traffic to use a secure scheme (https) + if acls match for routes that have the insecure option set to redirect. +*/}} +{{ define "/var/lib/haproxy/conf/os_route_http_redirect.map" -}} +{{ range $idx, $cfg := .State -}} +{{ if and (ne $cfg.Host "") (eq $cfg.InsecureEdgeTerminationPolicy "Redirect") -}} +{{generateRouteRegexp $cfg.Host $cfg.Path $cfg.IsWildcard}} {{$idx}} +{{ end -}} +{{ end -}} +{{ end -}}{{/* end redirect http host map template */}} + + +{{/* + os_tcp_be.map: contains a mapping of www.example.com -> . This map is used to discover the correct backend + by attaching a prefix (be_tcp: or be_secure:) by use_backend statements if acls are matched. +*/}} +{{ define "/var/lib/haproxy/conf/os_tcp_be.map" -}} +{{ range $idx, $cfg := .State -}} +{{ if and (eq $cfg.Path "") (and (ne $cfg.Host "") (matchValues (print $cfg.TLSTermination) "passthrough" "reencrypt")) -}} +{{generateRouteRegexp $cfg.Host "" $cfg.IsWildcard}} {{$idx}} +{{ end -}} +{{ end -}} +{{ end -}}{{/* end tcp host map template */}} + +{{/* + os_sni_passthrough.map: contains a mapping of routes that expect to have an sni header and should be passed + through to the host_be. Driven by the termination type of the ServiceAliasConfigs +*/}} +{{ define "/var/lib/haproxy/conf/os_sni_passthrough.map" -}} +{{ range $idx, $cfg := .State -}} +{{ if and (eq $cfg.Path "") (eq $cfg.TLSTermination "passthrough") -}} +{{generateRouteRegexp $cfg.Host "" $cfg.IsWildcard}} 1 +{{ end -}} +{{ end -}} +{{ end -}}{{/* end sni passthrough map template */}} + +{{/* + cert_config.map: contains a mapping of -> example.org + This map is used to present the appropriate cert + based on the sni header. + Note: It is sort of a reverse map for our case but the order + ": " is important as this allows us to use + wildcards and/or use a deny set with ! in the future. +*/}} +{{ define "/var/lib/haproxy/conf/cert_config.map" -}} +{{ $workingDir := .WorkingDir -}} +{{ range $idx, $cfg := .State -}} +{{ if and (ne $cfg.Host "") (matchValues (print $cfg.TLSTermination) "edge" "reencrypt") -}} +{{ $cert := index $cfg.Certificates $cfg.Host -}} +{{ if ne $cert.Contents "" -}} +{{$workingDir}}/certs/{{$idx}}.pem {{genCertificateHostName $cfg.Host $cfg.IsWildcard}} +{{ end -}} +{{ end -}} +{{ end -}} +{{ end }}{{/* end cert_config map template */}} diff --git a/reload-haproxy b/reload-haproxy new file mode 100755 index 0000000..b5b4e62 --- /dev/null +++ b/reload-haproxy @@ -0,0 +1,140 @@ +#!/bin/bash + +set -o nounset + +config_file=/var/lib/haproxy/conf/haproxy.config +pid_file=/var/lib/haproxy/run/haproxy.pid +old_pid="" +haproxy_conf_dir=/var/lib/haproxy/conf +readonly max_wait_time=30 +readonly timeout_opts="-m 1 --connect-timeout 1" +readonly numeric_re='^[0-9]+$' + +function haproxyHealthCheck() { + local wait_time=${MAX_RELOAD_WAIT_TIME:-$max_wait_time} + local port=${ROUTER_SERVICE_HTTP_PORT:-"80"} + local url="http://localhost:${port}" + local retries=0 + local start_ts=$(date +"%s") + local proxy_proto="${ROUTER_USE_PROXY_PROTOCOL-}" + + if ! [[ $wait_time =~ $numeric_re ]]; then + echo " - Invalid max reload wait time, using default $max_wait_time ..." + wait_time=$max_wait_time + fi + + local end_ts=$((start_ts + wait_time)) + + # test with proxy protocol on + if [[ "${proxy_proto}" == "TRUE" || "${proxy_proto}" == "true" ]]; then + echo " - Proxy protocol on, checking ${url} ..." + while true; do + local statusline=$(echo $'PROXY UNKNOWN\r\nGET / HTTP/1.1\r\n' | socat tcp-connect:localhost:${port} stdio | head -1) + + if [[ "$statusline" == *" 503 "* ]]; then + echo " - Health check ok : $retries retry attempt(s)." + return 0 + fi + + if [ $(date +"%s") -ge $end_ts ]; then + echo " - Exceeded max wait time ($wait_time) in health check - $retries retry attempt(s)." + return 1 + fi + + sleep 0.5 + retries=$((retries + 1)) + done + return 0 + fi + + echo " - Checking ${url} ..." + while true; do + local httpcode=$(curl $timeout_opts -s -o /dev/null -I -H "Host: " -w "%{http_code}" ${url}) + + if [ "$httpcode" == "503" ]; then + echo " - Health check ok : $retries retry attempt(s)." + return 0 + fi + + if [ $(date +"%s") -ge $end_ts ]; then + echo " - Exceeded max wait time ($wait_time) in health check - $retries retry attempt(s)." + return 1 + fi + + sleep 0.5 + retries=$((retries + 1)) + done +} + + +# How many times to retry removal of the iptables rules (if requested at all) +# It will sleep for 1/2 a second between attempts, so the time is retries / 2 secs +retries=20 + + +# sort the path based map files for the haproxy map_beg function +for mapfile in "$haproxy_conf_dir"/*.map; do + sort -r "$mapfile" -o "$mapfile" +done + +old_pids=$(ps -A -opid,args | grep haproxy | egrep -v -e 'grep|reload-haproxy' | awk '{print $1}' | tr '\n' ' ') + +reload_status=0 +installed_iptables=0 +if [ -n "$old_pids" ]; then + if $(set | grep DROP_SYN_DURING_RESTART= > /dev/null) && [[ "$DROP_SYN_DURING_RESTART" == 'true' || "$DROP_SYN_DURING_RESTART" == '1' ]]; then + # We install the syn eater so that connections that come in during the restart don't + # go onto the wrong socket, which is then closed. + ports=$(grep -E -o '^\s*bind\s+:[[:digit:]]+\w' "$config_file" | cut -f2 -d: | paste -d, -s) + if [ -n "$ports" ]; then + # If this doesn't insert, we don't care, we still want to reload + /usr/sbin/iptables -I INPUT -p tcp -m multiport --dports $ports --syn -j DROP \ + -m comment --comment "Eat SYNs while reloading haproxy" || : + installed_iptables=1 + + # The sleep is needed to let the socket drain before the new + # haproxy starts and binds to the same port. The value was + # determined by trial and error: I stopped seeing failures at + # 0.01 under load, so I put in a 10x margin. At worst, we may + # leave a connection in the old process' listen buffer that + # won't get handled, and they'll get a reset. I didn't want to + # set it too long, because that affects the overall time a + # reload takes which means that incoming connections aren't + # handled while the SYN eater is in place. + sleep 0.1 + fi + fi + + /usr/sbin/haproxy -f $config_file -p $pid_file -x /var/lib/haproxy/run/haproxy.sock -sf $old_pids + reload_status=$? + + if [[ "$installed_iptables" == 1 ]]; then + # We NEVER want to leave the syn eater in place after the reload or haproxy + # will never get new connections. So try to remove it twenty times, and if + # that fails, log the error and return failure so the pod logs a fatal error. + i=0 + while (( i++ < retries )) ; do + /usr/sbin/iptables -D INPUT -p tcp -m multiport --dports $ports --syn -j DROP \ + -m comment --comment "Eat SYNs while reloading haproxy" || : + + # Test the condition and end the loop if the rule has been removed + /usr/sbin/iptables -L INPUT | grep -F '/* Eat SYNs while reloading haproxy */' || break + + >&2 echo "Unable to remove SYN eating rule, attempt $i. Will retry..." + + # But sleep for a bit before retrying + sleep 0.5 + done + if (( i >= retries )); then + # We failed to remove the rule... log failure and exit to signal the caller + >&2 echo "Unable to remove the iptables SYN eating rule. Aborting after $retries retries" + exit 1 + fi + fi +else + /usr/sbin/haproxy -f $config_file -p $pid_file + reload_status=$? +fi + +[ $reload_status -ne 0 ] && exit $reload_status +haproxyHealthCheck