Enable TLS for Prometheus

This patchset enabled TLS path for Prometheus when it acts as
a server.  Note that TLS is not directly terminated at Prometheus.
TLS is terminated at apache proxy which in turn route request
to Prometheus.

Change-Id: I0db366b6237a34da2e9a31345d96ae8f63815fa2

prometheus/Chart.yaml

@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v2.25.0
 description: OpenStack-Helm Prometheus
 name: prometheus
-version: 0.1.6
+version: 0.1.7
 home: https://prometheus.io/
 sources:
   - https://github.com/prometheus/prometheus

prometheus/templates/bin/_helm-tests.sh.tpl

@@ -17,7 +17,7 @@ limitations under the License.
 set -ex
 
 function endpoints_up () {
-  endpoints_result=$(curl -K- <<< "--user ${PROMETHEUS_ADMIN_USERNAME}:${PROMETHEUS_ADMIN_PASSWORD}" \
+  endpoints_result=$(curl ${CACERT_OPTION} -K- <<< "--user ${PROMETHEUS_ADMIN_USERNAME}:${PROMETHEUS_ADMIN_PASSWORD}" \
     "${PROMETHEUS_ENDPOINT}/api/v1/query?query=up" \
     | python -c "import sys, json; print(json.load(sys.stdin)['status'])")
   if [ "$endpoints_result" = "success" ];
@@ -30,7 +30,7 @@ function endpoints_up () {
 }
 
 function get_targets () {
-  targets_result=$(curl -K- <<< "--user ${PROMETHEUS_ADMIN_USERNAME}:${PROMETHEUS_ADMIN_PASSWORD}" \
+  targets_result=$(curl ${CACERT_OPTION} -K- <<< "--user ${PROMETHEUS_ADMIN_USERNAME}:${PROMETHEUS_ADMIN_PASSWORD}" \
     "${PROMETHEUS_ENDPOINT}/api/v1/targets" \
     | python -c "import sys, json; print(json.load(sys.stdin)['status'])")
   if [ "$targets_result" = "success" ];
@@ -43,7 +43,7 @@ function get_targets () {
 }
 
 function get_alertmanagers () {
-  alertmanager=$(curl -K- <<< "--user ${PROMETHEUS_ADMIN_USERNAME}:${PROMETHEUS_ADMIN_PASSWORD}" \
+  alertmanager=$(curl ${CACERT_OPTION} -K- <<< "--user ${PROMETHEUS_ADMIN_USERNAME}:${PROMETHEUS_ADMIN_PASSWORD}" \
     "${PROMETHEUS_ENDPOINT}/api/v1/alertmanagers" \
     |  python -c "import sys, json; print(json.load(sys.stdin)['status'])")
   if [ "$alertmanager" = "success" ];

prometheus/templates/certificates.yaml

@@ -0,0 +1,17 @@
+{{/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.certificates -}}
+{{  dict "envAll" . "service" "monitoring" "type" "internal" | include "helm-toolkit.manifests.certificates" }}
+{{- end -}}

prometheus/templates/ingress-prometheus.yaml

@@ -13,6 +13,11 @@ limitations under the License.
 */}}
 
 {{- if and .Values.manifests.ingress .Values.network.prometheus.ingress.public }}
-{{- $ingressOpts := dict "envAll" . "backendService" "prometheus" "backendServiceType" "monitoring" "backendPort" "http" -}}
+{{- $envAll := . -}}
+{{- $ingressOpts := dict "envAll" $envAll "backendService" "prometheus" "backendServiceType" "monitoring" "backendPort" "https" -}}
+{{- $secretName := $envAll.Values.secrets.tls.monitoring.prometheus.internal -}}
+{{- if and .Values.manifests.certificates $secretName -}}
+{{- $_ := set $ingressOpts "certIssuer" .Values.endpoints.monitoring.host_fqdn_override.default.tls.issuerRef.name -}}
+{{- end -}}
 {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
 {{- end }}

prometheus/templates/pod-helm-tests.yaml

@@ -54,8 +54,13 @@ spec:
             secretKeyRef:
               name: {{ printf "%s-%s" $envAll.Release.Name "admin-user" | quote }}
               key: PROMETHEUS_ADMIN_PASSWORD
+
+{{- if .Values.manifests.certificates }}
+        - name: CACERT_OPTION
+          value: "--cacert /etc/prometheus/certs/ca.crt"
+{{- end }}
         - name: PROMETHEUS_ENDPOINT
-          value: {{ tuple "monitoring" "internal" "http" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+          value: {{ printf "%s://%s" (tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") (tuple "monitoring" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") }}
       volumeMounts:
         - name: pod-tmp
           mountPath: /tmp
@@ -63,6 +68,7 @@ spec:
           mountPath: /tmp/helm-tests.sh
           subPath: helm-tests.sh
           readOnly: true
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.monitoring.prometheus.internal "path" "/etc/prometheus/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 8 }}
   volumes:
     - name: pod-tmp
       emptyDir: {}
@@ -70,4 +76,5 @@ spec:
       configMap:
         name: {{ printf "%s-%s" $envAll.Release.Name "prometheus-bin" | quote }}
         defaultMode: 0555
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.monitoring.prometheus.internal | include "helm-toolkit.snippets.tls_volume" | indent 4 }}
 {{- end }}

prometheus/templates/service.yaml

@@ -28,8 +28,9 @@ metadata:
 {{- end }}
 spec:
   ports:
-  - name: http
+  - name: {{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }}
     port: {{ tuple "monitoring" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+    targetPort: {{ tuple "monitoring" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
     {{ if .Values.network.prometheus.node_port.enabled }}
     nodePort: {{ .Values.network.prometheus.node_port.port }}
     {{ end }}

prometheus/templates/statefulset.yaml

@@ -19,6 +19,7 @@ limitations under the License.
 {{- $authHeader := printf "%s:%s" $probeUser $probePass | b64enc }}
 httpGet:
   path: /status
+  scheme: {{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | upper }}
   port: {{ $probePort }}
   httpHeaders:
     - name: Authorization
@@ -133,8 +134,8 @@ spec:
             - /tmp/apache.sh
             - start
           ports:
-            - name: http
+            - name: {{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }}
-              containerPort: 80
+              containerPort: {{ tuple "monitoring" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
           env:
             - name: PROMETHEUS_PORT
               value: {{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
@@ -169,6 +170,7 @@ spec:
               mountPath: /usr/local/apache2/conf/httpd.conf
               subPath: httpd.conf
               readOnly: true
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.monitoring.prometheus.internal "path" "/etc/prometheus/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }}
         - name: prometheus
 {{ tuple $envAll "prometheus" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.prometheus | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
@@ -230,6 +232,7 @@ spec:
           secret:
             secretName: {{ printf "%s-%s" $envAll.Release.Name "prometheus-etc" | quote }}
             defaultMode: 0444
+{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.monitoring.prometheus.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }}
         - name: prometheus-bin
           configMap:
             name: {{ printf "%s-%s" $envAll.Release.Name "prometheus-bin" | quote }}

prometheus/values.yaml

@@ -261,6 +261,7 @@ secrets:
     monitoring:
       prometheus:
         public: prometheus-tls-public
+        internal: prometheus-tls-api
 
 tls_configs:
   # If client certificates are required to connect to metrics endpoints, they
@@ -292,6 +293,7 @@ storage:
   storage_class: general
 
 manifests:
+  certificates: false
   configmap_bin: true
   configmap_etc: true
   ingress: true

prometheus/values_overrides/tls.yaml

@@ -0,0 +1,250 @@
+---
+endpoints:
+  monitoring:
+    host_fqdn_override:
+      default:
+        tls:
+          secretName: prometheus-tls-api
+          issuerRef:
+            name: ca-issuer
+            kind: ClusterIssuer
+    scheme:
+      default: "https"
+    port:
+      http:
+        default: 443
+network:
+  prometheus:
+    ingress:
+      annotations:
+        nginx.ingress.kubernetes.io/backend-protocol: https
+conf:
+  httpd: |
+    ServerRoot "/usr/local/apache2"
+    Listen 443
+    LoadModule mpm_event_module modules/mod_mpm_event.so
+    LoadModule authn_file_module modules/mod_authn_file.so
+    LoadModule authn_core_module modules/mod_authn_core.so
+    LoadModule authz_host_module modules/mod_authz_host.so
+    LoadModule authz_groupfile_module modules/mod_authz_groupfile.so
+    LoadModule authz_user_module modules/mod_authz_user.so
+    LoadModule authz_core_module modules/mod_authz_core.so
+    LoadModule access_compat_module modules/mod_access_compat.so
+    LoadModule auth_basic_module modules/mod_auth_basic.so
+    LoadModule ldap_module modules/mod_ldap.so
+    LoadModule authnz_ldap_module modules/mod_authnz_ldap.so
+    LoadModule reqtimeout_module modules/mod_reqtimeout.so
+    LoadModule filter_module modules/mod_filter.so
+    LoadModule proxy_html_module modules/mod_proxy_html.so
+    LoadModule log_config_module modules/mod_log_config.so
+    LoadModule env_module modules/mod_env.so
+    LoadModule headers_module modules/mod_headers.so
+    LoadModule setenvif_module modules/mod_setenvif.so
+    LoadModule version_module modules/mod_version.so
+    LoadModule proxy_module modules/mod_proxy.so
+    LoadModule proxy_connect_module modules/mod_proxy_connect.so
+    LoadModule proxy_http_module modules/mod_proxy_http.so
+    LoadModule proxy_balancer_module modules/mod_proxy_balancer.so
+    LoadModule slotmem_shm_module modules/mod_slotmem_shm.so
+    LoadModule slotmem_plain_module modules/mod_slotmem_plain.so
+    LoadModule unixd_module modules/mod_unixd.so
+    LoadModule status_module modules/mod_status.so
+    LoadModule autoindex_module modules/mod_autoindex.so
+    LoadModule ssl_module modules/mod_ssl.so
+
+    <IfModule unixd_module>
+    User daemon
+    Group daemon
+    </IfModule>
+
+    <Directory />
+        AllowOverride none
+        Require all denied
+    </Directory>
+
+    <Files ".ht*">
+        Require all denied
+    </Files>
+
+    ErrorLog /dev/stderr
+
+    LogLevel warn
+
+    <IfModule log_config_module>
+        LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
+        LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" proxy
+        LogFormat "%h %l %u %t \"%r\" %>s %b" common
+
+        <IfModule logio_module>
+          LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
+        </IfModule>
+
+        SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded
+        CustomLog /dev/stdout common
+        CustomLog /dev/stdout combined
+        CustomLog /dev/stdout proxy env=forwarded
+    </IfModule>
+
+    <Directory "/usr/local/apache2/cgi-bin">
+        AllowOverride None
+        Options None
+        Require all granted
+    </Directory>
+
+    <IfModule headers_module>
+        RequestHeader unset Proxy early
+    </IfModule>
+
+    <IfModule proxy_html_module>
+    Include conf/extra/proxy-html.conf
+    </IfModule>
+
+    <VirtualHost *:443>
+      # Expose metrics to all users, as this is not sensitive information and
+      # circumvents the inability of Prometheus to interpolate environment vars
+      # in its configuration file
+      <Location /metrics>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/metrics
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/metrics
+          Satisfy Any
+          Allow from all
+      </Location>
+      # Expose the /federate endpoint to all users, as this is also not
+      # sensitive information and circumvents the inability of Prometheus to
+      # interpolate environment vars in its configuration file
+      <Location /federate>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/metrics
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/metrics
+          Satisfy Any
+          Allow from all
+      </Location>
+      # Restrict general user (LDAP) access to the /graph endpoint, as general trusted
+      # users should only be able to query Prometheus for metrics and not have access
+      # to information like targets, configuration, flags or build info for Prometheus
+      <Location />
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file ldap
+          AuthUserFile /usr/local/apache2/conf/.htpasswd
+          AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }}
+          AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }}
+          AuthLDAPURL {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
+          Require valid-user
+      </Location>
+      <Location /graph>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/graph
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/graph
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file ldap
+          AuthUserFile /usr/local/apache2/conf/.htpasswd
+          AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }}
+          AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }}
+          AuthLDAPURL {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
+          Require valid-user
+      </Location>
+      # Restrict access to the /config (dashboard) and /api/v1/status/config (http) endpoints
+      # to the admin user
+      <Location /config>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/config
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/config
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file
+          AuthUserFile /usr/local/apache2/conf/.htpasswd
+          Require valid-user
+      </Location>
+      <Location /api/v1/status/config>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/status/config
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/status/config
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file
+          AuthUserFile /usr/local/apache2/conf/.htpasswd
+          Require valid-user
+      </Location>
+      # Restrict access to the /flags (dashboard) and /api/v1/status/flags (http) endpoints
+      # to the admin user
+      <Location /flags>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/flags
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/flags
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file
+          AuthUserFile /usr/local/apache2/conf/.htpasswd
+          Require valid-user
+      </Location>
+      <Location /api/v1/status/flags>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/status/flags
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/status/flags
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file
+          AuthUserFile /usr/local/apache2/conf/.htpasswd
+          Require valid-user
+      </Location>
+      # Restrict access to the /status (dashboard) endpoint to the admin user
+      <Location /status>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/status
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/status
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file
+          AuthUserFile /usr/local/apache2/conf/.htpasswd
+          Require valid-user
+      </Location>
+      # Restrict access to the /rules (dashboard) endpoint to the admin user
+      <Location /rules>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/rules
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/rules
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file
+          AuthUserFile /usr/local/apache2/conf/.htpasswd
+          Require valid-user
+      </Location>
+      # Restrict access to the /targets (dashboard) and /api/v1/targets (http) endpoints
+      # to the admin user
+      <Location /targets>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/targets
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/targets
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file
+          AuthUserFile /usr/local/apache2/conf/.htpasswd
+          Require valid-user
+      </Location>
+      <Location /api/v1/targets>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/targets
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/targets
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file
+          AuthUserFile /usr/local/apache2/conf/.htpasswd
+          Require valid-user
+      </Location>
+      # Restrict access to the /api/v1/admin/tsdb/ endpoints (http) to the admin user.
+      # These endpoints are disabled by default, but are included here to ensure only
+      # an admin user has access to these endpoints when enabled
+      <Location /api/v1/admin/tsdb/>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/admin/tsdb/
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/admin/tsdb/
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file
+          AuthUserFile /usr/local/apache2/conf/.htpasswd
+          Require valid-user
+      </Location>
+      SSLEngine On
+      SSLProxyEngine on
+      SSLCertificateFile      /etc/prometheus/certs/tls.crt
+      SSLCertificateKeyFile   /etc/prometheus/certs/tls.key
+      SSLProtocol             all -SSLv3 -TLSv1 -TLSv1.1
+      SSLCipherSuite          ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256
+      SSLHonorCipherOrder     on
+    </VirtualHost>
+manifests:
+  certificates: true
+...

releasenotes/notes/prometheus.yaml

@@ -7,4 +7,5 @@ prometheus:
   - 0.1.4 Fix spacing inconsistencies with flags
   - 0.1.5 Fix spacing inconsistencies with flags
   - 0.1.6 Upgrade version to v2.25 fix/remove deprecated flags
+  - 0.1.7 Enable TLS for Prometheus
 ...