Add auth for prometheus

This adds authentication to Prometheus with an apache reverse
proxy, similar to elasticsearch, kibana and nagios. This adds an
admin user and password via htpasswd along with adding ldap
support.

This required modifying the grafana chart to configure the
prometheus datasource's basic auth credentials in the data sources
provisioning configuration file by checking whether basic auth is
enabled and injecting the username/password defined in the
corresponding endpoint definition.

This also modifies the nagios chart to use the authenticated
endpoint for prometheus, which is required for nagios to
successfully query the prometheus endpoint for its service
checking mechanism

Change-Id: Ia4ccc3c44a89b2c56594be1f4cc28ac07169bf8c

grafana/templates/secret-prom-creds.yaml

@@ -0,0 +1,32 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.secret_prom_creds }}
+{{- $envAll := . }}
+{{- $secretName := index $envAll.Values.secrets.prometheus.user }}
+
+{{- $prometheus_user := .Values.endpoints.monitoring.auth.user.username }}
+{{- $prometheus_password := .Values.endpoints.monitoring.auth.user.password }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $secretName }}
+type: Opaque
+data:
+  PROMETHEUS_USERNAME: {{ .Values.endpoints.monitoring.auth.user.username | b64enc }}
+  PROMETHEUS_PASSWORD: {{ .Values.endpoints.monitoring.auth.user.password | b64enc }}
+{{- end }}

grafana/templates/utils/_generate_datasources.tpl

@@ -26,6 +26,16 @@ limitations under the License.
 {{- $datasource_url := tuple $datasource "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}
 {{- $_ := set $config "url" $datasource_url }}
 {{- end }}
+{{- if and ($config.basicAuth) (empty $config.basicAuthUser) -}}
+{{- $datasource_endpoint := index $envAll.Values.endpoints $datasource -}}
+{{- $datasource_user :=  $datasource_endpoint.auth.user.username -}}
+{{- $_ := set $config "basicAuthUser" $datasource_user -}}
+{{- end }}
+{{- if and ($config.basicAuth) (empty $config.basicAuthPassword) -}}
+{{- $datasource_endpoint := index $envAll.Values.endpoints $datasource -}}
+{{- $datasource_password :=  $datasource_endpoint.auth.user.password -}}
+{{- $_ := set $config "basicAuthPassword" $datasource_password -}}
+{{- end }}
 {{- $__datasources := append $envAll.Values.__datasources $config }}
 {{- $_ := set $envAll.Values "__datasources" $__datasources }}
 {{- end }}

grafana/values.yaml

@@ -196,6 +196,10 @@ endpoints:
   monitoring:
     name: prometheus
     namespace: null
+    auth:
+      user:
+        username: admin
+        password: changeme
     hosts:
       default: prom-metrics
       public: prometheus
@@ -207,7 +211,7 @@ endpoints:
       default: http
     port:
       api:
-        default: 9090
+        default: 80
         public: 80
   ldap:
     hosts:
@@ -290,6 +294,8 @@ secrets:
     grafana:
       grafana:
         public: grafana-tls-public
+  prometheus:
+    user: prometheus-user-creds
 
 manifests:
   configmap_bin: true
@@ -306,6 +312,7 @@ manifests:
   secret_db_session: true
   secret_admin_creds: true
   secret_ingress_tls: true
+  secret_prom_creds: true
   service: true
   service_ingress: true
 
@@ -365,6 +372,7 @@ conf:
         access: proxy
         orgId: 1
         editable: true
+        basicAuth: true
   grafana:
     auth.ldap:
       enabled: true

nagios/templates/deployment.yaml

@@ -128,7 +128,7 @@ spec:
               containerPort: {{ tuple "nagios" "internal" "nagios" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
           env:
             - name: PROMETHEUS_SERVICE
-              value: {{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+              value: {{ tuple "monitoring" "internal" "admin" "http" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" }}
             - name: SNMP_NOTIF_PRIMARY_TARGET_WITH_PORT
               value: {{ $envAll.Values.conf.nagios.notification.snmp.primary_target }}
             - name: SNMP_NOTIF_SECONDARY_TARGET_WITH_PORT

nagios/values.yaml

@@ -77,6 +77,10 @@ endpoints:
         node: 5000
   monitoring:
     name: prometheus
+    auth:
+      admin:
+        username: admin
+        password: changeme
     hosts:
       default: prom-metrics
       public: prometheus
@@ -87,9 +91,8 @@ endpoints:
     scheme:
       default: http
     port:
-      api:
-        default: 9090
-        public: 80
+      http:
+        default: 80
   nagios:
     name: nagios
     namespace: null

prometheus/templates/bin/_apache.sh.tpl

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+set -ev
+
+COMMAND="${@:-start}"
+
+function start () {
+
+  if [ -f /etc/apache2/envvars ]; then
+     # Loading Apache2 ENV variables
+     source /etc/httpd/apache2/envvars
+  fi
+  # Apache gets grumpy about PID files pre-existing
+  rm -f /etc/httpd/logs/httpd.pid
+
+  if [ -f /usr/local/apache2/conf/.htpasswd ]; then
+    htpasswd -b /usr/local/apache2/conf/.htpasswd "$PROMETHEUS_ADMIN_USERNAME" "$PROMETHEUS_ADMIN_PASSWORD"
+  else
+    htpasswd -cb /usr/local/apache2/conf/.htpasswd "$PROMETHEUS_ADMIN_USERNAME" "$PROMETHEUS_ADMIN_PASSWORD"
+  fi
+
+  #Launch Apache on Foreground
+  exec httpd -DFOREGROUND
+}
+
+function stop () {
+  apachectl -k graceful-stop
+}
+
+$COMMAND

prometheus/templates/bin/_helm-tests.sh.tpl

@@ -19,7 +19,8 @@ limitations under the License.
 set -ex
 
 function endpoints_up () {
-  endpoints_result=$(curl "${PROMETHEUS_ENDPOINT}/api/v1/query?query=up" \
+  endpoints_result=$(curl -K- <<< "--user ${PROMETHEUS_ADMIN_USERNAME}:${PROMETHEUS_ADMIN_PASSWORD}" \
+    "${PROMETHEUS_ENDPOINT}/api/v1/query?query=up" \
     | python -c "import sys, json; print json.load(sys.stdin)['status']")
   if [ "$endpoints_result" = "success" ];
   then
@@ -31,7 +32,8 @@ function endpoints_up () {
 }
 
 function get_targets () {
-  targets_result=$(curl "${PROMETHEUS_ENDPOINT}/api/v1/targets" \
+  targets_result=$(curl -K- <<< "--user ${PROMETHEUS_ADMIN_USERNAME}:${PROMETHEUS_ADMIN_PASSWORD}" \
+    "${PROMETHEUS_ENDPOINT}/api/v1/targets" \
     | python -c "import sys, json; print json.load(sys.stdin)['status']")
   if [ "$targets_result" = "success" ];
   then
@@ -43,7 +45,8 @@ function get_targets () {
 }
 
 function get_alertmanagers () {
-  alertmanager=$(curl "${PROMETHEUS_ENDPOINT}/api/v1/alertmanagers" \
+  alertmanager=$(curl -K- <<< "--user ${PROMETHEUS_ADMIN_USERNAME}:${PROMETHEUS_ADMIN_PASSWORD}" \
+    "${PROMETHEUS_ENDPOINT}/api/v1/alertmanagers" \
     |  python -c "import sys, json; print json.load(sys.stdin)['status']")
   if [ "$alertmanager" = "success" ];
   then

prometheus/templates/configmap-bin.yaml

@@ -22,6 +22,8 @@ kind: ConfigMap
 metadata:
   name: prometheus-bin
 data:
+  apache.sh: |
+{{ tuple "bin/_apache.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   prometheus.sh: |
 {{ tuple "bin/_prometheus.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   helm-tests.sh: |

prometheus/templates/configmap-etc.yaml

@@ -28,16 +28,26 @@ limitations under the License.
 {{- $_ := set .Values.conf.prometheus.scrape_configs "rule_files" $envAll.Values.__rule_files -}}
 {{- end -}}
 
+{{- if not (empty $envAll.Values.conf.prometheus.scrape_configs.scrape_configs) }}
+{{- $_ := set $envAll.Values "__updated_scrape_configs" ( list ) }}
+{{- $promScrapeTarget := first $envAll.Values.conf.prometheus.scrape_configs.scrape_configs }}
+{{- if (empty $promScrapeTarget.basic_auth) }}
+{{- $_ := set $promScrapeTarget "basic_auth" $envAll.Values.endpoints.monitoring.auth.admin }}
+{{- end }}
+{{- end }}
+
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: prometheus-etc
 data:
-  prometheus.yml: |
+  prometheus.yml: |+
 {{ toYaml .Values.conf.prometheus.scrape_configs | indent 4 }}
 {{ range $key, $value := .Values.conf.prometheus.rules }}
   {{ $key }}.rules: |
 {{ toYaml $value | indent 4 }}
 {{ end }}
+#NOTE(srwilkers): this must be last, to work round helm ~2.7 bug.
+{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.httpd "key" "httpd.conf") | indent 2 }}
 {{- end }}

prometheus/templates/ingress-prometheus.yaml

@@ -15,6 +15,6 @@ limitations under the License.
 */}}
 
 {{- if and .Values.manifests.ingress .Values.network.prometheus.ingress.public }}
-{{- $ingressOpts := dict "envAll" . "backendService" "prometheus" "backendServiceType" "monitoring" "backendPort" "prom-metrics" -}}
+{{- $ingressOpts := dict "envAll" . "backendService" "prometheus" "backendServiceType" "monitoring" "backendPort" "http" -}}
 {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }}
 {{- end }}

prometheus/templates/pod-helm-tests.yaml

@@ -16,6 +16,7 @@ limitations under the License.
 
 {{- if .Values.manifests.helm_tests }}
 {{- $envAll := . }}
+{{- $promUserSecret := .Values.secrets.prometheus.admin }}
 ---
 apiVersion: v1
 kind: Pod
@@ -34,8 +35,18 @@ spec:
       command:
         - /tmp/helm-tests.sh
       env:
+        - name: PROMETHEUS_ADMIN_USERNAME
+          valueFrom:
+            secretKeyRef:
+              name: {{ $promUserSecret }}
+              key: PROMETHEUS_ADMIN_USERNAME
+        - name: PROMETHEUS_ADMIN_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: {{ $promUserSecret }}
+              key: PROMETHEUS_ADMIN_PASSWORD
         - name: PROMETHEUS_ENDPOINT
-          value: {{ tuple "monitoring" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
+          value: {{ tuple "monitoring" "internal" "http" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
       volumeMounts:
         - name: prometheus-bin
           mountPath: /tmp/helm-tests.sh

prometheus/templates/secret-prometheus.yaml

@@ -0,0 +1,29 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.secret_prometheus }}
+{{- $envAll := . }}
+{{- $secretName := index $envAll.Values.secrets.prometheus.admin }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ $secretName }}
+type: Opaque
+data:
+  PROMETHEUS_ADMIN_USERNAME: {{ .Values.endpoints.monitoring.auth.admin.username | b64enc }}
+  PROMETHEUS_ADMIN_PASSWORD: {{ .Values.endpoints.monitoring.auth.admin.password | b64enc }}
+{{- end }}

prometheus/templates/service.yaml

@@ -30,8 +30,8 @@ metadata:
 {{- end }}
 spec:
   ports:
-  - name: prom-metrics
-    port: {{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
+  - name: http
+    port: {{ tuple "monitoring" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
     {{ if .Values.network.prometheus.node_port.enabled }}
     nodePort: {{ .Values.network.prometheus.node_port.port }}
     {{ end }}

prometheus/templates/statefulset.yaml

@@ -19,6 +19,7 @@ limitations under the License.
 
 {{- $mounts_prometheus := .Values.pod.mounts.prometheus.prometheus }}
 {{- $mounts_prometheus_init := .Values.pod.mounts.prometheus.init_container }}
+{{- $promUserSecret := .Values.secrets.prometheus.admin }}
 
 {{- $serviceAccountName := printf "%s-%s" .Release.Name "prometheus"}}
 {{ tuple $envAll "prometheus" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
@@ -106,6 +107,37 @@ spec:
             - name: storage
               mountPath: /var/lib/prometheus/data
       containers:
+        - name: apache-proxy
+{{ tuple $envAll "apache_proxy" | include "helm-toolkit.snippets.image" | indent 10 }}
+{{ tuple $envAll $envAll.Values.pod.resources.apache_proxy | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
+          command:
+            - /tmp/apache.sh
+            - start
+          ports:
+            - name: http
+              containerPort: 80
+          env:
+            - name: PROMETHEUS_PORT
+              value: {{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}
+            - name: PROMETHEUS_ADMIN_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $promUserSecret }}
+                  key: PROMETHEUS_ADMIN_USERNAME
+            - name: PROMETHEUS_ADMIN_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $promUserSecret }}
+                  key: PROMETHEUS_ADMIN_PASSWORD
+          volumeMounts:
+            - name: prometheus-bin
+              mountPath: /tmp/apache.sh
+              subPath: apache.sh
+              readOnly: true
+            - name: prometheus-etc
+              mountPath: /usr/local/apache2/conf/httpd.conf
+              subPath: httpd.conf
+              readOnly: true
         - name: prometheus
 {{ tuple $envAll "prometheus" | include "helm-toolkit.snippets.image" | indent 10 }}
 {{ tuple $envAll $envAll.Values.pod.resources.prometheus | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
@@ -150,6 +182,8 @@ spec:
               mountPath: /var/lib/prometheus/data
 {{ if $mounts_prometheus.volumeMounts }}{{ toYaml $mounts_prometheus.volumeMounts | indent 12 }}{{ end }}
       volumes:
+        - name: pod-etc-apache
+          emptyDir: {}
         - name: etcprometheus
           emptyDir: {}
         - name: rulesprometheus

prometheus/values.yaml

@@ -19,6 +19,7 @@
 
 images:
   tags:
+    apache_proxy: docker.io/httpd:2.4
     prometheus: docker.io/prom/prometheus:v2.0.0
     helm_tests: docker.io/kolla/ubuntu-source-heat-engine:3.0.3
     dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
@@ -103,6 +104,10 @@ endpoints:
   monitoring:
     name: prometheus
     namespace: null
+    auth:
+      admin:
+        username: admin
+        password: changeme
     hosts:
       default: prom-metrics
       public: prometheus
@@ -122,7 +127,8 @@ endpoints:
     port:
       api:
         default: 9090
-        public: 80
+      http:
+        default: 80
   alerts:
     name: alertmanager
     namespace: null
@@ -142,6 +148,22 @@ endpoints:
         public: 80
       mesh:
         default: 6783
+  ldap:
+    hosts:
+      default: ldap
+    auth:
+      admin:
+        bind: "cn=admin,dc=cluster,dc=local"
+        password: password
+    host_fqdn_override:
+      default: null
+    path:
+      default: "/ou=People,dc=cluster,dc=local"
+    scheme:
+      default: ldap
+    port:
+      ldap:
+        default: 389
 
 dependencies:
   dynamic:
@@ -184,6 +206,8 @@ secrets:
     monitoring:
       prometheus:
         public: prometheus-tls-public
+  prometheus:
+    admin: prometheus-admin-creds
 
 storage:
   enabled: true
@@ -201,11 +225,203 @@ manifests:
   helm_tests: true
   job_image_repo_sync: true
   secret_ingress_tls: true
+  secret_prometheus: true
   service_ingress: true
   service: true
   statefulset_prometheus: true
 
 conf:
+  httpd: |
+    ServerRoot "/usr/local/apache2"
+
+    Listen 80
+
+    LoadModule mpm_event_module modules/mod_mpm_event.so
+    LoadModule authn_file_module modules/mod_authn_file.so
+    LoadModule authn_core_module modules/mod_authn_core.so
+    LoadModule authz_host_module modules/mod_authz_host.so
+    LoadModule authz_groupfile_module modules/mod_authz_groupfile.so
+    LoadModule authz_user_module modules/mod_authz_user.so
+    LoadModule authz_core_module modules/mod_authz_core.so
+    LoadModule access_compat_module modules/mod_access_compat.so
+    LoadModule auth_basic_module modules/mod_auth_basic.so
+    LoadModule ldap_module modules/mod_ldap.so
+    LoadModule authnz_ldap_module modules/mod_authnz_ldap.so
+    LoadModule reqtimeout_module modules/mod_reqtimeout.so
+    LoadModule filter_module modules/mod_filter.so
+    LoadModule proxy_html_module modules/mod_proxy_html.so
+    LoadModule log_config_module modules/mod_log_config.so
+    LoadModule env_module modules/mod_env.so
+    LoadModule headers_module modules/mod_headers.so
+    LoadModule setenvif_module modules/mod_setenvif.so
+    LoadModule version_module modules/mod_version.so
+    LoadModule proxy_module modules/mod_proxy.so
+    LoadModule proxy_connect_module modules/mod_proxy_connect.so
+    LoadModule proxy_http_module modules/mod_proxy_http.so
+    LoadModule proxy_balancer_module modules/mod_proxy_balancer.so
+    LoadModule slotmem_shm_module modules/mod_slotmem_shm.so
+    LoadModule slotmem_plain_module modules/mod_slotmem_plain.so
+    LoadModule unixd_module modules/mod_unixd.so
+    LoadModule status_module modules/mod_status.so
+    LoadModule autoindex_module modules/mod_autoindex.so
+
+    <IfModule unixd_module>
+    User daemon
+    Group daemon
+    </IfModule>
+
+    <Directory />
+        AllowOverride none
+        Require all denied
+    </Directory>
+
+    <Files ".ht*">
+        Require all denied
+    </Files>
+
+    ErrorLog /dev/stderr
+
+    LogLevel warn
+
+    <IfModule log_config_module>
+        LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
+        LogFormat "%h %l %u %t \"%r\" %>s %b" common
+
+        <IfModule logio_module>
+          LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
+        </IfModule>
+
+        CustomLog /dev/stdout common
+
+        CustomLog /dev/stdout combined
+    </IfModule>
+
+    <Directory "/usr/local/apache2/cgi-bin">
+        AllowOverride None
+        Options None
+        Require all granted
+    </Directory>
+
+    <IfModule headers_module>
+        RequestHeader unset Proxy early
+    </IfModule>
+
+    <IfModule proxy_html_module>
+    Include conf/extra/proxy-html.conf
+    </IfModule>
+
+    <VirtualHost *:80>
+      # Restrict general user (LDAP) access to the /graph endpoint, as general trusted
+      # users should only be able to query Prometheus for metrics and not have access
+      # to information like targets, configuration, flags or build info for Prometheus
+      <Location />
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file ldap
+          AuthUserFile /usr/local/apache2/conf/.htpasswd
+          AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }}
+          AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }}
+          AuthLDAPURL {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
+          Require valid-user
+      </Location>
+      <Location /graph>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/graph
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/graph
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file ldap
+          AuthUserFile /usr/local/apache2/conf/.htpasswd
+          AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }}
+          AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }}
+          AuthLDAPURL {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
+          Require valid-user
+      </Location>
+      # Restrict access to the /config (dashboard) and /api/v1/status/config (http) endpoints
+      # to the admin user
+      <Location /config>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/config
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/config
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file
+          Require valid-user
+      </Location>
+      <Location /api/v1/status/config>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/status/config
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/status/config
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file
+          Require valid-user
+      </Location>
+      # Restrict access to the /flags (dashboard) and /api/v1/status/flags (http) endpoints
+      # to the admin user
+      <Location /flags>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/flags
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/flags
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file
+          Require valid-user
+      </Location>
+      <Location /api/v1/status/flags>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/status/flags
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/status/flags
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file
+          Require valid-user
+      </Location>
+      # Restrict access to the /status (dashboard) endpoint to the admin user
+      <Location /status>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/status
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/status
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file
+          Require valid-user
+      </Location>
+      # Restrict access to the /rules (dashboard) endpoint to the admin user
+      <Location /rules>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/rules
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/rules
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file
+          Require valid-user
+      </Location>
+      # Restrict access to the /targets (dashboard) and /api/v1/targets (http) endpoints
+      # to the admin user
+      <Location /targets>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/targets
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/targets
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file
+          Require valid-user
+      </Location>
+      <Location /api/v1/targets>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/targets
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/targets
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file
+          Require valid-user
+      </Location>
+      # Restrict access to the /api/v1/admin/tsdb/ endpoints (http) to the admin user.
+      # These endpoints are disabled by default, but are included here to ensure only
+      # an admin user has access to these endpoints when enabled
+      <Location /api/v1/admin/tsdb/>
+          ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/admin/tsdb/
+          ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/admin/tsdb/
+          AuthName "Prometheus"
+          AuthType Basic
+          AuthBasicProvider file
+          Require valid-user
+      </Location>
+    </VirtualHost>
   prometheus:
     # Consumed by a prometheus helper function to generate the command line flags
     # for configuring the prometheus service
@@ -232,6 +448,57 @@ conf:
         scrape_interval: 60s
         evaluation_interval: 60s
       scrape_configs:
+        # NOTE(srwilkers): The job definition for Prometheus should always be
+        # listed first, so we can inject the basic auth username and password
+        # via the endpoints section
+        - job_name: 'prometheus-metrics'
+          kubernetes_sd_configs:
+          - role: endpoints
+          scrape_interval: 60s
+          relabel_configs:
+          - source_labels:
+              - __meta_kubernetes_service_name
+            action: keep
+            regex: "prom-metrics"
+          - source_labels:
+              - __meta_kubernetes_service_annotation_prometheus_io_scrape
+            action: keep
+            regex: true
+          - source_labels:
+              - __meta_kubernetes_service_annotation_prometheus_io_scheme
+            action: replace
+            target_label: __scheme__
+            regex: (https?)
+          - source_labels:
+              - __meta_kubernetes_service_annotation_prometheus_io_path
+            action: replace
+            target_label: __metrics_path__
+            regex: (.+)
+          - source_labels:
+              - __address__
+              - __meta_kubernetes_service_annotation_prometheus_io_port
+            action: replace
+            target_label: __address__
+            regex: ([^:]+)(?::\d+)?;(\d+)
+            replacement: $1:$2
+          - action: labelmap
+            regex: __meta_kubernetes_service_label_(.+)
+          - source_labels:
+              - __meta_kubernetes_namespace
+            action: replace
+            target_label: kubernetes_namespace
+          - source_labels:
+              - __meta_kubernetes_service_name
+            action: replace
+            target_label: instance
+          - source_labels:
+              - __meta_kubernetes_service_name
+            action: replace
+            target_label: kubernetes_name
+          - source_labels:
+              - __meta_kubernetes_service_name
+            target_label: job
+            replacement: ${1}
         - job_name: kubelet
           scheme: https
           # This TLS & bearer token file config is used to connect to the actual scrape
@@ -424,7 +691,7 @@ conf:
           - source_labels:
               - __meta_kubernetes_service_name
             action: drop
-            regex: "openstack-metrics"
+            regex: '(openstack-metrics|prom-metrics)'
           - source_labels:
               - __meta_kubernetes_service_annotation_prometheus_io_scrape
             action: keep