Merge "Add ldap support in elasticsearch/kibana apache proxies"

elasticsearch/templates/bin/_apache.sh.tpl

@@ -29,10 +29,10 @@ function start () {
   # Apache gets grumpy about PID files pre-existing
   rm -f /etc/httpd/logs/httpd.pid
 
-  if [ -f {{ .Values.conf.apache.htpasswd }} ]; then
+  if [ -f /usr/local/apache2/conf/.htpasswd ]; then
-    htpasswd -b {{ .Values.conf.apache.htpasswd }} $ELASTICSEARCH_USERNAME $ELASTICSEARCH_PASSWORD
+    htpasswd -b /usr/local/apache2/conf/.htpasswd $ELASTICSEARCH_USERNAME $ELASTICSEARCH_PASSWORD
   else
-    htpasswd -cb {{ .Values.conf.apache.htpasswd }} $ELASTICSEARCH_USERNAME $ELASTICSEARCH_PASSWORD
+    htpasswd -cb /usr/local/apache2/conf/.htpasswd $ELASTICSEARCH_USERNAME $ELASTICSEARCH_PASSWORD
   fi
 
   #Launch Apache on Foreground

elasticsearch/templates/deployment-client.yaml

@@ -129,6 +129,18 @@ spec:
                 secretKeyRef:
                   name: {{ $esUserSecret }}
                   key: ELASTICSEARCH_PASSWORD
+            - name: LDAP_URL
+              value: {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
+            - name: BIND_DN
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: BIND_DN
+            - name: BIND_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: BIND_PASSWORD
           volumeMounts:
             - name: elasticsearch-bin
               mountPath: /tmp/apache.sh

elasticsearch/templates/etc/_elasticsearch-host.conf.tpl

@@ -15,14 +15,20 @@ limitations under the License.
 */}}
 
 <VirtualHost *:80>
+  ProxyRequests On
+  ProxyPreserveHost On
   <Location />
       ProxyPass http://localhost:${ELASTICSEARCH_PORT}/
       ProxyPassReverse http://localhost:${ELASTICSEARCH_PORT}/
   </Location>
   <Proxy *>
+      AuthName "Elasticsearch"
       AuthType Basic
-      AuthName "Authentication Required for Elasticsearch"
+      AuthBasicProvider ldap file
-      AuthUserFile {{.Values.conf.apache.htpasswd | quote}}
+      AuthUserFile /usr/local/apache2/conf/.htpasswd
+      AuthLDAPBindDN ${BIND_DN}
+      AuthLDAPBindPassword ${BIND_PASSWORD}
+      AuthLDAPURL ${LDAP_URL}
       Require valid-user
   </Proxy>
 </VirtualHost>

elasticsearch/templates/etc/_httpd.conf.tpl

@@ -52,6 +52,8 @@ LoadModule authz_user_module modules/mod_authz_user.so
 LoadModule authz_core_module modules/mod_authz_core.so
 LoadModule access_compat_module modules/mod_access_compat.so
 LoadModule auth_basic_module modules/mod_auth_basic.so
+LoadModule ldap_module modules/mod_ldap.so
+LoadModule authnz_ldap_module modules/mod_authnz_ldap.so
 LoadModule reqtimeout_module modules/mod_reqtimeout.so
 LoadModule filter_module modules/mod_filter.so
 LoadModule proxy_html_module modules/mod_proxy_html.so

elasticsearch/templates/secret-elasticsearch.yaml

@@ -32,4 +32,6 @@ data:
   ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }}
   ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }}
   ELASTICSEARCH_URI: {{ $elasticsearch_uri | b64enc }}
+  BIND_DN: {{ .Values.endpoints.ldap.auth.admin.bind | b64enc }}
+  BIND_PASSWORD: {{ .Values.endpoints.ldap.auth.admin.password | b64enc }}
 {{- end }}

elasticsearch/values.yaml

@@ -170,8 +170,8 @@ secrets:
 
 conf:
   apache:
-    htpasswd: /usr/local/apache2/conf/.htpasswd
+    httpd: null
-  httpd:
+    elasticsearch_host: null
   init:
     max_map_count: 262144
   curator:
@@ -370,6 +370,22 @@ endpoints:
     port:
       metrics:
         default: 9108
+  ldap:
+    hosts:
+      default: ldap
+    auth:
+      admin:
+        bind: "cn=admin,dc=cluster,dc=local"
+        password: password
+    host_fqdn_override:
+      default: null
+    path:
+      default: "/ou=People,dc=cluster,dc=local"
+    scheme:
+      default: ldap
+    port:
+      ldap:
+        default: 389
 
 monitoring:
   prometheus:

kibana/templates/bin/_apache.sh.tpl

@@ -29,12 +29,6 @@ function start () {
   # Apache gets grumpy about PID files pre-existing
   rm -f /etc/httpd/logs/httpd.pid
 
-  if [ -f {{ .Values.conf.apache.htpasswd }} ]; then
-    htpasswd -b {{ .Values.conf.apache.htpasswd }} $KIBANA_USERNAME $KIBANA_PASSWORD
-  else
-    htpasswd -cb {{ .Values.conf.apache.htpasswd }} $KIBANA_USERNAME $KIBANA_PASSWORD
-  fi
-
   #Launch Apache on Foreground
   exec httpd -DFOREGROUND
 }

kibana/templates/deployment.yaml

@@ -67,6 +67,18 @@ spec:
                 secretKeyRef:
                   name: {{ $esUserSecret }}
                   key: ELASTICSEARCH_PASSWORD
+            - name: LDAP_URL
+              value: {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }}
+            - name: BIND_DN
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: BIND_DN
+            - name: BIND_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $esUserSecret }}
+                  key: BIND_PASSWORD
           volumeMounts:
             - name: kibana-bin
               mountPath: /tmp/apache.sh

kibana/templates/etc/_httpd.conf.tpl

@@ -52,6 +52,8 @@ LoadModule authz_user_module modules/mod_authz_user.so
 LoadModule authz_core_module modules/mod_authz_core.so
 LoadModule access_compat_module modules/mod_access_compat.so
 LoadModule auth_basic_module modules/mod_auth_basic.so
+LoadModule ldap_module modules/mod_ldap.so
+LoadModule authnz_ldap_module modules/mod_authnz_ldap.so
 LoadModule reqtimeout_module modules/mod_reqtimeout.so
 LoadModule filter_module modules/mod_filter.so
 LoadModule proxy_html_module modules/mod_proxy_html.so

kibana/templates/etc/_kibana-host.conf.tpl

@@ -20,9 +20,12 @@ limitations under the License.
       ProxyPassReverse http://localhost:${KIBANA_PORT}/
   </Location>
   <Proxy *>
+      AuthName "Kibana"
       AuthType Basic
-      AuthName "Authentication Required for Kibana"
+      AuthBasicProvider ldap
-      AuthUserFile {{.Values.conf.apache.htpasswd | quote}}
+      AuthLDAPBindDN ${BIND_DN}
+      AuthLDAPBindPassword ${BIND_PASSWORD}
+      AuthLDAPURL ${LDAP_URL}
       Require valid-user
   </Proxy>
 </VirtualHost>

kibana/templates/secret-elasticsearch-creds.yaml

@@ -26,4 +26,6 @@ type: Opaque
 data:
   ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }}
   ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }}
+  BIND_DN: {{ .Values.endpoints.ldap.auth.admin.bind | b64enc }}
+  BIND_PASSWORD: {{ .Values.endpoints.ldap.auth.admin.password | b64enc }}
 {{- end }}

kibana/values.yaml

@@ -93,8 +93,8 @@ dependencies:
 
 conf:
   apache:
-    htpasswd: /usr/local/apache2/conf/.htpasswd
+    httpd: null
-  httpd:
+    kibana_host: null
   kibana:
     elasticsearch:
       pingTimeout: 1500
@@ -168,6 +168,22 @@ endpoints:
         default: 5601
       http:
         default: 80
+  ldap:
+    hosts:
+      default: ldap
+    auth:
+      admin:
+        bind: "cn=admin,dc=cluster,dc=local"
+        password: password
+    host_fqdn_override:
+      default: null
+    path:
+      default: "/ou=People,dc=cluster,dc=local"
+    scheme:
+      default: ldap
+    port:
+      ldap:
+        default: 389
 
 network:
   kibana:

playbooks/osh-infra-dev-deploy.yaml

@@ -96,6 +96,12 @@
         ./tools/deployment/developer/110-elasticsearch.sh
       args:
         chdir: "{{ zuul.project.src_dir }}"
+    - name: Test LDAP Auth for Elasticsearch
+      shell: |
+        set -xe;
+        ./tools/deployment/developer/115-elasticsearch-ldap.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
     - name: Deploy Fluent-Logging
       shell: |
         set -xe;

playbooks/osh-infra-ldap-deploy.yaml

@@ -0,0 +1,58 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  tasks:
+    - name: Deploy Required packages
+      shell: |
+        set -xe;
+        ./tools/deployment/ldap/000-install-packages.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+    - name: Deploy Kubernetes
+      shell: |
+        set -xe;
+        ./tools/deployment/ldap/010-deploy-k8s.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+    - name: Deploy NFS for Logging, Monitoring and Alerting Components
+      shell: |
+        set -xe;
+        ./tools/deployment/ldap/020-lma-nfs-provisioner.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+    - name: Deploy LDAP
+      shell: |
+        set -xe;
+        ./tools/deployment/ldap/030-ldap.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+    - name: Deploy Elasticsearch
+      shell: |
+        set -xe;
+        ./tools/deployment/ldap/040-elasticsearch.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+    - name: Test Elasticsearch Access via LDAP
+      shell: |
+        set -xe;
+        ./tools/deployment/ldap/045-elasticsearch-ldap.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
+    - name: Deploy Kibana
+      shell: |
+        set -xe;
+        ./tools/deployment/ldap/050-kibana.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"

playbooks/osh-infra-multinode-deploy.yaml

@@ -80,6 +80,12 @@
         ./tools/deployment/multinode/110-elasticsearch.sh
       args:
         chdir: "{{ zuul.project.src_dir }}"
+    - name: Test LDAP Auth for Elasticsearch
+      shell: |
+        set -xe;
+        ./tools/deployment/multinode/115-elasticsearch-ldap.sh
+      args:
+        chdir: "{{ zuul.project.src_dir }}"
     - name: Deploy Fluent-Logging
       shell: |
         set -xe;

tools/deployment/common/115-elasticsearch-ldap.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+ELASTICSEARCH_ENDPOINT="elasticsearch-logging.openstack"
+
+#NOTE: Create index with specified LDAP user
+function create_index () {
+  index_result=$(curl -K- <<< "--user $1:$2" \
+  -XPUT "${ELASTICSEARCH_ENDPOINT}/$1_index?pretty" -H 'Content-Type: application/json' -d'
+  {
+    "settings" : {
+      "index" : {
+        "number_of_shards" : 3,
+        "number_of_replicas" : 2
+      }
+    }
+  }
+  ' | python -c "import sys, json; print json.load(sys.stdin)['acknowledged']")
+  if [ "$index_result" == "True" ];
+  then
+     echo "$1's index successfully created!";
+  else
+     echo "$1's index not created!";
+     exit 1;
+  fi
+}
+
+#NOTE: Insert test data with specified LDAP user
+function insert_test_data () {
+  insert_result=$(curl -K- <<< "--user $1:$2" \
+  -XPUT "${ELASTICSEARCH_ENDPOINT}/$1_index/sample_type/123/_create?pretty" -H 'Content-Type: application/json' -d'
+  {
+      "name" : "Elasticsearch",
+      "message" : "Test data text entry"
+  }
+  ' | python -c "import sys, json; print json.load(sys.stdin)['result']")
+  if [ "$insert_result" == "created" ]; then
+     sleep 20
+     echo "Test data inserted into $1's index!";
+  else
+     echo "Test data not inserted into $1's index!";
+     exit 1;
+  fi
+}
+
+#NOTE: Check hits on test data in specified LDAP user's index
+function check_hits () {
+  total_hits=$(curl -K- <<< "--user $1:$2" \
+  "${ELASTICSEARCH_ENDPOINT}/_search?pretty" -H 'Content-Type: application/json' -d'
+  {
+    "query" : {
+      "bool": {
+        "must": [
+          { "match": { "name": "Elasticsearch" }},
+          { "match": { "message": "Test data text entry" }}
+        ]
+      }
+    }
+  }
+  ' | python -c "import sys, json; print json.load(sys.stdin)['hits']['total']")
+  if [ "$total_hits" -gt 0 ]; then
+     echo "Successful hits on test data query on $1's index!"
+  else
+     echo "No hits on query for test data on $1's index!";
+     exit 1;
+  fi
+}
+
+create_index bob password
+create_index alice password
+
+insert_test_data bob password
+insert_test_data alice password
+
+check_hits bob password
+check_hits alice password

tools/deployment/developer/000-install-packages.sh

@@ -1 +0,0 @@
-../common/000-install-packages.sh

tools/deployment/developer/000-install-packages.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+sudo apt-get update
+sudo apt-get install --no-install-recommends -y \
+        ca-certificates \
+        git \
+        make \
+        nmap \
+        curl

tools/deployment/developer/005-deploy-k8s.sh

@@ -1 +0,0 @@
-../common/005-deploy-k8s.sh

tools/deployment/developer/005-deploy-k8s.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Copyright 2017 The Openstack-Helm Authors.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set -xe
+
+make dev-deploy setup-host
+make dev-deploy k8s

tools/deployment/developer/115-elasticsearch-ldap.sh

@@ -0,0 +1 @@
+../common/115-elasticsearch-ldap.sh

tools/deployment/multinode/115-elasticsearch-ldap.sh

@@ -0,0 +1 @@
+../common/115-elasticsearch-ldap.sh