Support deploying the Monasca Log API

Deploys the Monasca Log API with mod_wsgi + Apache.

Change-Id: I28f0aa31c59b0b6917be2b125b5f8a0d7a7035af
Partially-Implements: blueprint monasca-roles

ansible/group_vars/all.yml

@@ -236,6 +236,7 @@ memcached_port: "11211"
 mistral_api_port: "8989"
 
 monasca_api_port: "8070"
+monasca_log_api_port: "5607"
 
 mongodb_port: "27017"
 mongodb_web_port: "28017"
@@ -519,7 +520,7 @@ enable_destroy_images: "no"
 ####################
 
 elasticsearch_address: "{{ kolla_internal_fqdn }}"
-enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_freezer | bool or enable_osprofiler | bool or enable_skydive | bool else 'no' }}"
+enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_freezer | bool or enable_osprofiler | bool or enable_skydive | bool or enable_monasca | bool else 'no' }}"
 enable_kibana: "{{ 'yes' if enable_central_logging | bool else 'no' }}"
 
 ####################

ansible/inventory/all-in-one

@@ -440,6 +440,9 @@ murano
 [monasca-api:children]
 monasca
 
+[monasca-log-api:children]
+monasca
+
 # Ironic
 [ironic-api:children]
 ironic

ansible/inventory/multinode

@@ -449,6 +449,9 @@ murano
 [monasca-api:children]
 monasca
 
+[monasca-log-api:children]
+monasca
+
 # Ironic
 [ironic-api:children]
 ironic

ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2

@@ -1,7 +1,7 @@
 <match kolla.var.log.kolla.*.*.log>
     @type rewrite_tag_filter
     capitalize_regex_backreference yes
-    rewriterule1 programname ^(cinder-api-access|cloudkitty-api-access|gnocchi-api-access|horizon-access|keystone-apache-admin-access|keystone-apache-public-access|monasca-api-access|placement-api-access|panko-api-access)$ apache_access
+    rewriterule1 programname ^(cinder-api-access|cloudkitty-api-access|gnocchi-api-access|horizon-access|keystone-apache-admin-access|keystone-apache-public-access|monasca-api-access|monasca-log-api-access|placement-api-access|panko-api-access)$ apache_access
     rewriterule2 programname ^(aodh_wsgi_access|barbican-api|zun_api_wsgi_access|vitrage_wsgi_access)$ wsgi_access
     rewriterule3 programname ^(nova-api|nova-compute|nova-compute-ironic|nova-conductor|nova-consoleauth|nova-manage|nova-novncproxy|nova-scheduler|nova-placement-api|placement-api|privsep-helper)$ openstack_python
     rewriterule4 programname ^(sahara-api|sahara-engine)$ openstack_python

ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2

@@ -3,7 +3,7 @@
     capitalize_regex_backreference yes
   <rule>
     key     programname
-    pattern ^(cinder-api-access|cloudkitty-api-access|gnocchi-api-access|horizon-access|keystone-apache-admin-access|keystone-apache-public-access|monasca-api-access|placement-api-access|panko-api-access)$
+    pattern ^(cinder-api-access|cloudkitty-api-access|gnocchi-api-access|horizon-access|keystone-apache-admin-access|keystone-apache-public-access|monasca-api-access|monasca-log-api-access|placement-api-access|panko-api-access)$
     tag apache_access
   </rule>
   <rule>

ansible/roles/haproxy/templates/haproxy.cfg.j2

@@ -175,6 +175,13 @@ listen monasca_api_internal
   server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ monasca_api_port }} check inter 2000 rise 2 fall 5
 {% endfor %}
 
+listen monasca_log_api_internal
+  bind {{ kolla_internal_vip_address }}:{{ monasca_log_api_port }}
+  http-request del-header X-Forwarded-Proto if { ssl_fc }
+{% for host in groups['monasca-log-api'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ monasca_log_api_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
+
 {% if haproxy_enable_external_vip | bool %}
 listen monasca_api_external
   bind {{ kolla_external_vip_address }}:{{ monasca_api_port }} {{ tls_bind_info }}
@@ -183,6 +190,14 @@ listen monasca_api_external
 {% for host in groups['monasca-api'] %}
   server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ monasca_api_port }} check inter 2000 rise 2 fall 5
 {% endfor %}
+
+listen monasca_log_api_external
+  bind {{ kolla_external_vip_address }}:{{ monasca_log_api_port }} {{ tls_bind_info }}
+  http-request del-header X-Forwarded-Proto if { ssl_fc }
+  http-request set-header X-Forwarded-Proto https if { ssl_fc }
+{% for host in groups['monasca-log-api'] %}
+  server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ monasca_log_api_port }} check inter 2000 rise 2 fall 5
+{% endfor %}
 {% endif %}
 {% endif %}
 

ansible/roles/monasca/defaults/main.yml

@@ -9,6 +9,15 @@ monasca_services:
       - "{{ node_config_directory }}/monasca-api/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla"
+  monasca-log-api:
+    container_name: monasca_log_api
+    group: monasca-log-api
+    enabled: true
+    image: "{{ monasca_log_api_image_full }}"
+    volumes:
+      - "{{ node_config_directory }}/monasca-log-api/:{{ container_config_directory }}/:ro"
+      - "/etc/localtime:/etc/localtime:ro"
+      - "kolla_logs:/var/log/kolla"
 
 ####################
 # Databases
@@ -37,6 +46,10 @@ monasca_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ doc
 monasca_api_tag: "{{ monasca_tag }}"
 monasca_api_image_full: "{{ monasca_api_image }}:{{ monasca_api_tag }}"
 
+monasca_log_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ monasca_install_type }}-monasca-log-api"
+monasca_log_api_tag: "{{ monasca_tag }}"
+monasca_log_api_image_full: "{{ monasca_log_api_image }}:{{ monasca_log_api_tag }}"
+
 ####################
 # OpenStack
 ####################
@@ -56,4 +69,8 @@ monasca_api_admin_endpoint: "{{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{
 monasca_api_internal_endpoint: "{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ monasca_api_port }}/v2.0"
 monasca_api_public_endpoint: "{{ public_protocol }}://{{ kolla_external_fqdn }}:{{ monasca_api_port }}/v2.0"
 
+monasca_log_api_admin_endpoint: "{{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ monasca_log_api_port }}"
+monasca_log_api_internal_endpoint: "{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ monasca_log_api_port }}"
+monasca_log_api_public_endpoint: "{{ public_protocol }}://{{ kolla_external_fqdn }}:{{ monasca_log_api_port }}"
+
 monasca_logging_debug: "{{ openstack_logging_debug }}"

ansible/roles/monasca/handlers/main.yml

@@ -19,3 +19,24 @@
       or monasca_api_confs.changed | bool
       or monasca_api_wsgi_confs.changed | bool
       or monasca_api_container.changed | bool
+
+- name: Restart monasca-log-api container
+  vars:
+    service_name: "monasca-log-api"
+    service: "{{ monasca_services[service_name] }}"
+    config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}"
+    monasca_log_api_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}"
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes }}"
+  when:
+    - kolla_action != "config"
+    - inventory_hostname in groups[service.group]
+    - service.enabled | bool
+    - config_json.changed | bool
+      or monasca_log_api_confs.changed | bool
+      or monasca_log_api_wsgi_confs.changed | bool
+      or monasca_log_api_container.changed | bool

ansible/roles/monasca/tasks/config.yml

@@ -26,6 +26,7 @@
   with_dict: "{{ monasca_services }}"
   notify:
     - Restart monasca-api container
+    - Restart monasca-log-api container
 
 - name: Copying over monasca-api config
   vars:
@@ -63,6 +64,42 @@
   notify:
     - Restart monasca-api container
 
+- name: Copying over monasca-log-api config
+  vars:
+    service: "{{ monasca_services['monasca-log-api'] }}"
+  merge_configs:
+    sources:
+      - "{{ role_path }}/templates/monasca-log-api/{{ item }}.j2"
+      - "{{ node_custom_config }}/monasca/{{ item }}"
+      - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/{{ item }}"
+    dest: "{{ node_config_directory }}/monasca-log-api/{{ item }}"
+    mode: "0660"
+  become: true
+  register: monasca_log_api_confs
+  with_items:
+    - log-api.conf
+    - log-api-paste.ini
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  notify:
+    - Restart monasca-log-api container
+
+- name: Copying over monasca-log-api wsgi config
+  vars:
+    service: "{{ monasca_services['monasca-log-api'] }}"
+  template:
+    src: "{{ role_path }}/templates/monasca-log-api/wsgi-log-api.conf.j2"
+    dest: "{{ node_config_directory }}/monasca-log-api/wsgi-log-api.conf"
+    mode: "0660"
+  become: true
+  register: monasca_log_api_wsgi_confs
+  when:
+    - inventory_hostname in groups[service['group']]
+    - service.enabled | bool
+  notify:
+    - Restart monasca-log-api container
+
 - name: Check monasca containers
   kolla_docker:
     action: "compare_container"
@@ -78,3 +115,4 @@
   with_dict: "{{ monasca_services }}"
   notify:
     - Restart monasca-api container
+    - Restart monasca-log-api container

ansible/roles/monasca/tasks/deploy.yml

@@ -1,9 +1,11 @@
 ---
 - include: register.yml
-  when: inventory_hostname in groups['monasca-api']
+  when: inventory_hostname in groups['monasca-api'] or
+        inventory_hostname in groups['monasca-log-api']
 
 - include: config.yml
-  when: inventory_hostname in groups['monasca-api']
+  when: inventory_hostname in groups['monasca-api'] or
+        inventory_hostname in groups['monasca-log-api']
 
 - include: bootstrap.yml
   when: inventory_hostname in groups['monasca-api']
@@ -12,4 +14,5 @@
   meta: flush_handlers
 
 - include: check.yml
-  when: inventory_hostname in groups['monasca-api']
+  when: inventory_hostname in groups['monasca-api'] or
+        inventory_hostname in groups['monasca-log-api']

ansible/roles/monasca/tasks/precheck.yml

@@ -14,3 +14,14 @@
   when:
     - inventory_hostname in groups[monasca_services['monasca-api']['group']]
     - container_facts['monasca_api'] is not defined
+
+- name: Checking free port for monasca-log-api
+  wait_for:
+    host: "{{ api_interface_address }}"
+    port: "{{ monasca_log_api_port }}"
+    connect_timeout: 1
+    timeout: 1
+    state: stopped
+  when:
+    - inventory_hostname in groups[monasca_services['monasca-log-api']['group']]
+    - container_facts['monasca_log_api'] is not defined

ansible/roles/monasca/tasks/register.yml

@@ -17,6 +17,24 @@
     - {'interface': 'internal', 'url': '{{ monasca_api_internal_endpoint }}'}
     - {'interface': 'public', 'url': '{{ monasca_api_public_endpoint }}'}
 
+- name: Creating monasca-log-api service and endpoints
+  kolla_toolbox:
+    module_name: "kolla_keystone_service"
+    module_args:
+      service_name: "monasca-log-api"
+      service_type: "logging"
+      description: "Monasca logging as a service"
+      endpoint_region: "{{ openstack_region_name }}"
+      url: "{{ item.url }}"
+      interface: "{{ item.interface }}"
+      region_name: "{{ openstack_region_name }}"
+      auth: "{{ monasca_openstack_auth }}"
+  run_once: True
+  with_items:
+    - {'interface': 'admin', 'url': '{{ monasca_log_api_admin_endpoint }}'}
+    - {'interface': 'internal', 'url': '{{ monasca_log_api_internal_endpoint }}'}
+    - {'interface': 'public', 'url': '{{ monasca_log_api_public_endpoint }}'}
+
 - name: Creating the monasca keystone user
   kolla_toolbox:
     module_name: "kolla_keystone_user"

ansible/roles/monasca/templates/monasca-log-api/log-api-paste.ini.j2

@@ -0,0 +1,65 @@
+#
+# Copyright 2016-2017 FUJITSU LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[DEFAULT]
+name = main
+
+[composite:main]
+use = egg:Paste#urlmap
+/: la_version
+/healthcheck: la_healthcheck
+/v2.0: la_api_v2
+/v3.0: la_api_v3
+
+[pipeline:la_version]
+pipeline = error_trap versionapp
+
+[pipeline:la_healthcheck]
+pipeline = error_trap healthcheckapp
+
+[pipeline:la_api_v2]
+pipeline = error_trap request_id auth roles api_v2_app
+
+[pipeline:la_api_v3]
+pipeline = error_trap request_id auth roles api_v3_app
+
+[app:versionapp]
+paste.app_factory = monasca_log_api.app.api:create_version_app
+
+[app:healthcheckapp]
+paste.app_factory = monasca_log_api.app.api:create_healthcheck_app
+
+[app:api_v2_app]
+paste.app_factory = monasca_log_api.app.api:create_api_app
+set api_version=v2.0
+
+[app:api_v3_app]
+paste.app_factory = monasca_log_api.app.api:create_api_app
+set api_version=v3.0
+
+[filter:auth]
+paste.filter_factory = keystonemiddleware.auth_token:filter_factory
+
+[filter:roles]
+paste.filter_factory = monasca_log_api.middleware.role_middleware:RoleMiddleware.factory
+
+[filter:request_id]
+paste.filter_factory = oslo_middleware.request_id:RequestId.factory
+
+[filter:error_trap]
+paste.filter_factory = oslo_middleware.catch_errors:CatchErrors.factory
+

ansible/roles/monasca/templates/monasca-log-api/log-api.conf.j2

@@ -0,0 +1,35 @@
+[DEFAULT]
+log_file = monasca-log-api.log
+log_dir = /var/log/kolla/monasca
+debug = {{ monasca_logging_debug }}
+
+[kafka_healthcheck]
+kafka_url = {{ monasca_kafka }}
+
+[log_publisher]
+kafka_url = {{ monasca_kafka }}
+
+[monitoring]
+# TODO(dszumski): Deploy statsd to use this
+enable = false
+
+[roles_middleware]
+path = /v2.0/log, /v3.0/logs
+default_roles = {{ monasca_default_authorized_roles|join(', ') }}
+agent_roles = {{ monasca_agent_authorized_roles|join(', ') }}
+delegate_roles = {{ monasca_delegate_authorized_roles|join(', ') }}
+
+[keystone_authtoken]
+auth_uri = {{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_public_port }}
+auth_url = {{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_admin_port }}
+auth_type = password
+project_domain_id =  {{ default_project_domain_id }}
+user_domain_id =  {{ default_user_domain_id }}
+project_name = service
+username = {{ monasca_keystone_user }}
+password = {{ monasca_keystone_password }}
+service_token_roles_required=True
+
+memcache_security_strategy = ENCRYPT
+memcache_secret_key = {{ memcache_secret_key }}
+memcached_servers = {{ monasca_memcached_servers }}

ansible/roles/monasca/templates/monasca-log-api/monasca-log-api.json.j2

@@ -0,0 +1,32 @@
+{% set monasca_cmd = 'apache2' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd' %}
+{% set wsgi_conf_dir = 'apache2/conf-enabled' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd/conf.d' %}
+{
+    "command": "/usr/sbin/{{ monasca_cmd }} -DFOREGROUND",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/log-api.conf",
+            "dest": "/etc/monasca/log-api.conf",
+            "owner": "monasca",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/log-api-paste.ini",
+            "dest": "/etc/monasca/log-api-paste.ini",
+            "owner": "monasca",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/wsgi-log-api.conf",
+            "dest": "/etc/{{ wsgi_conf_dir }}/wsgi-config.conf",
+            "owner": "monasca",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/monasca",
+            "owner": "monasca:kolla",
+            "recurse": true
+        }
+    ]
+}

ansible/roles/monasca/templates/monasca-log-api/wsgi-log-api.conf.j2

@@ -0,0 +1,24 @@
+{% set python_path = '/usr/lib/python2.7/site-packages' if kolla_install_type == 'binary' else '/var/lib/kolla/venv/lib/python2.7/site-packages' %}
+{% set wsgi_path = '/usr/bin' if monasca_install_type == 'binary' else '/monasca-log/monasca_log_api/app' %}
+
+Listen {{ api_interface_address }}:{{ monasca_log_api_port }}
+
+TraceEnable off
+
+<VirtualHost *:{{ monasca_log_api_port }}>
+
+    ErrorLog "/var/log/kolla/monasca/monasca-log-api-error.log"
+    LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"" logformat
+    CustomLog "/var/log/kolla/monasca/monasca-log-api-access.log" logformat
+    WSGIApplicationGroup %{GLOBAL}
+    WSGIDaemonProcess monasca-log-api group=monasca processes={{ openstack_service_workers }} threads=1 user=monasca python-path={{ python_path }}
+    WSGIProcessGroup  monasca-log-api
+    WSGIScriptAlias / {{ wsgi_path }}/wsgi.py
+    WSGIPassAuthorization On
+    SetEnv no-gzip 1
+
+    <Directory "{{ wsgi_path }}">
+        Require all granted
+    </Directory>
+
+</VirtualHost>

releasenotes/notes/add-monasca-log-api-d47662a4e643cd7f.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - Add support for deploying the Monasca Log API which forms part of the
+    Monasca distributed monitoring and logging as a service platform.
+    See https://wiki.openstack.org/wiki/Monasca for more details.