diff --git a/ansible/library/kolla_docker.py b/ansible/library/kolla_docker.py
index dee7a0df2b..b9b1e724c2 100644
--- a/ansible/library/kolla_docker.py
+++ b/ansible/library/kolla_docker.py
@@ -51,6 +51,7 @@ options:
- compare_container
- compare_image
- create_volume
+ - ensure_image
- get_container_env
- get_container_state
- pull_image
@@ -886,6 +887,10 @@ class DockerWorker(object):
)
raise
+ def ensure_image(self):
+ if not self.check_image():
+ self.pull_image()
+
def generate_module():
# NOTE(jeffrey4l): add empty string '' to choices let us use
@@ -893,13 +898,20 @@ def generate_module():
argument_spec = dict(
common_options=dict(required=False, type='dict', default=dict()),
action=dict(required=True, type='str',
- choices=['compare_container', 'compare_image',
- 'create_volume', 'get_container_env',
- 'get_container_state', 'pull_image',
+ choices=['compare_container',
+ 'compare_image',
+ 'create_volume',
+ 'ensure_image',
+ 'get_container_env',
+ 'get_container_state',
+ 'pull_image',
'recreate_or_restart_container',
- 'remove_container', 'remove_image',
- 'remove_volume', 'restart_container',
- 'start_container', 'stop_container',
+ 'remove_container',
+ 'remove_image',
+ 'remove_volume',
+ 'restart_container',
+ 'start_container',
+ 'stop_container',
'stop_and_remove_container']),
api_version=dict(required=False, type='str', default='auto'),
auth_email=dict(required=False, type='str'),
@@ -947,6 +959,7 @@ def generate_module():
['action', 'compare_container', ['name']],
['action', 'compare_image', ['name']],
['action', 'create_volume', ['name']],
+ ['action', 'ensure_image', ['image']],
['action', 'get_container_env', ['name']],
['action', 'get_container_state', ['name']],
['action', 'recreate_or_restart_container', ['name']],
diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml
index 71fc70022c..6ff7651ba8 100644
--- a/ansible/roles/common/tasks/config.yml
+++ b/ansible/roles/common/tasks/config.yml
@@ -26,6 +26,32 @@
- "fluentd/filter"
when: enable_fluentd | bool
+- name: Ensure fluentd image is present for label check
+ vars:
+ service_name: "fluentd"
+ service: "{{ common_services[service_name] }}"
+ become: true
+ kolla_docker:
+ action: "ensure_image"
+ common_options: "{{ docker_common_options }}"
+ image: "{{ service.image }}"
+ when: enable_fluentd | bool
+
+- name: Fetch fluentd image labels
+ vars:
+ service_name: "fluentd"
+ service: "{{ common_services[service_name] }}"
+ become: true
+ docker_image_facts:
+ name: "{{ service.image }}"
+ register: fluentd_labels
+ when: enable_fluentd | bool
+
+- name: Set fluentd facts
+ set_fact:
+ fluentd_binary: "{% if fluentd_labels.images.0.ContainerConfig.Labels.fluentd_binary is not defined %}{% if kolla_base_distro in 'ubuntu' and ansible_architecture == 'x86_64' %}td-agent{% else %}fluentd{% endif %}{% else %}{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_binary }}{% endif %}"
+ when: enable_fluentd | bool
+
- name: Copying over config.json files for services
template:
src: "{{ item.key }}.json.j2"
@@ -199,6 +225,7 @@
- name: Copying over fluentd filter config files
vars:
customised_filter_files: "{{ find_custom_fluentd_filters.files | map(attribute='path') | map('basename') | list }}"
+ fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}"
template:
src: "conf/filter/{{ item.src }}.conf.j2"
dest: "{{ node_config_directory }}/fluentd/filter/{{ item.dest }}.conf"
@@ -207,7 +234,7 @@
with_items:
- src: 00-record_transformer
dest: 00-record_transformer
- - src: "{{ '01-rewrite-0.14' if kolla_base_distro in ['debian', 'ubuntu'] else '01-rewrite-0.12' }}"
+ - src: "{{ '01-rewrite-0.14' if fluentd_version == '0.14' else '01-rewrite-0.12' }}"
dest: 01-rewrite
when:
- enable_fluentd | bool
diff --git a/ansible/roles/common/templates/conf/input/00-global.conf.j2 b/ansible/roles/common/templates/conf/input/00-global.conf.j2
index 36374d409a..54b863f828 100644
--- a/ansible/roles/common/templates/conf/input/00-global.conf.j2
+++ b/ansible/roles/common/templates/conf/input/00-global.conf.j2
@@ -1,5 +1,4 @@
#jinja2: trim_blocks: False
-{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
{% set services = [
( 'aodh', enable_aodh ),
( 'barbican', enable_barbican ),
@@ -51,7 +50,7 @@
"/var/log/kolla/neutron/dnsmasq.log",
"/var/log/kolla/*/*-access.log",
"/var/log/kolla/*/*-error.log"]
- pos_file /var/run/{{ fluentd_dir }}/kolla-openstack.pos
+ pos_file /var/run/{{ fluentd_binary }}/kolla-openstack.pos
tag kolla.*
format multiline
format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/
diff --git a/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 b/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2
index 86ff6fa992..3933a38568 100644
--- a/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2
+++ b/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2
@@ -1,8 +1,7 @@
-{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/mariadb/mariadb.log
- pos_file /var/run/{{ fluentd_dir }}/mariadb.pos
+ pos_file /var/run/{{ fluentd_binary }}/mariadb.pos
tag infra.mariadb
format multiline
format_firstline /^\d{6}/
diff --git a/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2 b/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2
index 913f3755f5..4cc414d9bd 100644
--- a/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2
+++ b/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2
@@ -1,8 +1,7 @@
-{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/rabbitmq/rabbit.log
- pos_file /var/run/{{ fluentd_dir }}/rabbit.pos
+ pos_file /var/run/{{ fluentd_binary }}/rabbit.pos
tag infra.rabbit
format multiline
format_firstline /^\s*$/
diff --git a/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2 b/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2
index 91b2ae33a7..cf0fe83323 100644
--- a/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2
+++ b/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2
@@ -1,9 +1,8 @@
# Note (blallau): to manage Apache and WSGI log files
-{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/*/*-access.log,/var/log/kolla/*/*-error.log
- pos_file /var/run/{{ fluentd_dir }}/kolla-openstack-wsgi.pos
+ pos_file /var/run/{{ fluentd_binary }}/kolla-openstack-wsgi.pos
tag kolla.*
format /^(?<Payload>.*)$/
enable_watch_timer false
diff --git a/ansible/roles/common/templates/conf/input/05-libvirt.conf.j2 b/ansible/roles/common/templates/conf/input/05-libvirt.conf.j2
index ba4c40a434..040b940621 100644
--- a/ansible/roles/common/templates/conf/input/05-libvirt.conf.j2
+++ b/ansible/roles/common/templates/conf/input/05-libvirt.conf.j2
@@ -1,8 +1,7 @@
-{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/libvirt/libvirtd.log
- pos_file /var/run/{{ fluentd_dir }}/libvirt.pos
+ pos_file /var/run/{{ fluentd_binary }}/libvirt.pos
tag infra.libvirt
format /^(?<Timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}\+\d{4}): (?<Pid>\d+): (?<log_level>\S+) : (?<Payload>.*)?$/
time_key Timestamp
diff --git a/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2 b/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2
index 49c733c63f..3193227fd7 100644
--- a/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2
+++ b/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2
@@ -1,8 +1,7 @@
-{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/zookeeper/zookeeper.log
- pos_file /var/run/{{ fluentd_dir }}/zookeeper.pos
+ pos_file /var/run/{{ fluentd_binary }}/zookeeper.pos
tag infra.*
format multiline
format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} \S+ \S+ \S+ .*$/
diff --git a/ansible/roles/common/templates/conf/input/07-kafka.conf.j2 b/ansible/roles/common/templates/conf/input/07-kafka.conf.j2
index 22d22521a4..ad086ba899 100644
--- a/ansible/roles/common/templates/conf/input/07-kafka.conf.j2
+++ b/ansible/roles/common/templates/conf/input/07-kafka.conf.j2
@@ -1,8 +1,7 @@
-{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/kafka/controller.log, /var/log/kolla/kafka/server.log, /var/log/kolla/kafka/state-change.log
- pos_file /var/run/{{ fluentd_dir }}/kafka.pos
+ pos_file /var/run/{{ fluentd_binary }}/kafka.pos
tag infra.*
format multiline
format_firstline /^\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\] \S+ .*$/
diff --git a/ansible/roles/common/templates/conf/input/08-opendaylight.conf.j2 b/ansible/roles/common/templates/conf/input/08-opendaylight.conf.j2
index 13c6906825..43879c918f 100644
--- a/ansible/roles/common/templates/conf/input/08-opendaylight.conf.j2
+++ b/ansible/roles/common/templates/conf/input/08-opendaylight.conf.j2
@@ -1,8 +1,7 @@
-{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/opendaylight/karaf.log
- pos_file /var/run/{{ fluentd_dir }}/opendaylight.pos
+ pos_file /var/run/{{ fluentd_binary }}/opendaylight.pos
tag infra.opendaylight
format multiline
format_firstline /\d{4}-\d{2}-\d{2}/
diff --git a/ansible/roles/common/templates/conf/input/09-monasca.conf.j2 b/ansible/roles/common/templates/conf/input/09-monasca.conf.j2
index 0d3c153831..7da68f4b9d 100644
--- a/ansible/roles/common/templates/conf/input/09-monasca.conf.j2
+++ b/ansible/roles/common/templates/conf/input/09-monasca.conf.j2
@@ -1,8 +1,7 @@
-{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %}
<source>
@type tail
path /var/log/kolla/monasca/agent*.log
- pos_file /var/run/{{ fluentd_dir }}/monasca-agent.pos
+ pos_file /var/run/{{ fluentd_binary }}/monasca-agent.pos
tag kolla.*
format multiline
format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+ \| \S+ \| \S+ \| .*$/
@@ -13,7 +12,7 @@
<source>
@type tail
path /var/log/kolla/monasca/grafana.log
- pos_file /var/run/{{ fluentd_dir }}/monasca-grafana.pos
+ pos_file /var/run/{{ fluentd_binary }}/monasca-grafana.pos
tag infra.*
format multiline
format_firstline /^t=\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+\d{4} lvl=\S+ msg=.*$/
diff --git a/ansible/roles/common/templates/fluentd.json.j2 b/ansible/roles/common/templates/fluentd.json.j2
index 3ae1d5be0c..40d2d14fb5 100644
--- a/ansible/roles/common/templates/fluentd.json.j2
+++ b/ansible/roles/common/templates/fluentd.json.j2
@@ -1,18 +1,18 @@
-{% if kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64' %}
- {% set fluentd_cmd = '/usr/sbin/td-agent' %}
- {% set fluentd_dir = '/etc/td-agent' %}
- {% set fluentd_conf = 'td-agent.conf' %}
- {% set fluentd_user = 'td-agent' %}
-{% else %}
- {% if kolla_base_distro in ['ubuntu', 'debian'] %}
- {% set fluentd_cmd = '/usr/local/bin/fluentd -c /etc/fluentd/fluent.conf' %}
- {% else %}
- {% set fluentd_cmd = '/usr/bin/fluentd -c /etc/fluentd/fluent.conf' %}
- {% endif %}
- {% set fluentd_dir = '/etc/fluentd' %}
+{% set fluentd_user = fluentd_binary %}
+{% set fluentd_dir = '/etc/' ~ fluentd_binary %}
+
+{% if fluentd_binary == 'fluentd' %}
{% set fluentd_conf = 'fluent.conf' %}
- {% set fluentd_user = 'fluentd' %}
+ {% if kolla_base_distro in ['ubuntu', 'debian'] %}
+ {% set fluentd_cmd = '/usr/local/bin/fluentd -c ' ~ fluentd_dir ~ '/' ~ fluentd_conf %}
+ {% else %}
+ {% set fluentd_cmd = '/usr/bin/fluentd -c ' ~ fluentd_dir ~ '/' ~ fluentd_conf %}
+ {% endif %}
+{% elif fluentd_binary == 'td-agent' %}
+ {% set fluentd_conf = fluentd_binary ~ '.conf' %}
+ {% set fluentd_cmd = '/usr/sbin/td-agent' %}
{% endif %}
+
{
"command": "{{ fluentd_cmd }}",
"config_files": [
diff --git a/tests/test_kolla_docker.py b/tests/test_kolla_docker.py
index 35be4ecbab..31dba4633e 100644
--- a/tests/test_kolla_docker.py
+++ b/tests/test_kolla_docker.py
@@ -44,12 +44,21 @@ class ModuleArgsTest(base.BaseTestCase):
common_options=dict(required=False, type='dict', default=dict()),
action=dict(
required=True, type='str',
- choices=['compare_container', 'compare_image', 'create_volume',
- 'get_container_env', 'get_container_state',
- 'pull_image', 'recreate_or_restart_container',
- 'remove_container', 'remove_image', 'remove_volume',
- 'restart_container', 'start_container',
- 'stop_container', 'stop_and_remove_container']),
+ choices=['compare_container',
+ 'compare_image',
+ 'create_volume',
+ 'ensure_image',
+ 'get_container_env',
+ 'get_container_state',
+ 'pull_image',
+ 'recreate_or_restart_container',
+ 'remove_container',
+ 'remove_image',
+ 'remove_volume',
+ 'restart_container',
+ 'start_container',
+ 'stop_container',
+ 'stop_and_remove_container']),
api_version=dict(required=False, type='str', default='auto'),
auth_email=dict(required=False, type='str'),
auth_password=dict(required=False, type='str', no_log=True),
@@ -96,6 +105,7 @@ class ModuleArgsTest(base.BaseTestCase):
['action', 'compare_container', ['name']],
['action', 'compare_image', ['name']],
['action', 'create_volume', ['name']],
+ ['action', 'ensure_image', ['image']],
['action', 'get_container_env', ['name']],
['action', 'get_container_state', ['name']],
['action', 'recreate_or_restart_container', ['name']],