diff --git a/ansible/library/kolla_docker.py b/ansible/library/kolla_docker.py index dee7a0df2b..b9b1e724c2 100644 --- a/ansible/library/kolla_docker.py +++ b/ansible/library/kolla_docker.py @@ -51,6 +51,7 @@ options: - compare_container - compare_image - create_volume + - ensure_image - get_container_env - get_container_state - pull_image @@ -886,6 +887,10 @@ class DockerWorker(object): ) raise + def ensure_image(self): + if not self.check_image(): + self.pull_image() + def generate_module(): # NOTE(jeffrey4l): add empty string '' to choices let us use @@ -893,13 +898,20 @@ def generate_module(): argument_spec = dict( common_options=dict(required=False, type='dict', default=dict()), action=dict(required=True, type='str', - choices=['compare_container', 'compare_image', - 'create_volume', 'get_container_env', - 'get_container_state', 'pull_image', + choices=['compare_container', + 'compare_image', + 'create_volume', + 'ensure_image', + 'get_container_env', + 'get_container_state', + 'pull_image', 'recreate_or_restart_container', - 'remove_container', 'remove_image', - 'remove_volume', 'restart_container', - 'start_container', 'stop_container', + 'remove_container', + 'remove_image', + 'remove_volume', + 'restart_container', + 'start_container', + 'stop_container', 'stop_and_remove_container']), api_version=dict(required=False, type='str', default='auto'), auth_email=dict(required=False, type='str'), @@ -947,6 +959,7 @@ def generate_module(): ['action', 'compare_container', ['name']], ['action', 'compare_image', ['name']], ['action', 'create_volume', ['name']], + ['action', 'ensure_image', ['image']], ['action', 'get_container_env', ['name']], ['action', 'get_container_state', ['name']], ['action', 'recreate_or_restart_container', ['name']], diff --git a/ansible/roles/common/tasks/config.yml b/ansible/roles/common/tasks/config.yml index 71fc70022c..6ff7651ba8 100644 --- a/ansible/roles/common/tasks/config.yml +++ b/ansible/roles/common/tasks/config.yml @@ -26,6 +26,32 @@ - "fluentd/filter" when: enable_fluentd | bool +- name: Ensure fluentd image is present for label check + vars: + service_name: "fluentd" + service: "{{ common_services[service_name] }}" + become: true + kolla_docker: + action: "ensure_image" + common_options: "{{ docker_common_options }}" + image: "{{ service.image }}" + when: enable_fluentd | bool + +- name: Fetch fluentd image labels + vars: + service_name: "fluentd" + service: "{{ common_services[service_name] }}" + become: true + docker_image_facts: + name: "{{ service.image }}" + register: fluentd_labels + when: enable_fluentd | bool + +- name: Set fluentd facts + set_fact: + fluentd_binary: "{% if fluentd_labels.images.0.ContainerConfig.Labels.fluentd_binary is not defined %}{% if kolla_base_distro in 'ubuntu' and ansible_architecture == 'x86_64' %}td-agent{% else %}fluentd{% endif %}{% else %}{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_binary }}{% endif %}" + when: enable_fluentd | bool + - name: Copying over config.json files for services template: src: "{{ item.key }}.json.j2" @@ -199,6 +225,7 @@ - name: Copying over fluentd filter config files vars: customised_filter_files: "{{ find_custom_fluentd_filters.files | map(attribute='path') | map('basename') | list }}" + fluentd_version: "{{ fluentd_labels.images.0.ContainerConfig.Labels.fluentd_version | default('0.12') }}" template: src: "conf/filter/{{ item.src }}.conf.j2" dest: "{{ node_config_directory }}/fluentd/filter/{{ item.dest }}.conf" @@ -207,7 +234,7 @@ with_items: - src: 00-record_transformer dest: 00-record_transformer - - src: "{{ '01-rewrite-0.14' if kolla_base_distro in ['debian', 'ubuntu'] else '01-rewrite-0.12' }}" + - src: "{{ '01-rewrite-0.14' if fluentd_version == '0.14' else '01-rewrite-0.12' }}" dest: 01-rewrite when: - enable_fluentd | bool diff --git a/ansible/roles/common/templates/conf/input/00-global.conf.j2 b/ansible/roles/common/templates/conf/input/00-global.conf.j2 index 36374d409a..54b863f828 100644 --- a/ansible/roles/common/templates/conf/input/00-global.conf.j2 +++ b/ansible/roles/common/templates/conf/input/00-global.conf.j2 @@ -1,5 +1,4 @@ #jinja2: trim_blocks: False -{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %} {% set services = [ ( 'aodh', enable_aodh ), ( 'barbican', enable_barbican ), @@ -51,7 +50,7 @@ "/var/log/kolla/neutron/dnsmasq.log", "/var/log/kolla/*/*-access.log", "/var/log/kolla/*/*-error.log"] - pos_file /var/run/{{ fluentd_dir }}/kolla-openstack.pos + pos_file /var/run/{{ fluentd_binary }}/kolla-openstack.pos tag kolla.* format multiline format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3} \d+ \S+ \S+ \[(req-\S+ \S+ \S+ \S+ \S+ \S+|-)\]/ diff --git a/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 b/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 index 86ff6fa992..3933a38568 100644 --- a/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 +++ b/ansible/roles/common/templates/conf/input/02-mariadb.conf.j2 @@ -1,8 +1,7 @@ -{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %} @type tail path /var/log/kolla/mariadb/mariadb.log - pos_file /var/run/{{ fluentd_dir }}/mariadb.pos + pos_file /var/run/{{ fluentd_binary }}/mariadb.pos tag infra.mariadb format multiline format_firstline /^\d{6}/ diff --git a/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2 b/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2 index 913f3755f5..4cc414d9bd 100644 --- a/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2 +++ b/ansible/roles/common/templates/conf/input/03-rabbitmq.conf.j2 @@ -1,8 +1,7 @@ -{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %} @type tail path /var/log/kolla/rabbitmq/rabbit.log - pos_file /var/run/{{ fluentd_dir }}/rabbit.pos + pos_file /var/run/{{ fluentd_binary }}/rabbit.pos tag infra.rabbit format multiline format_firstline /^\s*$/ diff --git a/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2 b/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2 index 91b2ae33a7..cf0fe83323 100644 --- a/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2 +++ b/ansible/roles/common/templates/conf/input/04-openstack-wsgi.conf.j2 @@ -1,9 +1,8 @@ # Note (blallau): to manage Apache and WSGI log files -{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %} @type tail path /var/log/kolla/*/*-access.log,/var/log/kolla/*/*-error.log - pos_file /var/run/{{ fluentd_dir }}/kolla-openstack-wsgi.pos + pos_file /var/run/{{ fluentd_binary }}/kolla-openstack-wsgi.pos tag kolla.* format /^(?.*)$/ enable_watch_timer false diff --git a/ansible/roles/common/templates/conf/input/05-libvirt.conf.j2 b/ansible/roles/common/templates/conf/input/05-libvirt.conf.j2 index ba4c40a434..040b940621 100644 --- a/ansible/roles/common/templates/conf/input/05-libvirt.conf.j2 +++ b/ansible/roles/common/templates/conf/input/05-libvirt.conf.j2 @@ -1,8 +1,7 @@ -{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %} @type tail path /var/log/kolla/libvirt/libvirtd.log - pos_file /var/run/{{ fluentd_dir }}/libvirt.pos + pos_file /var/run/{{ fluentd_binary }}/libvirt.pos tag infra.libvirt format /^(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}\+\d{4}): (?\d+): (?\S+) : (?.*)?$/ time_key Timestamp diff --git a/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2 b/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2 index 49c733c63f..3193227fd7 100644 --- a/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2 +++ b/ansible/roles/common/templates/conf/input/06-zookeeper.conf.j2 @@ -1,8 +1,7 @@ -{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %} @type tail path /var/log/kolla/zookeeper/zookeeper.log - pos_file /var/run/{{ fluentd_dir }}/zookeeper.pos + pos_file /var/run/{{ fluentd_binary }}/zookeeper.pos tag infra.* format multiline format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} \S+ \S+ \S+ .*$/ diff --git a/ansible/roles/common/templates/conf/input/07-kafka.conf.j2 b/ansible/roles/common/templates/conf/input/07-kafka.conf.j2 index 22d22521a4..ad086ba899 100644 --- a/ansible/roles/common/templates/conf/input/07-kafka.conf.j2 +++ b/ansible/roles/common/templates/conf/input/07-kafka.conf.j2 @@ -1,8 +1,7 @@ -{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %} @type tail path /var/log/kolla/kafka/controller.log, /var/log/kolla/kafka/server.log, /var/log/kolla/kafka/state-change.log - pos_file /var/run/{{ fluentd_dir }}/kafka.pos + pos_file /var/run/{{ fluentd_binary }}/kafka.pos tag infra.* format multiline format_firstline /^\[\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\] \S+ .*$/ diff --git a/ansible/roles/common/templates/conf/input/08-opendaylight.conf.j2 b/ansible/roles/common/templates/conf/input/08-opendaylight.conf.j2 index 13c6906825..43879c918f 100644 --- a/ansible/roles/common/templates/conf/input/08-opendaylight.conf.j2 +++ b/ansible/roles/common/templates/conf/input/08-opendaylight.conf.j2 @@ -1,8 +1,7 @@ -{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %} @type tail path /var/log/kolla/opendaylight/karaf.log - pos_file /var/run/{{ fluentd_dir }}/opendaylight.pos + pos_file /var/run/{{ fluentd_binary }}/opendaylight.pos tag infra.opendaylight format multiline format_firstline /\d{4}-\d{2}-\d{2}/ diff --git a/ansible/roles/common/templates/conf/input/09-monasca.conf.j2 b/ansible/roles/common/templates/conf/input/09-monasca.conf.j2 index 0d3c153831..7da68f4b9d 100644 --- a/ansible/roles/common/templates/conf/input/09-monasca.conf.j2 +++ b/ansible/roles/common/templates/conf/input/09-monasca.conf.j2 @@ -1,8 +1,7 @@ -{% set fluentd_dir = 'td-agent' if (kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64') else 'fluentd' %} @type tail path /var/log/kolla/monasca/agent*.log - pos_file /var/run/{{ fluentd_dir }}/monasca-agent.pos + pos_file /var/run/{{ fluentd_binary }}/monasca-agent.pos tag kolla.* format multiline format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \S+ \| \S+ \| \S+ \| .*$/ @@ -13,7 +12,7 @@ @type tail path /var/log/kolla/monasca/grafana.log - pos_file /var/run/{{ fluentd_dir }}/monasca-grafana.pos + pos_file /var/run/{{ fluentd_binary }}/monasca-grafana.pos tag infra.* format multiline format_firstline /^t=\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+\d{4} lvl=\S+ msg=.*$/ diff --git a/ansible/roles/common/templates/fluentd.json.j2 b/ansible/roles/common/templates/fluentd.json.j2 index 3ae1d5be0c..40d2d14fb5 100644 --- a/ansible/roles/common/templates/fluentd.json.j2 +++ b/ansible/roles/common/templates/fluentd.json.j2 @@ -1,18 +1,18 @@ -{% if kolla_base_distro in ['ubuntu', 'debian'] and ansible_architecture == 'x86_64' %} - {% set fluentd_cmd = '/usr/sbin/td-agent' %} - {% set fluentd_dir = '/etc/td-agent' %} - {% set fluentd_conf = 'td-agent.conf' %} - {% set fluentd_user = 'td-agent' %} -{% else %} - {% if kolla_base_distro in ['ubuntu', 'debian'] %} - {% set fluentd_cmd = '/usr/local/bin/fluentd -c /etc/fluentd/fluent.conf' %} - {% else %} - {% set fluentd_cmd = '/usr/bin/fluentd -c /etc/fluentd/fluent.conf' %} - {% endif %} - {% set fluentd_dir = '/etc/fluentd' %} +{% set fluentd_user = fluentd_binary %} +{% set fluentd_dir = '/etc/' ~ fluentd_binary %} + +{% if fluentd_binary == 'fluentd' %} {% set fluentd_conf = 'fluent.conf' %} - {% set fluentd_user = 'fluentd' %} + {% if kolla_base_distro in ['ubuntu', 'debian'] %} + {% set fluentd_cmd = '/usr/local/bin/fluentd -c ' ~ fluentd_dir ~ '/' ~ fluentd_conf %} + {% else %} + {% set fluentd_cmd = '/usr/bin/fluentd -c ' ~ fluentd_dir ~ '/' ~ fluentd_conf %} + {% endif %} +{% elif fluentd_binary == 'td-agent' %} + {% set fluentd_conf = fluentd_binary ~ '.conf' %} + {% set fluentd_cmd = '/usr/sbin/td-agent' %} {% endif %} + { "command": "{{ fluentd_cmd }}", "config_files": [ diff --git a/tests/test_kolla_docker.py b/tests/test_kolla_docker.py index 35be4ecbab..31dba4633e 100644 --- a/tests/test_kolla_docker.py +++ b/tests/test_kolla_docker.py @@ -44,12 +44,21 @@ class ModuleArgsTest(base.BaseTestCase): common_options=dict(required=False, type='dict', default=dict()), action=dict( required=True, type='str', - choices=['compare_container', 'compare_image', 'create_volume', - 'get_container_env', 'get_container_state', - 'pull_image', 'recreate_or_restart_container', - 'remove_container', 'remove_image', 'remove_volume', - 'restart_container', 'start_container', - 'stop_container', 'stop_and_remove_container']), + choices=['compare_container', + 'compare_image', + 'create_volume', + 'ensure_image', + 'get_container_env', + 'get_container_state', + 'pull_image', + 'recreate_or_restart_container', + 'remove_container', + 'remove_image', + 'remove_volume', + 'restart_container', + 'start_container', + 'stop_container', + 'stop_and_remove_container']), api_version=dict(required=False, type='str', default='auto'), auth_email=dict(required=False, type='str'), auth_password=dict(required=False, type='str', no_log=True), @@ -96,6 +105,7 @@ class ModuleArgsTest(base.BaseTestCase): ['action', 'compare_container', ['name']], ['action', 'compare_image', ['name']], ['action', 'create_volume', ['name']], + ['action', 'ensure_image', ['image']], ['action', 'get_container_env', ['name']], ['action', 'get_container_state', ['name']], ['action', 'recreate_or_restart_container', ['name']],