Merge pull request #35 from markgoddard/custom-fluent-filters

Add support for configuration of custom fluentd filters

ansible/kolla-openstack.yml

@@ -185,7 +185,4 @@
       kolla_extra_neutron_ml2: "{{ kolla_extra_config.neutron_ml2 | default }}"
       kolla_extra_nova: "{{ kolla_extra_config.nova | default }}"
       kolla_extra_sahara: "{{ kolla_extra_config.sahara | default }}"
-      kolla_extra_fluentd_output_path: "{{ kayobe_config_path }}/kolla/config/fluentd/output"
+      kolla_extra_config_path: "{{ kayobe_config_path }}/kolla/config"
-      kolla_extra_glance_path: "{{ kayobe_config_path }}/kolla/config/glance"
-      kolla_extra_neutron_path: "{{ kayobe_config_path }}/kolla/config/neutron"
-      kolla_extra_nova_path: "{{ kayobe_config_path }}/kolla/config/nova"

ansible/roles/kolla-openstack/defaults/main.yml

@@ -1,13 +1,10 @@
 ---
+# Path to extra kolla-ansible configuration files.
+kolla_extra_config_path:
+
 # Directory where Kolla custom configuration files will be installed.
 kolla_node_custom_config_path:
 
-###############################################################################
-# Fluentd configuration.
-
-# Path to extra Fluentd output configuration files.
-kolla_extra_fluentd_output_path:
-
 ###############################################################################
 # Glance configuration.
 
@@ -18,9 +15,6 @@ kolla_enable_glance:
 # glance-registry.conf.
 kolla_extra_glance:
 
-# Path to extra Glance configuration files.
-kolla_extra_glance_path:
-
 ###############################################################################
 # Ironic configuration.
 
@@ -255,9 +249,6 @@ kolla_extra_neutron:
 # Free form extra configuration to append to ml2_conf.ini.
 kolla_extra_neutron_ml2:
 
-# Path to extra Neutron configuration files.
-kolla_extra_neutron_path:
-
 ###############################################################################
 # Nova configuration.
 
@@ -267,9 +258,6 @@ kolla_enable_nova:
 # Free form extra configuration to append to nova.conf.
 kolla_extra_nova:
 
-# Path to extra Nova configuration files.
-kolla_extra_nova_path:
-
 ###############################################################################
 # Sahara configuration.
 
@@ -278,3 +266,9 @@ kolla_enable_sahara:
 
 # Free form extra configuration to append to sahara.conf.
 kolla_extra_sahara:
+
+###############################################################################
+# Swift configuration.
+
+# Whether to enable swift.
+kolla_enable_swift:

ansible/roles/kolla-openstack/tasks/config.yml

@@ -1,16 +1,10 @@
 ---
 - name: Ensure the Kolla OpenStack configuration directores exist
   file:
-    path: "{{ kolla_node_custom_config_path }}/{{ item.name }}"
+    path: "{{ item.dest }}"
     state: directory
     mode: 0750
-  with_items:
+  with_items: "{{ kolla_openstack_custom_config }}"
-    - { name: fluentd/output, enabled: "{{ kolla_extra_fluentd_output_path != None }}" }
-    - { name: glance, enabled: "{{ kolla_enable_glance }}" }
-    - { name: ironic, enabled: "{{ kolla_enable_ironic }}" }
-    - { name: neutron, enabled: "{{ kolla_enable_neutron }}" }
-    - { name: nova, enabled: "{{ kolla_enable_nova }}" }
-    - { name: swift, enabled: "{{ kolla_enable_swift }}" }
   when: item.enabled | bool
 
 - name: Ensure the Kolla OpenStack configuration files exist
@@ -32,24 +26,6 @@
     - { src: sahara.conf.j2, dest: sahara.conf, enabled: "{{ kolla_enable_sahara }}" }
   when: item.enabled | bool
 
-- name: Ensure extra glance configuration files exist
-  template:
-    src: "{{ item }}"
-    dest: "{{ kolla_node_custom_config_path }}/glance/{{ item | basename }}"
-    mode: 0640
-  with_fileglob:
-    - "{{ kolla_extra_glance_path }}/*"
-  when: kolla_extra_glance_path != None
-
-- name: Ensure extra fluentd output configuration files exist
-  template:
-    src: "{{ item }}"
-    dest: "{{ kolla_node_custom_config_path }}/fluentd/output/{{ item | basename }}"
-    mode: 0640
-  with_fileglob:
-    - "{{ kolla_extra_fluentd_output_path }}/*.conf"
-  when: kolla_extra_fluentd_output_path != None
-
 - name: Ensure the ironic inspector kernel and ramdisk are downloaded
   get_url:
     url: "{{ item.url }}"
@@ -74,20 +50,50 @@
     - kolla_enable_ironic | bool
     - item.path != None
 
-- name: Ensure extra neutron configuration files exist
+# We support a fairly flexible mechanism of dropping config file templates into
-  template:
+# an 'extra' config directory, and passing these through to kolla-ansible. We
-    src: "{{ item }}"
+# look for matching files in the source directory to template, and also remove
-    dest: "{{ kolla_node_custom_config_path }}/neutron/{{ item | basename }}"
+# any unexpected files from the destination, to support removal of files.
-    mode: 0640
-  with_fileglob:
-    - "{{ kolla_extra_neutron_path }}/*"
-  when: kolla_extra_neutron_path != None
 
-- name: Ensure extra nova configuration files exist
+- name: Find extra configuration files
+  find:
+    path: "{{ item.src }}"
+    patterns: "{{ item.patterns }}"
+  with_items: "{{ kolla_openstack_custom_config }}"
+  register: find_src_result
+
+- name: Find previously generated extra configuration files
+  find:
+    path: "{{ item.dest }}"
+    patterns: "{{ item.patterns }}"
+  with_items: "{{ kolla_openstack_custom_config }}"
+  register: find_dest_result
+
+- name: Ensure extra configuration files exist
   template:
-    src: "{{ item }}"
+    src: "{{ item.1.path }}"
-    dest: "{{ kolla_node_custom_config_path }}/nova/{{ item | basename }}"
+    dest: "{{ item.0.item.dest }}/{{ item.1.path | basename }}"
     mode: 0640
-  with_fileglob:
+  with_subelements:
-    - "{{ kolla_extra_nova_path }}/*"
+    - "{{ find_src_result.results }}"
-  when: kolla_extra_nova_path != None
+    - files
+    - skip_missing: True
+  when: item.0.item.enabled | bool
+
+- name: Ensure unnecessary extra configuration files are absent
+  file:
+    path: "{{ item.1.path }}"
+    state: absent
+  with_subelements:
+    - "{{ find_dest_result.results }}"
+    - files
+    - skip_missing: True
+  when:
+    - not item.0.item.enabled or
+      item.1.path | basename not in src_files
+    - item.1.path | basename not in item.0.item.ignore | default([])
+  vars:
+    # Find the source result that corresponds to this one.
+    src_result: "{{ (find_src_result.results | selectattr('item', 'equalto', item.0.item) | list)[0] }}"
+    # Find the list of files in the source.
+    src_files: "{{ src_result.files | map(attribute='path') | map('basename') | list }}"

ansible/roles/kolla-openstack/vars/main.yml

@@ -0,0 +1,79 @@
+---
+# List of custom configuration directories.
+# Each item is a dict containing the following items:
+# src: Path to directory containing configuration file templates.
+# dest: Path to directory in which generated files will be created.
+# patterns: One or more file name patterns to match.
+# enabled: Whether these files should be templated.
+# ignore: Optional list of files to leave in the destination, even if disabled
+#         or unexpected.
+kolla_openstack_custom_config:
+  # Fluentd filters.
+  - src: "{{ kolla_extra_config_path }}//fluentd/filter"
+    dest: "{{ kolla_node_custom_config_path }}/fluentd/filter"
+    patterns: "*.conf"
+    enabled: True
+  # Fluentd outputs.
+  - src: "{{ kolla_extra_config_path }}/fluentd/output"
+    dest: "{{ kolla_node_custom_config_path }}/fluentd/output"
+    patterns: "*.conf"
+    enabled: True
+  # Glance.
+  - src: "{{ kolla_extra_config_path }}/glance"
+    dest: "{{ kolla_node_custom_config_path }}/glance"
+    patterns: "*"
+    enabled: "{{ kolla_enable_glance }}"
+  # Heat.
+  - src: "{{ kolla_extra_config_path }}/heat"
+    dest: "{{ kolla_node_custom_config_path }}/heat"
+    patterns: "*"
+    enabled: "{{ kolla_enable_heat }}"
+  # Ironic.
+  - src: "{{ kolla_extra_config_path }}/ironic"
+    dest: "{{ kolla_node_custom_config_path }}/ironic"
+    patterns: "*"
+    enabled: "{{ kolla_enable_ironic }}"
+    ignore:
+      # These are templated by kayobe, so don't remove them.
+      - ironic-agent.initramfs
+      - ironic-agent.kernel
+      - ironic-dnsmasq.conf
+      - pxelinux.default
+  # Keystone.
+  - src: "{{ kolla_extra_config_path }}/keystone"
+    dest: "{{ kolla_node_custom_config_path }}/keystone"
+    patterns: "*"
+    enabled: True
+  # Magnum.
+  - src: "{{ kolla_extra_config_path }}/magnum"
+    dest: "{{ kolla_node_custom_config_path }}/magnum"
+    patterns: "*"
+    enabled: "{{ kolla_enable_magnum }}"
+  # Murano.
+  - src: "{{ kolla_extra_config_path }}/murano"
+    dest: "{{ kolla_node_custom_config_path }}/murano"
+    patterns: "*"
+    enabled: "{{ kolla_enable_murano }}"
+  # Neutron.
+  - src: "{{ kolla_extra_config_path }}/neutron"
+    dest: "{{ kolla_node_custom_config_path }}/neutron"
+    patterns: "*"
+    enabled: "{{ kolla_enable_neutron }}"
+    ignore:
+      # These are templated by kayobe, so don't remove them.
+      - ml2_conf.ini
+  # Nova.
+  - src: "{{ kolla_extra_config_path }}/nova"
+    dest: "{{ kolla_node_custom_config_path }}/nova"
+    patterns: "*"
+    enabled: "{{ kolla_enable_nova }}"
+  # Sahara.
+  - src: "{{ kolla_extra_config_path }}/sahara"
+    dest: "{{ kolla_node_custom_config_path }}/sahara"
+    patterns: "*"
+    enabled: "{{ kolla_enable_sahara }}"
+  # Swift.
+  - src: "{{ kolla_extra_config_path }}/swift"
+    dest: "{{ kolla_node_custom_config_path }}/swift"
+    patterns: "*"
+    enabled: "{{ kolla_enable_swift }}"

doc/source/release-notes.rst

@@ -14,8 +14,9 @@ Features
 * Adds a ``--display`` argument to ``kayobe physical network configure``
   command.  This will output the candidate switch configuration without
   applying it.
-* Adds support for custom neutron and nova configuration files in
+* Adds support for configuration of custom fluentd filters, and additional
-  ``$KAYOBE_CONFIG_PATH/kolla/config/[neutron,nova]``.
+  config file templates for heat, ironic, keystone, magnum, murano, sahara, and
+  swift in ``$KAYOBE_CONFIG_PATH/kolla/config/<component>/``.
 
 Upgrade Notes
 -------------