From c53966f310f94f20c0f5586b999d732549dc6c3a Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Thu, 25 Jul 2019 11:03:28 -0500 Subject: [PATCH] Add the ability to parse list only files This change will allow config_template to parse list only files and extend or replace the content of the list based on the selected module options. Change-Id: I2a24033b0323bcc25bd1b50ffb4034441ab2f468 Signed-off-by: Kevin Carter --- action/config_template.py | 93 ++++++++++++------- ...enhance-yaml-parsing-105c8494fa2b5169.yaml | 7 ++ tests/files/test_list_only.yml.expected | 17 ++++ .../files/test_list_only_replace.yml.expected | 1 + tests/templates/test_list_only.yml | 21 +++++ tests/test-yaml.yml | 50 ++++++++++ tests/test.yml | 12 +++ 7 files changed, 168 insertions(+), 33 deletions(-) create mode 100644 releasenotes/notes/enhance-yaml-parsing-105c8494fa2b5169.yaml create mode 100644 tests/files/test_list_only.yml.expected create mode 100644 tests/files/test_list_only_replace.yml.expected create mode 100644 tests/templates/test_list_only.yml diff --git a/action/config_template.py b/action/config_template.py index e5ab7b8..4de2fea 100644 --- a/action/config_template.py +++ b/action/config_template.py @@ -588,35 +588,41 @@ class ActionModule(ActionBase): """Recursively merge new_items into base_items. :param base_items: ``dict`` - :param new_items: ``dict`` + :param new_items: ``dict`` || ``list`` :returns: ``dict`` """ - for key, value in new_items.items(): - if isinstance(value, dict): - base_items[key] = self._merge_dict( - base_items=base_items.get(key, {}), - new_items=value, - list_extend=list_extend - ) - elif (not isinstance(value, int) and - (',' in value or - ('\n' in value and not yml_multilines))): - base_items[key] = re.split(',|\n', value) - base_items[key] = [i.strip() for i in base_items[key] if i] - elif isinstance(value, list): - if isinstance(base_items.get(key), list) and list_extend: - base_items[key].extend(value) + if isinstance(new_items, dict): + for key, value in new_items.items(): + if isinstance(value, dict): + base_items[key] = self._merge_dict( + base_items=base_items.get(key, {}), + new_items=value, + list_extend=list_extend + ) + elif (not isinstance(value, int) and ( + ',' in value or ( + '\n' in value and not yml_multilines))): + base_items[key] = re.split(',|\n', value) + base_items[key] = [i.strip() for i in base_items[key] if i] + elif isinstance(value, list): + if isinstance(base_items.get(key), list) and list_extend: + base_items[key].extend(value) + else: + base_items[key] = value + elif isinstance(value, (tuple, set)): + if isinstance(base_items.get(key), tuple) and list_extend: + base_items[key] += tuple(value) + elif isinstance(base_items.get(key), list) and list_extend: + base_items[key].extend(list(value)) + else: + base_items[key] = value else: - base_items[key] = value - elif isinstance(value, (tuple, set)): - if isinstance(base_items.get(key), tuple) and list_extend: - base_items[key] += tuple(value) - elif isinstance(base_items.get(key), list) and list_extend: - base_items[key].extend(list(value)) - else: - base_items[key] = value + base_items[key] = new_items[key] + elif isinstance(new_items, list): + if list_extend: + base_items.extend(new_items) else: - base_items[key] = new_items[key] + base_items = new_items return base_items def _load_options_and_status(self, task_vars): @@ -812,7 +818,7 @@ class ActionModule(ActionBase): ) type_merger = getattr(self, CONFIG_TYPES.get(_vars['config_type'])) - resultant, config_dict_base = type_merger( + resultant, config_base = type_merger( config_overrides=_vars['config_overrides'], resultant=resultant, list_extend=_vars.get('list_extend', True), @@ -822,13 +828,13 @@ class ActionModule(ActionBase): ) changed = False + config_new = None if self._play_context.diff: slurpee = self._execute_module( module_name='slurp', module_args=dict(src=_vars['dest']), task_vars=task_vars ) - config_dict_new = dict() if 'content' in slurpee: dest_data = base64.b64decode( slurpee['content']).decode('utf-8') @@ -840,7 +846,7 @@ class ActionModule(ActionBase): ) type_merger = getattr(self, CONFIG_TYPES.get(_vars['config_type'])) - resultant_new, config_dict_new = type_merger( + _, config_new = type_merger( config_overrides={}, resultant=resultant_dest, list_extend=_vars.get('list_extend', True), @@ -851,11 +857,32 @@ class ActionModule(ActionBase): # Compare source+overrides with dest to look for changes and # build diff - cmp_dicts = DictCompare( - self.resultant_ini_as_dict(resultant_dict=config_dict_new), - self.resultant_ini_as_dict(resultant_dict=config_dict_base) - ) - mods, changed = cmp_dicts.get_changes() + if isinstance(config_base, dict): + if not config_new: + config_new = dict() + cmp_dicts = DictCompare( + self.resultant_ini_as_dict(resultant_dict=config_new), + self.resultant_ini_as_dict(resultant_dict=config_base) + ) + mods, changed = cmp_dicts.get_changes() + elif isinstance(config_base, list): + if not config_new: + config_new = list() + mods = { + 'added': [ + i for i in config_new + if i not in config_base + ], + 'removed': [ + i for i in config_base + if i not in config_new + ], + 'changed': [ + i for i in (config_base + config_new) + if i not in config_base or i not in config_new + ] + } + changed = len(mods['changed']) > 0 # Re-template the resultant object as it may have new data within it # as provided by an override variable. diff --git a/releasenotes/notes/enhance-yaml-parsing-105c8494fa2b5169.yaml b/releasenotes/notes/enhance-yaml-parsing-105c8494fa2b5169.yaml new file mode 100644 index 0000000..a5b90b5 --- /dev/null +++ b/releasenotes/notes/enhance-yaml-parsing-105c8494fa2b5169.yaml @@ -0,0 +1,7 @@ +--- +features: + - Option parsing in `config_template` has been extended to allow for array + only overrides. This enhancement will allow us to ingest files which + contain only an array. For this work, the `config_overrides` now accepts + both a hash and an array, and will merge, replace, or extend based on the + data-type. diff --git a/tests/files/test_list_only.yml.expected b/tests/files/test_list_only.yml.expected new file mode 100644 index 0000000..a0b45d2 --- /dev/null +++ b/tests/files/test_list_only.yml.expected @@ -0,0 +1,17 @@ +- post_restart_command: null + post_start_command: null + pre_restart_command: null + pre_start_command: null + process_name: /usr/sbin/libvirtd + restart_command: systemctl restart libvirt-bin + run_as_root: true + start_command: systemctl start libvirt-bin +- post_restart_command: null + post_start_command: null + pre_restart_command: null + pre_start_command: null + process_name: /usr/sbin/libvirtd-test + restart_command: systemctl restart libvirt-bin-test + run_as_root: true + start_command: systemctl start libvirt-bin-test +- things: stuff diff --git a/tests/files/test_list_only_replace.yml.expected b/tests/files/test_list_only_replace.yml.expected new file mode 100644 index 0000000..08ed37a --- /dev/null +++ b/tests/files/test_list_only_replace.yml.expected @@ -0,0 +1 @@ +- things: stuff diff --git a/tests/templates/test_list_only.yml b/tests/templates/test_list_only.yml new file mode 100644 index 0000000..bee9569 --- /dev/null +++ b/tests/templates/test_list_only.yml @@ -0,0 +1,21 @@ +- + # libvirt-bin + process_name: /usr/sbin/libvirtd + start_command: systemctl start libvirt-bin + pre_start_command: + post_start_command: + restart_command: systemctl restart libvirt-bin + pre_restart_command: + post_restart_command: + run_as_root: True + +- + # libvirt-bin + process_name: /usr/sbin/libvirtd-test + start_command: systemctl start libvirt-bin-test + pre_start_command: + post_start_command: + restart_command: systemctl restart libvirt-bin-test + pre_restart_command: + post_restart_command: + run_as_root: True diff --git a/tests/test-yaml.yml b/tests/test-yaml.yml index 4d578a6..bceee14 100644 --- a/tests/test-yaml.yml +++ b/tests/test-yaml.yml @@ -123,3 +123,53 @@ assert: that: - "(multiline_strs_file_expected.content | b64decode) == (multiline_strs_file.content | b64decode)" + + +# Test yaml list only files +- name: Test list only files in yaml (extend) + config_template: + src: "{{ playbook_dir }}/templates/test_list_only.yml" + dest: "/tmp/test_list_only.yml" + config_overrides: "{{ test_list_only_overrides }}" + config_type: yaml + list_extend: True + +- name: Read test_list_only.yml + slurp: + src: /tmp/test_list_only.yml + register: test_list_only_file + +- debug: + msg: "List only Yaml Strings - {{ test_list_only_file.content | b64decode }}" + +- debug: + msg: "List only Yaml Strings Expected - {{ test_list_only_file_expected.content | b64decode }}" + +- name: Compare files + assert: + that: + - "(test_list_only_file_expected.content | b64decode) == (test_list_only_file.content | b64decode)" + +- name: Test list only files in yaml (replace) + config_template: + src: "{{ playbook_dir }}/templates/test_list_only.yml" + dest: "/tmp/test_list_only.yml" + config_overrides: "{{ test_list_only_overrides }}" + config_type: yaml + list_extend: False + +- name: Read test_list_only.yml + slurp: + src: /tmp/test_list_only.yml + register: test_list_only_replace_file + +- debug: + msg: "List only Yaml Strings - {{ test_list_only_replace_file.content | b64decode }}" + +- debug: + msg: "List only Yaml Strings Expected - {{ test_list_only_replace_file_expected.content | b64decode }}" + +- name: Compare files + assert: + that: + - "(test_list_only_replace_file_expected.content | b64decode) == (test_list_only_replace_file.content | b64decode)" diff --git a/tests/test.yml b/tests/test.yml index 5891162..2e01a85 100644 --- a/tests/test.yml +++ b/tests/test.yml @@ -63,6 +63,16 @@ src: "{{ playbook_dir }}/files/test_multiline_strs.yml.expected" register: multiline_strs_file_expected + - name: Read expected test_list_only.yml + slurp: + src: "{{ playbook_dir }}/files/test_list_only.yml.expected" + register: test_list_only_file_expected + + - name: Read expected test_list_only.yml + slurp: + src: "{{ playbook_dir }}/files/test_list_only_replace.yml.expected" + register: test_list_only_replace_file_expected + - import_tasks: test-common-tasks.yml - import_tasks: test-common-tasks.yml @@ -149,6 +159,8 @@ new_multiline_str: | This should not be a list + test_list_only_overrides: + - things: stuff test_default_section_overrides: global: test2: 2