From 1769158a0e4ce66a8030ed66220268560c08ae0b Mon Sep 17 00:00:00 2001 From: Georgina Date: Wed, 24 Apr 2019 16:27:21 +0000 Subject: [PATCH] config_template does not respect yaml multiline as overrides Added a flag called yml_multilines which when used will not split on lines that have a newline UNLESS they have a comma. Change-Id: I90364403e215a67b320dfc7e67a85d47c774e634 Partial-Bug: #1819974 --- action/config_template.py | 38 ++++++++++++++------ tests/files/test_multiline_strs.yml.expected | 14 ++++++++ tests/templates/test_multiline_strs.yml | 6 ++++ tests/test-common-tasks.yml | 23 ++++++++++++ tests/test.yml | 9 +++++ 5 files changed, 79 insertions(+), 11 deletions(-) create mode 100644 tests/files/test_multiline_strs.yml.expected create mode 100644 tests/templates/test_multiline_strs.yml diff --git a/action/config_template.py b/action/config_template.py index fbf6508..00ddd8b 100644 --- a/action/config_template.py +++ b/action/config_template.py @@ -46,7 +46,6 @@ from ansible import __version__ as __ansible_version__ __metaclass__ = type - CONFIG_TYPES = { 'ini': 'return_config_overrides_ini', 'json': 'return_config_overrides_json', @@ -144,6 +143,7 @@ class ConfigTemplateParser(ConfigParser.RawConfigParser): self._comments = {} self.ignore_none_type = bool(kwargs.pop('ignore_none_type', True)) self.default_section = str(kwargs.pop('default_section', 'DEFAULT')) + self.yml_multilines = bool(kwargs.pop('yml_multilines', False)) ConfigParser.RawConfigParser.__init__(self, *args, **kwargs) def _write(self, fp, section, key, item, entry): @@ -398,7 +398,8 @@ class ActionModule(ActionBase): resultant, list_extend=True, ignore_none_type=True, - default_section='DEFAULT'): + default_section='DEFAULT', + yml_multilines=False): """Returns string value from a modified config file and dict of merged config @@ -414,7 +415,8 @@ class ActionModule(ActionBase): allow_no_value=True, dict_type=MultiKeyDict, ignore_none_type=ignore_none_type, - default_section=default_section + default_section=default_section, + yml_multilines=yml_multilines ) config.optionxform = str except Exception: @@ -498,7 +500,8 @@ class ActionModule(ActionBase): resultant, list_extend=True, ignore_none_type=True, - default_section='DEFAULT'): + default_section='DEFAULT', + yml_multilines=False): """Returns config json and dict of merged config Its important to note that file ordering will not be preserved as the @@ -525,7 +528,8 @@ class ActionModule(ActionBase): resultant, list_extend=True, ignore_none_type=True, - default_section='DEFAULT'): + default_section='DEFAULT', + yml_multilines=False): """Return config yaml and dict of merged config :param config_overrides: ``dict`` @@ -536,7 +540,8 @@ class ActionModule(ActionBase): merged_resultant = self._merge_dict( base_items=original_resultant, new_items=config_overrides, - list_extend=list_extend + list_extend=list_extend, + yml_multilines=yml_multilines ) return yaml.dump( merged_resultant, @@ -545,7 +550,11 @@ class ActionModule(ActionBase): width=1000, ), merged_resultant - def _merge_dict(self, base_items, new_items, list_extend=True): + def _merge_dict(self, + base_items, + new_items, + list_extend=True, + yml_multilines=False): """Recursively merge new_items into base_items. :param base_items: ``dict`` @@ -560,7 +569,8 @@ class ActionModule(ActionBase): list_extend=list_extend ) elif (not isinstance(value, int) and - (',' in value or '\n' in value)): + (',' in value or + ('\n' in value and not yml_multilines))): base_items[key] = re.split(',|\n', value) base_items[key] = [i.strip() for i in base_items[key] if i] elif isinstance(value, list): @@ -662,6 +672,8 @@ class ActionModule(ActionBase): default_section = self._task.args.get('default_section', 'DEFAULT') + yml_multilines = self._task.args.get('yml_multilines', False) + return True, dict( source=source, dest=user_dest, @@ -670,7 +682,8 @@ class ActionModule(ActionBase): searchpath=searchpath, list_extend=list_extend, ignore_none_type=ignore_none_type, - default_section=default_section + default_section=default_section, + yml_multilines=yml_multilines ) def run(self, tmp=None, task_vars=None): @@ -745,7 +758,8 @@ class ActionModule(ActionBase): resultant=resultant, list_extend=_vars.get('list_extend', True), ignore_none_type=_vars.get('ignore_none_type', True), - default_section=_vars.get('default_section', 'DEFAULT') + default_section=_vars.get('default_section', 'DEFAULT'), + yml_multilines=_vars.get('yml_multilines', False) ) changed = False @@ -773,7 +787,8 @@ class ActionModule(ActionBase): resultant=resultant_dest, list_extend=_vars.get('list_extend', True), ignore_none_type=_vars.get('ignore_none_type', True), - default_section=_vars.get('default_section', 'DEFAULT') + default_section=_vars.get('default_section', 'DEFAULT'), + yml_multilines=_vars.get('yml_multilines', False) ) # Compare source+overrides with dest to look for changes and @@ -822,6 +837,7 @@ class ActionModule(ActionBase): new_module_args.pop('list_extend', None) new_module_args.pop('ignore_none_type', None) new_module_args.pop('default_section', None) + new_module_args.pop('yml_multilines', None) # Content from config_template is converted to src new_module_args.pop('content', None) diff --git a/tests/files/test_multiline_strs.yml.expected b/tests/files/test_multiline_strs.yml.expected new file mode 100644 index 0000000..420c980 --- /dev/null +++ b/tests/files/test_multiline_strs.yml.expected @@ -0,0 +1,14 @@ +existing_multiline: 'This multiline string + + already exists and will + + be dumped out by PyYaml. + + ' +new_multiline_str: 'This should not + + be a list + + ' +test_category1: a +test_category2: b diff --git a/tests/templates/test_multiline_strs.yml b/tests/templates/test_multiline_strs.yml new file mode 100644 index 0000000..1e6877a --- /dev/null +++ b/tests/templates/test_multiline_strs.yml @@ -0,0 +1,6 @@ +test_category1: a +test_category2: b +existing_multiline: | + This multiline string + already exists and will + be dumped out by PyYaml. diff --git a/tests/test-common-tasks.yml b/tests/test-common-tasks.yml index 62ab179..d5b08ee 100644 --- a/tests/test-common-tasks.yml +++ b/tests/test-common-tasks.yml @@ -14,6 +14,7 @@ # limitations under the License. # # Test basic function of config_template + - name: Template test INI template config_template: src: "{{ playbook_dir }}/templates/test.ini" @@ -128,6 +129,28 @@ that: - "((hostvars_file.content | b64decode | from_yaml).test_hostvar) == (test_config_yml_hostvars_overrides.test_hostvar)" +# Values containing newlines should not be chopped into a list +# when yml_multilines is set to True +- name: Test multiline strings in yaml + config_template: + src: "{{ playbook_dir }}/templates/test_multiline_strs.yml" + dest: "/tmp/multiline_strs.yml" + config_overrides: "{{ test_multiline_strs_yml_overrides }}" + config_type: yaml + yml_multilines: True +- name: Read multiline_strs.yml + slurp: + src: /tmp/multiline_strs.yml + register: multiline_strs_file +- debug: + msg: "Multiline Yaml Strings - {{ multiline_strs_file.content | b64decode }}" +- debug: + msg: "Multiline Yaml Strings Expected - {{ multiline_strs_file_expected.content | b64decode }}" +- name: Compare files + assert: + that: + - "(multiline_strs_file_expected.content | b64decode) == (multiline_strs_file.content | b64decode)" + # Test multistropt ordering - name: Template MultiStrOpts using overrides config_template: diff --git a/tests/test.yml b/tests/test.yml index 43eb2ac..ac58306 100644 --- a/tests/test.yml +++ b/tests/test.yml @@ -58,6 +58,11 @@ src: "{{ playbook_dir }}/files/test_content_no_overrides.json.expected" register: content_no_overrides_file_expected + - name: Read expected test_multiline_strs.yml + slurp: + src: "{{ playbook_dir }}/files/test_multiline_strs.yml.expected" + register: multiline_strs_file_expected + - import_tasks: test-common-tasks.yml - import_tasks: test-common-tasks.yml @@ -161,6 +166,10 @@ - 4 test_config_yml_hostvars_overrides: test_hostvar: "{{ ansible_default_ipv4.address }}" + test_multiline_strs_yml_overrides: + new_multiline_str: | + This should not + be a list test_default_section_overrides: global: test2: 2