openstack/ansible-config_template
Code Issues Proposed changes

Enhance the config_template comment parser

Browse Source 
The config_template comment parser will now respect all comments in
INI files, as they are written. This will ensure spacing, paragraphs,
and other comments bits that may be in an INI file remain intact,
even when overriding options and sections. With this feature we'll
now be able to insert options in OSLO config generated files without
truncating or making a mess of the file structure.

This is an internal enhancement and requires no change from the
operator or from within any ansible task. To ensure enhanced comments
are working, new tests have been added which will run though all of
the `config_template` INI file functions using a mock service file.
Existing tests for the old comment structure has been removed.
This includes tests that were running redundant tasks for file
diffs or expected the only style, merged, comment layout. All of the
tests have been broken out into descriptive task files. This was
largely done for readability. It was difficult to see what tests
we had and how I needed to extend them to test the enhanced comments
functionality given the INI type tests were all thoughout the
`tests/test-common-tasks.yml`. Now that the files have been broken
out developers will easily be able to audit our tests making it
simple to extend.

Change-Id: Ia6cdc215c35fa9ac45b718c211616a9887a74e37
Signed-off-by: Kevin Carter <kecarter@redhat.com>
This commit is contained in:
Kevin Carter 2019-07-22 15:10:44 -05:00
parent 0d950310a4
commit 09c76e2380
No known key found for this signature in database
GPG Key ID: CE94BD890A47B20A
15 changed files with 782 additions and 634 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

345
action/config_template.py
View File

@ -22,20 +22,23 @@ try:
except ImportError:
import configparser as ConfigParser
import datetime
try:
from StringIO import StringIO
except ImportError:
from io import StringIO
import base64
import json
import os
import pwd
import re
import six
import time
import yaml
import tempfile as tmpfilelib
from collections import OrderedDict
from ansible.plugins.action import ActionBase
from ansible.module_utils._text import to_bytes, to_text
from ansible.module_utils.parsing.convert_bool import boolean
@ -53,13 +56,15 @@ CONFIG_TYPES = {
'yaml': 'return_config_overrides_yaml'
}
STRIP_MARKER = '__MARKER__'
class IDumper(AnsibleDumper):
def increase_indent(self, flow=False, indentless=False):
return super(IDumper, self).increase_indent(flow, False)
class MultiKeyDict(dict):
class MultiKeyDict(OrderedDict):
"""Dictionary class which supports duplicate keys.
This class allows for an item to be added into a standard python dictionary
however if a key is created more than once the dictionary will convert the
@ -77,6 +82,32 @@ class MultiKeyDict(dict):
... {'a': tuple(['1', '2']), 'c': {'a': 1}, 'b': ['a', 'b', 'c']}
"""
def index(self, key):
index_search = [
i for i, item in enumerate(self) if item.startswith(key)
]
if len(index_search) > 1:
raise SystemError('Index search returned more than one value')
return index_search[0]
def insert(self, index, key, value):
list(self)[index] # Validates the index
shadow = MultiKeyDict()
counter = 0
for k, v in self.items():
if counter == index:
shadow[k] = v
shadow[key] = value
else:
shadow[k] = v
counter += 1
else:
return shadow
def update(self, E=None, **kwargs):
for key, value in E.items():
super(MultiKeyDict, self).__setitem__(key, value)
def __setitem__(self, key, value):
if key in self:
if isinstance(self[key], tuple):
@ -89,7 +120,7 @@ class MultiKeyDict(dict):
items = tuple([str(self[key]), str(value)])
super(MultiKeyDict, self).__setitem__(key, items)
else:
return dict.__setitem__(self, key, value)
return super(MultiKeyDict, self).__setitem__(key, value)
class ConfigTemplateParser(ConfigParser.RawConfigParser):
@ -141,162 +172,144 @@ class ConfigTemplateParser(ConfigParser.RawConfigParser):
"""
def __init__(self, *args, **kwargs):
self._comments = {}
self.ignore_none_type = bool(kwargs.pop('ignore_none_type', True))
self.default_section = str(kwargs.pop('default_section', 'DEFAULT'))
self.yml_multilines = bool(kwargs.pop('yml_multilines', False))
self._comment_prefixes = kwargs.pop('comment_prefixes', '/')
self._empty_lines_in_values = kwargs.get('allow_no_value', True)
self._strict = kwargs.get('strict', False)
self._allow_no_value = self._empty_lines_in_values
ConfigParser.RawConfigParser.__init__(self, *args, **kwargs)
def set(self, section, option, value=None):
if not section or section == 'DEFAULT':
sectdict = self._defaults
use_defaults = True
else:
try:
sectdict = self._sections[section]
except KeyError:
raise SystemError('Section %s not found' % section)
else:
use_defaults = False
option = self.optionxform(option)
try:
index = sectdict.index('#%s' % option)
except (ValueError, IndexError):
sectdict[option] = value
else:
if use_defaults:
self._defaults = sectdict.insert(index, option, value)
else:
self._sections[section] = sectdict.insert(
index,
option,
value
)
def _write(self, fp, section, key, item, entry):
if section:
# If we are not ignoring a none type value, then print out
# the option name only if the value type is None.
if not self.ignore_none_type and item is None:
fp.write(key + '\n')
elif (item is not None) or (self._optcre == self.OPTCRE):
fp.write(entry)
else:
fp.write(entry)
return
fp.write(entry)
def _write_check(self, fp, key, value, section=False):
if isinstance(value, (tuple, set)):
for item in value:
item = str(item).replace('\n', '\n\t')
entry = "%s = %s\n" % (key, item)
self._write(fp, section, key, item, entry)
else:
if isinstance(value, list):
_value = [str(i.replace('\n', '\n\t')) for i in value]
entry = '%s = %s\n' % (key, ','.join(_value))
def _return_entry(option, item):
if item:
return "%s = %s\n" % (option, str(item).replace('\n', '\n\t'))
else:
entry = '%s = %s\n' % (key, str(value).replace('\n', '\n\t'))
return "%s\n" % option
key = key.split(STRIP_MARKER)[0]
if isinstance(value, (tuple, set)):
for i in sorted(value):
entry = _return_entry(option=key, item=i)
self._write(fp, section, key, i, entry)
elif isinstance(value, list):
_value = [str(i.replace('\n', '\n\t')) for i in value]
entry = '%s = %s\n' % (key, ','.join(_value))
self._write(fp, section, key, value, entry)
else:
entry = _return_entry(option=key, item=value)
self._write(fp, section, key, value, entry)
def write(self, fp):
def write(self, fp, **kwargs):
def _do_write(section_name, section, section_bool=False):
_write_comments(section_name)
fp.write("[%s]\n" % section_name)
for key, value in sorted(section.items()):
_write_comments(section_name, optname=key)
self._write_check(fp, key=key, value=value,
section=section_bool)
for key, value in section.items():
self._write_check(
fp,
key=key,
value=value,
section=section_bool
)
else:
fp.write("\n")
def _write_comments(section, optname=None):
comsect = self._comments.get(section, {})
if optname in comsect:
fp.write(''.join(comsect[optname]))
if self.default_section != 'DEFAULT' and self._sections.get(
self.default_section, False):
_do_write(self.default_section,
self._sections[self.default_section],
section_bool=True)
self._sections.pop(self.default_section)
if self._defaults:
if self.default_section != 'DEFAULT':
if not self._sections.get(self.default_section, False):
_do_write(
section_name=self.default_section,
section=self._sections[self.default_section],
section_bool=True
)
elif self._defaults:
_do_write('DEFAULT', self._defaults)
for section in sorted(self._sections):
_do_write(section, self._sections[section], section_bool=True)
for i in self._sections:
_do_write(i, self._sections[i], section_bool=True)
def _read(self, fp, fpname):
comments = []
cursect = None
def _temp_set():
_temp_item = [cursect[optname]]
cursect.update({optname: _temp_item})
optname = None
lineno = 0
e = None
while True:
line = fp.readline()
if not line:
break
lineno += 1
if line.strip() == '':
if comments:
comments.append('')
continue
if line.lstrip()[0] in '#;':
comments.append(line.lstrip())
continue
if line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR":
continue
if line[0].isspace() and cursect is not None and optname:
value = line.strip()
if value:
try:
if isinstance(cursect[optname], (tuple, set)):
_temp_item = list(cursect[optname])
del cursect[optname]
cursect[optname] = _temp_item
elif isinstance(cursect[optname], six.text_type):
_temp_item = [cursect[optname]]
del cursect[optname]
cursect[optname] = _temp_item
except NameError:
if isinstance(cursect[optname], (bytes, str)):
_temp_item = [cursect[optname]]
del cursect[optname]
cursect[optname] = _temp_item
cursect[optname].append(value)
else:
mo = self.SECTCRE.match(line)
if mo:
sectname = mo.group('header')
if sectname in self._sections:
cursect = self._sections[sectname]
elif sectname == 'DEFAULT':
cursect = self._defaults
else:
cursect = self._dict()
self._sections[sectname] = cursect
optname = None
comsect = self._comments.setdefault(sectname, {})
if comments:
# NOTE(flaper87): Using none as the key for
# section level comments
comsect[None] = comments
comments = []
elif cursect is None:
raise ConfigParser.MissingSectionHeaderError(
fpname,
lineno,
line
)
cursect = {}
marker_counter = 0
for lineno, line in enumerate(fp, start=0):
marker_counter += 1
mo_match = self.SECTCRE.match(line)
mo_optcre = self._optcre.match(line)
if mo_match:
sectname = mo_match.group('header')
if sectname in self._sections:
cursect = self._sections[sectname]
elif sectname == 'DEFAULT':
cursect = self._defaults
else:
mo = self._optcre.match(line)
if mo:
optname, vi, optval = mo.group('option', 'vi', 'value')
optname = self.optionxform(optname.rstrip())
if optval is not None:
if vi in ('=', ':') and ';' in optval:
pos = optval.find(';')
if pos != -1 and optval[pos - 1].isspace():
optval = optval[:pos]
optval = optval.strip()
if optval == '""':
optval = ''
cursect[optname] = optval
if comments:
comsect[optname] = comments
comments = []
else:
if not e:
e = ConfigParser.ParsingError(fpname)
e.append(lineno, repr(line))
if e:
raise e
all_sections = [self._defaults]
all_sections.extend(self._sections.values())
for options in all_sections:
for name, val in options.items():
if isinstance(val, list):
_temp_item = '\n'.join(val)
del options[name]
options[name] = _temp_item
cursect = self._dict()
self._sections[sectname] = cursect
elif mo_optcre:
optname, vi, optval = mo_optcre.group('option', 'vi', 'value')
optname = self.optionxform(optname.rstrip())
if optname and not optname.startswith('#') and optval:
if vi in ('=', ':') and ';' in optval:
pos = optval.find(';')
if pos != -1 and optval[pos - 1].isspace():
optval = optval[:pos]
optval = optval.strip()
if optval == '""':
optval = ''
else:
optname = '%s%s-%d' % (
optname,
STRIP_MARKER,
marker_counter
)
cursect[optname] = optval
else:
optname = '%s-%d' % (
STRIP_MARKER,
marker_counter
)
cursect[optname] = None
class DictCompare(object):
@ -322,6 +335,7 @@ class DictCompare(object):
... {'test1': {'current_val': 'vol1', 'new_val': 'val2'}
... }
"""
def __init__(self, base_dict, new_dict):
self.new_dict, self.base_dict = new_dict, base_dict
self.base_items, self.new_items = set(
@ -408,6 +422,15 @@ class ActionModule(ActionBase):
:param resultant: ``str`` || ``unicode``
:returns: ``str``, ``dict``
"""
def _add_section(section_name):
# Attempt to add a section to the config file passing if
# an error is raised that is related to the section
# already existing.
try:
config.add_section(section_name)
except (ConfigParser.DuplicateSectionError, ValueError):
pass
# If there is an exception loading the RawConfigParser The config obj
# is loaded again without the extra option. This is being done to
# support older python.
@ -417,14 +440,25 @@ class ActionModule(ActionBase):
dict_type=MultiKeyDict,
ignore_none_type=ignore_none_type,
default_section=default_section,
yml_multilines=yml_multilines
yml_multilines=yml_multilines,
comment_prefixes='/'
)
config.optionxform = str
except Exception:
config = ConfigTemplateParser(dict_type=MultiKeyDict)
config = ConfigTemplateParser(
allow_no_value=True,
dict_type=MultiKeyDict,
comment_prefixes='/'
)
config_object = StringIO(resultant)
config.readfp(config_object)
try:
config.read_file(config_object)
except AttributeError:
config.readfp(config_object)
if default_section != 'DEFAULT':
_add_section(section_name=default_section)
for section, items in config_overrides.items():
# If the items value is not a dictionary it is assumed that the
@ -432,20 +466,15 @@ class ActionModule(ActionBase):
if not isinstance(items, dict):
if isinstance(items, list):
items = ','.join(to_text(i) for i in items)
self._option_write(
config,
'DEFAULT',
default_section,
section,
items
)
else:
# Attempt to add a section to the config file passing if
# an error is raised that is related to the section
# already existing.
try:
config.add_section(section)
except (ConfigParser.DuplicateSectionError, ValueError):
pass
_add_section(section_name=section)
for key, value in items.items():
try:
self._option_write(config, section, key, value)
@ -459,10 +488,10 @@ class ActionModule(ActionBase):
else:
config_object.close()
config_dict_new = {}
config_dict_new = OrderedDict()
config_defaults = config.defaults()
for s in config.sections():
config_dict_new[s] = {}
config_dict_new[s] = OrderedDict()
for k, v in config.items(s):
if k not in config_defaults or config_defaults[k] != v:
config_dict_new[s][k] = v
@ -571,7 +600,7 @@ class ActionModule(ActionBase):
)
elif (not isinstance(value, int) and
(',' in value or
('\n' in value and not yml_multilines))):
('\n' in value and not yml_multilines))):
base_items[key] = re.split(',|\n', value)
base_items[key] = [i.strip() for i in base_items[key] if i]
elif isinstance(value, list):
@ -702,6 +731,21 @@ class ActionModule(ActionBase):
remote_src=remote_src
)
def resultant_ini_as_dict(self, resultant_dict, return_dict=None):
if not return_dict:
return_dict = {}
for key, value in resultant_dict.items():
if not value:
continue
key = key.split(STRIP_MARKER)[0]
if isinstance(value, (OrderedDict, MultiKeyDict, dict)):
return_dict[key] = self.resultant_ini_as_dict(value)
else:
return_dict[key] = value
return return_dict
def run(self, tmp=None, task_vars=None):
"""Run the method"""
@ -767,7 +811,6 @@ class ActionModule(ActionBase):
self._templar._available_variables
)
config_dict_base = {}
type_merger = getattr(self, CONFIG_TYPES.get(_vars['config_type']))
resultant, config_dict_base = type_merger(
config_overrides=_vars['config_overrides'],
@ -785,8 +828,7 @@ class ActionModule(ActionBase):
module_args=dict(src=_vars['dest']),
task_vars=task_vars
)
config_dict_new = {}
config_dict_new = dict()
if 'content' in slurpee:
dest_data = base64.b64decode(
slurpee['content']).decode('utf-8')
@ -809,7 +851,10 @@ class ActionModule(ActionBase):
# Compare source+overrides with dest to look for changes and
# build diff
cmp_dicts = DictCompare(config_dict_new, config_dict_base)
cmp_dicts = DictCompare(
self.resultant_ini_as_dict(resultant_dict=config_dict_new),
self.resultant_ini_as_dict(resultant_dict=config_dict_base)
)
mods, changed = cmp_dicts.get_changes()
# Re-template the resultant object as it may have new data within it

8
releasenotes/notes/enhance-comment-parser-6fcb40646cdad662.yaml Normal file
View File

@ -0,0 +1,8 @@
---
features:
- Config template comment parser will now respect all comments and spacing
throughout INI files. This will allow us to use OSLO config to generate
files which contain commentary about the various defaults. This enhancement
will allow operators to benefit from deployer comments and system commentary
all from within the on-disk files while keeping all of the flexibility the
`config_template` action plugin provides.

8
releasenotes/notes/enhance-option-insert-412e9032d8d6cb86.yaml Normal file
View File

@ -0,0 +1,8 @@
---
features:
- The `config_template` action plugin will now search for options within a
given section that may be commented within an INI file, and if a an option is
following the OSLO config pattern, '#OPTION_KEY...' pattern, `config_template`
will insert the override one line after the comment. This provides operators
the ability to see in service options and any comments regarding the option
within the configuration file on-disk.

46
tests/files/test_comment_configs.ini.expected Normal file
View File

@ -0,0 +1,46 @@
[DEFAULT]
#
# From nova.conf
#
#
# Availability zone for internal services. For more information, refer to the
# documentation. (string value)
#internal_service_availability_zone = internal
#
# Default availability zone for compute services. For more information, refer to
# the documentation. (string value)
#default_availability_zone = nova
default_availability_zone = zone1
#
# Default availability zone for instances. For more information, refer to the
# documentation. (string value)
#default_schedule_zone = <None>
# Length of generated instance admin passwords (integer value)
# Minimum value = 0
#password_length = 12
password_length = 100
#
# Time period to generate instance usages for. It is possible to define optional
# offset to given period by appending @ character followed by a number defining
# offset. For more information, refer to the documentation. (string value)
#instance_usage_audit_period = month
instance_usage_audit_period = blah blah blah
test = test1,test2
[SubSection]
#Comments and overrides in a subsection
#testopt1 = 9000
testopt1 = 9000
# This is another test opt
#testop2 = over 9000
[TestSection]
things = stuff

3
tests/files/test_default_section.ini.expected
View File

@ -1,8 +1,9 @@
[global]
test1 = 1
# This is a post option comment
test2 = 2
[section1]
setting1 = 1
setting2 = 2

6
tests/files/test_remote_src_multistropts.ini.expected
View File

@ -3,8 +3,10 @@ test = test1
test = test2
test = test3
[remote_src_section]
test = output
[testsection]
test = output
[remote_src_section]
test = output

38
tests/templates/test_comment_configs.ini Normal file
View File

@ -0,0 +1,38 @@
[DEFAULT]
#
# From nova.conf
#
#
# Availability zone for internal services. For more information, refer to the
# documentation. (string value)
#internal_service_availability_zone = internal
#
# Default availability zone for compute services. For more information, refer to
# the documentation. (string value)
#default_availability_zone = nova
#
# Default availability zone for instances. For more information, refer to the
# documentation. (string value)
#default_schedule_zone = <None>
# Length of generated instance admin passwords (integer value)
# Minimum value: 0
#password_length = 12
#
# Time period to generate instance usages for. It is possible to define optional
# offset to given period by appending @ character followed by a number defining
# offset. For more information, refer to the documentation. (string value)
#instance_usage_audit_period = month
[SubSection]
#Comments and overrides in a subsection
#testopt1 = 9000
testopt1 = 9000
# This is another test opt
#testop2 = over 9000

9
tests/templates/test_default_section.ini
View File

@ -1,5 +1,6 @@
[section1]
setting1=1
[global]
test1=1
test1 = 1
# This is a post option comment
[section1]
setting1 = 1

1
tests/templates/test_multistropts.ini
View File

@ -2,4 +2,3 @@
test = test1
test = test2
test = test3

16
tests/templates/test_with_comments.ini
View File

@ -1,16 +0,0 @@
# This comment tests bug 1755821
# A default section comment
# broken into multiple lines
[DEFAULT]
# This tests the py3 unicode bug #1763422
test_hosts =
+_unicode
1
string
[foo]
#This is a comment
baz = baz
[bar]

344
tests/test-common-tasks.yml
View File

@ -15,344 +15,6 @@
#
# Test basic function of config_template
- name: Template test INI template
config_template:
src: "{{ playbook_dir }}/templates/test.ini"
dest: "/tmp/test.ini"
config_overrides: "{{ test_config_ini_overrides }}"
config_type: "ini"
register: test_ini
notify: test_ini check diff
- name: Read test.ini
slurp:
src: /tmp/test.ini
register: ini_file
- debug:
msg: "ini - {{ ini_file.content | b64decode }}"
- name: Validate output
assert:
that:
- "(lookup('ini', 'new_key section=DEFAULT file=/tmp/test.ini')) == 'new_value'"
- "(lookup('ini', 'baz section=foo file=/tmp/test.ini')) == 'bar'"
# Test basic function of config_template with content instead of src
- name: Template test INI template
config_template:
content: "{{ lookup('file', playbook_dir + '/templates/test.ini') }}"
dest: "/tmp/test_with_content.ini"
config_overrides: "{{ test_config_ini_overrides }}"
config_type: "ini"
register: test_with_content_ini
notify: test_with_content_ini check diff
- name: Read test.ini
slurp:
src: /tmp/test_with_content.ini
register: ini_file_with_content
- debug:
msg: "ini - {{ ini_file_with_content.content | b64decode }}"
- name: Validate output
assert:
that:
- "(lookup('ini', 'new_key section=DEFAULT file=/tmp/test_with_content.ini')) == 'new_value'"
- "(lookup('ini', 'baz section=foo file=/tmp/test_with_content.ini')) == 'bar'"
# Test list additions in config_template
- name: Template test YML template
config_template:
src: "{{ playbook_dir }}/templates/test.yml"
dest: "/tmp/test_extend.yml"
config_overrides: "{{ test_config_yml_overrides }}"
config_type: "yaml"
list_extend: True
register: test_extend_yml
notify: test_extend_yml check diff
- name: Read test_extend.yml
slurp:
src: /tmp/test_extend.yml
register: extend_file
- debug:
msg: "extend - {{ extend_file.content | b64decode }}"
- debug:
msg: "extend.expected - {{ extend_file_expected.content | b64decode }}"
- name: Compare files
assert:
that:
- "(extend_file.content | b64decode) == (extend_file_expected.content | b64decode)"
# Test list replacement in config_template
- name: Template test YML template
config_template:
src: "{{ playbook_dir }}/templates/test.yml"
dest: "/tmp/test_no_extend.yml"
config_overrides: "{{ test_config_yml_overrides }}"
config_type: "yaml"
list_extend: False
register: test_no_extend_yml
notify: test_no_extend_yml check diff
- name: Read test_no_extend.yml
slurp:
src: /tmp/test_no_extend.yml
register: no_extend_file
- debug:
msg: "no_extend - {{ no_extend_file.content | b64decode }}"
- debug:
msg: "no_extend.expected - {{ no_extend_file_expected.content | b64decode }}"
- name: Compare files
assert:
that:
- "(no_extend_file.content | b64decode) == (no_extend_file_expected.content | b64decode)"
# Test dumping hostvars using config overrides
- name: Template test YML template with hostvars override
config_template:
src: "{{ playbook_dir }}/templates/test.yml"
dest: "/tmp/test_hostvars.yml"
config_overrides: "{{ test_config_yml_hostvars_overrides }}"
config_type: "yaml"
register: test_hostvars_yml
notify: test_hostvars_yml check diff
- name: Read test_hostvars.yml
slurp:
src: /tmp/test_hostvars.yml
register: hostvars_file
- debug:
msg: "hostvars - {{ (hostvars_file.content | b64decode | from_yaml).test_hostvar }}"
- debug:
msg: "hostvars.expected - {{ test_config_yml_hostvars_overrides.test_hostvar }}"
- name: Compare files
assert:
that:
- "((hostvars_file.content | b64decode | from_yaml).test_hostvar) == (test_config_yml_hostvars_overrides.test_hostvar)"
# Values containing newlines should not be chopped into a list
# when yml_multilines is set to True
- name: Test multiline strings in yaml
config_template:
src: "{{ playbook_dir }}/templates/test_multiline_strs.yml"
dest: "/tmp/multiline_strs.yml"
config_overrides: "{{ test_multiline_strs_yml_overrides }}"
config_type: yaml
yml_multilines: True
- name: Read multiline_strs.yml
slurp:
src: /tmp/multiline_strs.yml
register: multiline_strs_file
- debug:
msg: "Multiline Yaml Strings - {{ multiline_strs_file.content | b64decode }}"
- debug:
msg: "Multiline Yaml Strings Expected - {{ multiline_strs_file_expected.content | b64decode }}"
- name: Compare files
assert:
that:
- "(multiline_strs_file_expected.content | b64decode) == (multiline_strs_file.content | b64decode)"
# Test multistropt ordering
- name: Template MultiStrOpts using overrides
config_template:
src: test_multistropts.ini
dest: /tmp/test_multistropts.ini
config_overrides:
testsection:
test: output
config_type: ini
- name: Create expected MultiStrOpts file
copy:
src: files/test_multistropts.ini.expected
dest: /tmp/test_multistropts.ini.expected
- name: Read test_multistropts.ini
slurp:
src: /tmp/test_multistropts.ini
register: multistropts_file
- name: Read test_multistropts.ini.expected
slurp:
src: /tmp/test_multistropts.ini.expected
register: multistropts_expected_file
- name: Set content facts
set_fact:
_multistropts_file: "{{ (multistropts_file.content | b64decode).strip() }}"
_multistropts_expected_file: "{{ (multistropts_expected_file.content | b64decode).strip() }}"
- name: Show rendered file
debug:
msg: "multistropts rendered - {{ _multistropts_file }}"
- name: Show expected file
debug:
msg: "multistropts expected - {{ _multistropts_expected_file }}"
- name: Compare files
assert:
that:
- _multistropts_file == _multistropts_expected_file
# Test remote_src
- name: Template remote source using overrides
config_template:
src: /tmp/test_multistropts.ini
dest: /tmp/test_remote_src_multistropts.ini
remote_src: true
config_overrides:
remote_src_section:
test: output
config_type: ini
- name: Create expected MultiStrOpts file
copy:
src: files/test_remote_src_multistropts.ini.expected
dest: /tmp/test_remote_src_multistropts.ini.expected
- name: Read test_remote_src_multistropts.ini
slurp:
src: /tmp/test_remote_src_multistropts.ini
register: multistropts_file
- name: Read test_remote_src_multistropts.ini.expected
slurp:
src: /tmp/test_remote_src_multistropts.ini.expected
register: multistropts_expected_file
- name: Set content facts
set_fact:
_remote_src_file: "{{ (multistropts_file.content | b64decode).strip() }}"
_remote_src_expected_file: "{{ (multistropts_expected_file.content | b64decode).strip() }}"
- name: Show rendered file
debug:
msg: "multistropts rendered - {{ _remote_src_file }}"
- name: Show expected file
debug:
msg: "multistropts expected - {{ _remote_src_expected_file }}"
- name: Compare files
assert:
that:
- _remote_src_file == _remote_src_expected_file
# Test content attribute with a dictionary input and config_type equal to 'json'
- name: Template test JSON template with content attribute
config_template:
dest: "/tmp/test_content_no_overrides.json"
config_overrides: {}
config_type: "json"
content: "{{ lookup('file', playbook_dir ~ '/templates/test.json') | from_json }}"
register: test_content_no_overrides_json
notify: test_content_no_overrides_json check diff
- name: Read test_content_no_overrides.json
slurp:
src: /tmp/test_content_no_overrides.json
register: content_no_overrides_file
- debug:
msg: "content_no_overrides.json - {{ content_no_overrides_file.content | b64decode | from_json }}"
- debug:
msg: "content_no_overrides.json.expected - {{ content_no_overrides_file_expected.content | b64decode | from_json }}"
# NOTE (alextricity25): The config_template module doesn't use ordered dicts when reading and writing json
# data, so we can't guarantee that the string literal of both file's content will be the same. Instead, we compare
# the content after transforming it into a dictionary.
- name: Compare file content
assert:
that:
- "(content_no_overrides_file.content | b64decode | from_json) == (content_no_overrides_file_expected.content | b64decode | from_json)"
# Test the ignore_none_type attribute when set to False
- name: Template test with ignore_none_type set to false
config_template:
src: "{{ playbook_dir }}/templates/test_ignore_none_type.ini"
dest: "/tmp/test_ignore_none_type.ini"
config_overrides: "{{ test_config_ini_overrides }}"
config_type: "ini"
ignore_none_type: False
register: test_ignore_none_type_ini
notify: test_ignore_none_type_ini check diff
- name: Read test_ignore_none_type.ini
slurp:
src: /tmp/test_ignore_none_type.ini
register: test_ignore_none_type
- debug:
msg: "test_ignore_none_type.ini - {{ test_ignore_none_type.content | b64decode }}"
- name: Validate output has valueless options printed out
assert:
that:
- "{{ test_ignore_none_type.content | b64decode | search('(?m)^india$') }}"
- "{{ test_ignore_none_type.content | b64decode | search('(?m)^juliett kilo$') }}"
# Test basic function of config_template
- name: Template test INI comments
config_template:
src: "{{ playbook_dir }}/templates/test_with_comments.ini"
dest: "/tmp/test_with_comments.ini"
config_overrides: "{{ test_config_ini_overrides }}"
config_type: "ini"
tags: test
register: test_with_comments_ini
notify: test_with_comments_ini check diff
- name: Read test.ini
slurp:
src: /tmp/test_with_comments.ini
register: ini_file
tags: test
- debug:
msg: "ini - {{ ini_file.content | b64decode }}"
- name: Validate output
tags: test
assert:
that:
- "(lookup('ini', 'new_key section=DEFAULT file=/tmp/test_with_comments.ini')) == 'new_value'"
- "(lookup('ini', 'baz section=foo file=/tmp/test_with_comments.ini')) == 'bar'"
- "{{ ini_file.content | b64decode | search('# This comment tests bug 1755821')}}"
- "{{ not(ini_file.content | b64decode | search(' # This comment tests bug 1755821'))}}"
- "{{ ini_file.content | b64decode | search('#This is a comment')}}"
- "{{ ini_file.content | b64decode | search('# A default section comment\n# broken into multiple lines\n\\[DEFAULT\\]')}}"
- name: Template multiple times to assert no changes
config_template:
src: "{{ playbook_dir }}/templates/test_with_comments.ini"
dest: "/tmp/test_with_comments.ini"
config_type: "ini"
config_overrides: "{{ item[1] }}"
register: template_changed
failed_when: template_changed is changed
with_nested:
- [ 0, 1, 2 ]
- [ "{{ test_config_ini_overrides }}" ]
- name: Put down default_section_expected file
copy:
src: "{{ playbook_dir }}/files/test_default_section.ini.expected"
dest: "/tmp/test_default_section.ini"
- name: Template using default_section
config_template:
src: "{{ playbook_dir }}/templates/test_default_section.ini"
dest: "/tmp/test_default_section.ini"
config_type: "ini"
config_overrides: "{{ test_default_section_overrides }}"
default_section: "global"
register: template_changed
failed_when: template_changed is changed
- name: Write ini for testing diff output
config_template:
src: "{{ playbook_dir }}/templates/test_diff.ini"
dest: "/tmp/test_diff.ini"
config_type: "ini"
config_overrides: {}
- name: Test ini with additions and changed
config_template:
src: "{{ playbook_dir }}/templates/test_diff.ini"
dest: "/tmp/test_diff.ini"
config_type: "ini"
config_overrides: "{{ test_diff_overrides }}"
register: test_diff_ini
notify: test_diff_ini check diff
- name: Test ini with removes
config_template:
src: "{{ playbook_dir }}/templates/test_diff_remove.ini"
dest: "/tmp/test_diff.ini"
config_type: "ini"
config_overrides: "{{ test_diff_overrides }}"
register: test_diff_remove_ini
notify: test_diff_remove_ini check diff
- import_tasks: test-ini.yml
- import_tasks: test-yaml.yml
- import_tasks: test-json.yml

286
tests/test-ini.yml Normal file
View File

@ -0,0 +1,286 @@
---
# Copyright 2018, Rackspace US
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Test basic function of config_template
# Test basic ini template
- name: Template test INI template
config_template:
src: "{{ playbook_dir }}/templates/test.ini"
dest: "/tmp/test.ini"
config_overrides: "{{ test_config_ini_overrides }}"
config_type: "ini"
- name: Read test.ini
slurp:
src: /tmp/test.ini
register: ini_file
- debug:
msg: "ini - {{ ini_file.content | b64decode }}"
- name: Validate output
assert:
that:
- "(lookup('ini', 'new_key section=DEFAULT file=/tmp/test.ini')) == 'new_value'"
- "(lookup('ini', 'baz section=foo file=/tmp/test.ini')) == 'bar'"
# Test basic function of config_template with content instead of src
- name: Template test INI template
config_template:
content: "{{ lookup('file', playbook_dir + '/templates/test.ini') }}"
dest: "/tmp/test_with_content.ini"
config_overrides: "{{ test_config_ini_overrides }}"
config_type: "ini"
- name: Read test.ini
slurp:
src: /tmp/test_with_content.ini
register: ini_file_with_content
- debug:
msg: "ini - {{ ini_file_with_content.content | b64decode }}"
- name: Validate output
assert:
that:
- "(lookup('ini', 'new_key section=DEFAULT file=/tmp/test_with_content.ini')) == 'new_value'"
- "(lookup('ini', 'baz section=foo file=/tmp/test_with_content.ini')) == 'bar'"
# Test multistropt ordering
- name: Template MultiStrOpts using overrides
config_template:
src: test_multistropts.ini
dest: /tmp/test_multistropts.ini
config_overrides:
testsection:
test: output
config_type: ini
- name: Create expected MultiStrOpts file
copy:
src: files/test_multistropts.ini.expected
dest: /tmp/test_multistropts.ini.expected
- name: Read test_multistropts.ini
slurp:
src: /tmp/test_multistropts.ini
register: multistropts_file
- name: Read test_multistropts.ini.expected
slurp:
src: /tmp/test_multistropts.ini.expected
register: multistropts_expected_file
- name: Set content facts
set_fact:
_multistropts_file: "{{ (multistropts_file.content | b64decode).strip() }}"
_multistropts_expected_file: "{{ (multistropts_expected_file.content | b64decode).strip() }}"
- name: Show rendered file
debug:
msg: "multistropts rendered - {{ _multistropts_file }}"
- name: Show expected file
debug:
msg: "multistropts expected - {{ _multistropts_expected_file }}"
- name: Compare files
assert:
that:
- _multistropts_file == _multistropts_expected_file
# Test remote_src
- name: Template remote source using overrides
config_template:
src: /tmp/test_multistropts.ini
dest: /tmp/test_remote_src_multistropts.ini
remote_src: true
config_overrides:
remote_src_section:
test: output
config_type: ini
- name: Create expected MultiStrOpts file
copy:
src: files/test_remote_src_multistropts.ini.expected
dest: /tmp/test_remote_src_multistropts.ini.expected
- name: Read test_remote_src_multistropts.ini
slurp:
src: /tmp/test_remote_src_multistropts.ini
register: multistropts_file
- name: Read test_remote_src_multistropts.ini.expected
slurp:
src: /tmp/test_remote_src_multistropts.ini.expected
register: multistropts_expected_file
- name: Set content facts
set_fact:
_remote_src_file: "{{ (multistropts_file.content | b64decode).strip() }}"
_remote_src_expected_file: "{{ (multistropts_expected_file.content | b64decode).strip() }}"
- name: Show rendered file
debug:
msg: "multistropts rendered - {{ _remote_src_file }}"
- name: Show expected file
debug:
msg: "multistropts expected - {{ _remote_src_expected_file }}"
- name: Compare files
assert:
that:
- _remote_src_file == _remote_src_expected_file
# Test the ignore_none_type attribute when set to False
- name: Template test with ignore_none_type set to false
config_template:
src: "{{ playbook_dir }}/templates/test_ignore_none_type.ini"
dest: "/tmp/test_ignore_none_type.ini"
config_overrides: "{{ test_config_ini_overrides }}"
config_type: "ini"
ignore_none_type: False
- name: Read test_ignore_none_type.ini
slurp:
src: /tmp/test_ignore_none_type.ini
register: test_ignore_none_type
- debug:
msg: "test_ignore_none_type.ini - {{ test_ignore_none_type.content | b64decode }}"
- name: Validate output has valueless options printed out
assert:
that:
- "{{ test_ignore_none_type.content | b64decode | search('(?m)^india$') }}"
- "{{ test_ignore_none_type.content | b64decode | search('(?m)^juliett kilo$') }}"
# Test enhanced comments
- name: Template test INI template
config_template:
content: "{{ lookup('file', playbook_dir + '/templates/test_comment_configs.ini') }}"
dest: "/tmp/test_comment_configs.ini"
config_overrides: "{{ test_enhanced_comments_ini_overrides }}"
config_type: "ini"
- name: Create expected enhanced comments file
copy:
src: files/test_comment_configs.ini.expected
dest: /tmp/test_comment_configs.ini.expected
- name: Read test_comment_configs.ini
slurp:
src: /tmp/test_comment_configs.ini
register: test_comment_configs
- name: Read test_comment_configs.ini.expected
slurp:
src: /tmp/test_comment_configs.ini.expected
register: test_comment_configs_expected
- name: Set content facts
set_fact:
_enhanced_comments_file: "{{ (test_comment_configs.content | b64decode).strip() }}"
_enhanced_comments_expected_file: "{{ (test_comment_configs_expected.content | b64decode).strip() }}"
- name: Show rendered file
debug:
msg: "multistropts rendered - {{ _enhanced_comments_file }}"
- name: Show expected file
debug:
msg: "multistropts expected - {{ _enhanced_comments_expected_file }}"
- name: Compare files
assert:
that:
- _enhanced_comments_file == _enhanced_comments_expected_file
# Test setting a default_section
- name: Template using default_section
config_template:
src: "{{ playbook_dir }}/templates/test_default_section.ini"
dest: "/tmp/test_default_section.ini"
config_type: "ini"
config_overrides: "{{ test_default_section_overrides }}"
default_section: "global"
- name: Put down default_section_expected file
copy:
src: "{{ playbook_dir }}/files/test_default_section.ini.expected"
dest: "/tmp/test_default_section.ini.expected"
- name: Read test_default_section.ini
slurp:
src: "/tmp/test_default_section.ini"
register: test_default_section
- name: Read test_default_section.ini.expected
slurp:
src: "/tmp/test_default_section.ini.expected"
register: test_default_section_expected
- name: Set content facts
set_fact:
_test_default_section_file: "{{ (test_default_section.content | b64decode).strip() }}"
_test_default_section_expected_file: "{{ (test_default_section_expected.content | b64decode).strip() }}"
- name: Show rendered file
debug:
msg: "default rendered - {{ _test_default_section_file }}"
- name: Show expected file
debug:
msg: "default expected - {{ _test_default_section_expected_file }}"
- name: Compare files
assert:
that:
- _test_default_section_file == _test_default_section_expected_file
# Check output diff
- name: Write ini for testing diff output
config_template:
src: "{{ playbook_dir }}/templates/test_diff.ini"
dest: "/tmp/test_diff.ini"
config_type: "ini"
config_overrides: {}
- name: Test ini with additions and changed
config_template:
src: "{{ playbook_dir }}/templates/test_diff.ini"
dest: "/tmp/test_diff.ini"
config_type: "ini"
config_overrides: "{{ test_diff_overrides }}"
register: test_diff_ini
notify: test_diff_ini check diff
- name: Test ini with removes
config_template:
src: "{{ playbook_dir }}/templates/test_diff_remove.ini"
dest: "/tmp/test_diff.ini"
config_type: "ini"
config_overrides: "{{ test_diff_overrides }}"
register: test_diff_remove_ini
notify: test_diff_remove_ini check diff

45
tests/test-json.yml Normal file
View File

@ -0,0 +1,45 @@
---
# Copyright 2018, Rackspace US
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Test basic function of config_template
# Test content attribute with a dictionary input and config_type equal to 'json'
- name: Template test JSON template with content attribute
config_template:
dest: "/tmp/test_content_no_overrides.json"
config_overrides: {}
config_type: "json"
content: "{{ lookup('file', playbook_dir ~ '/templates/test.json') | from_json }}"
register: test_content_no_overrides_json
notify: test_content_no_overrides_json check diff
- name: Read test_content_no_overrides.json
slurp:
src: /tmp/test_content_no_overrides.json
register: content_no_overrides_file
- debug:
msg: "content_no_overrides.json - {{ content_no_overrides_file.content | b64decode | from_json }}"
- debug:
msg: "content_no_overrides.json.expected - {{ content_no_overrides_file_expected.content | b64decode | from_json }}"
# NOTE (alextricity25): The config_template module doesn't use ordered dicts when reading and writing json
# data, so we can't guarantee that the string literal of both file's content will be the same. Instead, we compare
# the content after transforming it into a dictionary.
- name: Compare file content
assert:
that:
- "(content_no_overrides_file.content | b64decode | from_json) == (content_no_overrides_file_expected.content | b64decode | from_json)"

125
tests/test-yaml.yml Normal file
View File

@ -0,0 +1,125 @@
---
# Copyright 2018, Rackspace US
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Test basic function of config_template
# Test list additions in config_template
- name: Template test YML template
config_template:
src: "{{ playbook_dir }}/templates/test.yml"
dest: "/tmp/test_extend.yml"
config_overrides: "{{ test_config_yml_overrides }}"
config_type: "yaml"
list_extend: True
register: test_extend_yml
notify: test_extend_yml check diff
- name: Read test_extend.yml
slurp:
src: /tmp/test_extend.yml
register: extend_file
- debug:
msg: "extend - {{ extend_file.content | b64decode }}"
- debug:
msg: "extend.expected - {{ extend_file_expected.content | b64decode }}"
- name: Compare files
assert:
that:
- "(extend_file.content | b64decode) == (extend_file_expected.content | b64decode)"
# Test list replacement in config_template
- name: Template test YML template
config_template:
src: "{{ playbook_dir }}/templates/test.yml"
dest: "/tmp/test_no_extend.yml"
config_overrides: "{{ test_config_yml_overrides }}"
config_type: "yaml"
list_extend: False
register: test_no_extend_yml
notify: test_no_extend_yml check diff
- name: Read test_no_extend.yml
slurp:
src: /tmp/test_no_extend.yml
register: no_extend_file
- debug:
msg: "no_extend - {{ no_extend_file.content | b64decode }}"
- debug:
msg: "no_extend.expected - {{ no_extend_file_expected.content | b64decode }}"
- name: Compare files
assert:
that:
- "(no_extend_file.content | b64decode) == (no_extend_file_expected.content | b64decode)"
# Test dumping hostvars using config overrides
- name: Template test YML template with hostvars override
config_template:
src: "{{ playbook_dir }}/templates/test.yml"
dest: "/tmp/test_hostvars.yml"
config_overrides: "{{ test_config_yml_hostvars_overrides }}"
config_type: "yaml"
register: test_hostvars_yml
notify: test_hostvars_yml check diff
- name: Read test_hostvars.yml
slurp:
src: /tmp/test_hostvars.yml
register: hostvars_file
- debug:
msg: "hostvars - {{ (hostvars_file.content | b64decode | from_yaml).test_hostvar }}"
- debug:
msg: "hostvars.expected - {{ test_config_yml_hostvars_overrides.test_hostvar }}"
- name: Compare files
assert:
that:
- "((hostvars_file.content | b64decode | from_yaml).test_hostvar) == (test_config_yml_hostvars_overrides.test_hostvar)"
# Values containing newlines should not be chopped into a list
# when yml_multilines is set to True
- name: Test multiline strings in yaml
config_template:
src: "{{ playbook_dir }}/templates/test_multiline_strs.yml"
dest: "/tmp/multiline_strs.yml"
config_overrides: "{{ test_multiline_strs_yml_overrides }}"
config_type: yaml
yml_multilines: True
- name: Read multiline_strs.yml
slurp:
src: /tmp/multiline_strs.yml
register: multiline_strs_file
- debug:
msg: "Multiline Yaml Strings - {{ multiline_strs_file.content | b64decode }}"
- debug:
msg: "Multiline Yaml Strings Expected - {{ multiline_strs_file_expected.content | b64decode }}"
- name: Compare files
assert:
that:
- "(multiline_strs_file_expected.content | b64decode) == (multiline_strs_file.content | b64decode)"

134
tests/test.yml
View File

@ -69,16 +69,6 @@
delegate_to: container1
handlers:
- name: test_ini check diff
assert:
that:
- test_ini.diff[0].prepared|from_json == diff_ini
- name: test_with_content_ini check diff
assert:
that:
- test_with_content_ini.diff[0].prepared|from_json == diff_ini
- name: test_extend_yml check diff
assert:
that:
@ -99,17 +89,6 @@
that:
- test_content_no_overrides_json.diff[0].prepared|from_json == diff_content_no_overrides_json
- name: test_ignore_none_type_ini check diff
assert:
that:
- test_ignore_none_type_ini.diff[0].prepared|from_json == diff_ignore_none_type_ini
- name: test_with_comments_ini check diff
tags: test
assert:
that:
- test_with_comments_ini.diff[0].prepared|from_json == diff_with_comments_ini
- name: test_diff_ini check diff
tags: test
assert:
@ -130,6 +109,8 @@
baz: "bar"
section1:
key1: "String1"
key10: 10
key11: 11
key2: "string2"
key3: "string3"
key4: "string4"
@ -138,8 +119,10 @@
key7: 1
key8: 2
key9: 3
key10: 10
key11: 11
section10:
key1: 1
section11:
key1: 1
section2:
key1: "value1"
section3:
@ -156,10 +139,6 @@
key1: 1
section9:
key1: 1
section10:
key1: 1
section11:
key1: 1
test_config_yml_overrides:
list_one:
- four
@ -180,53 +159,20 @@
baz: "hotel"
section3:
alfa: "bravo"
diff_with_comments_ini:
added:
DEFAULT:
new_key: "new_value"
test_hosts: "\n+_unicode\n1\nstring"
bar: {}
foo:
baz: "bar"
section1:
key1: "String1"
key10: "10"
key11: "11"
key2: "string2"
key3: "string3"
key4: "string4"
key5: "string5"
key6: "string6"
key7: "1"
key8: "2"
key9: "3"
section10:
key1: "1"
section11:
key1: "1"
section2:
key1: "value1"
section3:
key1: "value1"
section4:
key1: "value1"
section5:
key1: "value1"
section6:
key1: "value1"
section7:
key1: "value1"
section8:
key1: "1"
section9:
key1: "1"
changed: {}
removed: {}
test_enhanced_comments_ini_overrides:
DEFAULT:
default_availability_zone: zone1
instance_usage_audit_period: blah blah blah
password_length: 100
test:
- test1
- test2
TestSection:
things: stuff
diff_ini:
added:
DEFAULT:
new_key: "new_value"
bar: {}
foo:
baz: "bar"
section1:
@ -298,54 +244,6 @@
test_hostvar: "{{ ansible_default_ipv4.address }}"
changed: {}
removed: {}
diff_ignore_none_type_ini:
added:
DEFAULT:
new_key: "new_value"
alfa:
bravo: "charlie"
delta: "echo"
foo:
baz: "bar"
foxtrot:
golf: "hotel"
india: null
juliett kilo: null
lima: "mike"
section1:
key1: "String1"
key10: "10"
key11: "11"
key2: "string2"
key3: "string3"
key4: "string4"
key5: "string5"
key6: "string6"
key7: "1"
key8: "2"
key9: "3"
section10:
key1: "1"
section11:
key1: "1"
section2:
key1: "value1"
section3:
key1: "value1"
section4:
key1: "value1"
section5:
key1: "value1"
section6:
key1: "value1"
section7:
key1: "value1"
section8:
key1: "1"
section9:
key1: "1"
changed: {}
removed: {}
diff_content_no_overrides_json:
added:
alfa: "bravo"