openstack/swift
Code Issues Proposed changes

func tests: fix etag-quoter insertion in pipeline

Browse Source 
Previously the in-process functional test setup attempted to insert
etag-quoter in the proxy pipeline by replacing the substring 'cache
listing_formats' with 'cache etag-quoter listing_formats'. However,
the pipeline had already been modified to have 'cache domain-remap
listing_formats', so the etag-quoter replacement failed silently.

This patch modifies the etag-quoter replacement to match just
'cache'. Some helper functions are also introduced to make the
pipeline modification more uniform and to consistently verify the
intended modifications have been achieved, or otherwise raise an
exception.

Change-Id: I640837a4da985bcbe48f9ae5eeb56385b11034eb
This commit is contained in:
Alistair Coles 2024-09-25 12:57:37 +01:00
parent 81547cb67d
commit 86081b7822
1 changed files with 126 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

245
test/functional/__init__.py
View File

@ -14,6 +14,10 @@
# limitations under the License.
from __future__ import print_function
import configparser
import contextlib
import mock
import os
import six
@ -35,7 +39,7 @@ from shutil import rmtree
from tempfile import mkdtemp
from unittest import SkipTest
from six.moves.configparser import ConfigParser, NoSectionError
from six.moves.configparser import ConfigParser
from six.moves import http_client
from six.moves.http_client import HTTPException
@ -131,24 +135,60 @@ def _debug(msg):
_info('DEBUG: ' + msg)
@contextlib.contextmanager
def _modify_conf_file(src_conf_file, dest_conf_file):
conf = ConfigParser()
conf.read(src_conf_file)
try:
yield conf
except configparser.Error as err:
msg = 'Error modifying conf file %s: %s' % (src_conf_file, err)
raise InProcessException(msg)
else:
with open(dest_conf_file, 'w') as fp:
conf.write(fp)
def _cleanup_pipeline(pipeline_str):
# return single-whitespace delimited and padded string
# e.g. "x y " -> " x y "
return ' %s ' % ' '.join([x for x in pipeline_str.split(' ') if x])
def _modify_pipeline_substring(conf, old_substr, new_substr):
section = 'pipeline:main'
old_pipeline = _cleanup_pipeline(conf.get(section, 'pipeline'))
old_substr = _cleanup_pipeline(old_substr)
new_substr = _cleanup_pipeline(new_substr)
if old_substr not in old_pipeline:
raise InProcessException(
'Failed to replace pipeline substring: old="%s", new="%s", '
'pipeline="%s"' % (old_substr, new_substr, old_pipeline))
if new_substr in old_pipeline:
_info('WARNING: "%s" is already in pipeline, not replacing'
% new_substr)
new_pipeline = old_pipeline
else:
new_pipeline = old_pipeline.replace(old_substr, new_substr)
_debug('Replaced pipeline substring "%s" with "%s"'
% (old_substr, new_substr))
# remove padding
new_pipeline = new_pipeline.strip(' ')
conf.set(section, 'pipeline', new_pipeline)
_debug('Proxy pipeline is now "%s"' % new_pipeline)
def _in_process_setup_swift_conf(swift_conf_src, testdir):
# override swift.conf contents for in-process functional test runs
conf = ConfigParser()
conf.read(swift_conf_src)
try:
test_conf_file = os.path.join(testdir, 'swift.conf')
with _modify_conf_file(swift_conf_src, test_conf_file) as conf:
section = 'swift-hash'
conf.set(section, 'swift_hash_path_suffix', 'inprocfunctests')
conf.set(section, 'swift_hash_path_prefix', 'inprocfunctests')
section = 'swift-constraints'
max_file_size = (8 * 1024 * 1024) + 2 # 8 MB + 2
conf.set(section, 'max_file_size', str(max_file_size))
except NoSectionError:
msg = 'Conf file %s is missing section %s' % (swift_conf_src, section)
raise InProcessException(msg)
test_conf_file = os.path.join(testdir, 'swift.conf')
with open(test_conf_file, 'w') as fp:
conf.write(fp)
return test_conf_file
@ -195,39 +235,36 @@ def _in_process_setup_ring(swift_conf, conf_src_dir, testdir):
in process testing, and renaming it to suit policy-0
Otherwise, create a default ring file.
"""
conf = ConfigParser()
conf.read(swift_conf)
sp_prefix = 'storage-policy:'
with _modify_conf_file(swift_conf, swift_conf) as conf:
sp_prefix = 'storage-policy:'
try:
# policy index 0 will be created if no policy exists in conf
policies = parse_storage_policies(conf)
except PolicyError as e:
raise InProcessException(e)
try:
# policy index 0 will be created if no policy exists in conf
policies = parse_storage_policies(conf)
except PolicyError as e:
raise InProcessException(e)
# clear all policies from test swift.conf before adding test policy back
for policy in policies:
conf.remove_section(sp_prefix + str(policy.idx))
# clear all policies from test swift.conf before adding test policy
# back
for policy in policies:
conf.remove_section(sp_prefix + str(policy.idx))
if policy_specified:
policy_to_test = policies.get_by_name(policy_specified)
if policy_to_test is None:
raise InProcessException('Failed to find policy name "%s"'
% policy_specified)
_info('Using specified policy %s' % policy_to_test.name)
else:
policy_to_test = policies.default
_info('Defaulting to policy %s' % policy_to_test.name)
if policy_specified:
policy_to_test = policies.get_by_name(policy_specified)
if policy_to_test is None:
raise InProcessException('Failed to find policy name "%s"'
% policy_specified)
_info('Using specified policy %s' % policy_to_test.name)
else:
policy_to_test = policies.default
_info('Defaulting to policy %s' % policy_to_test.name)
# make policy_to_test be policy index 0 and default for the test config
sp_zero_section = sp_prefix + '0'
conf.add_section(sp_zero_section)
for (k, v) in policy_to_test.get_info(config=True).items():
conf.set(sp_zero_section, k, str(v))
conf.set(sp_zero_section, 'default', 'True')
with open(swift_conf, 'w') as fp:
conf.write(fp)
# make policy_to_test be policy index 0 and default for the test config
sp_zero_section = sp_prefix + '0'
conf.add_section(sp_zero_section)
for (k, v) in policy_to_test.get_info(config=True).items():
conf.set(sp_zero_section, k, str(v))
conf.set(sp_zero_section, 'default', 'True')
# look for a source ring file
ring_file_src = ring_file_test = 'object.ring.gz'
@ -284,6 +321,18 @@ def _in_process_setup_ring(swift_conf, conf_src_dir, testdir):
return obj_sockets
def _load_etag_quoter(proxy_conf_file, swift_conf_file, **kwargs):
_debug('Ensuring etag-quoter is in proxy pipeline')
test_conf_file = os.path.join(_testdir, 'proxy-server.conf')
with _modify_conf_file(proxy_conf_file, test_conf_file) as conf:
_modify_pipeline_substring(
conf,
"cache",
"cache etag-quoter")
return test_conf_file, swift_conf_file
def _load_encryption(proxy_conf_file, swift_conf_file, **kwargs):
"""
Load encryption configuration and override proxy-server.conf contents.
@ -302,34 +351,20 @@ def _load_encryption(proxy_conf_file, swift_conf_file, **kwargs):
# conf during wsgi load_app.
# Therefore we must modify the [pipeline:main] section.
conf = ConfigParser()
conf.read(proxy_conf_file)
try:
section = 'pipeline:main'
pipeline = conf.get(section, 'pipeline')
pipeline = pipeline.replace(
test_conf_file = os.path.join(_testdir, 'proxy-server.conf')
with _modify_conf_file(proxy_conf_file, test_conf_file) as conf:
_modify_pipeline_substring(
conf,
"proxy-logging proxy-server",
"keymaster encryption proxy-logging proxy-server")
pipeline = pipeline.replace(
"cache listing_formats",
"cache etag-quoter listing_formats")
conf.set(section, 'pipeline', pipeline)
root_secret = base64.b64encode(os.urandom(32))
if not six.PY2:
root_secret = root_secret.decode('ascii')
conf.set('filter:keymaster', 'encryption_root_secret', root_secret)
conf.set('filter:versioned_writes', 'allow_object_versioning', 'true')
conf.set('filter:etag-quoter', 'enable_by_default', 'true')
except NoSectionError as err:
msg = 'Error problem with proxy conf file %s: %s' % \
(proxy_conf_file, err)
raise InProcessException(msg)
test_conf_file = os.path.join(_testdir, 'proxy-server.conf')
with open(test_conf_file, 'w') as fp:
conf.write(fp)
return test_conf_file, swift_conf_file
return _load_etag_quoter(test_conf_file, swift_conf_file)
def _load_ec_as_default_policy(proxy_conf_file, swift_conf_file, **kwargs):
@ -342,29 +377,26 @@ def _load_ec_as_default_policy(proxy_conf_file, swift_conf_file, **kwargs):
"""
_debug('Setting configuration for default EC policy')
conf = ConfigParser()
conf.read(swift_conf_file)
# remove existing policy sections that came with swift.conf-sample
for section in list(conf.sections()):
if section.startswith('storage-policy'):
conf.remove_section(section)
# add new policy 0 section for an EC policy
conf.add_section('storage-policy:0')
ec_policy_spec = {
'name': 'ec-test',
'policy_type': 'erasure_coding',
'ec_type': 'liberasurecode_rs_vand',
'ec_num_data_fragments': 2,
'ec_num_parity_fragments': 1,
'ec_object_segment_size': 1048576,
'default': True
}
with _modify_conf_file(swift_conf_file, swift_conf_file) as conf:
# remove existing policy sections that came with swift.conf-sample
for section in list(conf.sections()):
if section.startswith('storage-policy'):
conf.remove_section(section)
# add new policy 0 section for an EC policy
conf.add_section('storage-policy:0')
ec_policy_spec = {
'name': 'ec-test',
'policy_type': 'erasure_coding',
'ec_type': 'liberasurecode_rs_vand',
'ec_num_data_fragments': 2,
'ec_num_parity_fragments': 1,
'ec_object_segment_size': 1048576,
'default': True
}
for k, v in ec_policy_spec.items():
conf.set('storage-policy:0', k, str(v))
for k, v in ec_policy_spec.items():
conf.set('storage-policy:0', k, str(v))
with open(swift_conf_file, 'w') as fp:
conf.write(fp)
return proxy_conf_file, swift_conf_file
@ -390,33 +422,19 @@ def _load_domain_remap_staticweb(proxy_conf_file, swift_conf_file, **kwargs):
# DEFAULTS options) then it prevents pipeline being loaded into the local
# conf during wsgi load_app.
# Therefore we must modify the [pipeline:main] section.
conf = ConfigParser()
conf.read(proxy_conf_file)
try:
section = 'pipeline:main'
old_pipeline = conf.get(section, 'pipeline')
pipeline = old_pipeline.replace(
" tempauth ",
" tempauth staticweb ")
pipeline = pipeline.replace(
" listing_formats ",
" domain_remap listing_formats ")
if pipeline == old_pipeline:
raise InProcessException(
"Failed to insert domain_remap and staticweb into pipeline: %s"
% old_pipeline)
conf.set(section, 'pipeline', pipeline)
test_conf_file = os.path.join(_testdir, 'proxy-server.conf')
with _modify_conf_file(proxy_conf_file, test_conf_file) as conf:
_modify_pipeline_substring(
conf,
"tempauth",
"tempauth staticweb")
_modify_pipeline_substring(
conf,
"listing_formats",
"domain_remap listing_formats")
# set storage_domain in domain_remap middleware to match test config
section = 'filter:domain_remap'
conf.set(section, 'storage_domain', storage_domain)
except NoSectionError as err:
msg = 'Error problem with proxy conf file %s: %s' % \
(proxy_conf_file, err)
raise InProcessException(msg)
test_conf_file = os.path.join(_testdir, 'proxy-server.conf')
with open(test_conf_file, 'w') as fp:
conf.write(fp)
return test_conf_file, swift_conf_file
@ -439,26 +457,15 @@ def _load_s3api(proxy_conf_file, swift_conf_file, **kwargs):
# conf during wsgi load_app.
# Therefore we must modify the [pipeline:main] section.
conf = ConfigParser()
conf.read(proxy_conf_file)
try:
section = 'pipeline:main'
pipeline = conf.get(section, 'pipeline')
pipeline = pipeline.replace(
test_conf_file = os.path.join(_testdir, 'proxy-server.conf')
with _modify_conf_file(proxy_conf_file, test_conf_file) as conf:
_modify_pipeline_substring(
conf,
"tempauth",
"s3api tempauth")
conf.set(section, 'pipeline', pipeline)
conf.set('filter:s3api', 's3_acl', 'true')
conf.set('filter:versioned_writes', 'allow_object_versioning', 'true')
except NoSectionError as err:
msg = 'Error problem with proxy conf file %s: %s' % \
(proxy_conf_file, err)
raise InProcessException(msg)
test_conf_file = os.path.join(_testdir, 'proxy-server.conf')
with open(test_conf_file, 'w') as fp:
conf.write(fp)
return test_conf_file, swift_conf_file