From 6b626f2f986af0f99fae4c9e8efc9579fa47acf2 Mon Sep 17 00:00:00 2001 From: Alistair Coles Date: Mon, 19 Jun 2017 14:46:14 +0100 Subject: [PATCH] Experimental swift-ring-composer CLI to build composite rings Provides a simple, experimental, CLI tool to generate a composite ring from a list of component builder files. For example: swift-ring-composer compose \ --output Commands available: - compose: compose a list of builder file to a composite ring - show: show the metadata for a composite ring Co-Authored-By: Kota Tsuyuzaki Co-Authored-By: Matthew Oliver Change-Id: I25a79e71c13af352e19e4358f60545265b51584f --- bin/swift-ring-composer | 22 +++ doc/manpages/swift-ring-composer.1 | 40 ++++++ doc/source/overview_ring.rst | 5 + setup.cfg | 1 + swift/cli/ringcomposer.py | 183 +++++++++++++++++++++++++ test/unit/cli/test_ringcomposer.py | 211 +++++++++++++++++++++++++++++ 6 files changed, 462 insertions(+) create mode 100755 bin/swift-ring-composer create mode 100644 doc/manpages/swift-ring-composer.1 create mode 100644 swift/cli/ringcomposer.py create mode 100644 test/unit/cli/test_ringcomposer.py diff --git a/bin/swift-ring-composer b/bin/swift-ring-composer new file mode 100755 index 0000000000..fccc9016c8 --- /dev/null +++ b/bin/swift-ring-composer @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# Copyright (c) 2017 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import sys + +from swift.cli.ringcomposer import main + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/doc/manpages/swift-ring-composer.1 b/doc/manpages/swift-ring-composer.1 new file mode 100644 index 0000000000..8d029ff334 --- /dev/null +++ b/doc/manpages/swift-ring-composer.1 @@ -0,0 +1,40 @@ +.TH swift-ring-composer "1" "June 2018" "Linux" "OpenStack Swift" +.SH NAME +.B swift-ring-composer +\- manual page for swift-ring-composer + +.SH SYNOPSIS +.LP +.B swift-ring-composer +[\-h] {show,compose} ... + +.SH DESCRIPTION +This is a tool for building a composite ring file from other existing ring +builder files. The component ring builders must all have the same partition +power. Each device must only be used in a single component builder. Each +region must only be used in a single component builder. +.PP +.B NOTE: +This tool is for experimental use and may be removed in future versions of Swift. +.PP +.SS "positional arguments:" +.TP + +Name of composite builder file +.SS "optional arguments:" +.TP +\fB\-h\fR, \fB\-\-help\fR +show this help message and exit +.SH "COMMANDS" +.TP +.SS "\fBshow\fR [-h]" +show composite ring builder metadata +.TP +.SS "\fBcompose\fR [-h] [ [ ...] --output [--force]" +compose composite ring +.PP +.SH DOCUMENTATION +.LP +More in depth documentation about the swift ring and also OpenStack Swift as a +whole can be found at +.BI https://swift.openstack.org diff --git a/doc/source/overview_ring.rst b/doc/source/overview_ring.rst index 3daca009d6..58ccc95499 100644 --- a/doc/source/overview_ring.rst +++ b/doc/source/overview_ring.rst @@ -361,6 +361,11 @@ Composite Rings --------------- .. automodule:: swift.common.ring.composite_builder +********************************** +swift-ring-composer (Experimental) +********************************** +.. automodule:: swift.cli.ringcomposer + --------------------- Ring Builder Analyzer --------------------- diff --git a/setup.cfg b/setup.cfg index 1e5fdbeeb3..5b6c5be1bb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -61,6 +61,7 @@ scripts = bin/swift-recon-cron bin/swift-ring-builder bin/swift-ring-builder-analyzer + bin/swift-ring-composer [extras] kms_keymaster = diff --git a/swift/cli/ringcomposer.py b/swift/cli/ringcomposer.py new file mode 100644 index 0000000000..90cd3a25a2 --- /dev/null +++ b/swift/cli/ringcomposer.py @@ -0,0 +1,183 @@ +# Copyright (c) 2017 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +``swift-ring-composer`` is an experimental tool for building a composite ring +file from other existing component ring builder files. Its CLI, name or +implementation may change or be removed altogether in future versions of Swift. + +Currently its interface is similar to that of the ``swift-ring-builder``. The +command structure takes the form of:: + + swift-ring-composer + +where ```` is a special builder which stores a json +blob of composite ring metadata. This metadata describes the component +``RingBuilder``'s used in the composite ring, their order and version. + +There are currently 2 sub-commands: ``show`` and ``compose``. The ``show`` +sub-command takes no additional arguments and displays the current contents of +of the composite builder file:: + + swift-ring-composer show + +The ``compose`` sub-command is the one that actually stitches the component +ring builders together to create both the composite ring file and composite +builder file. The command takes the form:: + + swift-ring-composer compose \\ + [ .. ] --output \\ + [--force] + +There may look like there is a lot going on there but it's actually quite +simple. The ``compose`` command takes in the list of builders to stitch +together and the filename for the composite ring file via the ``--output`` +option. The ``--force`` option overrides checks on the ring composition. + +To change ring devices, first add or remove devices from the component ring +builders and then use the ``compose`` sub-command to create a new composite +ring file. + +.. note:: + + ``swift-ring-builder`` cannot be used to inspect the generated composite + ring file because there is no conventional builder file corresponding to + the composite ring file name. You can either programmatically look inside + the composite ring file using the swift ring classes or create a temporary + builder file from the composite ring file using:: + + swift-ring-builder write_builder + + Do not use this builder file to manage ring devices. + +For further details use:: + + swift-ring-composer -h +""" +from __future__ import print_function +import argparse +import json +import os +import sys + +from swift.common.ring.composite_builder import CompositeRingBuilder + +EXIT_SUCCESS = 0 +EXIT_ERROR = 2 + +WARNING = """ +NOTE: This tool is for experimental use and may be + removed in future versions of Swift. +""" + +DESCRIPTION = """ +This is a tool for building a composite ring file from other existing ring +builder files. The component ring builders must all have the same partition +power. Each device must only be used in a single component builder. Each region +must only be used in a single component builder. +""" + + +def _print_to_stderr(msg): + print(msg, file=sys.stderr) + + +def _print_err(msg, err): + _print_to_stderr('%s\nOriginal exception message:\n%s' % (msg, err)) + + +def show(composite_builder, args): + print(json.dumps(composite_builder.to_dict(), indent=4, sort_keys=True)) + return EXIT_SUCCESS + + +def compose(composite_builder, args): + composite_builder = composite_builder or CompositeRingBuilder() + try: + ring_data = composite_builder.compose( + args.builder_files, force=args.force, require_modified=True) + except Exception as err: + _print_err( + 'An error occurred while composing the ring.', err) + return EXIT_ERROR + try: + ring_data.save(args.output) + except Exception as err: + _print_err( + 'An error occurred while writing the composite ring file.', err) + return EXIT_ERROR + try: + composite_builder.save(args.composite_builder_file) + except Exception as err: + _print_err( + 'An error occurred while writing the composite builder file.', err) + return EXIT_ERROR + return EXIT_SUCCESS + + +def main(arguments=None): + if arguments is not None: + argv = arguments + else: + argv = sys.argv + + parser = argparse.ArgumentParser(description=DESCRIPTION) + parser.add_argument( + 'composite_builder_file', + metavar='composite_builder_file', type=str, + help='Name of composite builder file') + + subparsers = parser.add_subparsers( + help='subcommand help', title='subcommands') + + # show + show_parser = subparsers.add_parser( + 'show', help='show composite ring builder metadata') + show_parser.set_defaults(func=show) + + # compose + compose_parser = subparsers.add_parser( + 'compose', help='compose composite ring', + usage='%(prog)s [-h] ' + '[builder_file builder_file [builder_file ...] ' + '--output ring_file [--force]') + bf_help = ('Paths to component ring builder files to include in composite ' + 'ring') + compose_parser.add_argument('builder_files', metavar='builder_file', + nargs='*', type=str, help=bf_help) + compose_parser.add_argument('--output', metavar='output_file', type=str, + required=True, help='Name of output ring file') + compose_parser.add_argument( + '--force', action='store_true', + help='Force new composite ring file to be written') + compose_parser.set_defaults(func=compose) + + _print_to_stderr(WARNING) + args = parser.parse_args(argv[1:]) + composite_builder = None + if args.func != compose or os.path.exists(args.composite_builder_file): + try: + composite_builder = CompositeRingBuilder.load( + args.composite_builder_file) + except Exception as err: + _print_err( + 'An error occurred while loading the composite builder file.', + err) + exit(EXIT_ERROR) + + exit(args.func(composite_builder, args)) + + +if __name__ == '__main__': + main() diff --git a/test/unit/cli/test_ringcomposer.py b/test/unit/cli/test_ringcomposer.py new file mode 100644 index 0000000000..3022d3702f --- /dev/null +++ b/test/unit/cli/test_ringcomposer.py @@ -0,0 +1,211 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json + +import os +import shutil +import tempfile +import unittest + +import six +from mock import mock + +from swift.cli import ringcomposer +from swift.common.ring import RingBuilder + + +class TestCommands(unittest.TestCase): + + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + self.composite_builder_file = os.path.join(self.tmpdir, + 'composite.builder') + self.composite_ring_file = os.path.join(self.tmpdir, + 'composite.ring') + + def tearDown(self): + shutil.rmtree(self.tmpdir) + + def _write_stub_builder(self, region): + replicas = 3 + builder = RingBuilder(8, replicas, 1) + for i in range(replicas): + dev = {'weight': 100, + 'region': '%d' % region, + 'zone': '1', + 'ip': '10.0.0.%d' % region, + 'port': '3600', + 'device': 'sdb%d' % i} + builder.add_dev(dev) + builder.rebalance() + builder_file = os.path.join(self.tmpdir, '%d.builder' % region) + builder.save(builder_file) + return builder, builder_file + + def _run_composer(self, args): + mock_stdout = six.StringIO() + mock_stderr = six.StringIO() + with mock.patch("sys.stdout", mock_stdout): + with mock.patch("sys.stderr", mock_stderr): + with self.assertRaises(SystemExit) as cm: + ringcomposer.main(args) + return (cm.exception.code, + mock_stdout.getvalue(), + mock_stderr.getvalue()) + + def test_unknown_command(self): + args = ('', self.composite_builder_file, 'unknown') + exit_code, stdout, stderr = self._run_composer(args) + self.assertEqual(2, exit_code) + self.assertIn('invalid choice', stderr) + + args = ('', 'non-existent-file', 'unknown') + exit_code, stdout, stderr = self._run_composer(args) + self.assertEqual(2, exit_code) + self.assertIn('invalid choice', stderr) + + def test_bad_composite_builder_file(self): + cmds = (('', self.composite_builder_file, 'show'), + ('', self.composite_builder_file, 'compose', + 'b1_file', 'b2_file', '--output', self.composite_ring_file)) + for cmd in cmds: + try: + with open(self.composite_builder_file, 'wb') as fd: + fd.write('not json') + exit_code, stdout, stderr = self._run_composer(cmd) + self.assertEqual(2, exit_code) + self.assertIn('An error occurred while loading the composite ' + 'builder file', stderr) + self.assertIn( + 'File does not contain valid composite ring data', stderr) + except AssertionError as err: + self.fail('Failed testing command %r due to: %s' % (cmd, err)) + + def test_compose(self): + b1, b1_file = self._write_stub_builder(1) + b2, b2_file = self._write_stub_builder(2) + args = ('', self.composite_builder_file, 'compose', b1_file, b2_file, + '--output', self.composite_ring_file) + exit_code, stdout, stderr = self._run_composer(args) + self.assertEqual(0, exit_code) + self.assertTrue(os.path.exists(self.composite_builder_file)) + self.assertTrue(os.path.exists(self.composite_ring_file)) + + def test_compose_existing(self): + b1, b1_file = self._write_stub_builder(1) + b2, b2_file = self._write_stub_builder(2) + args = ('', self.composite_builder_file, 'compose', b1_file, b2_file, + '--output', self.composite_ring_file) + exit_code, stdout, stderr = self._run_composer(args) + self.assertEqual(0, exit_code) + os.unlink(self.composite_ring_file) + # no changes - expect failure + args = ('', self.composite_builder_file, 'compose', + '--output', self.composite_ring_file) + exit_code, stdout, stderr = self._run_composer(args) + self.assertEqual(2, exit_code) + self.assertFalse(os.path.exists(self.composite_ring_file)) + # --force should force output + args = ('', self.composite_builder_file, 'compose', + '--output', self.composite_ring_file, '--force') + exit_code, stdout, stderr = self._run_composer(args) + self.assertEqual(0, exit_code) + self.assertTrue(os.path.exists(self.composite_ring_file)) + + def test_compose_insufficient_component_builder_files(self): + b1, b1_file = self._write_stub_builder(1) + args = ('', self.composite_builder_file, 'compose', b1_file, + '--output', self.composite_ring_file) + exit_code, stdout, stderr = self._run_composer(args) + self.assertEqual(2, exit_code) + self.assertIn('An error occurred while composing the ring', stderr) + self.assertIn('Two or more component builders are required', stderr) + self.assertFalse(os.path.exists(self.composite_builder_file)) + self.assertFalse(os.path.exists(self.composite_ring_file)) + + def test_compose_nonexistent_component_builder_file(self): + b1, b1_file = self._write_stub_builder(1) + bad_file = os.path.join(self.tmpdir, 'non-existent-file') + args = ('', self.composite_builder_file, 'compose', b1_file, bad_file, + '--output', self.composite_ring_file) + exit_code, stdout, stderr = self._run_composer(args) + self.assertIn('An error occurred while composing the ring', stderr) + self.assertIn('Ring Builder file does not exist', stderr) + self.assertEqual(2, exit_code) + self.assertFalse(os.path.exists(self.composite_builder_file)) + self.assertFalse(os.path.exists(self.composite_ring_file)) + + def test_compose_fails_to_write_composite_ring_file(self): + b1, b1_file = self._write_stub_builder(1) + b2, b2_file = self._write_stub_builder(2) + args = ('', self.composite_builder_file, 'compose', b1_file, b2_file, + '--output', self.composite_ring_file) + with mock.patch('swift.common.ring.RingData.save', + side_effect=IOError('io error')): + exit_code, stdout, stderr = self._run_composer(args) + self.assertEqual(2, exit_code) + self.assertIn( + 'An error occurred while writing the composite ring file', stderr) + self.assertIn('io error', stderr) + self.assertFalse(os.path.exists(self.composite_builder_file)) + self.assertFalse(os.path.exists(self.composite_ring_file)) + + def test_compose_fails_to_write_composite_builder_file(self): + b1, b1_file = self._write_stub_builder(1) + b2, b2_file = self._write_stub_builder(2) + args = ('', self.composite_builder_file, 'compose', b1_file, b2_file, + '--output', self.composite_ring_file) + func = 'swift.common.ring.composite_builder.CompositeRingBuilder.save' + with mock.patch(func, side_effect=IOError('io error')): + exit_code, stdout, stderr = self._run_composer(args) + self.assertEqual(2, exit_code) + self.assertIn( + 'An error occurred while writing the composite builder file', + stderr) + self.assertIn('io error', stderr) + self.assertFalse(os.path.exists(self.composite_builder_file)) + self.assertTrue(os.path.exists(self.composite_ring_file)) + + def test_show(self): + b1, b1_file = self._write_stub_builder(1) + b2, b2_file = self._write_stub_builder(2) + args = ('', self.composite_builder_file, 'compose', b1_file, b2_file, + '--output', self.composite_ring_file) + exit_code, stdout, stderr = self._run_composer(args) + self.assertEqual(0, exit_code) + args = ('', self.composite_builder_file, 'show') + exit_code, stdout, stderr = self._run_composer(args) + self.assertEqual(0, exit_code) + expected = {'component_builder_files': {b1.id: b1_file, + b2.id: b2_file}, + 'components': [ + {'id': b1.id, + 'replicas': b1.replicas, + # added replicas devices plus rebalance + 'version': b1.replicas + 1}, + {'id': b2.id, + 'replicas': b2.replicas, + # added replicas devices plus rebalance + 'version': b2.replicas + 1}], + 'version': 1 + } + self.assertEqual(expected, json.loads(stdout)) + + def test_show_nonexistent_composite_builder_file(self): + args = ('', 'non-existent-file', 'show') + exit_code, stdout, stderr = self._run_composer(args) + self.assertEqual(2, exit_code) + self.assertIn( + 'An error occurred while loading the composite builder file', + stderr) + self.assertIn("No such file or directory: 'non-existent-file'", stderr)