From ffe658de9391a0179f4df884a68e73000e43b083 Mon Sep 17 00:00:00 2001 From: Tony Breeds Date: Thu, 22 Sep 2016 20:51:04 +1000 Subject: [PATCH] Add the concept of a version map for constraints generation We need to generate constraints for all python versions we use. This is python 2.7 (trusty and xenial), 3.4 (trusty) and 3.5(xenial). There isn't a supportable way to install all 3 versions on a single node. Add a version-map that allows us to clone the output of a freeze between python versions. This assumes that the python version in question are compatible during the transition window. It's possible to run with: ... --version-map 3.4:3.5 --version-map 3.5:3.4 ... This will mean that we generate constraints consistently between versions regardless of which version /usr/bin/python3 is. Change-Id: I55a3597ea2f2bf48e30dc564a4bf4b2ef7f178b3 --- openstack_requirements/cmds/generate.py | 32 +++++++++++ openstack_requirements/tests/test_generate.py | 53 +++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/openstack_requirements/cmds/generate.py b/openstack_requirements/cmds/generate.py index d1b2ad2499..3e62cdd63e 100644 --- a/openstack_requirements/cmds/generate.py +++ b/openstack_requirements/cmds/generate.py @@ -11,6 +11,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + +import copy import optparse import os.path import subprocess @@ -131,6 +134,18 @@ def _combine_freezes(freezes, blacklist=None): yield '%s===%s\n' % (package, list(versions.keys())[0]) +def _clone_versions(freezes, options): + for freeze_data in freezes: + versions = [v for v, d in freezes] + version, freeze = freeze_data + if (version in options.version_map and + options.version_map[version] not in versions): + print("Duplicating %s freeze to %s" % + (version, options.version_map[version]), + file=sys.stderr) + freezes.append((options.version_map[version], copy.copy(freeze))) + + # -- untested UI glue from here down. def _validate_options(options): @@ -154,6 +169,15 @@ def _validate_options(options): raise Exception( "Blacklist file %(path)s not found." % dict(path=options.blacklist)) + version_map = {} + for map_entry in options.version_map: + if ':' not in map_entry: + raise Exception( + "Invalid version-map entry %(map_entry)s" + % dict(map_entry=map_entry)) + src, dst = map_entry.split(':') + version_map[src] = dst + options.version_map = version_map def _parse_blacklist(path): @@ -175,12 +199,20 @@ def main(argv=None, stdout=None): parser.add_option( "-b", dest="blacklist", help="Filename of a list of package names to exclude.") + parser.add_option( + "--version-map", dest='version_map', default=[], action='append', + help=('Add a : seperated list of versions to clone. To \'clone\' ' + 'a freeze geberated by python3.4 to python3.5 specify 3.4:3.5. ' + 'This is intented as as a way to transition between python ' + 'versions when it\'s not possible to have all versions ' + 'installed')) options, args = parser.parse_args(argv) if stdout is None: stdout = sys.stdout _validate_options(options) freezes = [ _freeze(options.requirements, python) for python in options.pythons] + _clone_versions(freezes, options) blacklist = _parse_blacklist(options.blacklist) stdout.writelines(_combine_freezes(freezes, blacklist)) stdout.flush() diff --git a/openstack_requirements/tests/test_generate.py b/openstack_requirements/tests/test_generate.py index 8cf7a15d3e..9a33124336 100644 --- a/openstack_requirements/tests/test_generate.py +++ b/openstack_requirements/tests/test_generate.py @@ -98,3 +98,56 @@ class TestCombine(testtools.TestCase): ['enum===1.5.0\n'], list(generate._combine_freezes( [freeze_27], blacklist=blacklist))) + + +class Namespace(object): + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + +class TestClone(testtools.TestCase): + + def test_py34_clone_py35(self): + # Simulate an environment where we have python 3.4 data and need to + # clone that to python 3.5 + options = Namespace(version_map={'3.4': '3.5', '3.5': '3.4'}) + freeze_27 = ('2.7', [('dnspython', '1.15.0')]) + freeze_34 = ('3.4', [('dnspython3', '1.12.0')]) + freeze_35 = ('3.5', [('dnspython3', '1.12.0')]) + + freezes = [freeze_27, freeze_34] + expected_freezes = [freeze_27, freeze_34, freeze_35] + + generate._clone_versions(freezes, options) + + self.assertEqual(expected_freezes, freezes) + + def test_py34_noclone_py35(self): + # Simulate an environment where we have python 3.4 and python 3.5 data + # so there is no need to clone. + options = Namespace(version_map={'3.4': '3.5', '3.5': '3.4'}) + freeze_27 = ('2.7', [('dnspython', '1.15.0')]) + freeze_34 = ('3.4', [('dnspython3', '1.12.0')]) + freeze_35 = ('3.5', [('other-pkg', '1.0.0')]) + + freezes = [freeze_27, freeze_34, freeze_35] + expected_freezes = [freeze_27, freeze_34, freeze_35] + + generate._clone_versions(freezes, options) + + self.assertEqual(expected_freezes, freezes) + + def test_py35_clone_py34(self): + # Simulate an environment where we have python 3.5 data and need to + # clone that to python 3.4 + options = Namespace(version_map={'3.4': '3.5', '3.5': '3.4'}) + freeze_27 = ('2.7', [('dnspython', '1.15.0')]) + freeze_34 = ('3.4', [('dnspython3', '1.12.0')]) + freeze_35 = ('3.5', [('dnspython3', '1.12.0')]) + + freezes = [freeze_27, freeze_35] + expected_freezes = [freeze_27, freeze_35, freeze_34] + + generate._clone_versions(freezes, options) + + self.assertEqual(expected_freezes, freezes)