# Copyright 2012 OpenStack Foundation # Copyright 2013 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. """ A simple script to update the requirements files from a global set of allowable requirements. The script can be called like this: $> python update.py ../myproj Any requirements listed in the target files will have their versions updated to match the global requirements. Requirements not in the global files will be dropped. """ import optparse import os import os.path import sys import six from openstack_requirements import project from openstack_requirements import requirement _setup_py_text = """# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. # THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT import setuptools # In python < 2.7.4, a lazy loading of package `pbr` will break # setuptools if some other modules registered functions in `atexit`. # solution from: http://bugs.python.org/issue15881#msg170215 try: import multiprocessing # noqa except ImportError: pass setuptools.setup( setup_requires=['pbr>=2.0.0'], pbr=True) """ # Pure -- class Change(object): def __init__(self, name, old, new): self.name = name self.old = old.strip() self.new = new.strip() def __repr__(self): return "%-30.30s -> %s" % (self.old, self.new) def _check_setup_py(proj): actions = [] # If it doesn't have a setup.py, then we don't want to update it if 'setup.py' not in proj: return actions # If it doesn't use pbr, we don't want to update it. elif 'pbr' not in proj['setup.py']: return actions # We don't update pbr's setup.py because it can't use itself. if 'setup.cfg' in proj and 'name = pbr' in proj['setup.cfg']: return actions actions.append(project.Verbose("Syncing setup.py")) actions.append(project.File('setup.py', _setup_py_text)) return actions def _sync_requirements_file( source_reqs, dest_sequence, dest_label, softupdate, hacking, non_std_reqs, blacklist={}): actions = [] dest_reqs = requirement.to_dict(dest_sequence) changes = [] output_requirements = [] processed_packages = set() for req, req_line in dest_sequence: # Skip the instructions header if req_line in requirement._REQS_HEADER: continue elif req is None: # Unparsable lines. output_requirements.append( requirement.Requirement('', '', '', '', req_line.rstrip())) continue elif not req.package: # Comment-only lines output_requirements.append(req) continue elif req.package.lower() in processed_packages: continue processed_packages.add(req.package.lower()) # Special cases: # projects need to align hacking version on their own time if req.package == "hacking" and not hacking: output_requirements.append(req) continue # the overall blacklist is similarly synced by projects as # needed if req.package in blacklist: output_requirements.append(req) continue reference = source_reqs.get(req.package.lower()) if reference: actual = dest_reqs.get(req.package.lower()) for req, ref in six.moves.zip_longest(actual, reference): if not req: # More in globals changes.append(Change(ref[0].package, '', ref[1])) elif not ref: # less in globals changes.append(Change(req[0].package, req[1], '')) elif req[0] != ref[0]: # NOTE(jamielennox): extras are allowed to be specified in # a project's requirements and the version be updated and # extras maintained. Create a new ref object the same as # the original but with the req's extras. merged_ref = requirement.Requirement(ref[0].package, ref[0].location, ref[0].specifiers, ref[0].markers, ref[0].comment, req[0].extras) ref = (merged_ref, merged_ref.to_line()) if req[0] != ref[0]: # A change on this entry changes.append(Change(req[0].package, req[1], ref[1])) if ref: output_requirements.append(ref[0]) elif softupdate: # under softupdate we pass through anything unknown packages, # this is intended for ecosystem projects that want to stay in # sync with existing requirements, but also add their own above # and beyond. output_requirements.append(req) else: # What do we do if we find something unexpected? # # In the default cause we should die horribly, because # the point of global requirements was a single lever # to control all the pip installs in the gate. # # However, we do have other projects using # devstack jobs that might have legitimate reasons to # override. For those we support NON_STANDARD_REQS=1 # environment variable to turn this into a warning only. # However this drops the unknown requirement. actions.append(project.Error( "'%s' is not in global-requirements.txt or blacklist.txt" % req.package)) # always print out what we did if we did a thing if changes: actions.append(project.StdOut( "Version change for: %s\n" % ", ".join([x.name for x in changes]))) actions.append(project.StdOut("Updated %s:\n" % dest_label)) for change in changes: actions.append(project.StdOut(" %s\n" % change)) return actions, requirement.Requirements(output_requirements) def _copy_requires( suffix, softupdate, hacking, proj, global_reqs, non_std_reqs, blacklist={}): """Copy requirements files.""" actions = [] for source, content in sorted(proj['requirements'].items()): dest_path = os.path.join(proj['root'], source) # this is specifically for global-requirements gate jobs so we don't # modify the git tree if suffix: dest_path = "%s.%s" % (dest_path, suffix) dest_name = "%s.%s" % (source, suffix) else: dest_name = source dest_sequence = list(requirement.to_reqs(content)) actions.append(project.Verbose("Syncing %s" % dest_path)) _actions, reqs = _sync_requirements_file( global_reqs, dest_sequence, dest_path, softupdate, hacking, non_std_reqs, blacklist) actions.extend(_actions) actions.append(project.File(dest_name, requirement.to_content(reqs))) extras = project.extras(proj) output_extras = {} for extra, content in sorted(extras.items()): dest_name = 'extra-%s' % extra dest_path = "%s[%s]" % (proj['root'], extra) dest_sequence = list(requirement.to_reqs(content)) actions.append(project.Verbose("Syncing extra [%s]" % extra)) _actions, reqs = _sync_requirements_file( global_reqs, dest_sequence, dest_path, softupdate, hacking, non_std_reqs, blacklist) actions.extend(_actions) output_extras[extra] = reqs dest_path = 'setup.cfg' if suffix: dest_path = "%s.%s" % (dest_path, suffix) actions.append(project.File( dest_path, project.merge_setup_cfg(proj['setup.cfg'], output_extras))) return actions def _process_project( project, global_reqs, suffix, softupdate, hacking, non_std_reqs, blacklist={}): """Project a project. :return: The actions to take as a result. """ actions = _copy_requires( suffix, softupdate, hacking, project, global_reqs, non_std_reqs, blacklist) actions.extend(_check_setup_py(project)) return actions # IO -- def main(argv=None, stdout=None, _worker=None): parser = optparse.OptionParser() parser.add_option("-o", "--output-suffix", dest="suffix", default="", help="output suffix for updated files (i.e. .global)") parser.add_option("-s", "--soft-update", dest="softupdate", action="store_true", help="Pass through extra requirements without warning.") parser.add_option("-H", "--hacking", dest="hacking", action="store_true", help="Include the hacking project.") parser.add_option("-v", "--verbose", dest="verbose", action="store_true", help="Add further verbosity to output") parser.add_option("--source", dest="source", default=".", help="Dir where global-requirements.txt is located.") options, args = parser.parse_args(argv) if len(args) != 1: print("Must specify directory to update") raise Exception("Must specify one and only one directory to update.") if not os.path.isdir(args[0]): print("%s is not a directory." % (args[0])) raise Exception("%s is not a directory." % (args[0])) if stdout is None: stdout = sys.stdout if _worker is None: _worker = _do_main non_std_reqs = os.getenv('NON_STANDARD_REQS', '0') == '1' _worker( args[0], options.source, options.suffix, options.softupdate, options.hacking, stdout, options.verbose, non_std_reqs) def _do_main( root, source, suffix, softupdate, hacking, stdout, verbose, non_std_reqs): """No options or environment variable access from here on in.""" proj = project.read(root) global_req_content = open( os.path.join(source, 'global-requirements.txt'), 'rt').read() global_reqs = requirement.parse(global_req_content) blacklist_content = open( os.path.join(source, 'blacklist.txt'), 'rt').read() blacklist = requirement.parse(blacklist_content) actions = _process_project( proj, global_reqs, suffix, softupdate, hacking, non_std_reqs, blacklist) project.write(proj, actions, stdout=stdout, verbose=verbose) if __name__ == "__main__": main()