Preserve order and comments when updating

This removes reordering of the requirements
when updatin the files, as preserving the
order makes reviewing significantly easier.

Also preserve comments around the requirements.

In addition, search for missing test-requirements
also in the tools/pip-requires, to avoid accidentally
dropping them.

Change-Id: I3c51783a42db9907da9d9a98b42b52b5f639bcc8

update.py

@@ -34,6 +34,17 @@ import sys
 from pip import req
 
 
+def _parse_pip(pip):
+
+    install_require = req.InstallRequirement.from_line(pip)
+    if install_require.editable:
+        return pip
+    elif install_require.url:
+        return pip
+    else:
+        return install_require.req.key
+
+
 def _parse_reqs(filename):
 
     reqs = dict()
@@ -43,17 +54,11 @@ def _parse_reqs(filename):
         pip = pip.strip()
         if pip.startswith("#") or len(pip) == 0:
             continue
-        install_require = req.InstallRequirement.from_line(pip)
-        if install_require.editable:
-            reqs[pip] = pip
-        elif install_require.url:
-            reqs[pip] = pip
-        else:
-            reqs[install_require.req.key] = pip
+        reqs[_parse_pip(pip)] = pip
     return reqs
 
 
-def _copy_requires(req, source_path, dest_dir):
+def _copy_requires(req, source_paths, dest_dir):
     """Copy requirements files."""
 
     dest_path = os.path.join(dest_dir, req)
@@ -62,34 +67,48 @@ def _copy_requires(req, source_path, dest_dir):
         # This can happen, we try all paths
         return
 
-    source_reqs = _parse_reqs(source_path)
-    dest_reqs = _parse_reqs(dest_path)
-    dest_keys = [key.lower() for key in dest_reqs.keys()]
-    dest_keys.sort()
+    source_reqs = dict()
+    for s in source_paths:
+        source_reqs.update(_parse_reqs(s))
+
+    with open(dest_path, 'r') as dest_reqs_file:
+        dest_reqs = dest_reqs_file.readlines()
 
     print "Syncing %s" % req
+
     with open(dest_path, 'w') as new_reqs:
-        new_reqs.write("# This file is managed by openstack-depends\n")
-        for old_require in dest_keys:
+        for old_require in dest_reqs:
+            # copy comments
+            if old_require[0] == '#':
+                new_reqs.write(old_require)
+                continue
+            old_require = old_require.rstrip().lower()
+            if not old_require:
+                continue
+            old_pip = _parse_pip(old_require)
+
             # Special cases:
             # projects need to align pep8 version on their own time
-            if "pep8" in old_require:
-                new_reqs.write("%s\n" % dest_reqs[old_require])
+            if "pep8" in old_pip:
+                new_reqs.write("%s\n" % old_require)
+                continue
+            # versions of our stuff from tarballs.openstack.org are ok
+            if "http://tarballs.openstack.org/" in old_pip:
+                new_reqs.write("%s\n" % old_require)
                 continue
 
-            # versions of our stuff from tarballs.openstack.org are ok
-            if old_require in source_reqs or \
-                    "http://tarballs.openstack.org" in old_require:
-                new_reqs.write("%s\n" % source_reqs[old_require])
+            if old_pip in source_reqs:
+                new_reqs.write("%s\n" % source_reqs[old_pip])
 
 
 def main(argv):
-
     for req in ('tools/pip-requires', 'requirements.txt'):
-        _copy_requires(req, 'tools/pip-requires', argv[0])
+        _copy_requires(req, ['tools/pip-requires'], argv[0])
 
     for req in ('tools/test-requires', 'test-requirements.txt'):
-        _copy_requires(req, 'tools/test-requires', argv[0])
+        _copy_requires(req,
+                       ['tools/pip-requires', 'tools/test-requires'],
+                       argv[0])
 
 
 if __name__ == "__main__":