Add Packages file parsing lookup

This is a lookup in case you need to parse a ubuntu repo Packages file
for a list of packages. It's useful in osa if you want to have a mirror
of an ubuntu repo, and output a list of its packages, with their
location and checksum.

You can use the output dict to feed another tool (for example
downloading with aria).

Change-Id: I10ba72f0b64267ae6afe57317e135d782445e934

lookup/packages_file.py

@@ -0,0 +1,79 @@
+# Copyright 2017, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# (c) 2017, Jean-Philippe Evrard <jean-philippe.evrard@rackspace.co.uk>
+
+# Take a path to a debian mirror release file, and outputs a dict with
+# package names, each of the packages holding the following:
+# - package version
+# - checksums
+# - Relative location to pool folder
+#
+# example:
+# get_url:
+#   url: http://rpc-repo.rackspace.com/apt-mirror/integrated/dists/r14.0.0rc1-trusty/main/binary-amd64/Packages
+#   dest: /tmp/trusty-amd64-Packages
+# debug:
+#   var: item
+# with_packages_file:
+#    - /tmp/trusty-amd64-Packages
+
+import os
+
+from ansible.plugins.lookup import LookupBase
+from ansible.errors import AnsibleLookupError
+
+IMPORTANT_FIELDS = ['Version', 'Filename', 'MD5sum', 'SHA1', 'SHA256']
+
+try:
+    from __main__ import display
+except ImportError:
+    from ansible.utils.display import Display
+    display = Display()
+
+def parse_fields(line):
+    for field in IMPORTANT_FIELDS:
+        if line.startswith(field + ":"):
+            return (field, line.split(":")[1].strip())
+
+
+class LookupModule(LookupBase):
+    def run(self, terms, variables=None, **kwargs):
+        ret = []
+        for term in terms:
+            pkg_details = {}
+            with open(term, 'r') as f:
+                for line in f:
+                    #non empty line means pkg data
+                    if line.strip():
+                        if line.startswith('Package:'):
+                            currentpkg = line.split(":")[1].strip()
+                            pkg_details[currentpkg] = {}
+                        elif line.startswith('Provides:'):
+                            pkg_details[line.split(":")[1].strip()] = pkg_details[currentpkg]
+                        else:
+                            # Now doing package data
+                            parsed = parse_fields(line)
+                            if parsed:
+                                pkg_details[currentpkg][parsed[0]] = parsed[1]
+                    else:
+                        currentpkg=""
+            ret.append(pkg_details)
+        return ret
+
+# For debug purposes
+if __name__ == '__main__':
+    import sys
+    import json
+    print(json.dumps(LookupModule().run(terms=sys.argv[1:]), indent=4, sort_keys=True))

tests/files/Packages

@@ -0,0 +1,44 @@
+Package: 0ad
+Priority: optional
+Section: universe/games
+Installed-Size: 9487
+Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
+Original-Maintainer: Debian Games Team <pkg-games-devel@lists.alioth.debian.org>
+Architecture: amd64
+Version: 0.0.16-2~ubuntu14.04.1
+Depends: 0ad-data (>= 0.0.16), 0ad-data (<= 0.0.16-2~ubuntu14.04.1), 0ad-data-common (>= 0.0.16), 0ad-data-common (<= 0.0.16-2~ubuntu14.04.1), libboost-filesystem1.54.0, libc6 (>= 2.15), libcurl3-gnutls (>= 7.16.2), libenet2a (>= 1.3.7+ds), libgcc1 (>= 1:4.1.1), libgl1-mesa-glx | libgl1, libgloox11, libicu52 (>= 52~m1-1~), libjpeg8 (>= 8c), libminiupnpc8 (>= 1.6), libmozjs-24-0, libnvtt2, libopenal1 (>= 1:1.13), libpng12-0 (>= 1.2.13-4), libsdl1.2debian (>= 1.2.11), libstdc++6 (>= 4.6), libvorbisfile3 (>= 1.1.2), libwxbase3.0-0 (>= 3.0.0), libwxgtk3.0-0 (>= 3.0.0), libx11-6, libxcursor1 (>> 1.1.2), libxml2 (>= 2.9.0), zlib1g (>= 1:1.2.0)
+Pre-Depends: dpkg (>= 1.15.6~)
+Filename: pool/main/0/0ad/0ad_0.0.16-2~ubuntu14.04.1_amd64.deb
+Size: 2599794
+MD5sum: 7e5f2ba5e1a95e47753eeb962af32e26
+SHA1: 3cf898d4595092daa274e6cd8d9afd0332b0afbe
+SHA256: f4602a90a305abeacb4a48bbfd7d609aa7cbb3ed2ab9127ae30ef64a4be88378
+Description: Real-time strategy game of ancient warfare
+Origin: Ubuntu
+Description-Md5: d943033bedada21853d2ae54a2578a7b
+Homepage: http://play0ad.com/
+Bugs: https://bugs.launchpad.net/ubuntu/+filebug
+
+Package: aide
+Priority: optional
+Section: admin
+Installed-Size: 1581
+Maintainer: Ubuntu Developers <ubuntu-devel-discuss@lists.ubuntu.com>
+Original-Maintainer: Aide Maintainers <pkg-aide-maintainers@lists.alioth.debian.org>
+Architecture: amd64
+Version: 0.16~a2.git20130520-2ubuntu0.1
+Provides: aide-binary
+Recommends: aide-common (= 0.16~a2.git20130520-2ubuntu0.1)
+Conflicts: aide-dynamic, aide-xen
+Filename: pool/main/a/aide/aide_0.16~a2.git20130520-2ubuntu0.1_amd64.deb
+Size: 576282
+MD5sum: 53ef6f122eb5d4da6546b7d84de40141
+SHA1: 62cfec54ee4654b3a3390430d545961a52184a9f
+SHA256: 527e0d4e26edc274c18f255d2f44cd1d48b3681e37883e8d1aec2225bfc0b9ed
+Description: Advanced Intrusion Detection Environment - static binary
+Built-Using: acl (= 2.2.52-1), attr (= 1:2.4.47-1ubuntu1), e2fsprogs (= 1.42.9-3ubuntu1), eglibc (= 2.19-0ubuntu6.3), libselinux (= 2.2.2-1ubuntu0.1), mhash (= 0.9.9.9-4), zlib (= 1:1.2.8.dfsg-1ubuntu1)
+Homepage: http://aide.sourceforge.net
+Origin: Ubuntu
+Supported: 5y
+Description-Md5: 5cf26f0af8c95254f9ec7920c1214cbe
+Bugs: https://bugs.launchpad.net/ubuntu/+filebug

tests/test-lookups.yml

@@ -0,0 +1,40 @@
+---
+# Copyright 2017, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# (c) 2017, Jean-Philippe Evrard <jean-philippe.evrard@rackspace.co.uk>
+
+
+- name: Test lookups
+  hosts: localhost
+  connection: local
+  gather_facts: no
+  tasks:
+    - name: Ensure Package list contains the files listed in Package
+      set_fact:
+        packages_dict: "{{ lookup('packages_file', playbook_dir ~ '/files/Packages') }}"
+    - name: Validate that Filename, MD5sum, SHA1, SHA256 and Version are properly generated
+      assert:
+        that:
+          - packages_dict['0ad']['Filename'] == "pool/main/0/0ad/0ad_0.0.16-2~ubuntu14.04.1_amd64.deb"
+          - packages_dict['0ad']['MD5sum'] == "7e5f2ba5e1a95e47753eeb962af32e26"
+          - packages_dict['0ad']['SHA1'] == "3cf898d4595092daa274e6cd8d9afd0332b0afbe"
+          - packages_dict['0ad']['SHA256'] == "f4602a90a305abeacb4a48bbfd7d609aa7cbb3ed2ab9127ae30ef64a4be88378"
+          - packages_dict['0ad']['Version'] == "0.0.16-2~ubuntu14.04.1"
+    - name: Validate that a provides package is listed as a package too
+      assert:
+        that:
+          - packages_dict['aide'] is defined
+          - packages_dict['aide-binary'] is defined
+          - packages_dict['aide'] == packages_dict['aide-binary']

tests/test.yml

@@ -18,4 +18,6 @@
 
 - include: test-filters.yml
 
+- include: test-lookups.yml
+
 - include: test-strategy-tagfilter.yml