Add ability to make tarballs from Git clones

This feature allows to specify added tarballs source
by git repository and reference (tag/branch).

Change-Id: Ie7e8a5a5852421a9d99e6371f22ad3638fe077b6
Partially-Implements: blueprint build-script

kolla/cmd/build.py

@@ -26,11 +26,13 @@ import requests
 import shutil
 import signal
 import sys
+import tarfile
 import tempfile
 from threading import Thread
 import time
 
 import docker
+import git
 import jinja2
 from requests.exceptions import ConnectionError
 
@@ -77,20 +79,51 @@ class WorkerThread(Thread):
                 self.queue.task_done()
                 break
 
-    def process_source(self, source, dest_dir):
+    def process_source(self, image):
+        source = image['source']
+        dest_dir = image['path']
+        dest_tar = os.path.join(dest_dir, source['dest'])
+
         if source.get('type') == 'url':
+            LOG.debug("Getting tarball from " + source['source'])
             r = requests.get(source['source'])
 
             if r.status_code == 200:
-                with open(os.path.join(dest_dir, source['dest']), 'wb') as f:
+                with open(dest_tar, 'wb') as f:
                     f.write(r.content)
             else:
                 LOG.error(
                     'Failed to download tarball: status_code {}'.format(
                         r.status_code))
+                image['status'] = "error"
+                return
 
-            # Set time on destination tarball to epoch 0
+        elif source.get('type') == 'git':
-            os.utime(os.path.join(dest_dir, source['dest']), (0, 0))
+            clone_dir = os.path.splitext(dest_tar)[0] + \
+                '-' + source['reference']
+            try:
+                LOG.debug("Cloning from " + source['source'])
+                git.Git().clone(source['source'], clone_dir)
+                LOG.debug("Git checkout by reference " + source['reference'])
+                git.Git(clone_dir).checkout(source['reference'])
+            except Exception as e:
+                LOG.error("Failed to get source from git")
+                LOG.error("Error: " + str(e))
+                # clean-up clone folder to retry
+                shutil.rmtree(clone_dir)
+                image['status'] = "error"
+                return
+
+            with tarfile.open(dest_tar, 'w') as tar:
+                tar.add(clone_dir, arcname=os.path.basename(clone_dir))
+
+        else:
+            LOG.error("Wrong source type: " + source.get('type'))
+            image['status'] = "error"
+            return
+
+        # Set time on destination tarball to epoch 0
+        os.utime(os.path.join(dest_dir, source['dest']), (0, 0))
 
     def builder(self, image):
         LOG.info('Processing: {}'.format(image['name']))
@@ -104,8 +137,10 @@ class WorkerThread(Thread):
             image['status'] = "parent_error"
             return
 
-        if 'source' in image:
+        if 'source' in image and 'source' in image['source']:
-            self.process_source(image['source'], image['path'])
+            self.process_source(image)
+            if image['status'] == "error":
+                return
 
         # Pull the latest image for the base distro only
         pull = True if image['parent'] is None else False
@@ -204,17 +239,19 @@ class KollaWorker(object):
 
     def __init__(self, args):
         def find_base_dir():
-            if os.path.basename(sys.path[0]) == 'tests':
+            script_path = os.path.dirname(os.path.realpath(sys.argv[0]))
-                return os.path.join(sys.path[0], '..')
+            if os.path.basename(script_path) == 'cmd':
-            if os.path.basename(sys.path[0]) == 'cmd':
+                return os.path.join(script_path, '..', '..')
-                return os.path.join(sys.path[0], '..', '..')
+            if os.path.basename(script_path) == 'bin':
-            if os.path.basename(sys.path[0]) == 'bin':
                 return '/usr/share/kolla'
+            if os.path.exists(os.path.join(script_path, 'tests')):
+                return script_path
             raise KollaDirNotFoundException(
                 'I do not know where your Kolla directory is'
             )
 
-        self.base_dir = find_base_dir()
+        self.base_dir = os.path.abspath(find_base_dir())
+        LOG.debug("Kolla base directory: " + self.base_dir)
         self.images_dir = os.path.join(self.base_dir, 'docker')
         self.templates_dir = os.path.join(self.base_dir, 'docker_templates')
         self.namespace = args['namespace']
@@ -225,7 +262,7 @@ class KollaWorker(object):
         self.tag = args['tag']
         self.prefix = self.base + '-' + self.type_ + '-'
         self.config = ConfigParser.SafeConfigParser()
-        self.config.read(os.path.join(sys.path[0], self.base_dir, 'build.ini'))
+        self.config.read(os.path.join(self.base_dir, 'build.ini'))
         self.include_header = args['include_header']
         self.regex = args['regex']
 
@@ -409,6 +446,10 @@ class KollaWorker(object):
                                                                 'location')
                     image['source']['dest'] = self.config.get(image['name'],
                                                               'dest_filename')
+                    if image['source']['type'] == 'git':
+                        image['source']['reference'] = \
+                            self.config.get(image['name'], 'reference')
+
                 except ConfigParser.NoSectionError:
                     LOG.debug('No config found for {}'.format(image['name']))
                     pass

requirements.txt

@@ -1,3 +1,5 @@
 pbr<2.0,>=1.4
 docker-py>=1.1.0  # Apache-2.0
 Jinja2>=2.6  # BSD License (3 clause)
+gitdb>=0.6.4 # BSD License (3 clause)
+GitPython>=1.0.1 # BSD License (3 clause)