Add support for purging obsolete, managed jobs

Make it possible to get rid of no-longer-managed jobs and so
automatically clean up any such jobs.  This is optional and defaults
to off.

Change-Id: I059b811ddaaaae3105e7835e927549d9c0bbca15
Fixes: https://bugs.launchpad.net/openstack-ci/+bug/1183716

jenkins_jobs/builder.py

@@ -30,6 +30,7 @@ import itertools
 from jenkins_jobs.errors import JenkinsJobsException
 
 logger = logging.getLogger(__name__)
+MAGIC_MANAGE_STRING = "<!-- Managed by Jenkins Job Builder -->"
 
 
 def deep_format(obj, paramdict):
@@ -202,6 +203,8 @@ class YamlParser(object):
 
     def getXMLForJob(self, data):
         kind = data.get('project-type', 'freestyle')
+        data["description"] = data.get("description", "") + \
+            self.get_managed_string()
         for ep in pkg_resources.iter_entry_points(
                 group='jenkins_jobs.projects', name=kind):
             Mod = ep.load()
@@ -217,6 +220,11 @@ class YamlParser(object):
             if hasattr(module, 'gen_xml'):
                 module.gen_xml(self, xml, data)
 
+    def get_managed_string(self):
+        # The \n\n is not hard coded, because they get stripped if the
+        # project does not otherwise have a description.
+        return "\n\n" + MAGIC_MANAGE_STRING
+
 
 class ModuleRegistry(object):
     def __init__(self, config):
@@ -391,6 +399,16 @@ class Jenkins(object):
     def get_jobs(self):
         return self.jenkins.get_jobs()
 
+    def is_managed(self, job_name):
+        xml = self.jenkins.get_job_config(job_name)
+        out = XML.fromstring(xml)
+        try:
+            description = out.find(".//description").text
+            return description.endswith(MAGIC_MANAGE_STRING)
+        except AttributeError:
+            pass
+        return False
+
 
 class Builder(object):
     def __init__(self, jenkins_url, jenkins_user, jenkins_password,
@@ -405,6 +423,15 @@ class Builder(object):
         if(self.cache.is_cached(name)):
             self.cache.set(name, '')
 
+    def delete_old_managed(self, keep):
+        jobs = self.jenkins.get_jobs()
+        for job in jobs:
+            if job['name'] not in keep and \
+                    self.jenkins.is_managed(job['name']):
+                logger.info("Removing obsolete jenkins job {0}"
+                            .format(job['name']))
+                self.delete_job(job['name'])
+
     def delete_all_jobs(self):
         jobs = self.jenkins.get_jobs()
         for job in jobs:
@@ -451,3 +478,4 @@ class Builder(object):
                 self.cache.set(job.name, md5)
             else:
                 logger.debug("'{0}' has not changed".format(job.name))
+        return parser.jobs

jenkins_jobs/cmd.py

@@ -35,6 +35,9 @@ def main():
     parser_update = subparser.add_parser('update')
     parser_update.add_argument('path', help='Path to YAML file or directory')
     parser_update.add_argument('names', help='name(s) of job(s)', nargs='*')
+    parser_update.add_argument('--delete-old', help='Delete obsolete jobs',
+                               action='store_true',
+                               dest='delete_old', default=False,)
     parser_test = subparser.add_parser('test')
     parser_test.add_argument('path', help='Path to YAML file or directory')
     parser_test.add_argument('-o', dest='output_dir',
@@ -106,7 +109,9 @@ def main():
     elif options.command == 'update':
         logger.info("Updating jobs in {0} ({1})".format(
             options.path, options.names))
-        builder.update_job(options.path, options.names)
+        jobs = builder.update_job(options.path, options.names)
+        if options.delete_old:
+            builder.delete_old_managed(keep=[x.name for x in jobs])
     elif options.command == 'test':
         builder.update_job(options.path, options.name,
                            output_dir=options.output_dir)