From b5a8b7035ed8b80e288807b6f478001b5f759d37 Mon Sep 17 00:00:00 2001 From: Tomas Janousek Date: Fri, 3 Aug 2018 22:59:50 +0200 Subject: [PATCH] Request multiple folder levels at once in get_all_jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On our Jenkins instance with almost a hundred folders, JJB update stalls for quite a while because it calls get_all_jobs. When invoked locally at the Jenkins master, it's a matter of seconds, on a fast broadband link and VPN, it's 2 minutes, and on a train it's easily 10 minutes. But there's trick! curl \ --show-error --silent --fail \ --user : --negotiate \ --get \ --data-urlencode \ tree=jobs\[url\,name\,jobs\[url\,name\,jobs\[url\,name\,\ jobs\[url\,name\,jobs\[url\,name\,jobs\[url\,name\,\ jobs\[url\,name\,jobs\[url\,name\,jobs\[url\,name\,\ jobs\[url\,name\,jobs\]\]\]\]\]\]\]\]\]\] https://jenkins.example.com/api/json This returns almost instantly. And it gets better: if we fail to correctly guess the nesting level necessary, Jenkins returns …, "jobs": [{}, {}, …], … so we can easily detect that we need to recurse deeper. Change-Id: I7268259149e4bc8939c512a112c7e6ec1908224f --- jenkins/__init__.py | 100 ++++++++++++++++---------------------- tests/jobs/base.py | 16 ++++++ tests/jobs/test_get.py | 2 +- tests/jobs/test_getall.py | 25 ++++++++++ 4 files changed, 85 insertions(+), 58 deletions(-) diff --git a/jenkins/__init__.py b/jenkins/__init__.py index d5c5e8e..9717f58 100755 --- a/jenkins/__init__.py +++ b/jenkins/__init__.py @@ -97,7 +97,8 @@ INFO = 'api/json' PLUGIN_INFO = 'pluginManager/api/json?depth=%(depth)s' CRUMB_URL = 'crumbIssuer/api/json' WHOAMI_URL = 'me/api/json?depth=%(depth)s' -JOBS_QUERY = '?tree=jobs[url,color,name,jobs]' +JOBS_QUERY = '?tree=%s' +JOBS_QUERY_TREE = 'jobs[url,color,name,%s]' JOB_INFO = '%(folder_url)sjob/%(short_name)s/api/json?depth=%(depth)s' JOB_NAME = '%(folder_url)sjob/%(short_name)s/api/json?tree=name' ALL_BUILDS = '%(folder_url)sjob/%(short_name)s/api/json?tree=allBuilds[number,url]' @@ -474,17 +475,21 @@ class Jenkins(object): raise JenkinsException( "Could not parse JSON info for job[%s]" % name) - def get_job_info_regex(self, pattern, depth=0, folder_depth=0): + def get_job_info_regex(self, pattern, depth=0, folder_depth=0, + folder_depth_per_request=10): '''Get a list of jobs information that contain names which match the regex pattern. :param pattern: regex pattern, ``str`` :param depth: JSON depth, ``int`` :param folder_depth: folder level depth to search ``int`` + :param folder_depth_per_request: Number of levels to fetch at once, + ``int``. See :func:`get_all_jobs`. :returns: List of jobs info, ``list`` ''' result = [] - jobs = self.get_all_jobs(folder_depth) + jobs = self.get_all_jobs(folder_depth=folder_depth, + folder_depth_per_request=folder_depth_per_request) for job in jobs: if re.search(pattern, job['name']): result.append(self.get_job_info(job['name'], depth=depth)) @@ -942,7 +947,7 @@ class Jenkins(object): return plugins_data - def get_jobs(self, folder_depth=0, view_name=None): + def get_jobs(self, folder_depth=0, folder_depth_per_request=10, view_name=None): """Get list of jobs. Each job is a dictionary with 'name', 'url', 'color' and 'fullname' @@ -955,6 +960,8 @@ class Jenkins(object): :param folder_depth: Number of levels to search, ``int``. By default 0, which will limit search to toplevel. None disables the limit. + :param folder_depth_per_request: Number of levels to fetch at once, + ``int``. See :func:`get_all_jobs`. :param view_name: Name of a Jenkins view for which to retrieve jobs, ``str``. By default, the job list is not limited to a specific view. @@ -976,9 +983,10 @@ class Jenkins(object): if view_name: return self._get_view_jobs(name=view_name) else: - return self.get_all_jobs(folder_depth=folder_depth) + return self.get_all_jobs(folder_depth=folder_depth, + folder_depth_per_request=folder_depth_per_request) - def get_all_jobs(self, folder_depth=None): + def get_all_jobs(self, folder_depth=None, folder_depth_per_request=10): """Get list of all jobs recursively to the given folder depth. Each job is a dictionary with 'name', 'url', 'color' and 'fullname' @@ -986,46 +994,37 @@ class Jenkins(object): :param folder_depth: Number of levels to search, ``int``. By default None, which will search all levels. 0 limits to toplevel. + :param folder_depth_per_request: Number of levels to fetch at once, + ``int``. By default 10, which is usually enough to fetch all jobs + using a single request and still easily fits into an HTTP request. :returns: list of jobs, ``[ { str: str} ]`` .. note:: - On instances with many folders it may be more efficient to use the - run_script method to retrieve all jobs instead. + On instances with many folders it would not be efficient to fetch + each folder separately, hence `folder_depth_per_request` levels + are fetched at once using the ``tree`` query parameter:: - Example:: + ?tree=jobs[url,color,name,jobs[...,jobs[...,jobs[...,jobs]]]] - server.run_script(\"\"\" - import groovy.json.JsonBuilder; + If there are more folder levels than the query asks for, Jenkins + returns empty [#]_ objects at the deepest level:: - // get all projects excluding matrix configuration - // as they are simply part of a matrix project. - // there may be better ways to get just jobs - items = Jenkins.instance.getAllItems(AbstractProject); - items.removeAll { - it instanceof hudson.matrix.MatrixConfiguration - }; + {"name": "folder", "url": "...", "jobs": [{}, {}, ...]} - def json = new JsonBuilder() - def root = json { - jobs items.collect { - [ - name: it.name, - url: Jenkins.instance.getRootUrl() + it.getUrl(), - color: it.getIconColor().toString(), - fullname: it.getFullName() - ] - } - } - - // use json.toPrettyString() if viewing - println json.toString() - \"\"\") + This makes it possible to detect when additional requests are + needed. + .. [#] Actually recent Jenkins includes a ``_class`` field + everywhere, but it's missing the requested fields. """ - jobs_list = [] + jobs_query = 'jobs' + for _ in range(folder_depth_per_request): + jobs_query = JOBS_QUERY_TREE % jobs_query + jobs_query = JOBS_QUERY % jobs_query - jobs = [(0, [], self.get_info(query=JOBS_QUERY)['jobs'])] + jobs_list = [] + jobs = [(0, [], self.get_info(query=jobs_query)['jobs'])] for lvl, root, lvl_jobs in jobs: if not isinstance(lvl_jobs, list): lvl_jobs = [lvl_jobs] @@ -1036,13 +1035,16 @@ class Jenkins(object): if u'fullname' not in job: job[u'fullname'] = '/'.join(path) jobs_list.append(job) - if 'jobs' in job: # folder + if 'jobs' in job and isinstance(job['jobs'], list): # folder if folder_depth is None or lvl < folder_depth: - url_path = ''.join(['/job/' + p for p in path]) - jobs.append( - (lvl + 1, path, - self.get_info(url_path, - query=JOBS_QUERY)['jobs'])) + children = job['jobs'] + # once folder_depth_per_request is reached, Jenkins + # returns empty objects + if any('url' not in child for child in job['jobs']): + url_path = ''.join(['/job/' + p for p in path]) + children = self.get_info(url_path, + query=jobs_query)['jobs'] + jobs.append((lvl + 1, path, children)) return jobs_list def copy_job(self, from_name, to_name): @@ -1161,22 +1163,6 @@ class Jenkins(object): '''Get the number of jobs on the Jenkins server :returns: Total number of jobs, ``int`` - - .. note:: - - On instances with many folders it may be more efficient to use the - run_script method to retrieve the total number of jobs instead. - - Example:: - - # get all projects excluding matrix configuration - # as they are simply part of a matrix project. - server.run_script( - "print(Hudson.instance.getAllItems(" - " hudson.model.AbstractProject).count{" - " !(it instanceof hudson.matrix.MatrixConfiguration)" - " })") - ''' return len(self.get_all_jobs()) diff --git a/tests/jobs/base.py b/tests/jobs/base.py index 1ad980e..5913271 100644 --- a/tests/jobs/base.py +++ b/tests/jobs/base.py @@ -72,3 +72,19 @@ class JenkinsGetJobsTestBase(JenkinsJobsTestBase): {'name': 'my_job', 'color': 'blue', 'url': 'http://...'} ]} ] + + jobs_in_folder_deep_query = [ + {'jobs': [ + {'name': 'top_folder', 'url': 'http://...', 'jobs': [ + {'name': 'middle_folder', 'url': 'http://...', 'jobs': [ + {'name': 'bottom_folder', 'url': 'http://...', + 'jobs': [{}, {}]} + ]} + ]} + ]}, + # top_folder/middle_folder/bottom_folder jobs + {'jobs': [ + {'name': 'my_job1', 'color': 'blue', 'url': 'http://...'}, + {'name': 'my_job2', 'color': 'blue', 'url': 'http://...'} + ]} + ] diff --git a/tests/jobs/test_get.py b/tests/jobs/test_get.py index 36c599b..43de9f8 100644 --- a/tests/jobs/test_get.py +++ b/tests/jobs/test_get.py @@ -18,7 +18,7 @@ class JenkinsGetJobsTest(JenkinsGetJobsTestBase): job_info_to_return = {u'jobs': jobs} jenkins_mock.return_value = json.dumps(job_info_to_return) - job_info = self.j.get_jobs() + job_info = self.j.get_jobs(folder_depth_per_request=1) jobs[u'fullname'] = jobs[u'name'] self.assertEqual(job_info, [jobs]) diff --git a/tests/jobs/test_getall.py b/tests/jobs/test_getall.py index 1daf84c..ffef71a 100644 --- a/tests/jobs/test_getall.py +++ b/tests/jobs/test_getall.py @@ -120,3 +120,28 @@ class JenkinsGetAllJobsTest(JenkinsGetJobsTestBase): ] self.assertEqual(expected_request_urls, self.got_request_urls(jenkins_mock)) + + @patch.object(jenkins.Jenkins, 'jenkins_open') + def test_deep_query(self, jenkins_mock): + jenkins_mock.side_effect = map( + json.dumps, self.jobs_in_folder_deep_query) + + jobs_info = self.j.get_all_jobs() + + expected_fullnames = [ + u"top_folder", + u"top_folder/middle_folder", + u"top_folder/middle_folder/bottom_folder", + u"top_folder/middle_folder/bottom_folder/my_job1", + u"top_folder/middle_folder/bottom_folder/my_job2" + ] + self.assertEqual(len(expected_fullnames), len(jobs_info)) + got_fullnames = [job[u"fullname"] for job in jobs_info] + self.assertEqual(expected_fullnames, got_fullnames) + + expected_request_urls = [ + self.make_url('api/json'), + self.make_url('job/top_folder/job/middle_folder/job/bottom_folder/api/json') + ] + self.assertEqual(expected_request_urls, + self.got_request_urls(jenkins_mock))