From c569dc02d31e4d4bc58430f4dad290f3de96d182 Mon Sep 17 00:00:00 2001 From: Wayne Warren Date: Thu, 31 Dec 2015 14:59:16 -0800 Subject: [PATCH] Move load_files method to YamlParser from Builder. This is the first step in removing jenkins_modules.parser.YamlParser references from jenkins_jobs.builder.Builder entirely, necessary because load_files is actually a YamlParser behavior, not a Builder behavior. Change-Id: I1cad99b4cdb44af25ba398837f7f328cfcbd5bbb --- jenkins_jobs/builder.py | 60 ++++--------------------------- jenkins_jobs/parser.py | 51 ++++++++++++++++++++++++++ tests/localyaml/test_localyaml.py | 6 ++-- 3 files changed, 61 insertions(+), 56 deletions(-) diff --git a/jenkins_jobs/builder.py b/jenkins_jobs/builder.py index e263c13e4..582a5fd66 100644 --- a/jenkins_jobs/builder.py +++ b/jenkins_jobs/builder.py @@ -242,57 +242,6 @@ class Builder(object): self._plugins_list = self.jenkins.get_plugins_info() return self._plugins_list - def load_files(self, fn): - self.parser = YamlParser(self.jjb_config, self.plugins_list) - - # handle deprecated behavior, and check that it's not a file like - # object as these may implement the '__iter__' attribute. - if not hasattr(fn, '__iter__') or hasattr(fn, 'read'): - logger.warning( - 'Passing single elements for the `fn` argument in ' - 'Builder.load_files is deprecated. Please update your code ' - 'to use a list as support for automatic conversion will be ' - 'removed in a future version.') - fn = [fn] - - files_to_process = [] - for path in fn: - if not hasattr(path, 'read') and os.path.isdir(path): - files_to_process.extend([os.path.join(path, f) - for f in os.listdir(path) - if (f.endswith('.yml') - or f.endswith('.yaml'))]) - else: - files_to_process.append(path) - - # symlinks used to allow loading of sub-dirs can result in duplicate - # definitions of macros and templates when loading all from top-level - unique_files = [] - for f in files_to_process: - if hasattr(f, 'read'): - unique_files.append(f) - continue - rpf = os.path.realpath(f) - if rpf not in unique_files: - unique_files.append(rpf) - else: - logger.warning("File '%s' already added as '%s', ignoring " - "reference to avoid duplicating yaml " - "definitions." % (f, rpf)) - - for in_file in unique_files: - # use of ask-for-permissions instead of ask-for-forgiveness - # performs better when low use cases. - if hasattr(in_file, 'name'): - fname = in_file.name - else: - fname = in_file - logger.debug("Parsing YAML file {0}".format(fname)) - if hasattr(in_file, 'read'): - self.parser.parse_fp(in_file) - else: - self.parser.parse(in_file) - def delete_old_managed(self, keep=None): jobs = self.jenkins.get_jobs() deleted_jobs = 0 @@ -313,8 +262,10 @@ class Builder(object): return deleted_jobs def delete_job(self, jobs_glob, fn=None): + self.parser = YamlParser(self.jjb_config, self.plugins_list) + if fn: - self.load_files(fn) + self.parser.load_files(fn) self.parser.expandYaml([jobs_glob]) jobs = [j['name'] for j in self.parser.jobs] else: @@ -348,7 +299,10 @@ class Builder(object): def update_jobs(self, input_fn, jobs_glob=None, output=None, n_workers=None): orig = time.time() - self.load_files(input_fn) + + self.parser = YamlParser(self.jjb_config, self.plugins_list) + self.parser.load_files(input_fn) + self.parser.expandYaml(jobs_glob) self.parser.generateXML() step = time.time() diff --git a/jenkins_jobs/parser.py b/jenkins_jobs/parser.py index ba947e79c..ff1906f46 100644 --- a/jenkins_jobs/parser.py +++ b/jenkins_jobs/parser.py @@ -20,6 +20,7 @@ import fnmatch import io import itertools import logging +import os import pkg_resources from jenkins_jobs.constants import MAGIC_MANAGE_STRING @@ -82,6 +83,56 @@ class YamlParser(object): self.registry = ModuleRegistry(jjb_config, plugins_info) + def load_files(self, fn): + + # handle deprecated behavior, and check that it's not a file like + # object as these may implement the '__iter__' attribute. + if not hasattr(fn, '__iter__') or hasattr(fn, 'read'): + logger.warning( + 'Passing single elements for the `fn` argument in ' + 'Builder.load_files is deprecated. Please update your code ' + 'to use a list as support for automatic conversion will be ' + 'removed in a future version.') + fn = [fn] + + files_to_process = [] + for path in fn: + if not hasattr(path, 'read') and os.path.isdir(path): + files_to_process.extend([os.path.join(path, f) + for f in os.listdir(path) + if (f.endswith('.yml') + or f.endswith('.yaml'))]) + else: + files_to_process.append(path) + + # symlinks used to allow loading of sub-dirs can result in duplicate + # definitions of macros and templates when loading all from top-level + unique_files = [] + for f in files_to_process: + if hasattr(f, 'read'): + unique_files.append(f) + continue + rpf = os.path.realpath(f) + if rpf not in unique_files: + unique_files.append(rpf) + else: + logger.warning("File '%s' already added as '%s', ignoring " + "reference to avoid duplicating yaml " + "definitions." % (f, rpf)) + + for in_file in unique_files: + # use of ask-for-permissions instead of ask-for-forgiveness + # performs better when low use cases. + if hasattr(in_file, 'name'): + fname = in_file.name + else: + fname = in_file + logger.debug("Parsing YAML file {0}".format(fname)) + if hasattr(in_file, 'read'): + self.parse_fp(in_file) + else: + self.parse(in_file) + def parse_fp(self, fp): # wrap provided file streams to ensure correct encoding used data = local_yaml.load(utils.wrap_stream(fp), search_path=self.path) diff --git a/tests/localyaml/test_localyaml.py b/tests/localyaml/test_localyaml.py index a40002f78..9f5d6923d 100644 --- a/tests/localyaml/test_localyaml.py +++ b/tests/localyaml/test_localyaml.py @@ -21,8 +21,8 @@ from testtools import ExpectedException from testtools import TestCase from yaml.composer import ComposerError -from jenkins_jobs import builder from jenkins_jobs.config import JJBConfig +from jenkins_jobs.parser import YamlParser from tests.base import get_scenarios from tests.base import JsonTestCase from tests.base import LoggingFixture @@ -83,5 +83,5 @@ class TestCaseLocalYamlIncludeAnchors(LoggingFixture, TestCase): jjb_config.jenkins['password'] = 'password' jjb_config.builder['plugins_info'] = [] jjb_config.validate() - b = builder.Builder(jjb_config) - b.load_files([os.path.join(self.fixtures_path, f) for f in files]) + j = YamlParser(jjb_config) + j.load_files([os.path.join(self.fixtures_path, f) for f in files])