From 449bb24f8f40866eb57404b8e4c5e77b529a996b Mon Sep 17 00:00:00 2001 From: Darragh Bailey Date: Wed, 2 Dec 2015 11:12:24 +0000 Subject: [PATCH] Infer and build entry points from component modules Use the component type entry points to infer modules to scan corresponding modules and automatically construct entry points for any functions associated with yaml entries in JJB definitions. This removes the need for new JJB modules added directly to the project to also need to be listed in the setup.cfg to be reachable. Simply adding the needed functionality to the correct file ensures that it is available. Change-Id: I2a0fd8f6720ddfd8582b58e47556a87ed1ad553d --- jenkins_jobs/modules/parameters.py | 2 +- jenkins_jobs/modules/publishers.py | 8 +- jenkins_jobs/modules/wrappers.py | 6 +- jenkins_jobs/registry.py | 50 +++++- setup.cfg | 248 ----------------------------- 5 files changed, 53 insertions(+), 261 deletions(-) diff --git a/jenkins_jobs/modules/parameters.py b/jenkins_jobs/modules/parameters.py index 923ba2a9e..1eac97740 100644 --- a/jenkins_jobs/modules/parameters.py +++ b/jenkins_jobs/modules/parameters.py @@ -620,7 +620,7 @@ def matrix_combinations_param(parser, xml_parent, data): def copyartifact_build_selector_param(parser, xml_parent, data): - """yaml: copyartifact-build-selector-param + """yaml: copyartifact-build-selector Control via a build parameter, which build the copyartifact plugin should copy when it is configured to use 'build-param'. Requires the Jenkins diff --git a/jenkins_jobs/modules/publishers.py b/jenkins_jobs/modules/publishers.py index 1220b47cc..a73dfb56e 100644 --- a/jenkins_jobs/modules/publishers.py +++ b/jenkins_jobs/modules/publishers.py @@ -120,9 +120,9 @@ def blame_upstream(parser, xml_parent, data): def jclouds(parser, xml_parent, data): - """yaml: JClouds Cloud Storage Settings - provides a way to store artifacts on JClouds supported storage providers. - Requires the Jenkins `JClouds Plugin. + """yaml: jclouds + JClouds Cloud Storage Settings provides a way to store artifacts on + JClouds supported storage providers. Requires the Jenkins `JClouds Plugin. `_ JClouds Cloud Storage Settings must be configured for the Jenkins instance. @@ -2773,7 +2773,7 @@ def html_publisher(parser, xml_parent, data): def rich_text_publisher(parser, xml_parent, data): - """yaml: rich_text_publisher + """yaml: rich-text-publisher This plugin puts custom rich text message to the Build pages and Job main page. diff --git a/jenkins_jobs/modules/wrappers.py b/jenkins_jobs/modules/wrappers.py index fbcee538a..6d12000d3 100644 --- a/jenkins_jobs/modules/wrappers.py +++ b/jenkins_jobs/modules/wrappers.py @@ -1675,7 +1675,7 @@ def android_emulator(parser, xml_parent, data): def artifactory_maven(parser, xml_parent, data): - """ yaml: artifactory-maven + """yaml: artifactory-maven Wrapper for non-Maven projects. Requires the :jenkins-wiki:`Artifactory Plugin ` @@ -1720,7 +1720,7 @@ def artifactory_maven(parser, xml_parent, data): def artifactory_generic(parser, xml_parent, data): - """ yaml: artifactory-generic + """yaml: artifactory-generic Wrapper for non-Maven projects. Requires the :jenkins-wiki:`Artifactory Plugin ` @@ -1797,7 +1797,7 @@ def artifactory_generic(parser, xml_parent, data): def artifactory_maven_freestyle(parser, xml_parent, data): - """ yaml: artifactory-maven-freestyle + """yaml: artifactory-maven-freestyle Wrapper for Free Stype projects. Requires the Artifactory plugin. Requires :jenkins-wiki:`Artifactory Plugin ` diff --git a/jenkins_jobs/registry.py b/jenkins_jobs/registry.py index d1c9845a9..3c06beed8 100644 --- a/jenkins_jobs/registry.py +++ b/jenkins_jobs/registry.py @@ -19,6 +19,7 @@ import logging import operator import pkg_resources import re +import types from jenkins_jobs.errors import JenkinsJobsException from jenkins_jobs.formatter import deep_format @@ -47,7 +48,7 @@ class ModuleRegistry(object): self.modules.append(mod) self.modules.sort(key=operator.attrgetter('sequence')) if mod.component_type is not None: - self.modules_by_component_type[mod.component_type] = mod + self.modules_by_component_type[mod.component_type] = entrypoint @staticmethod def _get_plugins_info_dict(plugins_list): @@ -142,8 +143,8 @@ class ModuleRegistry(object): raise JenkinsJobsException("Unknown component type: " "'{0}'.".format(component_type)) - component_list_type = self.modules_by_component_type[component_type] \ - .component_list_type + entry_point = self.modules_by_component_type[component_type] + component_list_type = entry_point.load().component_list_type if isinstance(component, dict): # The component is a singleton dictionary of name: dict(args) @@ -168,8 +169,45 @@ class ModuleRegistry(object): # Look for a component function defined in an entry point eps = ModuleRegistry.entry_points_cache.get(component_list_type) if eps is None: - module_eps = list(pkg_resources.iter_entry_points( - group='jenkins_jobs.{0}'.format(component_list_type))) + module_eps = [] + # auto build entry points by inferring from base component_types + mod = pkg_resources.EntryPoint( + "__all__", entry_point.module_name, dist=entry_point.dist) + + Mod = mod.load() + func_eps = [Mod.__dict__.get(a) for a in dir(Mod) + if isinstance(Mod.__dict__.get(a), + types.FunctionType)] + for func_ep in func_eps: + try: + # extract entry point based on docstring + name_line = func_ep.__doc__.split('\n') + if not name_line[0].startswith('yaml:'): + logger.debug("Ignoring '%s' as an entry point" % + name_line) + continue + ep_name = name_line[0].split(' ')[1] + except (AttributeError, IndexError): + # AttributeError by docstring not being defined as + # a string to have split called on it. + # IndexError raised by name_line not containing anything + # after the 'yaml:' string. + logger.debug("Not including func '%s' as an entry point" + % func_ep.__name__) + continue + + module_eps.append( + pkg_resources.EntryPoint( + ep_name, entry_point.module_name, + dist=entry_point.dist, attrs=(func_ep.__name__,))) + logger.debug( + "Adding auto EP '%s=%s:%s'" % + (ep_name, entry_point.module_name, func_ep.__name__)) + + # load from explicitly defined entry points + module_eps.extend(list(pkg_resources.iter_entry_points( + group='jenkins_jobs.{0}'.format(component_list_type)))) + eps = {} for module_ep in module_eps: if module_ep.name in eps: @@ -177,8 +215,10 @@ class ModuleRegistry(object): "Duplicate entry point found for component type: " "'{0}', '{0}'," "name: '{1}'".format(component_type, name)) + eps[module_ep.name] = module_ep + # cache both sets of entry points ModuleRegistry.entry_points_cache[component_list_type] = eps logger.debug("Cached entry point group %s = %s", component_list_type, eps) diff --git a/setup.cfg b/setup.cfg index 0a4c08749..c8792b816 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,271 +45,23 @@ jenkins_jobs.projects = multijob=jenkins_jobs.modules.project_multijob:MultiJob workflow=jenkins_jobs.modules.project_workflow:Workflow jenkins_jobs.builders = - ant=jenkins_jobs.modules.builders:ant - artifact-resolver=jenkins_jobs.modules.builders:artifact_resolver - batch=jenkins_jobs.modules.builders:batch - beaker=jenkins_jobs.modules.builders:beaker - builders-from=jenkins_jobs.modules.builders:builders_from - change-assembly-version=jenkins_jobs.modules.builders:change_assembly_version - cloudformation=jenkins_jobs.modules.builders:cloudformation - cmake=jenkins_jobs.modules.builders:cmake - conditional-step=jenkins_jobs.modules.builders:conditional_step - config-file-provider=jenkins_jobs.modules.builders:config_file_provider - copyartifact=jenkins_jobs.modules.builders:copyartifact - critical-block-start=jenkins_jobs.modules.builders:critical_block_start - critical-block-end=jenkins_jobs.modules.builders:critical_block_end - dsl=jenkins_jobs.modules.builders:dsl - github-notifier=jenkins_jobs.modules.builders:github_notifier - gradle=jenkins_jobs.modules.builders:gradle - grails=jenkins_jobs.modules.builders:grails - groovy=jenkins_jobs.modules.builders:groovy - inject=jenkins_jobs.modules.builders:inject - managed-script=jenkins_jobs.modules.builders:managed_script - maven-builder=jenkins_jobs.modules.builders:maven_builder - maven-target=jenkins_jobs.modules.builders:maven_target - msbuild=jenkins_jobs.modules.builders:msbuild - multijob=jenkins_jobs.modules.builders:multijob - openshift-builder=jenkins_jobs.modules.builders:openshift_builder - openshift-build-verify=jenkins_jobs.modules.builders:openshift_build_verify - openshift-deployer=jenkins_jobs.modules.builders:openshift_deployer - openshift-dep-verify=jenkins_jobs.modules.builders:openshift_dep_verify - openshift-img-tagger=jenkins_jobs.modules.builders:openshift_img_tagger - openshift-scaler=jenkins_jobs.modules.builders:openshift_scaler - openshift-svc-verify=jenkins_jobs.modules.builders:openshift_svc_verify - powershell=jenkins_jobs.modules.builders:powershell - python=jenkins_jobs.modules.builders:python raw=jenkins_jobs.modules.general:raw - runscope=jenkins_jobs.modules.builders:runscope - sbt=jenkins_jobs.modules.builders:sbt - shell=jenkins_jobs.modules.builders:shell - shining-panda=jenkins_jobs.modules.builders:shining_panda - sonar=jenkins_jobs.modules.builders:sonar - sonatype-clm=jenkins_jobs.modules.builders:sonatype_clm - ssh-builder=jenkins_jobs.modules.builders:ssh_builder - system-groovy=jenkins_jobs.modules.builders:system_groovy - trigger-builds=jenkins_jobs.modules.builders:trigger_builds - trigger-remote=jenkins_jobs.modules.builders:trigger_remote jenkins_jobs.reporters = - email=jenkins_jobs.modules.reporters:email - findbugs=jenkins_jobs.modules.reporters:findbugs raw=jenkins_jobs.modules.general:raw jenkins_jobs.properties = - authenticated-build=jenkins_jobs.modules.properties:authenticated_build - authorization=jenkins_jobs.modules.properties:authorization - batch-tasks=jenkins_jobs.modules.properties:batch_tasks - build-blocker=jenkins_jobs.modules.properties:build_blocker - builds-chain-fingerprinter=jenkins_jobs.modules.properties:builds_chain_fingerprinter - copyartifact=jenkins_jobs.modules.properties:copyartifact - delivery-pipeline=jenkins_jobs.modules.properties:delivery_pipeline - extended-choice=jenkins_jobs.modules.properties:extended_choice - github=jenkins_jobs.modules.properties:github - heavy-job=jenkins_jobs.modules.properties:heavy_job - inject=jenkins_jobs.modules.properties:inject - least-load=jenkins_jobs.modules.properties:least_load - ownership=jenkins_jobs.modules.properties:ownership - priority-sorter=jenkins_jobs.modules.properties:priority_sorter - promoted-build=jenkins_jobs.modules.properties:promoted_build raw=jenkins_jobs.modules.general:raw - rebuild=jenkins_jobs.modules.properties:rebuild - sidebar=jenkins_jobs.modules.properties:sidebar - slave-utilization=jenkins_jobs.modules.properties:slave_utilization - throttle=jenkins_jobs.modules.properties:throttle - zeromq-event=jenkins_jobs.modules.properties:zeromq_event jenkins_jobs.parameters = - bool=jenkins_jobs.modules.parameters:bool_param - choice=jenkins_jobs.modules.parameters:choice_param - copyartifact-build-selector=jenkins_jobs.modules.parameters:copyartifact_build_selector_param - dynamic-choice=jenkins_jobs.modules.parameters:dynamic_choice_param - dynamic-choice-scriptler=jenkins_jobs.modules.parameters:dynamic_choice_scriptler_param - dynamic-string=jenkins_jobs.modules.parameters:dynamic_string_param - dynamic-string-scriptler=jenkins_jobs.modules.parameters:dynamic_string_scriptler_param - extended-choice=jenkins_jobs.modules.parameters:extended_choice_param - file=jenkins_jobs.modules.parameters:file_param - label=jenkins_jobs.modules.parameters:label_param - matrix-combinations=jenkins_jobs.modules.parameters:matrix_combinations_param - node=jenkins_jobs.modules.parameters:node_param - password=jenkins_jobs.modules.parameters:password_param raw=jenkins_jobs.modules.general:raw - run=jenkins_jobs.modules.parameters:run_param - string=jenkins_jobs.modules.parameters:string_param - svn-tags=jenkins_jobs.modules.parameters:svn_tags_param - text=jenkins_jobs.modules.parameters:text_param - validating-string=jenkins_jobs.modules.parameters:validating_string_param -jenkins_jobs.metadata = - date=jenkins_jobs.modules.metadata:date_metadata - number=jenkins_jobs.modules.metadata:number_metadata - string=jenkins_jobs.modules.metadata:string_metadata jenkins_jobs.notifications = - http=jenkins_jobs.modules.notifications:http_endpoint raw=jenkins_jobs.modules.general:raw jenkins_jobs.publishers = - aggregate-flow-tests=jenkins_jobs.modules.publishers:aggregate_flow_tests - aggregate-tests=jenkins_jobs.modules.publishers:aggregate_tests - archive=jenkins_jobs.modules.publishers:archive - artifact-deployer=jenkins_jobs.modules.publishers:artifact_deployer - artifactory=jenkins_jobs.modules.publishers:artifactory - blame-upstream=jenkins_jobs.modules.publishers:blame_upstream - build-publisher=jenkins_jobs.modules.publishers:build_publisher - campfire=jenkins_jobs.modules.publishers:campfire - checkstyle=jenkins_jobs.modules.publishers:checkstyle - cifs=jenkins_jobs.modules.publishers:cifs - cigame=jenkins_jobs.modules.publishers:cigame - claim-build=jenkins_jobs.modules.publishers:claim_build - clamav=jenkins_jobs.modules.publishers:clamav - clone-workspace=jenkins_jobs.modules.publishers:clone_workspace - cloudformation=jenkins_jobs.modules.publishers:cloudformation - cloverphp=jenkins_jobs.modules.publishers:cloverphp - cobertura=jenkins_jobs.modules.publishers:cobertura - conditional-publisher=jenkins_jobs.modules.publishers:conditional_publisher - copy-to-master=jenkins_jobs.modules.publishers:copy_to_master - coverage=jenkins_jobs.modules.publishers:coverage - cppcheck=jenkins_jobs.modules.publishers:cppcheck - cucumber-reports=jenkins_jobs.modules.publishers:cucumber_reports - cucumber-testresult=jenkins_jobs.modules.publishers:cucumber_testresult - description-setter=jenkins_jobs.modules.publishers:description_setter - disable-failed-job=jenkins_jobs.modules.publishers:disable_failed_job - display-upstream-changes=jenkins_jobs.modules.publishers:display_upstream_changes - downstream-ext=jenkins_jobs.modules.publishers:downstream_ext - doxygen=jenkins_jobs.modules.publishers:doxygen - dry=jenkins_jobs.modules.publishers:dry - email-ext=jenkins_jobs.modules.publishers:email_ext - email=jenkins_jobs.modules.publishers:email - emotional-jenkins=jenkins_jobs.modules.publishers:emotional_jenkins - findbugs=jenkins_jobs.modules.publishers:findbugs - fingerprint=jenkins_jobs.modules.publishers:fingerprint - fitnesse=jenkins_jobs.modules.publishers:fitnesse - flowdock=jenkins_jobs.modules.publishers:flowdock - ftp=jenkins_jobs.modules.publishers:ftp - gatling=jenkins_jobs.modules.publishers:gatling - git=jenkins_jobs.modules.publishers:git - github-notifier=jenkins_jobs.modules.publishers:github_notifier - google-cloud-storage=jenkins_jobs.modules.publishers:google_cloud_storage - groovy-postbuild=jenkins_jobs.modules.publishers:groovy_postbuild - hipchat=jenkins_jobs.modules.publishers:hipchat - html-publisher=jenkins_jobs.modules.publishers:html_publisher - image-gallery=jenkins_jobs.modules.publishers:image_gallery - ircbot=jenkins_jobs.modules.publishers:ircbot - jabber=jenkins_jobs.modules.publishers:jabber - jacoco=jenkins_jobs.modules.publishers:jacoco - javadoc=jenkins_jobs.modules.publishers:javadoc - jclouds=jenkins_jobs.modules.publishers:jclouds - jdepend=jenkins_jobs.modules.publishers:jdepend - jira=jenkins_jobs.modules.publishers:jira - join-trigger=jenkins_jobs.modules.publishers:join_trigger - junit=jenkins_jobs.modules.publishers:junit - logparser=jenkins_jobs.modules.publishers:logparser - logstash=jenkins_jobs.modules.publishers:logstash - maven-deploy=jenkins_jobs.modules.publishers:maven_deploy - naginator=jenkins_jobs.modules.publishers:naginator - performance=jenkins_jobs.modules.publishers:performance - phabricator=jenkins_jobs.modules.publishers:phabricator - pipeline=jenkins_jobs.modules.publishers:pipeline - plot=jenkins_jobs.modules.publishers:plot - pmd=jenkins_jobs.modules.publishers:pmd - post-tasks=jenkins_jobs.modules.publishers:post_tasks - postbuildscript=jenkins_jobs.modules.publishers:postbuildscript raw=jenkins_jobs.modules.general:raw - rich-text-publisher=jenkins_jobs.modules.publishers:rich_text_publisher - robot=jenkins_jobs.modules.publishers:robot - ruby-metrics=jenkins_jobs.modules.publishers:ruby_metrics - rundeck=jenkins_jobs.modules.publishers:rundeck - s3=jenkins_jobs.modules.publishers:s3 - scan-build=jenkins_jobs.modules.publishers:scan_build - scoverage=jenkins_jobs.modules.publishers:scoverage - scp=jenkins_jobs.modules.publishers:scp - shining-panda=jenkins_jobs.modules.publishers:shining_panda - sitemonitor=jenkins_jobs.modules.publishers:sitemonitor - sloccount=jenkins_jobs.modules.publishers:sloccount - sonar=jenkins_jobs.modules.publishers:sonar - ssh=jenkins_jobs.modules.publishers:ssh - stash=jenkins_jobs.modules.publishers:stash - tap=jenkins_jobs.modules.publishers:tap - testng=jenkins_jobs.modules.publishers:testng - text-finder=jenkins_jobs.modules.publishers:text_finder - trigger-parameterized-builds=jenkins_jobs.modules.publishers:trigger_parameterized_builds - trigger=jenkins_jobs.modules.publishers:trigger - valgrind=jenkins_jobs.modules.publishers:valgrind - violations=jenkins_jobs.modules.publishers:violations - warnings=jenkins_jobs.modules.publishers:warnings - whitesource=jenkins_jobs.modules.publishers:whitesource - workspace-cleanup=jenkins_jobs.modules.publishers:workspace_cleanup - xml-summary=jenkins_jobs.modules.publishers:xml_summary - xunit=jenkins_jobs.modules.publishers:xunit - zulip=jenkins_jobs.modules.publishers:zulip jenkins_jobs.scm = - cvs=jenkins_jobs.modules.scm:cvs - git=jenkins_jobs.modules.scm:git - hg=jenkins_jobs.modules.scm:hg - openshift-img-streams=jenkins_jobs.modules.scm:openshift_img_streams raw=jenkins_jobs.modules.general:raw - repo=jenkins_jobs.modules.scm:repo - store=jenkins_jobs.modules.scm:store - svn=jenkins_jobs.modules.scm:svn - tfs=jenkins_jobs.modules.scm:tfs - workspace=jenkins_jobs.modules.scm:workspace jenkins_jobs.triggers = - bitbucket=jenkins_jobs.modules.triggers:bitbucket - build-result=jenkins_jobs.modules.triggers:build_result - gerrit=jenkins_jobs.modules.triggers:gerrit - github=jenkins_jobs.modules.triggers:github - github-pull-request=jenkins_jobs.modules.triggers:github_pull_request - gitlab=jenkins_jobs.modules.triggers:gitlab - gitlab-merge-request=jenkins_jobs.modules.triggers:gitlab_merge_request - groovy-script=jenkins_jobs.modules.triggers:groovy_script - ivy=jenkins_jobs.modules.triggers:ivy - monitor-files=jenkins_jobs.modules.triggers:monitor_files - monitor-folders=jenkins_jobs.modules.triggers:monitor_folders - pollscm=jenkins_jobs.modules.triggers:pollscm - pollurl=jenkins_jobs.modules.triggers:pollurl raw=jenkins_jobs.modules.general:raw - reverse=jenkins_jobs.modules.triggers:reverse - script=jenkins_jobs.modules.triggers:script - timed=jenkins_jobs.modules.triggers:timed jenkins_jobs.wrappers = - android-emulator=jenkins_jobs.modules.wrappers:android_emulator - ansicolor=jenkins_jobs.modules.wrappers:ansicolor - artifactory-generic=jenkins_jobs.modules.wrappers:artifactory_generic - artifactory-maven=jenkins_jobs.modules.wrappers:artifactory_maven - artifactory-maven-freestyle=jenkins_jobs.modules.wrappers:artifactory_maven_freestyle - build-name=jenkins_jobs.modules.wrappers:build_name - build-user-vars=jenkins_jobs.modules.wrappers:build_user_vars - ci-skip=jenkins_jobs.modules.wrappers:ci_skip - config-file-provider=jenkins_jobs.modules.wrappers:config_file_provider - copy-to-slave=jenkins_jobs.modules.wrappers:copy_to_slave - credentials-binding=jenkins_jobs.modules.wrappers:credentials_binding - custom-tools=jenkins_jobs.modules.wrappers:custom_tools - delivery-pipeline=jenkins_jobs.modules.wrappers:delivery_pipeline - env-file=jenkins_jobs.modules.wrappers:env_file - env-script=jenkins_jobs.modules.wrappers:env_script - exclusion=jenkins_jobs.modules.wrappers:exclusion - inject-ownership-variables=jenkins_jobs.modules.wrappers:inject_ownership_variables - inject-passwords=jenkins_jobs.modules.wrappers:inject_passwords - inject=jenkins_jobs.modules.wrappers:inject - jclouds=jenkins_jobs.modules.wrappers:jclouds - job-log-logger=jenkins_jobs.modules.wrappers:job_log_logger - live-screenshot=jenkins_jobs.modules.wrappers:live_screenshot - locks=jenkins_jobs.modules.wrappers:locks - logfilesize=jenkins_jobs.modules.wrappers:logfilesize - logstash=jenkins_jobs.modules.wrappers:logstash - m2-repository-cleanup=jenkins_jobs.modules.wrappers:m2_repository_cleanup - mask-passwords=jenkins_jobs.modules.wrappers:mask_passwords - matrix-tie-parent=jenkins_jobs.modules.wrappers:matrix_tie_parent - mongo-db=jenkins_jobs.modules.wrappers:mongo_db - nodejs-installator=jenkins_jobs.modules.wrappers:nodejs_installator - pathignore=jenkins_jobs.modules.wrappers:pathignore - port-allocator=jenkins_jobs.modules.wrappers:port_allocator - pre-scm-buildstep=jenkins_jobs.modules.wrappers:pre_scm_buildstep raw=jenkins_jobs.modules.general:raw - rbenv=jenkins_jobs.modules.wrappers:rbenv - release=jenkins_jobs.modules.wrappers:release - rvm-env=jenkins_jobs.modules.wrappers:rvm_env - sauce-ondemand=jenkins_jobs.modules.wrappers:sauce_ondemand - ssh-agent-credentials=jenkins_jobs.modules.wrappers:ssh_agent_credentials - timeout=jenkins_jobs.modules.wrappers:timeout - timestamps=jenkins_jobs.modules.wrappers:timestamps - workspace-cleanup=jenkins_jobs.modules.wrappers:workspace_cleanup - xvfb=jenkins_jobs.modules.wrappers:xvfb - xvnc=jenkins_jobs.modules.wrappers:xvnc jenkins_jobs.modules = builders=jenkins_jobs.modules.builders:Builders general=jenkins_jobs.modules.general:General