From ddb310c5999c0002b66f3baca66cfd6e26859322 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Mon, 25 Apr 2022 09:51:20 -0700 Subject: [PATCH] Retire this project We've shutdown the subunit2sql processors and don't need to manage them with puppet any more. Depends-On: https://review.opendev.org/c/openstack/project-config/+/839235 Change-Id: Id72f597c28807684fcfc8795a93ba0dc1b7e403c --- .gitignore | 2 - Gemfile | 15 - LICENSE | 202 ---------- README.md | 11 +- Rakefile | 8 - files/run_migrations.sh | 3 - files/subunit-gearman-worker.py | 349 ------------------ manifests/init.pp | 123 ------ manifests/server.pp | 72 ---- manifests/worker.pp | 136 ------- metadata.json | 12 - spec/acceptance/basic_spec.rb | 36 -- spec/acceptance/fixtures/default.pp | 30 -- spec/acceptance/fixtures/postconditions.pp | 3 - spec/acceptance/nodesets/default.yml | 11 - spec/acceptance/nodesets/nodepool-centos7.yml | 10 - spec/acceptance/nodesets/nodepool-trusty.yml | 10 - spec/acceptance/nodesets/nodepool-xenial.yml | 10 - templates/jenkins-subunit-worker.init.erb | 158 -------- templates/subunit2sql-my.cnf.erb | 5 - templates/subunit2sql.conf.erb | 5 - 21 files changed, 7 insertions(+), 1204 deletions(-) delete mode 100644 .gitignore delete mode 100644 Gemfile delete mode 100644 LICENSE delete mode 100644 Rakefile delete mode 100644 files/run_migrations.sh delete mode 100644 files/subunit-gearman-worker.py delete mode 100644 manifests/init.pp delete mode 100644 manifests/server.pp delete mode 100644 manifests/worker.pp delete mode 100644 metadata.json delete mode 100644 spec/acceptance/basic_spec.rb delete mode 100644 spec/acceptance/fixtures/default.pp delete mode 100644 spec/acceptance/fixtures/postconditions.pp delete mode 100644 spec/acceptance/nodesets/default.yml delete mode 100644 spec/acceptance/nodesets/nodepool-centos7.yml delete mode 100644 spec/acceptance/nodesets/nodepool-trusty.yml delete mode 100644 spec/acceptance/nodesets/nodepool-xenial.yml delete mode 100755 templates/jenkins-subunit-worker.init.erb delete mode 100644 templates/subunit2sql-my.cnf.erb delete mode 100644 templates/subunit2sql.conf.erb diff --git a/.gitignore b/.gitignore deleted file mode 100644 index dade81e..0000000 --- a/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -Gemfile.lock -.bundled_gems/ diff --git a/Gemfile b/Gemfile deleted file mode 100644 index 019213a..0000000 --- a/Gemfile +++ /dev/null @@ -1,15 +0,0 @@ -source 'https://rubygems.org' - -if File.exists?('/home/zuul/src/git.openstack.org/openstack-infra/puppet-openstack_infra_spec_helper') - gem_checkout_method = {:path => '/home/zuul/src/git.openstack.org/openstack-infra/puppet-openstack_infra_spec_helper'} -else - gem_checkout_method = {:git => 'https://git.openstack.org/openstack-infra/puppet-openstack_infra_spec_helper'} -end -gem_checkout_method[:require] = false - -group :development, :test, :system_tests do - gem 'puppet-openstack_infra_spec_helper', - gem_checkout_method -end - -# vim:ft=ruby diff --git a/LICENSE b/LICENSE deleted file mode 100644 index d645695..0000000 --- a/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/README.md b/README.md index d340ce8..3265f02 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ -# OpenStack Subunit2sql Module +This project is no longer maintained. -## Overview +The contents of this repository are still available in the Git +source code management system. To see the contents of this +repository before it reached its end of life, please check out the +previous commit with "git checkout HEAD^1". -Installs and configures subunit2sql - a tool to convert subunit results into -a SQL database. +For any further questions, please email +service-discuss@lists.openstack.org or join #opendev on OFTC. diff --git a/Rakefile b/Rakefile deleted file mode 100644 index ff1f0d7..0000000 --- a/Rakefile +++ /dev/null @@ -1,8 +0,0 @@ -require 'rubygems' -require 'puppetlabs_spec_helper/rake_tasks' -require 'puppet-lint/tasks/puppet-lint' -PuppetLint.configuration.fail_on_warnings = true -PuppetLint.configuration.send('disable_80chars') -PuppetLint.configuration.send('disable_autoloader_layout') -PuppetLint.configuration.send('disable_class_inherits_from_params_class') -PuppetLint.configuration.send('disable_class_parameter_defaults') diff --git a/files/run_migrations.sh b/files/run_migrations.sh deleted file mode 100644 index 0d72116..0000000 --- a/files/run_migrations.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -(time subunit2sql-db-manage --config-file /etc/subunit2sql.conf upgrade head) > /var/log/subunit2sql_migration.log 2>&1 & diff --git a/files/subunit-gearman-worker.py b/files/subunit-gearman-worker.py deleted file mode 100644 index e6b3107..0000000 --- a/files/subunit-gearman-worker.py +++ /dev/null @@ -1,349 +0,0 @@ -#!/usr/bin/python2 -# -# Copyright 2013 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import argparse -import daemon -import gear -import gzip -import io -import json -import logging -import os -import socket -import time -import urllib2 -import yaml - -import paho.mqtt.publish as publish -from subunit2sql import read_subunit -from subunit2sql import shell - - -try: - import daemon.pidlockfile as pidfile_mod -except ImportError: - import daemon.pidfile as pidfile_mod - - -def semi_busy_wait(seconds): - # time.sleep() may return early. If it does sleep() again and repeat - # until at least the number of seconds specified has elapsed. - start_time = time.time() - while True: - time.sleep(seconds) - cur_time = time.time() - seconds = seconds - (cur_time - start_time) - if seconds <= 0.0: - return - - -class FilterException(Exception): - pass - - -class SubunitRetriever(object): - def __init__(self, gearman_worker, filters, subunit2sql_conf, mqtt=None): - super(SubunitRetriever, self).__init__() - self.gearman_worker = gearman_worker - self.filters = filters - # Initialize subunit2sql settings - self.config = subunit2sql_conf - shell.cli_opts() - extensions = shell.get_extensions() - shell.parse_args([], [self.config]) - self.extra_targets = shell.get_targets(extensions) - self.mqtt = mqtt - - def _uniquify_name(self, meta): - job_name = meta.pop('build_name') - if 'tempest' not in job_name and 'dsvm' not in job_name: - project = meta['project'].split('/')[-1] - if project not in job_name: - job_name = '-'.join([job_name, project]) - if job_name.startswith('legacy-'): - job_name = job_name[7:] - meta['build_name'] = job_name - return meta - - def _write_to_db(self, subunit): - subunit_v2 = subunit.pop('subunit') - # Set run metadata from gearman - log_url = subunit.pop('log_url', None) - subunit = self._uniquify_name(subunit) - if log_url: - log_dir = os.path.dirname(log_url) - - # log_dir should be the top-level directory containing a job run, - # but the subunit file may be nested in 0 - 2 subdirectories (top, - # logs/, or logs/old/), so we need to safely correct the path here - log_base = os.path.basename(log_dir) - if log_base == 'logs': - log_dir = os.path.dirname(log_dir) - elif log_base == 'old': - log_dir = os.path.dirname(os.path.dirname(log_dir)) - - shell.CONF.set_override('artifacts', log_dir) - shell.CONF.set_override('run_meta', subunit) - # Parse subunit stream and store in DB - if subunit_v2.closed: - logging.debug('Trying to convert closed subunit v2 stream: %s to ' - 'SQL' % subunit_v2) - else: - logging.debug('Converting Subunit V2 stream: %s to SQL' % - subunit_v2) - stream = read_subunit.ReadSubunit(subunit_v2, - targets=self.extra_targets, - use_wall_time=True) - results = stream.get_results() - start_time = sorted( - [results[x]['start_time'] for x in results if x != 'run_time'])[0] - shell.CONF.set_override('run_at', start_time.isoformat()) - shell.process_results(results) - subunit_v2.close() - - def run(self): - while True: - try: - self._handle_event() - except: - logging.exception("Exception retrieving log event.") - - def _handle_event(self): - fields = {} - source_url = '' - job = self.gearman_worker.getJob() - try: - arguments = json.loads(job.arguments.decode('utf-8')) - source_url = arguments['source_url'] - retry = arguments['retry'] - event = arguments['event'] - logging.debug("Handling event: " + json.dumps(event)) - fields = event.get('fields') or event.get('@fields') - if fields.pop('build_status') != 'ABORTED': - # Handle events ignoring aborted builds. These builds are - # discarded by zuul. - subunit_io = self._retrieve_subunit_v2(source_url, retry) - if not subunit_io: - raise Exception('Unable to retrieve subunit stream') - else: - if subunit_io.closed: - logging.debug("Pushing closed subunit file: %s" % - subunit_io) - else: - logging.debug("Pushing subunit file: %s" % subunit_io) - out_event = fields.copy() - out_event["subunit"] = subunit_io - self._write_to_db(out_event) - job.sendWorkComplete() - if self.mqtt: - msg = json.dumps({ - 'build_uuid': out_event.get('build_uuid'), - 'source_url': source_url, - 'status': 'success', - }) - self.mqtt.publish_single(msg, out_event.get('project'), - out_event.get('build_change')) - except Exception as e: - logging.exception("Exception handling log event.") - job.sendWorkException(str(e).encode('utf-8')) - if self.mqtt: - msg = json.dumps({ - 'build_uuid': fields.get('build_uuid'), - 'source_url': source_url, - 'status': 'failed', - }) - self.mqtt.publish_single(msg, fields.get('project'), - fields.get('build_change')) - - def _retrieve_subunit_v2(self, source_url, retry): - # TODO (clarkb): This should check the content type instead of file - # extension for determining if gzip was used. - gzipped = False - raw_buf = b'' - try: - gzipped, raw_buf = self._get_subunit_data(source_url, retry) - except urllib2.HTTPError as e: - if e.code == 404: - logging.info("Unable to retrieve %s: HTTP error 404" % - source_url) - else: - logging.exception("Unable to get log data.") - return None - except Exception: - # Silently drop fatal errors when retrieving logs. - # TODO (clarkb): Handle these errors. - # Perhaps simply add a log message to raw_buf? - logging.exception("Unable to get log data.") - return None - if gzipped: - logging.debug("Decompressing gzipped source file.") - raw_strIO = io.BytesIO(raw_buf) - f = gzip.GzipFile(fileobj=raw_strIO) - buf = io.BytesIO(f.read()) - raw_strIO.close() - f.close() - else: - logging.debug("Decoding source file.") - buf = io.BytesIO(raw_buf) - return buf - - def _get_subunit_data(self, source_url, retry): - gzipped = False - try: - # TODO(clarkb): We really should be using requests instead - # of urllib2. urllib2 will automatically perform a POST - # instead of a GET if we provide urlencoded data to urlopen - # but we need to do a GET. The parameters are currently - # hardcoded so this should be ok for now. - logging.debug("Retrieving: " + source_url + ".gz") - req = urllib2.Request(source_url + ".gz") - req.add_header('Accept-encoding', 'gzip') - r = urllib2.urlopen(req) - except urllib2.URLError: - try: - # Fallback on GETting unzipped data. - logging.debug("Retrieving: " + source_url) - r = urllib2.urlopen(source_url) - except: - logging.exception("Unable to retrieve source file.") - raise - except: - logging.exception("Unable to retrieve source file.") - raise - if ('gzip' in r.info().get('Content-Type', '') or - 'gzip' in r.info().get('Content-Encoding', '')): - gzipped = True - - raw_buf = r.read() - # Hack to read all of Jenkins console logs as they upload - return gzipped, raw_buf - - -class PushMQTT(object): - def __init__(self, hostname, base_topic, port=1883, client_id=None, - keepalive=60, will=None, auth=None, tls=None, qos=0): - self.hostname = hostname - self.port = port - self.client_id = client_id - self.keepalive = 60 - self.will = will - self.auth = auth - self.tls = tls - self.qos = qos - self.base_topic = base_topic - - def _generate_topic(self, project, job_id): - return '/'.join([self.base_topic, project, job_id]) - - def publish_single(self, msg, project, job_id): - topic = self._generate_topic(project, job_id) - publish.single(topic, msg, hostname=self.hostname, - port=self.port, client_id=self.client_id, - keepalive=self.keepalive, will=self.will, - auth=self.auth, tls=self.tls, qos=self.qos) - - -class Server(object): - def __init__(self, config, debuglog): - # Config init. - self.config = config - self.gearman_host = self.config['gearman-host'] - self.gearman_port = self.config['gearman-port'] - mqtt_host = self.config.get('mqtt-host') - mqtt_port = self.config.get('mqtt-port', 1883) - mqtt_user = self.config.get('mqtt-user') - mqtt_pass = self.config.get('mqtt-pass') - mqtt_topic = self.config.get('mqtt-topic', 'gearman-subunit') - mqtt_ca_certs = self.config.get('mqtt-ca-certs') - mqtt_certfile = self.config.get('mqtt-certfile') - mqtt_keyfile = self.config.get('mqtt-keyfile') - self.mqtt = None - if mqtt_host: - auth = None - if mqtt_user: - auth = {'username': mqtt_user} - if mqtt_pass: - auth['password'] = mqtt_pass - tls = None - if mqtt_ca_certs: - tls = {'ca_certs': mqtt_ca_certs, 'certfile': mqtt_certfile, - 'keyfile': mqtt_keyfile} - - self.mqtt = PushMQTT(mqtt_host, mqtt_topic, port=mqtt_port, - auth=auth, tls=tls) - - # Pythong logging output file. - self.debuglog = debuglog - self.retriever = None - self.filter_factories = [] - - def setup_logging(self): - if self.debuglog: - logging.basicConfig(format='%(asctime)s %(message)s', - filename=self.debuglog, level=logging.DEBUG) - else: - # Prevent leakage into the logstash log stream. - logging.basicConfig(level=logging.CRITICAL) - logging.debug("Log pusher starting.") - - def setup_retriever(self): - hostname = socket.gethostname() - gearman_worker = gear.Worker(hostname + b'-pusher') - gearman_worker.addServer(self.gearman_host, - self.gearman_port) - gearman_worker.registerFunction(b'push-subunit') - subunit2sql_conf = self.config['config'] - self.retriever = SubunitRetriever(gearman_worker, - self.filter_factories, - subunit2sql_conf, - mqtt=self.mqtt) - - def main(self): - self.setup_retriever() - self.retriever.run() - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("-c", "--config", required=True, - help="Path to yaml config file.") - parser.add_argument("-d", "--debuglog", - help="Enable debug log. " - "Specifies file to write log to.") - parser.add_argument("--foreground", action='store_true', - help="Run in the foreground.") - parser.add_argument("-p", "--pidfile", - default="/var/run/jenkins-subunit-pusher/" - "jenkins-subunit-gearman-worker.pid", - help="PID file to lock during daemonization.") - args = parser.parse_args() - - with open(args.config, 'r') as config_stream: - config = yaml.load(config_stream) - server = Server(config, args.debuglog) - - if args.foreground: - server.setup_logging() - server.main() - else: - pidfile = pidfile_mod.TimeoutPIDLockFile(args.pidfile, 10) - with daemon.DaemonContext(pidfile=pidfile): - server.setup_logging() - server.main() - - -if __name__ == '__main__': - main() diff --git a/manifests/init.pp b/manifests/init.pp deleted file mode 100644 index 31cdd05..0000000 --- a/manifests/init.pp +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright 2012-2013 Hewlett-Packard Development Company, L.P. -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# == Class: subunit2sql -# -class subunit2sql ( -) { - include ::pip - - package {'python-mysqldb': - ensure => present, - } - - package {'python-psycopg2': - ensure => present, - } - - package {'python-netifaces': - ensure => present, - } - - package { 'python-subunit': - ensure => latest, - provider => openstack_pip, - require => Class['pip'], - } - - exec { 'install-subunit2sql-safely': - command => 'pip install --upgrade --upgrade-strategy=only-if-needed subunit2sql', - path => '/usr/local/bin:/usr/bin:/bin/', - # This checks the current installed subunit2sql version with pip list and - # the latest version of subunit2sql on pypi with pip search and if they are - # different then we know we need to upgrade to reconcile the local version - # with the upstream version. - # - # We do this using this check here rather than a pip package resource so we - # can override pip's default upgrade strategy in order to avoid replacing - # deps we've preinstalled from system packages because they lack wheels on - # PyPI and must be otherwise rebuilt from sdist instead (specifically - # netifaces). - onlyif => '/bin/bash -c "test \\"$(pip list --format columns | sed -ne \'s/^subunit2sql\s\+\([[:digit:].]*\)\s*$/\1/p\')\\" != \\"$(pip search \'subunit2sql$\' | sed -ne \'s/^subunit2sql (\(.*\)).*$/\1/p\')\\""', - require => [ - Class['pip'], - Package['python-mysqldb'], - Package['python-psycopg2'], - Package['python-netifaces'] - ], - } - - package { 'os-performance-tools': - ensure => latest, - provider => openstack_pip, - require => [ - Class['pip'] - ], - } - - package { 'testtools': - ensure => latest, - provider => openstack_pip, - require => Class['pip'], - } - - if ! defined(Package['python-daemon']) { - package { 'python-daemon': - ensure => present, - } - } - - if ! defined(Package['python-zmq']) { - package { 'python-zmq': - ensure => present, - } - } - - if ! defined(Package['gear']) { - package { 'gear': - ensure => latest, - provider => openstack_pip, - require => Class['pip'], - } - } - - if ! defined(Package['statsd']) { - package { 'statsd': - # NOTE(cmurphy) If this is not pinned, the openstack_pip provider will - # attempt to install latest and conflict with the <3 cap from - # os-performance-tools. Unpin this when os-performance-tools raises its - # cap. - ensure => '2.1.2', - provider => openstack_pip, - require => Class['pip'] - } - } - - file { '/usr/local/bin/subunit-gearman-worker.py': - ensure => present, - owner => 'root', - group => 'root', - mode => '0755', - source => 'puppet:///modules/subunit2sql/subunit-gearman-worker.py', - require => [ - Package['python-daemon'], - Package['python-zmq'], - Package['gear'], - Exec['install-subunit2sql-safely'], - Package['python-subunit'], - Package['testtools'] - ], - } -} diff --git a/manifests/server.pp b/manifests/server.pp deleted file mode 100644 index 2916ccf..0000000 --- a/manifests/server.pp +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2012-2013 Hewlett-Packard Development Company, L.P. -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# == Class: subunit2sql -# -class subunit2sql::server ( - $db_pass, - $db_host, - $db_dialect = 'mysql', - $db_user = 'subunit2sql', - $db_port = '3306', - $db_name = 'subunit2sql', - $expire_age = '186', - $expire_cron_minute = '0', - $expire_cron_hour = '3', - $expire_cron_weekday = '7', -) { - - file { '/etc/subunit2sql.conf': - ensure => present, - owner => 'root', - group => 'root', - mode => '0555', - content => template('subunit2sql/subunit2sql.conf.erb'), - } - - file {'/etc/subunit2sql-my.cnf': - ensure => present, - owner => 'root', - group => 'root', - mode => '0400', - content => template('subunit2sql/subunit2sql-my.cnf.erb'), - } - - file {'/usr/local/bin/run_migrations.sh': - ensure => present, - owner => 'root', - group => 'root', - mode => '0555', - source => 'puppet:///modules/subunit2sql/run_migrations.sh', - require => File['/etc/subunit2sql.conf'] - } - - exec { 'upgrade_subunit2sql_db': - command => '/usr/local/bin/run_migrations.sh', - require => File['/usr/local/bin/run_migrations.sh'], - subscribe => Exec['install-subunit2sql-safely'], - refreshonly => true, - timeout => 0, - } - - cron { 'subunit2sql-prune': - ensure => present, - command => "subunit2sql-db-manage --config-file /etc/subunit2sql.conf expire --expire-age ${expire_age} >> /var/log/subunit2sql_migration.log 2>&1 & ", - minute => $expire_cron_minute, - hour => $expire_cron_hour, - weekday => $expire_cron_weekday, - environment => 'PATH=/usr/local/bin:/usr/bin:/bin/', - } -} diff --git a/manifests/worker.pp b/manifests/worker.pp deleted file mode 100644 index 566964f..0000000 --- a/manifests/worker.pp +++ /dev/null @@ -1,136 +0,0 @@ -# Copyright 2012-2013 Hewlett-Packard Development Company, L.P. -# Copyright 2013 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# == Class: subunit_processor::worker -# -define subunit2sql::worker ( - $config_file, - $db_pass, - $db_host, - $db_dialect = 'mysql', - $db_user = 'subunit2sql', - $db_port = '3306', - $db_name = 'subunit2sql', -) { - $suffix = "-${name}" - - if ! defined(File['/etc/logstash']) { - file { '/etc/logstash': - ensure => absent, - } - } - if ! defined(Package['paho-mqtt']) { - package {'paho-mqtt': - ensure => latest, - provider => openstack_pip, - require => Class['pip'], - } - } - - if ! defined(User['subunit']) { - user { 'subunit': - ensure => present, - system => true, - } - } - - if ! defined(File['/etc/subunit2sql']) { - file { '/etc/subunit2sql': - ensure => directory, - owner => 'root', - group => 'root', - mode => '0644', - } - } - -if ! defined(File['/var/log/subunit2sql']) { - file { '/var/log/subunit2sql': - ensure => directory, - owner => 'subunit', - group => 'subunit', - mode => '0644', - require => User['subunit'], - } - } - if ! defined(File['/etc/subunit2sql/subunit2sql.conf']) { - file { '/etc/subunit2sql/subunit2sql.conf': - ensure => present, - owner => 'root', - group => 'root', - mode => '0555', - content => template('subunit2sql/subunit2sql.conf.erb'), - require => File['/etc/subunit2sql'], - } - } - - file { "/etc/subunit2sql/jenkins-subunit-worker${suffix}.yaml": - ensure => present, - owner => 'root', - group => 'root', - mode => '0555', - source => $config_file, - require => File['/etc/subunit2sql'], - } - - file { "/etc/init.d/jenkins-subunit-worker${suffix}": - ensure => present, - owner => 'root', - group => 'root', - mode => '0555', - content => template('subunit2sql/jenkins-subunit-worker.init.erb'), - require => [ - File['/usr/local/bin/subunit-gearman-worker.py'], - File["/etc/subunit2sql/jenkins-subunit-worker${suffix}.yaml"], - ], - } - - if ($::operatingsystem == 'Ubuntu') and ($::operatingsystemrelease >= '16.04') { - # This is a hack to make sure that systemd is aware of the new service - # before we attempt to start it. - exec { "jenkins-subunit-worker${suffix}-systemd-daemon-reload": - command => '/bin/systemctl daemon-reload', - before => Service["jenkins-subunit-worker${suffix}"], - subscribe => File["/etc/init.d/jenkins-subunit-worker${suffix}"], - refreshonly => true, - } - } - - service { "jenkins-subunit-worker${suffix}": - enable => true, - hasrestart => true, - subscribe => [ - File["/etc/subunit2sql/jenkins-subunit-worker${suffix}.yaml"], - Exec['install-subunit2sql-safely'], - ], - require => [ - File['/etc/subunit2sql'], - File["/etc/init.d/jenkins-subunit-worker${suffix}"], - ], - } - - include ::logrotate - logrotate::file { "subunit-worker${suffix}-debug.log": - log => "/var/log/subunit2sql/subunit-worker${suffix}-debug.log", - options => [ - 'compress', - 'copytruncate', - 'missingok', - 'rotate 7', - 'daily', - 'notifempty', - ], - require => Service["jenkins-subunit-worker${suffix}"], - } -} diff --git a/metadata.json b/metadata.json deleted file mode 100644 index bc4a5eb..0000000 --- a/metadata.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "openstackinfra-subunit2sql", - "version": "0.0.1", - "author": "OpenStack CI", - "summary": "Puppet module for Subunit2sql", - "license": "Apache 2.0", - "source": "https://git.openstack.org/openstack-infra/puppet-subunit2sql.git", - "project_page": "http://docs.openstack.org/infra/system-config/", - "issues_url": "https://storyboard.openstack.org/#!/project/798", - "dependencies": [ - ] -} diff --git a/spec/acceptance/basic_spec.rb b/spec/acceptance/basic_spec.rb deleted file mode 100644 index 9184ea4..0000000 --- a/spec/acceptance/basic_spec.rb +++ /dev/null @@ -1,36 +0,0 @@ -require 'puppet-openstack_infra_spec_helper/spec_helper_acceptance' - -describe 'subunit2sql', :if => ['debian', 'ubuntu'].include?(os[:family]) do - - def pp_path - base_path = File.dirname(__FILE__) - File.join(base_path, 'fixtures') - end - - def puppet_manifest - manifest_path = File.join(pp_path, 'default.pp') - File.read(manifest_path) - end - - def postconditions_puppet_manifest - manifest_path = File.join(pp_path, 'postconditions.pp') - File.read(manifest_path) - end - - it 'should work with no errors' do - apply_manifest(puppet_manifest, catch_failures: true) - end - - it 'should be idempotent' do - apply_manifest(puppet_manifest, catch_changes: true) - end - - it 'should start' do - apply_manifest(postconditions_puppet_manifest, catch_failures: true) - end - - describe service('jenkins-subunit-worker-A') do - it { should be_running } - end - -end diff --git a/spec/acceptance/fixtures/default.pp b/spec/acceptance/fixtures/default.pp deleted file mode 100644 index 2ad4530..0000000 --- a/spec/acceptance/fixtures/default.pp +++ /dev/null @@ -1,30 +0,0 @@ -$content = "gearman-host: logstash.openstack.org -gearman-port: 4730 -config: /etc/subunit2sql/subunit2sql.conf -mqtt-host: firehose.openstack.org -mqtt-port: 8883 -mqtt-topic: gearman-subunit/localhost -mqtt-user: 'mqtt_user' -mqtt-pass: 'mqtt_pass' -mqtt-ca-certs: /etc/subunit2sql/mqtt-root-CA.pem.crt" - -file { '/etc/subunit2sql/subunit-woker.yaml': - ensure => file, - owner => 'root', - group => 'root', - mode => '0555', - content => $content, -} - -include 'subunit2sql' - -class { 'subunit2sql::server': - db_host => 'subunit2sql_db_host', - db_pass => 'subunit2sql_db_pass', -} - -subunit2sql::worker { 'A': - config_file => '/etc/subunit2sql/subunit-woker.yaml', - db_host => $subunit2sql_db_host, - db_pass => $subunit2sql_db_pass, -} diff --git a/spec/acceptance/fixtures/postconditions.pp b/spec/acceptance/fixtures/postconditions.pp deleted file mode 100644 index 647a712..0000000 --- a/spec/acceptance/fixtures/postconditions.pp +++ /dev/null @@ -1,3 +0,0 @@ -service { 'jenkins-subunit-worker-A': - ensure => running, -} diff --git a/spec/acceptance/nodesets/default.yml b/spec/acceptance/nodesets/default.yml deleted file mode 100644 index 3bb3e62..0000000 --- a/spec/acceptance/nodesets/default.yml +++ /dev/null @@ -1,11 +0,0 @@ -HOSTS: - ubuntu-server-1404-x64: - roles: - - master - platform: ubuntu-14.04-amd64 - box: puppetlabs/ubuntu-14.04-64-nocm - box_url: https://vagrantcloud.com/puppetlabs/ubuntu-14.04-64-nocm - hypervisor: vagrant -CONFIG: - log_level: debug - type: git diff --git a/spec/acceptance/nodesets/nodepool-centos7.yml b/spec/acceptance/nodesets/nodepool-centos7.yml deleted file mode 100644 index c552874..0000000 --- a/spec/acceptance/nodesets/nodepool-centos7.yml +++ /dev/null @@ -1,10 +0,0 @@ -HOSTS: - centos-70-x64: - roles: - - master - platform: el-7-x86_64 - hypervisor: none - ip: 127.0.0.1 -CONFIG: - type: foss - set_env: false diff --git a/spec/acceptance/nodesets/nodepool-trusty.yml b/spec/acceptance/nodesets/nodepool-trusty.yml deleted file mode 100644 index 9fc624e..0000000 --- a/spec/acceptance/nodesets/nodepool-trusty.yml +++ /dev/null @@ -1,10 +0,0 @@ -HOSTS: - ubuntu-14.04-amd64: - roles: - - master - platform: ubuntu-14.04-amd64 - hypervisor: none - ip: 127.0.0.1 -CONFIG: - type: foss - set_env: false diff --git a/spec/acceptance/nodesets/nodepool-xenial.yml b/spec/acceptance/nodesets/nodepool-xenial.yml deleted file mode 100644 index 99dd318..0000000 --- a/spec/acceptance/nodesets/nodepool-xenial.yml +++ /dev/null @@ -1,10 +0,0 @@ -HOSTS: - ubuntu-16.04-amd64: - roles: - - master - platform: ubuntu-16.04-amd64 - hypervisor: none - ip: 127.0.0.1 -CONFIG: - type: foss - set_env: false diff --git a/templates/jenkins-subunit-worker.init.erb b/templates/jenkins-subunit-worker.init.erb deleted file mode 100755 index bbb3288..0000000 --- a/templates/jenkins-subunit-worker.init.erb +++ /dev/null @@ -1,158 +0,0 @@ -#! /bin/sh -### BEGIN INIT INFO -# Provides: jenkins-subunit-worker<%= @suffix %> -# Required-Start: $remote_fs $syslog -# Required-Stop: $remote_fs $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Jenkins Subunit Worker -# Description: Service to push Jenkins subunit files into a SQL DB. -### END INIT INFO - -# Do NOT "set -e" - -# PATH should only include /usr/* if it runs after the mountnfs.sh script -PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin -DESC="Jenkins Subunit Worker" -NAME=jenkins-subunit-worker<%= @suffix %> -DAEMON=/usr/local/bin/subunit-gearman-worker.py -PIDFILE=/var/run/$NAME/$NAME.pid -DAEMON_ARGS="-c /etc/subunit2sql/jenkins-subunit-worker<%= @suffix %>.yaml -d /var/log/subunit2sql/subunit-worker<%= @suffix %>-debug.log -p $PIDFILE" -SCRIPTNAME=/etc/init.d/$NAME -USER=subunit - -# Exit if the package is not installed -[ -x "$DAEMON" ] || exit 0 - -# Read configuration variable file if it is present -[ -r /etc/default/$NAME ] && . /etc/default/$NAME - -# Load the VERBOSE setting and other rcS variables -. /lib/init/vars.sh - -# Define LSB log_* functions. -# Depend on lsb-base (>= 3.0-6) to ensure that this file is present. -. /lib/lsb/init-functions - -# -# Function that starts the daemon/service -# -do_start() -{ - # Return - # 0 if daemon has been started - # 1 if daemon was already running - # 2 if daemon could not be started - - mkdir -p /var/run/$NAME - chown $USER /var/run/$NAME - start-stop-daemon --start --quiet --pidfile $PIDFILE -c $USER --exec $DAEMON --test > /dev/null \ - || return 1 - start-stop-daemon --start --quiet --pidfile $PIDFILE -c $USER --exec $DAEMON -- \ - $DAEMON_ARGS \ - || return 2 - # Add code here, if necessary, that waits for the process to be ready - # to handle requests from services started subsequently which depend - # on this one. As a last resort, sleep for some time. -} - -# -# Function that stops the daemon/service -# -do_stop() -{ - # Return - # 0 if daemon has been stopped - # 1 if daemon was already stopped - # 2 if daemon could not be stopped - # other if a failure occurred - start-stop-daemon --stop --signal 9 --pidfile $PIDFILE - RETVAL="$?" - [ "$RETVAL" = 2 ] && return 2 - rm -f /var/run/$NAME/* - return "$RETVAL" -} - -# -# Function that stops the daemon/service -# -#do_graceful_stop() -#{ -# PID=`cat $PIDFILE` -# kill -USR1 $PID -# -# # wait until really stopped -# if [ -n "${PID:-}" ]; then -# i=0 -# while kill -0 "${PID:-}" 2> /dev/null; do -# if [ $i -eq '0' ]; then -# echo -n " ... waiting " -# else -# echo -n "." -# fi -# i=$(($i+1)) -# sleep 1 -# done -# fi -# -# rm -f /var/run/$NAME/* -#} - -# -# Function that sends a SIGHUP to the daemon/service -# -#do_reload() { -# # -# # If the daemon can reload its configuration without -# # restarting (for example, when it is sent a SIGHUP), -# # then implement that here. -# # -# start-stop-daemon --stop --signal 1 --quiet --pidfile $PIDFILE --name zuul-server -# return 0 -#} - -case "$1" in - start) - [ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME" - do_start - case "$?" in - 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; - 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; - esac - ;; - stop) - [ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME" - do_stop - case "$?" in - 0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;; - 2) [ "$VERBOSE" != no ] && log_end_msg 1 ;; - esac - ;; - status) - status_of_proc "$DAEMON" "$NAME" && exit 0 || exit $? - ;; -# reload) -# # -# # If do_reload() is not implemented then leave this commented out -# # and leave 'force-reload' as an alias for 'restart'. -# # -# log_daemon_msg "Reloading $DESC" "$NAME" -# do_reload -# log_end_msg $? -# ;; - restart|force-reload) - # - # If the "reload" option is implemented then remove the - # 'force-reload' alias - # - log_daemon_msg "Restarting $DESC" "$NAME" - do_stop - do_start - ;; - *) - echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 - exit 3 - ;; -esac - -: diff --git a/templates/subunit2sql-my.cnf.erb b/templates/subunit2sql-my.cnf.erb deleted file mode 100644 index 6638511..0000000 --- a/templates/subunit2sql-my.cnf.erb +++ /dev/null @@ -1,5 +0,0 @@ -[client] -host=<%= @db_host %> -user=<%= @db_user %> -password=<%= @db_pass %> -port=<%= @db_port %> diff --git a/templates/subunit2sql.conf.erb b/templates/subunit2sql.conf.erb deleted file mode 100644 index 2e63e88..0000000 --- a/templates/subunit2sql.conf.erb +++ /dev/null @@ -1,5 +0,0 @@ -[DEFAULT] -disable-microsecond-data-migration = True - -[database] -connection = <%= @db_dialect %>://<%= @db_user %>:<%= @db_pass %>@<%= @db_host %>:<%= @db_port %>/<%= @db_name %>