From c3c0182c72a3849bdcfc9a48342570bc05960693 Mon Sep 17 00:00:00 2001 From: Dmitry Sutyagin Date: Mon, 12 Dec 2016 19:01:56 -0800 Subject: [PATCH] Add: dynamic import of analyze modules modules should be put into timmy/analyze_modules directory subdirectories are also supported __example__.py explains the internals and can be used as a template Change-Id: I4d32f2467dfe1885e71340bd24e8e587aed667c1 --- setup.py | 5 +- specs/python-timmy.spec | 5 +- timmy/analyze.py | 84 ++++++-------------------- timmy/analyze_health.py | 21 +++++++ timmy/analyze_modules/__example__.py | 76 ++++++++++++++++++++++++ timmy/analyze_modules/__init__.py | 0 timmy/analyze_modules/df.py | 88 ++++++++++++++++++++++++++++ timmy/env.py | 2 +- 8 files changed, 212 insertions(+), 69 deletions(-) create mode 100644 timmy/analyze_health.py create mode 100644 timmy/analyze_modules/__example__.py create mode 100644 timmy/analyze_modules/__init__.py create mode 100644 timmy/analyze_modules/df.py diff --git a/setup.py b/setup.py index 0e1267b..f344ea1 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,10 @@ setup(name=pname, 'operations: two-way data transfer, log collection, ' 'remote command execution'), long_description=open('README.md').read(), - packages=[pname, '%s.modules' % pname, '%s_data' % pname], + packages=[pname, + '%s.analyze_modules' % pname, + '%s.modules' % pname, + '%s_data' % pname], install_requires=['pyyaml'], include_package_data=True, entry_points={'console_scripts': ['%s=%s.cli:main' % (pname, pname)]}, diff --git a/specs/python-timmy.spec b/specs/python-timmy.spec index 2a93ac5..c6bee92 100644 --- a/specs/python-timmy.spec +++ b/specs/python-timmy.spec @@ -4,7 +4,7 @@ %global pypi_name timmy Name: python-%{pypi_name} -Version: 1.25.2 +Version: 1.25.3 Release: 1%{?dist}~mos0 Summary: Log collector tool for OpenStack Fuel @@ -107,6 +107,9 @@ popd %changelog +* Mon Dec 12 2016 Dmitry Sutyagin - 1.25.3 +- Add: dynamic import of analyze modules + * Thu Dec 9 2016 Aleksandr Dobdin - 1.25.2 - Add: fuel network template download script diff --git a/timmy/analyze.py b/timmy/analyze.py index fc68089..ba26310 100644 --- a/timmy/analyze.py +++ b/timmy/analyze.py @@ -15,7 +15,9 @@ # License for the specific language governing permissions and limitations # under the License. +from timmy.analyze_health import GREEN, UNKNOWN, YELLOW, RED from timmy.env import project_name +import imp import logging import os import sys @@ -25,70 +27,20 @@ logger = logging.getLogger(project_name) def analyze(node_manager): - col_msg = 'Column "%s" not found in output of "%s" from node "%s"' - green = 0 - unknown = 1 - yellow = 2 - red = 3 + def is_module(f): + return f.endswith('.py') and not f.startswith('__') - def parse_df_m(data, script, node): - column_use = "Use%" - full = 100 - near_full = 80 - health = green - details = [] - if column_use not in data[0]: - logger.warning(col_msg % (column_use, script, node.repr)) - health = unknown - index = data[0].split().index(column_use) - prepend_str = '' # workaround for data which spans 2 lines - index_shift = 0 - for line in data[2:]: - if len(line.split()) <= index: - prepend_str = line.rstrip() - index_shift = len(line.split()) - continue - value = int(line.split()[index - index_shift][:-1]) - if value >= full: - health = red - details.append(prepend_str + line) - elif value >= near_full: - health = yellow if health < yellow else health - details.append(prepend_str + line) - prepend_str = '' - index_shift = 0 - return health, details + fn_mapping = {} + modules_dir = 'analyze_modules' + modules_path = os.path.join(os.path.dirname(__file__), modules_dir) + module_paths = m = [] + for item in os.walk(modules_path): + m.extend([os.sep.join([item[0], f]) for f in item[2] if is_module(f)]) + for module_path in module_paths: + module_name = os.path.basename(module_path) + module = imp.load_source(module_name, module_path) + module.register(fn_mapping) - def parse_df_i(data, script, node): - column_use = "IUse%" - full = 100 - near_full = 80 - health = green - details = [] - if column_use not in data[0]: - logger.warning(col_msg % (column_use, script, node.repr)) - health = unknown - index = data[0].split().index(column_use) - prepend_str = '' # workaround for data which spans 2 lines - index_shift = 0 - for line in data[2:]: - if len(line.split()) <= index: - prepend_str = line.rstrip() - index_shift = len(line.split()) - continue - if "%" in line.split()[index - index_shift]: - value = int(line.split()[index - index_shift][:-1]) - if value >= full: - health = red - details.append(prepend_str + line) - elif value >= near_full: - health = yellow if health < yellow else health - details.append(prepend_str + line) - prepend_str = '' - return health, details - - fn_mapping = {"df-m": parse_df_m, - "df-i": parse_df_i} results = {} for node in node_manager.nodes.values(): if not node.mapscr: @@ -112,10 +64,10 @@ def analyze(node_manager): def analyze_print_results(node_manager): - code_colors = {3: ["RED", "\033[91m"], - 2: ["YELLOW", "\033[93m"], - 0: ["GREEN", "\033[92m"], - 1: ["BLUE", "\033[94m"]} + code_colors = {GREEN: ["GREEN", "\033[92m"], + UNKNOWN: ["UNKNOWN", "\033[94m"], + YELLOW: ["YELLOW", "\033[93m"], + RED: ["RED", "\033[91m"]} color_end = "\033[0m" print("Nodes health analysis:") for node, result in node_manager.analyze_results.items(): diff --git a/timmy/analyze_health.py b/timmy/analyze_health.py new file mode 100644 index 0000000..7f31c43 --- /dev/null +++ b/timmy/analyze_health.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +GREEN = 0 +UNKNOWN = 1 +YELLOW = 2 +RED = 3 diff --git a/timmy/analyze_modules/__example__.py b/timmy/analyze_modules/__example__.py new file mode 100644 index 0000000..1ba8e33 --- /dev/null +++ b/timmy/analyze_modules/__example__.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +''' +please import and use health constants from analyze_health +GREEN - no issues +UNKNOWN - cannot determine / cannot parse output +YELLOW - condition is bad but not critical / impactful +RED - critical / impactful condition + + +if you want to write log messages, add the following lines: +from timmy.env import project_name +import logging + + +logger = logging.getLogger(project_name) +''' +from timmy.analyze_health import GREEN, UNKNOWN, YELLOW, RED +from timmy.env import project_name +import logging + + +logger = logging.getLogger(project_name) + + +def register(function_mapping): + ''' + this function is mandatory and it's name must be "register" + it should have 1 argument which is a dict + it should update the dict with a relation between script names and + analyzing functions + more than one script can be mapped by a single module + see script names in timmy_data/rq/scripts folder + ''' + function_mapping['script-basename'] = parsing_function + + +def parsing_function(data, script, node): + ''' + each analyzing function should have 3 arguments: + data - list of strings aquired by reading the output file + script - path to the script file + node - node object + + return should contain 2 values: + health - set to one of the imported constants according to the analysis + details - a list of strings - an explanatory message or + lines which were indicative of the issue + ''' + health = UNKNOWN + line = data[0] # in this example we only look at the first line + details = [line] + if line.find('error'): + health = RED + details.append('This is very bad! Do something NOW!!!') + elif line.find('warning'): + health = YELLOW + details.append('Who cares if it is not RED, right? :)') + elif line.find('ok'): + health = GREEN + return health, details diff --git a/timmy/analyze_modules/__init__.py b/timmy/analyze_modules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/timmy/analyze_modules/df.py b/timmy/analyze_modules/df.py new file mode 100644 index 0000000..dc579b6 --- /dev/null +++ b/timmy/analyze_modules/df.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from timmy.analyze_health import GREEN, UNKNOWN, YELLOW, RED +from timmy.env import project_name +import logging + + +logger = logging.getLogger(project_name) + +col_msg = 'Column "%s" not found in output of "%s" from node "%s"' + + +def register(function_mapping): + function_mapping['df-m'] = parse_df_m + function_mapping['df-i'] = parse_df_i + + +def parse_df_m(data, script, node): + column_use = "Use%" + full = 100 + near_full = 80 + health = GREEN + details = [] + if column_use not in data[0]: + logger.warning(col_msg % (column_use, script, node.repr)) + health = UNKNOWN + index = data[0].split().index(column_use) + prepend_str = '' # workaround for data which spans 2 lines + index_shift = 0 + for line in data[2:]: + if len(line.split()) <= index: + prepend_str = line.rstrip() + index_shift = len(line.split()) + continue + value = int(line.split()[index - index_shift][:-1]) + if value >= full: + health = RED + details.append(prepend_str + line) + elif value >= near_full: + health = YELLOW if health < YELLOW else health + details.append(prepend_str + line) + prepend_str = '' + index_shift = 0 + return health, details + + +def parse_df_i(data, script, node): + column_use = "IUse%" + full = 100 + near_full = 80 + health = GREEN + details = [] + if column_use not in data[0]: + logger.warning(col_msg % (column_use, script, node.repr)) + health = UNKNOWN + index = data[0].split().index(column_use) + prepend_str = '' # workaround for data which spans 2 lines + index_shift = 0 + for line in data[2:]: + if len(line.split()) <= index: + prepend_str = line.rstrip() + index_shift = len(line.split()) + continue + if "%" in line.split()[index - index_shift]: + value = int(line.split()[index - index_shift][:-1]) + if value >= full: + health = RED + details.append(prepend_str + line) + elif value >= near_full: + health = YELLOW if health < YELLOW else health + details.append(prepend_str + line) + prepend_str = '' + return health, details diff --git a/timmy/env.py b/timmy/env.py index b09a674..5f9e8fd 100644 --- a/timmy/env.py +++ b/timmy/env.py @@ -16,7 +16,7 @@ # under the License. project_name = 'timmy' -version = '1.25.2' +version = '1.25.3' if __name__ == '__main__': import sys