diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..ae9ac22 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,11 @@ +[run] +omit = + *__init__* + *test_* + *conftest.py* + +[report] +omit = + *__init__* + *test_* + *conftest.py* diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..f7b3e5a --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 100 +ignore = H101,H102 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 89eaef4..c5f7bca 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ example/adminer .eggs .idea __pycache__ -*.egg-info/ \ No newline at end of file +*.egg-info/ +*.coverage \ No newline at end of file diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..ed962e9 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,652 @@ +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold under which the program will exit with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= + +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.9 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +source-roots= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Regular expression matching correct type alias names. If left empty, type +# alias names will be checked with the set naming style. +#typealias-rgx= + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + asyncSetUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + use-implicit-booleaness-not-comparison-to-string, + use-implicit-booleaness-not-comparison-to-zero, + wrong-import-order, + too-many-branches, + too-many-statements, + fixme, + consider-using-enumerate, + broad-exception-caught, + subprocess-run-check, + no-else-return, + no-else-raise, + too-many-locals, + consider-using-dict-items, + anomalous-backslash-in-string, + too-many-lines, + invalid-name, + inconsistent-return-statements, + too-many-instance-attributes, + useless-object-inheritance, + consider-using-with + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable= + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +notes-rgx= + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= + +# Set the output format. Available formats are: text, parseable, colorized, +# json2 (improved json format), json (old json format) and msvs (visual +# studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format= + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[SIMILARITIES] + +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=yes + +# Signatures are removed from the similarity computation +ignore-signatures=yes + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. No available dictionaries : You need to install +# both the python package and the system dependency for enchant to work. +spelling-dict= + +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..4c81913 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,10 @@ +{ + "editor.rulers": [ + 100 + ], + "python.testing.pytestArgs": [ + "tests" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} \ No newline at end of file diff --git a/.zuul.yaml b/.zuul.yaml index efdbc11..b1f57a6 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -7,13 +7,13 @@ - app-gen-tool-tox-py39 - app-gen-tool-tox-flake8 - app-gen-tool-tox-pylint - - app-gen-tool-tox-bandit + - app-gen-tool-tox-coverage gate: jobs: - app-gen-tool-tox-py39 - app-gen-tool-tox-flake8 - app-gen-tool-tox-pylint - - app-gen-tool-tox-bandit + - app-gen-tool-tox-coverage - job: name: app-gen-tool-tox-py39 @@ -28,6 +28,20 @@ python_version: 3.9 tox_extra_args: -c tox.ini +- job: + name: app-gen-tool-tox-coverage + parent: tox-py39 + description: | + Run coverage test for app-gen-tool + nodeset: debian-bullseye + pre-run: playbooks/app-gen-tool-tox-coverage/pre.yaml + files: + - ./* + vars: + tox_envlist: coverage + python_version: 3.9 + tox_extra_args: -c tox.ini + - job: name: app-gen-tool-tox-flake8 parent: tox diff --git a/README.md b/README.md index 6128415..e1ff5b7 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Below you will find the steps to deploy an application as a **StarlingX App**. - [Metadata File Configuration](#metadata-file-configuration) - [App Setup configuration](#app-setup-configuration) - [Run the StarlingX App Generator](#run-the-starlingx-app-generator) - - [Flux Packaging](#flux-packaging) + - [FluxCD Packaging](#fluxcd-packaging) - [FluxCD Manifest](#fluxcd-manifest) - [Plugins](#plugins) - [Metadata](#metadata) @@ -24,7 +24,7 @@ Below you will find the steps to deploy an application as a **StarlingX App**. - [Metadata](#metadata-1) - [Tarballs](#tarballs-1) - [Customizing the application](#customizing-the-application) - - [Flux Manifest](#flux-manifest) + - [FluxCD Manifest](#fluxcd-manifest) - [FluxCD](#fluxcd) - [Plugins](#plugins-1) - [Other files](#other-files) @@ -357,13 +357,13 @@ In order to allow such customization, the generator provides additional functions to modify specific files in the package. ```shell -starlingx-app-generator -i app_manifest.yaml -t [-o ./output] [--overwrite] [--no-package]|[--package-only] +starlingx-app-generator -i app_manifest.yaml -t [-o ./output] [--overwrite] [--no-package]|[--package-only] ``` Where: - `-i/--input`: path to the `app_manifest.yaml` configuration file. -- `-t/--type`: type of packaging, needs to choose between armada, flux or both. +- `-t/--type`: type of packaging, needs to choose between armada, fluxcd or both. - `-o/--output`: output folder. Defaults to a new folder with the app name in the current directory. - `--overwrite`: deletes existing output folder before starting. diff --git a/app_gen_tool/application.py b/app_gen_tool/application.py index b1107e2..d6b175e 100644 --- a/app_gen_tool/application.py +++ b/app_gen_tool/application.py @@ -1,26 +1,33 @@ -import os - +"""Application Module defines application generation methods.""" import hashlib +import os import re import shutil -import subprocess +import subprocess # nosec import sys import tarfile import yaml from urllib import request -from app_gen_tool import constants from app_gen_tool.common import to_camel_case +from app_gen_tool import constants -class Application(): - def __init__(self, app_data: dict, app_type: str): +class Application(object): + """Define Application Object. + + TODO: Future work. Split into base class, and the child class for armada and fluxcd, + respectively. + """ + + def __init__(self, app_data: dict, app_type: str, output_folder: str): # Initialize application config - self._app = {} + self._app: dict = {} self._app_type = app_type self._app = app_data['appManifestFile-config'] - self._temp_app_dir = constants.TEMP_USER_DIR + self.get_app_name() + '/' + self._temp_app_dir = os.path.join(constants.TEMP_USER_DIR, self.get_app_name()) + self.output_folder: str = os.path.join(output_folder, self.get_app_name()) self.APP_NAME = self._app['appName'] self.APP_NAME_WITH_UNDERSCORE = self._app['appName'].replace('-', '_') @@ -54,26 +61,23 @@ class Application(): # Initialize setup data self.plugin_setup = app_data['setupFile-config'] - # Initialize metadata self.metadata = app_data['metadataFile-config'] - - # TODO: Validate values - def _validate_app_values(self, app_data): + def _validate_app_values(self, app_data): # pylint: disable=unused-argument return True # TODO: Validate values - def _validate_manifest_values(self, manifest_data): + def _validate_manifest_values(self, manifest_data): # pylint: disable=unused-argument return True # TODO: Validate values - def _validate_chartgroup_values(self, chartgroup_data): + def _validate_chartgroup_values(self, chartgroup_data): # pylint: disable=unused-argument return True # TODO: Validate values - def _validate_chart_values(self, chart_data): + def _validate_chart_values(self, chart_data): # pylint: disable=unused-argument return True def _validate_app_attributes(self): @@ -88,22 +92,31 @@ class Application(): return True - - # Subprocess that check charts informations def check_charts(self): + """Subprocess that checks chart information.""" charts = self._chart for chart in charts: - manifest_data = dict() - chart_file_data = dict() + + manifest_data: dict = {} + chart_file_data: dict = {} + chart_yaml_file = os.path.join(chart['path'], "Chart.yaml") + manifest_data['name'], manifest_data['version'] = chart['name'], chart['version'] + if chart['_pathType'] == 'dir': try: - chart_metadata_f = open(f'{chart["path"]}/Chart.yaml', 'r') + chart_metadata_f = \ + open( + chart_yaml_file, + 'r', + encoding='utf-8' + ) except Exception as e: print(f'ERROR: {e}') sys.exit(1) chart_file_lines = chart_metadata_f.readlines() - chart_file_lines = [l for l in chart_file_lines if len(l) > 0 and l[0] != '#'] + chart_file_lines = \ + [cline for cline in chart_file_lines if len(cline) > 0 and cline[0] != '#'] chart_metadata_f.close() for line in chart_file_lines: line = line.rstrip('\n') @@ -114,43 +127,74 @@ class Application(): chart_file_data['name'] = line_data[-1] elif 'version:' in line_data[0]: chart_file_data['version'] = line_data[-1] + # To-do chart type different from dir for key in manifest_data: err_str = '' if key not in chart_file_data: - err_str = f'"{key}" is present in app-manifest.yaml but not in {chart["path"]}/Chart.yaml' + err_str = \ + f'"{key}" is present in app-manifest.yaml but not in {chart_yaml_file}' raise KeyError(err_str) if manifest_data[key] != chart_file_data[key]: - err_str = f'"{key}" has different values in app-manifest.yaml and {chart["path"]}/Chart.yaml' + err_str = \ + f'"{key}" has different values in app-manifest.yaml and {chart_yaml_file}' raise ValueError(err_str) - def get_app_name(self): + """Return name of app.""" return self._app['appName'] + def _print_helm_version(self) -> None: + + print("Getting which version of helm is in use.") + + cmd_lint = ['helm', 'version'] + subproc = subprocess.run( + cmd_lint, + env=os.environ.copy(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) - # Sub-process of app generation - # generate application helm-charts tarball - # - def _package_helm_chart(self, chart, chart_dir): - path = chart['path'] - # lint helm chart - cmd_lint = ['helm', 'lint', path] - subproc = subprocess.run(cmd_lint, env=os.environ.copy(), \ - stdout=subprocess.PIPE, stderr=subprocess.PIPE) if subproc.returncode == 0: - print(str(subproc.stdout, encoding = 'utf-8')) + print(str(subproc.stdout, encoding='utf-8')) else: - print(str(subproc.stderr, encoding = 'utf-8')) + print(str(subproc.stdout, encoding='utf-8')) + print(str(subproc.stderr, encoding='utf-8')) + return False + + def _package_helm_chart(self, chart, chart_dir) -> bool: + """Sub-process of app generation. Generate application helm-charts tarball.""" + # lint helm chart + print(f"Linting Helm Chart: {chart['path']}") + print(f"Chart Directory: {chart_dir}") + + cmd_lint = ['helm', 'lint', chart['path']] + subproc = subprocess.run( + cmd_lint, + env=os.environ.copy(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + + if subproc.returncode == 0: + print(str(subproc.stdout, encoding='utf-8')) + else: + print(str(subproc.stdout, encoding='utf-8')) + print(str(subproc.stderr, encoding='utf-8')) return False # package helm chart - cmd_package = ['helm', 'package', path, \ - '--destination=' + chart_dir] - subproc = subprocess.run(cmd_package, env=os.environ.copy(), \ - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + cmd_package = ['helm', 'package', chart['path'], '--destination=' + chart_dir] + subproc = subprocess.run( + cmd_package, + env=os.environ.copy(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + if subproc.returncode == 0: - output = str(subproc.stdout, encoding = 'utf-8') + output = str(subproc.stdout, encoding='utf-8') print(output) # capture tarball name for words in output.split('/'): @@ -159,12 +203,13 @@ class Application(): else: print(subproc.stderr) return False - return True + return True # pyyaml does not support writing yaml block with initial indent # add initial indent for yaml block substitution def _write_yaml_to_manifest(self, key, src, init_indent): + target = {} # add heading key target[key] = src @@ -178,10 +223,10 @@ class Application(): # restore ending '\n' return '\n'.join(lines) + '\n' - def _substitute_values(self, in_line, dicts): + out_line = in_line - pattern = re.compile('\$.+?\$') + pattern = re.compile(r'\$.+?\$') results = pattern.findall(out_line) if results: for result in results: @@ -202,10 +247,9 @@ class Application(): else: return out_line, True - def _substitute_blocks(self, in_line, dicts): out_line = in_line - result = re.search('@\S+\|\d+@',out_line) + result = re.search(r'@\S+\|\d+@', out_line) if result: block_key = result.group().strip('@').split('|') key = block_key[0].lower() @@ -217,58 +261,68 @@ class Application(): return out_line - # Fetch info from helm chart to fill # the values that needs to be substituted # Below info are needed: # - waitLabelKey # - chartArcname - def _fetch_info_from_chart(self, chart_idx): + a_chart = self._chart[chart_idx] - bin_fetch_script = constants.APP_GEN_PY_PATH + '/' + constants.BIN_FETCH_CHART_INFO + bin_fetch_script = os.path.join(constants.APP_GEN_PY_PATH, constants.BIN_FETCH_CHART_INFO) # waitLabelKey # search for the key of label which indicates '.Release.Name' # within deployment, statefulset, daemonset yaml file cmd = [bin_fetch_script, 'waitlabel', a_chart['path']] - subproc = subprocess.run(cmd, env=os.environ.copy(), \ - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + subproc = subprocess.run( + cmd, + env=os.environ.copy(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + if subproc.returncode == 0: - output = str(subproc.stdout, encoding = 'utf-8') + output = str(subproc.stdout, encoding='utf-8') if output.strip(): a_chart['waitLabelKey'] = output.strip() + if 'waitLabelKey' not in a_chart: - print("The label which indicates .Release.Name is not found in %s" % a_chart['name']) + print(f"The label which indicates .Release.Name is not found in {a_chart['name']}.") return False # chartArcname is the helm chart name in Chart.yaml # it is used as tarball arcname during helm package cmd = [bin_fetch_script, 'chartname', a_chart['path']] - subproc = subprocess.run(cmd, env=os.environ.copy(), \ - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + subproc = subprocess.run( + cmd, + env=os.environ.copy(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + if subproc.returncode == 0: - output = str(subproc.stdout, encoding = 'utf-8') + output = str(subproc.stdout, encoding='utf-8') if output.strip(): a_chart['chartArcname'] = output.strip() + if 'chartArcname' not in a_chart: - print("The name within Chart.yaml of chart %s folder is not found" % a_chart['name']) + print(f"The name within Chart.yaml of chart {a_chart['name']} folder is not found") return False return True - - # Sub-process of app generation - # lint and package helm chart # TODO: sub-chart dependency check - # def _gen_helm_chart_tarball(self, chart, chart_dir): + """Sub-process of app generation. lint and package helm chart""" ret = False - path = '' - print('Processing chart %s...' % chart['name']) + chart_path = '' + print(f'Processing chart {chart["name"]}...') # check pathtype of the chart if chart['_pathType'] == 'git': - gitname = '' # download git + + print("Processing chart _pathType git...") + if not os.path.exists(self._temp_app_dir): os.makedirs(self._temp_app_dir) # if the git folder exists, check git name and use that folder @@ -277,12 +331,18 @@ class Application(): saved_pwd = os.getcwd() os.chdir(self._temp_app_dir) cmd = ['git', 'clone', chart['path']] - subproc = subprocess.run(cmd, env=os.environ.copy(), \ - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + subproc = subprocess.run( + cmd, + env=os.environ.copy(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + if subproc.returncode != 0: - output = str(subproc.stderr, encoding = 'utf-8') + output = str(subproc.stderr, encoding='utf-8') print(output) - print('Error: git clone %s failed' % chart['_gitname']) + print(f'Error: git clone {chart["_gitname"]} failed') os.chdir(saved_pwd) return False os.chdir(saved_pwd) @@ -291,17 +351,29 @@ class Application(): saved_pwd = os.getcwd() os.chdir(self._temp_app_dir + chart['_gitname']) cmd = ['git', 'pull'] - subproc = subprocess.run(cmd, env=os.environ.copy(), \ - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + subproc = subprocess.run( + cmd, + env=os.environ.copy(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + if subproc.returncode != 0: - output = str(subproc.stderr, encoding = 'utf-8') + output = str(subproc.stderr, encoding='utf-8') print(output) - print('Error: git pull for %s failed' % chart['_gitname']) + print(f'Error: git pull for {chart["_gitname"]} failed') os.chdir(saved_pwd) return False + os.chdir(saved_pwd) - path = self._temp_app_dir + chart['_gitname'] + '/' + chart['subpath'] + + chart_path = os.path.join(self._temp_app_dir, chart['_gitname'], chart['subpath']) + elif chart['_pathType'] == 'tarball': + + print("Processing chart _pathType tarball...") + if not os.path.exists(self._temp_app_dir): os.makedirs(self._temp_app_dir) try: @@ -310,14 +382,16 @@ class Application(): # download tarball tarpath = self._temp_app_dir + chart['_tarname'] + '.tgz' if not os.path.exists(tarpath): - res = request.urlopen(chart['path']) + res = request.urlopen(chart['path']) # nosec with open(tarpath, 'wb') as f: f.write(res.read()) else: tarpath = chart['path'] + # extract tarball chart_tar = tarfile.open(tarpath, 'r:gz') chart_files = chart_tar.getnames() + # get tar arcname for packaging helm chart process # TODO: compatible with the case that there is no arcname chart['_tarArcname'] = chart_files[0].split('/')[0] @@ -326,43 +400,49 @@ class Application(): chart_tar.extract(chart_file, self._temp_app_dir) chart_tar.close() except Exception as e: - print('Error: %s' % e) + print(f'Error: {e}') return False - path = self._temp_app_dir + chart['_tarArcname'] + '/' + chart['subpath'] + + chart_path = os.path.join(self._temp_app_dir, chart['_tarArcname'], chart['subpath']) + elif chart['_pathType'] == 'dir': - path = chart['path'] + + print("Processing chart _pathType dir...") + + chart_path = chart['path'] # update chart path - # remove ending '/' - chart['path'] = path.rstrip('/') + # remove ending '/', if it exists... + chart['path'] = chart_path.rstrip('/') # lint and package ret = self._package_helm_chart(chart, chart_dir) return ret - - # Sub-process of app generation - # generate application manifest file - # def _gen_armada_manifest(self): + """Sub-process of app generation. Generate application manifest file.""" # check manifest file existance - manifest_file = self._app['outputArmadaDir'] + '/' + self._app['appName'] + '.yaml' + app_yaml_file = f"{self._app['appName']}.yaml" + manifest_file = os.path.join(self._app['outputArmadaDir'], app_yaml_file) if os.path.exists(manifest_file): os.remove(manifest_file) # update schema path to abspath - chart_template = constants.APP_GEN_PY_PATH + '/' + constants.ARMADA_CHART_TEMPLATE - chartgroup_template = constants.APP_GEN_PY_PATH + '/' + constants.ARMADA_CHARTGROUP_TEMPLATE - manifest_template = constants.APP_GEN_PY_PATH + '/' + constants.ARMADA_MANIFEST_TEMPLATE + chart_template = os.path.join(constants.APP_GEN_PY_PATH, constants.ARMADA_CHART_TEMPLATE) + chartgroup_template = \ + os.path.join(constants.APP_GEN_PY_PATH, constants.ARMADA_CHARTGROUP_TEMPLATE) + manifest_template = \ + os.path.join(constants.APP_GEN_PY_PATH, constants.ARMADA_MANIFEST_TEMPLATE) # generate chart schema try: - with open(chart_template, 'r') as f: + with open(chart_template, 'r', encoding='utf-8') as f: chart_schema = f.readlines() except FileNotFoundError: - print('File %s not found' % chart_template) + print(f'File {chart_template} not found') return False - with open(manifest_file, 'a') as f: + + with open(manifest_file, 'a', encoding='utf-8') as f: # iterate each armada_chart for idx in range(len(self._chart)): a_chart = self._chart[idx] @@ -379,12 +459,13 @@ class Application(): # generate chartgroup schema try: - with open(chartgroup_template, 'r') as f: + with open(chartgroup_template, 'r', encoding='utf-8') as f: chartgroup_schema = f.readlines() except FileNotFoundError: - print('File %s not found' % chartgroup_template) + print(f'File {chartgroup_template} not found') return False - with open(manifest_file, 'a') as f: + + with open(manifest_file, 'a', encoding='utf-8') as f: # iterate each chartgroup for chartgroup in self._chartgroup: for line in chartgroup_schema: @@ -397,12 +478,12 @@ class Application(): # generate manifest schema try: - with open(manifest_template, 'r') as f: + with open(manifest_template, 'r', encoding="utf-8") as f: manifest_schema = f.readlines() except FileNotFoundError: - print('File %s not found' % manifest_template) + print(f'File {manifest_template} not found') return False - with open(manifest_file, 'a') as f: + with open(manifest_file, 'a', encoding="utf-8") as f: # only one manifest in an application manifest = self._manifest # substitute values @@ -416,23 +497,49 @@ class Application(): return True - - # Sub-process of app generation - # generate application fluxcd manifest files - # - def _gen_fluxcd_manifest(self): + def _gen_fluxcd_manifest(self): # pylint: disable=too-many-return-statements + """Sub-process of app generation. Generate application fluxcd manifest files.""" # check manifest file existance - flux_dir = self._app['outputManifestDir'] + fluxcd_dir = self._app['outputManifestDir'] # update schema path to abspath - kustomization_template = constants.APP_GEN_PY_PATH + '/' + constants.FLUX_KUSTOMIZATION_TEMPLATE + kustomization_template = \ + os.path.join(constants.APP_GEN_PY_PATH, constants.FLUXCD_KUSTOMIZATION_TEMPLATE) - base_helmrepo_template = constants.APP_GEN_PY_PATH + '/' + constants.FLUX_BASE_TEMPLATES + '/helmrepository.template' - base_kustom_template = constants.APP_GEN_PY_PATH + '/' + constants.FLUX_BASE_TEMPLATES + '/kustomization.template' - base_namespace_template = constants.APP_GEN_PY_PATH + '/' + constants.FLUX_BASE_TEMPLATES + '/namespace.template' + base_helmrepo_template = \ + os.path.join( + constants.APP_GEN_PY_PATH, + constants.FLUXCD_BASE_TEMPLATES, + 'helmrepository.template' + ) - manifest_helmrelease_template = constants.APP_GEN_PY_PATH + '/' + constants.FLUX_MANIFEST_TEMPLATE + '/helmrelease.template' - manifest_kustomization_template = constants.APP_GEN_PY_PATH + '/' + constants.FLUX_MANIFEST_TEMPLATE + '/kustomization.template' + base_kustom_template = \ + os.path.join( + constants.APP_GEN_PY_PATH, + constants.FLUXCD_BASE_TEMPLATES, + 'kustomization.template' + ) + + base_namespace_template = \ + os.path.join( + constants.APP_GEN_PY_PATH, + constants.FLUXCD_BASE_TEMPLATES, + 'namespace.template' + ) + + manifest_helmrelease_template = \ + os.path.join( + constants.APP_GEN_PY_PATH, + constants.FLUXCD_MANIFEST_TEMPLATE, + 'helmrelease.template' + ) + + manifest_kustomization_template = \ + os.path.join( + constants.APP_GEN_PY_PATH, + constants.FLUXCD_MANIFEST_TEMPLATE, + 'kustomization.template' + ) manifest = self._app chartgroup = self._listcharts @@ -440,13 +547,13 @@ class Application(): # generate kustomization file try: - with open(kustomization_template, 'r') as f: + with open(kustomization_template, 'r', encoding="utf-8") as f: kustomization_schema = f.readlines() except FileNotFoundError: - print('File %s not found' % kustomization_template) + print(f'File {kustomization_template} not found') return False - kustom_file = flux_dir + 'kustomization.yaml' - with open(kustom_file, 'a') as f: + kustom_file = fluxcd_dir + 'kustomization.yaml' + with open(kustom_file, 'a', encoding="utf-8") as f: # substitute values for line in kustomization_schema: # substitute template values to manifest values @@ -458,13 +565,13 @@ class Application(): # generate base/namespace file try: - with open(base_namespace_template, 'r') as f: + with open(base_namespace_template, 'r', encoding="utf-8") as f: base_namespace_schema = f.readlines() except FileNotFoundError: - print('File %s not found' % base_namespace_template) + print(f'File {base_namespace_template} not found') return False - base_namespace_file = flux_dir + 'base/namespace.yaml' - with open(base_namespace_file, 'a') as f: + base_namespace_file = os.path.join(fluxcd_dir, 'base', 'namespace.yaml') + with open(base_namespace_file, 'a', encoding="utf-8") as f: # substitute values for line in base_namespace_schema: # substitute template values to manifest values @@ -478,25 +585,27 @@ class Application(): # generate base/helmrepository file # Both yaml files don't need to add informations from the input file try: - with open(base_kustom_template, 'r') as f: + with open(base_kustom_template, 'r', encoding="utf-8") as f: base_kustom_schema = f.readlines() except FileNotFoundError: - print('File %s not found' % base_kustom_template) + print(f'File {base_kustom_template} not found') return False - base_kustom_file = flux_dir + 'base/kustomization.yaml' - with open(base_kustom_file, 'a') as f: + + base_kustom_file = os.path.join(fluxcd_dir, 'base', 'kustomization.yaml') + with open(base_kustom_file, 'a', encoding="utf-8") as f: for line in base_kustom_schema: out_line = line f.write(out_line) try: - with open(base_helmrepo_template, 'r') as f: + with open(base_helmrepo_template, 'r', encoding="utf-8") as f: base_helmrepo_schema = f.readlines() except FileNotFoundError: - print('File %s not found' % base_helmrepo_template) + print(f'File {base_helmrepo_template} not found') return False - base_helmrepo_file = flux_dir + 'base/helmrepository.yaml' - with open(base_helmrepo_file, 'a') as f: + + base_helmrepo_file = os.path.join(fluxcd_dir, 'base', 'helmrepository.yaml') + with open(base_helmrepo_file, 'a', encoding="utf-8") as f: for line in base_helmrepo_schema: out_line = line f.write(out_line) @@ -507,13 +616,16 @@ class Application(): # generate manifest/helmrelease file try: - with open(manifest_helmrelease_template, 'r') as f: + with open(manifest_helmrelease_template, 'r', encoding="utf-8") as f: manifest_helmrelease_schema = f.readlines() except FileNotFoundError: - print('File %s not found' % manifest_helmrelease_template) + print(f'File {manifest_helmrelease_template} not found') return False - manifest_helmrelease_file = flux_dir + a_chart['name'] + '/helmrelease.yaml' - with open(manifest_helmrelease_file, 'a') as f: + + manifest_helmrelease_file = \ + os.path.join(fluxcd_dir, a_chart['name'], 'helmrelease.yaml') + + with open(manifest_helmrelease_file, 'a', encoding="utf-8") as f: # fetch chart specific info for line in manifest_helmrelease_schema: # substitute template values to chart values @@ -525,13 +637,16 @@ class Application(): # generate manifest/kustomizaion file try: - with open(manifest_kustomization_template, 'r') as f: + with open(manifest_kustomization_template, 'r', encoding="utf-8") as f: manifest_kustomization_schema = f.readlines() except FileNotFoundError: - print('File %s not found' % manifest_kustomization_template) + print(f'File {manifest_kustomization_template} not found') return False - manifest_kustomization_file = flux_dir + a_chart['name'] + '/kustomization.yaml' - with open(manifest_kustomization_file, 'a') as f: + + manifest_kustomization_file = \ + os.path.join(fluxcd_dir, a_chart['name'], 'kustomization.yaml') + + with open(manifest_kustomization_file, 'a', encoding="utf-8") as f: # fetch chart specific info for line in manifest_kustomization_schema: # substitute template values to chart values @@ -541,28 +656,32 @@ class Application(): out_line = self._substitute_blocks(line, a_chart) f.write(out_line) + # file names + system_overrides_name = f"{a_chart['name']}-system-overrides.yaml" + static_overrides_name = f"{a_chart['name']}-static-overrides.yaml" + # generate an empty manifest/system-overrides file - system_override_file = flux_dir + '/' + a_chart['name'] + '/' + a_chart['name'] + '-system-overrides.yaml' - open(system_override_file, 'w').close() + system_override_file = os.path.join(fluxcd_dir, a_chart['name'], system_overrides_name) + open(system_override_file, 'w', encoding="utf-8").close() # generate a manifest/static-overrides file - static_override_file = flux_dir + '/' + a_chart['name'] + '/' + a_chart['name'] + '-static-overrides.yaml' - open(static_override_file, 'w').close() + static_override_file = os.path.join(fluxcd_dir, a_chart['name'], static_overrides_name) + open(static_override_file, 'w', encoding="utf-8").close() return True - # Sub-process of app generation # generate application plugin files - # def _gen_plugins(self): - plugin_dir = self._app['outputPluginDir'] + plugin_dir = self._app['outputPluginDir'] - common_template = constants.APP_GEN_PY_PATH + '/' + constants.FLUX_COMMON_TEMPLATE - helm_template = constants.APP_GEN_PY_PATH + '/' + constants.FLUX_HELM_TEMPLATE - kustomize_template = constants.APP_GEN_PY_PATH + '/' + constants.FLUX_KUSTOMIZE_TEMPLATE - lifecycle_template = constants.APP_GEN_PY_PATH + '/' + constants.FLUX_LIFECYCLE_TEMPLATE + common_template = os.path.join(constants.APP_GEN_PY_PATH, constants.FLUXCD_COMMON_TEMPLATE) + helm_template = os.path.join(constants.APP_GEN_PY_PATH, constants.FLUXCD_HELM_TEMPLATE) + kustomize_template = \ + os.path.join(constants.APP_GEN_PY_PATH, constants.FLUXCD_KUSTOMIZE_TEMPLATE) + lifecycle_template = \ + os.path.join(constants.APP_GEN_PY_PATH, constants.FLUXCD_LIFECYCLE_TEMPLATE) appname = 'k8sapp_' + self.APP_NAME_WITH_UNDERSCORE namespace = self._app['namespace'] @@ -571,101 +690,108 @@ class Application(): # generate Common files try: - with open(common_template, 'r') as f: + with open(common_template, 'r', encoding="utf-8") as f: common_schema = f.read() except FileNotFoundError: - print('File %s not found' % common_template) + print(f'File {common_template} not found') return False - common_file = plugin_dir + '/' + appname + '/common/constants.py' + + common_file = os.path.join(plugin_dir, appname, 'common', 'constants.py') output = common_schema.format(appname=appname, name=name, namespace=namespace) - with open(common_file, "w") as f: + with open(common_file, "w", encoding="utf-8") as f: f.write(output) self.create_init_file(self._app['outputCommonDir']) # Generate Helm files try: - with open(helm_template, 'r') as f: + with open(helm_template, 'r', encoding="utf-8") as f: helm_schema = f.read() except FileNotFoundError: - print('File %s not found' % helm_template) + print(f'File {helm_template} not found') return False for idx in range(len(chart)): a_chart = chart[idx] - helm_file = plugin_dir + '/' + appname + '/helm/' + a_chart['name'].replace(" ", "_").replace("-", "_") + '.py' + updated_chart_name = a_chart['name'].replace(" ", "_").replace("-", "_") + '.py' + helm_file = os.path.join(plugin_dir, appname, 'helm', updated_chart_name) - name = a_chart['name'].replace('-', ' ').title().replace(' ','') + name = a_chart['name'].replace('-', ' ').title().replace(' ', '') namespace = a_chart['namespace'] output = helm_schema.format(appname=appname, name=name) - with open(helm_file, "w") as f: + with open(helm_file, "w", encoding="utf-8") as f: f.write(output) self.create_init_file(self._app['outputHelmDir']) # Generate Kustomize files try: - with open(kustomize_template, 'r') as f: + with open(kustomize_template, 'r', encoding="utf-8") as f: kustomize_schema = f.read() except FileNotFoundError: - print('File %s not found' % kustomize_template) + print(f'File {kustomize_template} not found') return False - kustomize_file = plugin_dir + '/' + appname + '/kustomize/kustomize_' + self.APP_NAME_WITH_UNDERSCORE + '.py' + + kustomize_file_name = f"kustomie_{self.APP_NAME_WITH_UNDERSCORE}.py" + kustomize_file = os.path.join(plugin_dir, appname, 'kustomize', kustomize_file_name) output = kustomize_schema.format(appname=appname, appnameStriped=self.APP_NAME_CAMEL_CASE) - with open(kustomize_file, "w") as f: + with open(kustomize_file, "w", encoding="utf-8") as f: f.write(output) self.create_init_file(self._app['outputKustomizeDir']) # Generate Lifecycle files try: - with open(lifecycle_template, 'r') as f: + with open(lifecycle_template, 'r', encoding="utf-8") as f: lifecycle_schema = f.read() except FileNotFoundError: - print('File %s not found' % lifecycle_template) + print(f'File {lifecycle_template} not found') return False - lifecycle_file = plugin_dir + '/' + appname + '/lifecycle/lifecycle_' + self.APP_NAME_WITH_UNDERSCORE + '.py' + + lifecycle_file_name = f"lifecycle_{self.APP_NAME_WITH_UNDERSCORE}.py" + lifecycle_file = os.path.join(plugin_dir, appname, 'lifecycle', lifecycle_file_name) output = lifecycle_schema.format(appnameStriped=self.APP_NAME_CAMEL_CASE) - with open(lifecycle_file, "w") as f: + with open(lifecycle_file, "w", encoding="utf-8") as f: f.write(output) self.create_init_file(self._app['outputLifecycleDir']) # Generate setup.py - setupPy_file = plugin_dir + '/setup.py' - file = f"""import setuptools\n\nsetuptools.setup(\n setup_requires=['pbr>=2.0.0'],\n pbr=True)""" + setup_py_file = plugin_dir + '/setup.py' + setup_py_file = os.path.join(plugin_dir, 'setup.py') + file_content = \ + '"""import setuptools\n\nsetuptools.setup(\n ' \ + 'setup_requires=["pbr>=2.0.0"],\n pbr=True)"""' - with open(setupPy_file, 'w') as f: - f.write(file) + with open(setup_py_file, 'w', encoding="utf-8") as f: + f.write(file_content) + f.close() # Generate setup.cfg file self.write_app_setup() - self.create_init_file(plugin_dir) - dir = plugin_dir + '/' + appname - self.create_init_file(dir) + directory = os.path.join(plugin_dir, appname) + self.create_init_file(directory) return True + def create_init_file(self, app_path: str): + """Subprocess that creates __init__.py file.""" + init_file = os.path.join(app_path, "__init__.py") + open(init_file, 'w', encoding="utf-8").close() - # Subprocess that creates __init__.py file - def create_init_file(self, path): - init_file = path + '/__init__.py' - open(init_file, 'w').close() - - - #Subprocess that writes the setup.cfg file def write_app_setup(self): + """Subprocess that writes the setup.cfg file.""" def split_and_format_value(value) -> str: - if type(value) == str: + if isinstance(value, str): return ''.join([f'\t{lin}\n' for lin in value.split('\n')]) else: return ''.join([f'\t{lin}\n' for lin in value]) @@ -681,13 +807,14 @@ class Application(): yml_data['metadata']['name'] = f'k8sapp-{self.APP_NAME}' yml_data['metadata']['summary'] = f'StarlingX sysinv extensions for {self.APP_NAME}' yml_data['metadata'] = dict(sorted(yml_data['metadata'].items(), key=expected_order)) + out = '' for label in yml_data: out += f'[{label}]\n' for key, val in yml_data[label].items(): if label == 'metadata' and val is None: raise ValueError(f'You should\'ve written a value for: {key}') - elif type(val) != list: + elif isinstance(val, list): out += f'{key} = {val}\n' else: out += f'{key} =\n' @@ -702,60 +829,64 @@ class Application(): out += '[entry_points]\nsystemconfig.helm_applications =\n\t' \ f'{self.APP_NAME} = systemconfig.helm_plugins.{self.APP_NAME_WITH_UNDERSCORE}\n\n' \ f'systemconfig.helm_plugins.{self.APP_NAME_WITH_UNDERSCORE} =\n' - for i, plug in enumerate(plugins_names): - out += f'\t{i+1:03d}_{plug} = k8sapp_{self.APP_NAME_WITH_UNDERSCORE}.helm.{plug.replace("-","_")}' - out += f':{plug.replace("-", " ").title().replace(" ", "")}Helm\n' - out += '\n' - out += 'systemconfig.fluxcd.kustomize_ops =\n' \ - f'\t{self.APP_NAME} = k8sapp_{self.APP_NAME_WITH_UNDERSCORE}.kustomize.kustomize_' \ - f'{self.APP_NAME_WITH_UNDERSCORE}:{self.APP_NAME_CAMEL_CASE}FluxCDKustomizeOperator\n\n' \ - 'systemconfig.app_lifecycle =\n' \ - f'\t{self.APP_NAME} = k8sapp_{self.APP_NAME_WITH_UNDERSCORE}.lifecycle.lifecycle_' \ - f'{self.APP_NAME_WITH_UNDERSCORE}:{self.APP_NAME_CAMEL_CASE}AppLifecycleOperator\n\n' - out += '[bdist_wheel]\nuniversal = 1' - with open(f'{self._app["outputPluginDir"]}/setup.cfg', 'w+') as f: - f.write(out) + for i, plug in enumerate(plugins_names): + + out += \ + f'\t{i+1:03d}_{plug} = k8sapp_{self.APP_NAME_WITH_UNDERSCORE}.' \ + f'helm.{plug.replace("-","_")}' + + out += f':{plug.replace("-", " ").title().replace(" ", "")}Helm\n' + + out += '\n' + out += \ + 'systemconfig.fluxcd.kustomize_ops =\n' \ + f'\t{self.APP_NAME} = k8sapp_{self.APP_NAME_WITH_UNDERSCORE}.kustomize.kustomize_' \ + f'{self.APP_NAME_WITH_UNDERSCORE}:{self.APP_NAME_CAMEL_CASE}' \ + 'FluxCDKustomizeOperator\n\n' \ + 'systemconfig.app_lifecycle =\n' \ + f'\t{self.APP_NAME} = k8sapp_{self.APP_NAME_WITH_UNDERSCORE}.lifecycle.lifecycle_' \ + f'{self.APP_NAME_WITH_UNDERSCORE}:{self.APP_NAME_CAMEL_CASE}AppLifecycleOperator\n\n' + out += '[bdist_wheel]\nuniversal = 1' + with open(f'{self._app["outputPluginDir"]}/setup.cfg', 'w+', encoding="utf-8") as f: + f.write(out) # Sub-process of app generation # generate application metadata - # - def _gen_metadata(self, package_type, output): - """ - gets the keys and values defined in the input yaml and writes the metadata.yaml app file. + def _gen_metadata(self, package_type, output_directory): + """Get the keys and values defined in the input yaml and writes the metadata.yaml app file. + """ yml_data = self.metadata app_name, app_version = self._app['appName'], self._app['appVersion'] - file = output + '/metadata.yaml' + metadata_file = os.path.join(output_directory, 'metadata.yaml') + try: - with open(file, 'w') as f: + with open(metadata_file, 'w', encoding="utf-8") as f: f.write(f'app_name: {app_name}\napp_version: {app_version}') - if package_type == 'flux': - with open(file, 'a') as f: + if package_type == 'fluxcd': + with open(metadata_file, 'a', encoding="utf-8") as f: f.write('\nhelm_repo: stx-platform\n') if yml_data is not None: yaml.safe_dump(yml_data, f) - except: + except Exception: return False return True - # Sub-process of app generation # generate application sha256 file - # def _gen_sha256(self, in_file): with open(in_file, 'rb') as f: out_sha256 = hashlib.sha256(f.read()).hexdigest() return out_sha256 - # Sub-process of app generation # generate plugin wheels - # def _gen_plugin_wheels(self): + dirplugins = self._app['outputPluginDir'] store_cwd = os.getcwd() @@ -771,67 +902,76 @@ class Application(): dirplugins] try: - subprocess.call(command, stderr=subprocess.STDOUT) - except: + subprocess.call(command, stderr=subprocess.STDOUT, shell=False) + except Exception: return False - files = [ - f'{dirplugins}/ChangeLog', - f'{dirplugins}/AUTHORS'] - for file in files: - if os.path.exists(file): - os.remove(file) + files_to_remove = [ + os.path.join(dirplugins, 'ChangeLog'), + os.path.join(dirplugins, "AUTHORS") + ] - dirs = [ - f'{dirplugins}/build/', - f'{dirplugins}/k8sapp_{self.APP_NAME_WITH_UNDERSCORE}.egg-info/'] - for dir in dirs: - if os.path.exists(dir): - shutil.rmtree(dir) + for remove_file in files_to_remove: + if os.path.exists(remove_file): + os.remove(remove_file) + + directories_to_remove = [ + os.path.join(dirplugins, 'build'), + os.path.join(dirplugins, f"k8sapp_{self.APP_NAME_WITH_UNDERSCORE}.egg-info") + ] + + for directory in directories_to_remove: + if os.path.exists(directory): + shutil.rmtree(directory) os.chdir(store_cwd) return True - # Sub-process of app generation # generate application checksum file and tarball - # def _gen_checksum_and_app_tarball(self, output): + store_cwd = os.getcwd() + app_files = [] os.chdir(output) + # gen checksum # check checksum file existance checksum_file = 'checksum.sha256' if os.path.exists(checksum_file): os.remove(checksum_file) - app_files = [] + try: - for parent, dirnames, filenames in os.walk('./'): + for parent, _, filenames in os.walk('./'): for filename in filenames: if filename[-3:] != '.py' and filename[-4:] != '.cfg': app_files.append(os.path.join(parent, filename)) except Exception as e: - print('Error: %s' % e) + print(f'Error: {e}') + try: - with open(checksum_file, 'a') as f: + with open(checksum_file, 'a', encoding="utf-8") as f: for target_file in sorted(app_files): f.write(self._gen_sha256(target_file) + ' *' + target_file + '\n') except Exception as e: - print('Error: %s' % e) + print(f'Error: {e}') app_files.append('./' + checksum_file) # gen application tarball tarname = f"{self._app['appName']}-{self._app['appVersion']}.tgz" - t = tarfile.open(tarname, 'w:gz') - for target_file in app_files: - t.add(target_file) - t.close() + + with tarfile.open(tarname, 'w:gz') as t: + + for target_file in app_files: + t.add(target_file) + + t.close() + os.chdir(store_cwd) return tarname - def _create_flux_dir(self, output_dir): if not os.path.exists(self._app['outputFluxChartDir']): @@ -840,10 +980,14 @@ class Application(): os.makedirs(self._app['outputFluxBaseDir']) for idx in range(len(self._chart)): chart = self._chart[idx] - self._app['outputFluxManifestDir'] = output_dir + '/FluxCD/fluxcd-manifests/' + chart['name'] + self._app['outputFluxManifestDir'] = os.path.join( + output_dir, + 'FluxCD', + 'fluxcd-manifests', + chart['name'] + ) os.makedirs(self._app['outputFluxManifestDir']) - def _create_plugins_dir(self): if not os.path.exists(self._app['outputPluginDir']): @@ -857,7 +1001,6 @@ class Application(): if not os.path.exists(self._app['outputLifecycleDir']): os.makedirs(self._app['outputLifecycleDir']) - def _create_armada_dir(self): if not os.path.exists(self._app['outputArmadaDir']): @@ -865,25 +1008,25 @@ class Application(): if not os.path.exists(self._app['outputArmadaChartDir']): os.makedirs(self._app['outputArmadaChartDir']) - - # Generate armada application, including: - # 1. Check chart values - # 2. Create Armada directory - # 3. Generate helm chart tarballs - # 4. Generate armada manifest - # 5. Generate metadata file - # 6. Generage checksum file - # 7. Package Armada application - # def gen_armada_app(self, output_dir, no_package, package_only): + """Generate armada application, including: + + 1. Check chart values + 2. Create Armada directory + 3. Generate helm chart tarballs + 4. Generate armada manifest + 5. Generate metadata file + 6. Generage checksum file + 7. Package Armada application + """ ret = False if not self._validate_app_attributes(): print('Error: Some of the app attributes are not valid!') return ret self._app['outputDir'] = output_dir - self._app['outputArmadaDir'] = output_dir + '/Armada' - self._app['outputArmadaChartDir'] = output_dir + '/Armada/charts' + self._app['outputArmadaDir'] = os.path.join(output_dir, 'Armada') + self._app['outputArmadaChartDir'] = os.path.join(output_dir, 'Armada', 'charts') # 1 - Validate input file and helm chart data self.check_charts() @@ -894,13 +1037,12 @@ class Application(): # 3. Generating helm chart tarball for chart in self._chart: - ret = self._gen_helm_chart_tarball( - chart, self._app['outputArmadaChartDir']) + ret = self._gen_helm_chart_tarball(chart, self._app['outputArmadaChartDir']) if ret: - print('Helm chart %s tarball generated!' % chart['name']) + print(f'Helm chart {chart["name"]} tarball generated!') print('') else: - print('Generating tarball for helm chart: %s error!' % chart['name']) + print(f'Generating tarball for helm chart: {chart["name"]} error!') return ret # 4. Generating armada manifest @@ -924,7 +1066,7 @@ class Application(): ret = self._gen_checksum_and_app_tarball(self._app['outputArmadaDir']) if ret: print('Checksum generated!') - print('Armada App tarball generated at %s/%s' % (self._app['outputArmadaDir'], ret)) + print(f'Armada App tarball generated at {self._app["outputArmadaDir"]}/{ret}') print('') else: print('Checksum and App tarball generation failed!') @@ -932,30 +1074,41 @@ class Application(): return ret + def gen_fluxcd_app(self, output_dir, no_package, package_only): + """Function to call all process fot the creation of the FluxCd app tarball - # Function to call all process fot the creation of the FluxCd app tarball - # 1 - Validate input file and helm chart data - # 2 - Create application directories - # 3 - Generate FluxCD Manifests - # 4 - Generate application plugins - # 5 - Generate application metadata - # 6 - Package helm-charts - # 7 - Package plugins in wheel format - # 8 - Generate checksum - # 9 - Package entire application - def gen_flux_app(self, output_dir, no_package, package_only): + 1 - Validate input file and helm chart data + 2 - Create application directories + 3 - Generate FluxCD Manifests + 4 - Generate application plugins + 5 - Generate application metadata + 6 - Package helm-charts + 7 - Package plugins in wheel format + 8 - Generate checksum + 9 - Package entire application + """ + app_name_fixed = self._app['appName'].replace(" ", "_").replace("-", "_") + updated_app_name = f"k8sapp_{app_name_fixed}" self._app['outputDir'] = output_dir - self._app['outputFluxCDDir'] = output_dir + '/FluxCD' - self._app['outputFluxChartDir'] = output_dir + '/FluxCD/charts/' - self._app['outputManifestDir'] = output_dir + '/FluxCD/fluxcd-manifests/' - self._app['outputFluxBaseDir'] = output_dir + '/FluxCD/fluxcd-manifests/base/' + self._app['outputFluxCDDir'] = os.path.join(output_dir, 'FluxCD') + self._app['outputFluxChartDir'] = os.path.join(output_dir, 'FluxCD', 'charts') + self._app['outputManifestDir'] = os.path.join(output_dir, 'FluxCD', 'fluxcd-manifests') + self._app['outputFluxBaseDir'] = \ + os.path.join(output_dir, 'FluxCD', 'fluxcd-manifests', 'base') - self._app['outputPluginDir'] = output_dir + '/FluxCD/plugins' - self._app['outputHelmDir'] = output_dir + '/FluxCD/plugins/k8sapp_' + self._app['appName'].replace(" ", "_").replace("-", "_") + '/helm/' - self._app['outputCommonDir'] = output_dir + '/FluxCD/plugins/k8sapp_' + self._app['appName'].replace(" ", "_").replace("-", "_") + '/common/' - self._app['outputKustomizeDir'] = output_dir + '/FluxCD/plugins/k8sapp_' + self._app['appName'].replace(" ", "_").replace("-", "_") + '/kustomize/' - self._app['outputLifecycleDir'] = output_dir + '/FluxCD/plugins/k8sapp_' + self._app['appName'].replace(" ", "_").replace("-", "_") + '/lifecycle/' + self._app['outputPluginDir'] = os.path.join(output_dir, 'FluxCD', 'plugins') + self._app['outputHelmDir'] = \ + os.path.join(output_dir, 'FluxCD', 'plugins', updated_app_name, 'helm') + self._app['outputCommonDir'] = \ + os.path.join(output_dir, 'FluxCD', 'plugins', updated_app_name, 'common') + self._app['outputKustomizeDir'] = \ + os.path.join(output_dir, 'FluxCD', 'plugins', updated_app_name, 'kustomize') + self._app['outputLifecycleDir'] = \ + os.path.join(output_dir, 'FluxCD', 'plugins', updated_app_name, 'lifecycle') + + # 0 - Print out helm version. + self._print_helm_version() # 1 - Validate input file and helm chart data self.check_charts() @@ -983,7 +1136,7 @@ class Application(): return ret # 5 - Generate application metadata - ret = self._gen_metadata('flux', self._app['outputFluxCDDir']) + ret = self._gen_metadata('fluxcd', self._app['outputFluxCDDir']) if ret: print('FluxCD Metadata generated!') else: @@ -996,10 +1149,10 @@ class Application(): for chart in self._chart: ret = self._gen_helm_chart_tarball(chart, self._app['outputFluxChartDir']) if ret: - print('Helm chart %s tarball generated!' % chart['name']) + print(f'Helm chart {chart["name"]} tarball generated!') print('') else: - print('Generating tarball for helm chart: %s error!' % chart['name']) + print(f'Generating tarball for helm chart: {chart["name"]} error!') return ret # 7 - Package plugins in wheel format @@ -1015,15 +1168,15 @@ class Application(): ret = self._gen_checksum_and_app_tarball(self._app['outputFluxCDDir']) if ret: print('Checksum generated!') - print('FluxCD App tarball generated at %s/%s' % (self._app['outputFluxCDDir'], ret)) + print(f"FluxCD App tarball generated at {self._app['outputFluxCDDir']}/{ret}") print('') else: print('Checksum and App tarball generation failed!') return ret - # For debug def print_app_data(self): + """Debug Print App Data.""" print(self._app) print(self._manifest) print(self._chartgroup) diff --git a/app_gen_tool/cmd/generator.py b/app_gen_tool/cmd/generator.py index 9c69e7e..403edc3 100644 --- a/app_gen_tool/cmd/generator.py +++ b/app_gen_tool/cmd/generator.py @@ -1,11 +1,14 @@ -#!/usr/bin/env python +"""Entry point for app generator CLI.""" import sys from app_gen_tool.generator import main as gen_main + def main(): + """Main Method for CLI App Generator Tool.""" gen_main(sys.argv[1:]) + # Entry point here kept to allow for debug/testing. if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/app_gen_tool/common.py b/app_gen_tool/common.py index 8094bf0..c303bc1 100644 --- a/app_gen_tool/common.py +++ b/app_gen_tool/common.py @@ -1,2 +1,6 @@ -def to_camel_case(s): - return s[0].lower() + s.title().replace('_','')[1:] if s else s +"""Module for common methods accross entire codebase.""" + + +def to_camel_case(s: str) -> str: + """Convert String to camel case.""" + return s[0].lower() + s.title().replace('_', '')[1:] if s else s diff --git a/app_gen_tool/constants.py b/app_gen_tool/constants.py index 06e093c..c4fdab8 100644 --- a/app_gen_tool/constants.py +++ b/app_gen_tool/constants.py @@ -1,21 +1,23 @@ -## Variables for armada packaging +"""Declare Constants for Repository Here.""" +# Variables for armada packaging import getpass import os -ARMADA_CHART_TEMPLATE = 'template_armada/armada-chart.template' -ARMADA_CHARTGROUP_TEMPLATE = 'template_armada/armada-chartgroup.template' -ARMADA_MANIFEST_TEMPLATE = 'template_armada/armada-manifest.template' -BIN_FETCH_CHART_INFO = 'scripts/fetch_chart_info.sh' +# Variables for Aramada packaging +ARMADA_CHART_TEMPLATE = os.path.join('templates_armada', 'armada-chart.template') +ARMADA_CHARTGROUP_TEMPLATE = os.path.join('templates_armada', 'armada-chartgroup.template') +ARMADA_MANIFEST_TEMPLATE = os.path.join('templates_armada', 'armada-manifest.template') -## Variables for FluxCD packaging -FLUX_KUSTOMIZATION_TEMPLATE = 'templates_flux/kustomization.template' -FLUX_BASE_TEMPLATES = 'templates_flux/base/' -FLUX_MANIFEST_TEMPLATE = 'templates_flux/fluxcd-manifest' -FLUX_COMMON_TEMPLATE = 'templates_plugins/common.template' -FLUX_HELM_TEMPLATE = 'templates_plugins/helm.template' -FLUX_KUSTOMIZE_TEMPLATE = 'templates_plugins/kustomize.template' -FLUX_LIFECYCLE_TEMPLATE = 'templates_plugins/lifecycle.template' +# Variables for FluxCD packaging +FLUXCD_KUSTOMIZATION_TEMPLATE = os.path.join('templates_flux', 'kustomization.template') +FLUXCD_BASE_TEMPLATES = os.path.join('templates_flux', 'base') +FLUXCD_MANIFEST_TEMPLATE = os.path.join('templates_flux', 'fluxcd-manifest') +FLUXCD_COMMON_TEMPLATE = os.path.join('templates_plugins', 'common.template') +FLUXCD_HELM_TEMPLATE = os.path.join('templates_plugins', 'helm.template') +FLUXCD_KUSTOMIZE_TEMPLATE = os.path.join('templates_plugins', 'kustomize.template') +FLUXCD_LIFECYCLE_TEMPLATE = os.path.join('templates_plugins', 'lifecycle.template') - -TEMP_USER_DIR = '/tmp/' + getpass.getuser() + '/' +# Other variables +BIN_FETCH_CHART_INFO = os.path.join('scripts', 'fetch_chart_info.sh') +TEMP_USER_DIR = os.path.join('tmp', getpass.getuser()) APP_GEN_PY_PATH = os.path.split(os.path.realpath(__file__))[0] diff --git a/app_gen_tool/generator.py b/app_gen_tool/generator.py index 5802adc..6a87012 100644 --- a/app_gen_tool/generator.py +++ b/app_gen_tool/generator.py @@ -1,25 +1,29 @@ +"""Generator Main Module.""" import getopt import os -import sys import re import shutil +import sys import yaml from app_gen_tool.application import Application + def parse_yaml(yaml_in): - yaml_data='' + """Pare generator input yaml file.""" + yaml_data = '' try: - with open(yaml_in) as f: + with open(yaml_in, 'r', encoding='utf-8') as f: yaml_data = yaml.safe_load(f) except FileNotFoundError: - print('Error: %s no found' % yaml_in ) - except Exception as e: + print('Error: {yaml_in} not found') + except Exception: print('Error: Invalid yaml file') return yaml_data -def check_manifest(manifest_data): +def check_manifest(manifest_data): # pylint: disable=too-many-return-statements + """Check generator input yaml file for correct inputs.""" # TODO: check more mandatory key/values in manifest yaml # check app values @@ -35,15 +39,6 @@ def check_manifest(manifest_data): print('Error: \'appVersion\' is missing.') return False - # # check manifest values - # if 'manifest' not in manifest_data['appManifestFile-config']: - # print('Error: \'manifest\'is missing.') - # return False - - # if 'releasePrefix' not in manifest_data['manifest']: - # print('Error: Manifest attribute \'releasePrefix\' is missing.') - # return False - # check chartGroup values if 'chartGroup' not in manifest_data['appManifestFile-config']: print('Error: \'chartGroup\' is missing.') @@ -67,45 +62,63 @@ def check_manifest(manifest_data): # check chart path, supporting: dir, git, tarball if 'path' not in chart: - print('Error: Chart attribute \'path\' is missing in chart %s.' % chart['name']) + print(f'Error: Chart attribute \'path\' is missing in chart {chart["name"]}.') return False else: # TODO: To support branches/tags in git repo if chart['path'].endswith('.git'): if 'subpath' not in chart: - print('Error: Chart attribute \'subpath\' is missing in chart %s.' % chart['name']) + print( + 'Error: Chart attribute \'subpath\' is missing in ' + f'chart {chart["name"]}.' + ) return False chart['_pathType'] = 'git' - gitname = re.search('[^/]+(?=\.git$)',chart['path']).group() + gitname = re.search(r'[^/]+(?=\.git$)', chart['path']).group() if gitname: chart['_gitname'] = gitname else: - print('Error: Invalid \'path\' in chart %s.' % chart['name']) - print(' only \'local dir\', \'.git\', \'.tar.gz\', \'.tgz\' are supported') + print(f'Error: Invalid \'path\' in chart {chart["name"]}.') + print( + ' only \'local dir\', \'.git\', \'.tar.gz\', \'.tgz\' are supported' + ) return False elif chart['path'].endswith('.tar.gz') or chart['path'].endswith('.tgz'): if 'subpath' not in chart: - print('Error: Chart attribute \'subpath\' is missing in chart %s.' % chart['name']) + print( + 'Error: Chart attribute \'subpath\' is missing in ' + f'chart {chart["name"]}.' + ) return False chart['_pathType'] = 'tarball' - tarname = re.search('[^/]+(?=\.tgz)|[^/]+(?=\.tar\.gz)',chart['path']).group() + tarname = \ + re.search( + r'[^/]+(?=\.tgz)|[^/]+(?=\.tar\.gz)', + chart['path'] + ).group() if tarname: chart['_tarname'] = tarname else: - print('Error: Invalid \'path\' in chart %s.' % chart['name']) - print(' only \'local dir\', \'.git\', \'.tar.gz\', \'.tgz\' are supported') + print(f'Error: Invalid \'path\' in chart {chart["name"]}.') + print( + ' only \'local dir\', \'.git\', \'.tar.gz\', \'.tgz\' are supported' + ) return False else: if not os.path.isdir(chart['path']): - print('Error: Invalid \'path\' in chart %s.' % chart['name']) - print(' only \'local dir\', \'.git\', \'.tar.gz\', \'.tgz\' are supported') + print(f'Error: Invalid \'path\' in chart {chart["name"]}.') + print( + ' only \'local dir\', \'.git\', \'.tar.gz\', \'.tgz\' are supported' + ) return False chart['_pathType'] = 'dir' - return True + return True + def print_help(): + """Print CLI Helm Menu.""" print('StarlingX User Application Generator') print('') print('Usage:') @@ -124,7 +137,7 @@ def print_help(): def check_input_file(app_data) -> bool: - """ Checks if input file passed by user is valid""" + """Check if input file passed by user is valid.""" if not app_data: print('Parse yaml error') return False @@ -135,8 +148,7 @@ def check_input_file(app_data) -> bool: def check_app_directory(app_out, overwrite, package_only) -> bool: - """ Checks if the user gave enough information to modify or create - app folder directory.""" + """Checks if the user gave enough information to modify or create app folder directory.""" if os.path.exists(app_out) and not overwrite and not package_only: print(f'Output folder {app_out} exists, please remove it, use ' @@ -157,13 +169,15 @@ def create_app_directories(app_out, overwrite): def generate_app(app, package_type, no_package, package_only): - if package_type == 'armada' or package_type == 'both': + """Generate app based on application packaging type.""" + if package_type == 'armada' or package_type == 'both': # pylint: disable=consider-using-in app.gen_armada_app(app.output_folder, no_package, package_only) - if package_type == 'fluxcd' or package_type == 'both': - app.gen_flux_app(app.output_folder, no_package, package_only) + if package_type == 'fluxcd' or package_type == 'both': # pylint: disable=consider-using-in + app.gen_fluxcd_app(app.output_folder, no_package, package_only) def main(argv): + """Main Method with argument parsing.""" input_file = '' output_folder = '.' package_type = '' @@ -171,8 +185,20 @@ def main(argv): package_only = False no_package = False try: - options, args = getopt.getopt(argv, 'hi:o:t:', \ - ['help', 'input=', 'output=', 'type=', 'overwrite', 'no-package', 'package-only']) + options, _ = \ + getopt.getopt( + argv, + 'hi:o:t:', + [ + 'help', + 'input=', + 'output=', + 'type=', + 'overwrite', + 'no-package', + 'package-only' + ] + ) except getopt.GetoptError: print('Error: Invalid argument') sys.exit(1) @@ -191,7 +217,7 @@ def main(argv): no_package = True if option in ('--package-only'): package_only = True - + if overwrite and package_only: print('Error: Selecting both overwrite and package-only is not allowed' '. Please consult our README if further clarification is needed') @@ -207,11 +233,13 @@ def main(argv): if not check_input_file(app_data): sys.exit(1) - app = Application(app_data, package_type) - app.output_folder = os.path.abspath(output_folder) + '/' + \ - app.get_app_name() + output_folder = os.path.abspath(output_folder) + + app = Application(app_data, package_type, output_folder) if not check_app_directory(app.output_folder, overwrite, package_only): sys.exit(1) + create_app_directories(app.output_folder, overwrite) + generate_app(app, package_type, no_package, package_only) diff --git a/bandit.yaml b/bandit.yaml new file mode 100644 index 0000000..6e43d9f --- /dev/null +++ b/bandit.yaml @@ -0,0 +1 @@ +skips: [B603] \ No newline at end of file diff --git a/playbooks/app-gen-tool-tox-coverage/pre.yaml b/playbooks/app-gen-tool-tox-coverage/pre.yaml new file mode 100644 index 0000000..4c80c2e --- /dev/null +++ b/playbooks/app-gen-tool-tox-coverage/pre.yaml @@ -0,0 +1,4 @@ +- hosts: all + roles: + - role: ensure-helm + helm_version: "3.12.2" \ No newline at end of file diff --git a/test-requirements.txt b/test-requirements.txt index f278010..a5b872e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,4 +2,8 @@ hacking>=1.1.0,<=2.0.0 # Apache-2.0 bashate >= 0.2 PyYAML >= 3.1.0 shellcheck-py;python_version>="3.0" # MIT -yamllint<1.26.1;python_version>="3.0" # GPLv2 \ No newline at end of file +yamllint<1.26.1;python_version>="3.0" # GPLv2 +pylint +bandit +pytest +coverage \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/resources/adminer-0.2.1.tgz b/tests/unit/resources/adminer-0.2.1.tgz new file mode 100644 index 0000000..02bffe0 Binary files /dev/null and b/tests/unit/resources/adminer-0.2.1.tgz differ diff --git a/tests/unit/resources/app-test.yaml b/tests/unit/resources/app-test.yaml new file mode 100644 index 0000000..b68fa87 --- /dev/null +++ b/tests/unit/resources/app-test.yaml @@ -0,0 +1,46 @@ +--- +## App Manifest Configuration +appManifestFile-config: + appName: app-adminer + appVersion: 1.0-1 + namespace: default + chart: + - name: adminer + version: 0.2.1 + path: ./tests/unit/resources/adminer/adminer + chartGroup: + - name: adminer + chart_names: + - adminer +################################################# +## App Metadata Configuration +# for further details about possible configurations on this file, please +# visit the link: https://wiki.openstack.org/wiki/StarlingX/Containers/StarlingXAppsInternals#metadata.yaml +metadataFile-config: + # the following configurations are optional + # uncomment and configure properly the ones you need for your application metadata + upgrades: + auto_update: true + + supported_k8s_version: + minimum: 1.21.8 + maximum: 1.26.1 + + k8s_upgrades: + auto_update: true + timing: pre + + maintain_user_overrides: true + + +################################################# +## App Setup Configuration +# if you wish to see a setup.cfg example, please see the link +# https://opendev.org/starlingx/app-dell-storage/src/branch/master/python3-k8sapp-dell-storage/k8sapp_dell_storage/setup.cfg +setupFile-config: + metadata: + author: StarlingX + author_email: starlingx-discuss@lists.starlingx.io + url: https://www.starlingx.io/ + classifier: + - "Environment :: OpenStack" \ No newline at end of file diff --git a/tests/unit/test_application_class.py b/tests/unit/test_application_class.py new file mode 100644 index 0000000..6774c09 --- /dev/null +++ b/tests/unit/test_application_class.py @@ -0,0 +1,64 @@ +import os +import shutil +import tarfile + +from app_gen_tool.application import Application +from app_gen_tool.generator import parse_yaml, create_app_directories, generate_app, check_input_file + +def _extract(tar_url, extract_path='.'): + tar = tarfile.open(tar_url, 'r') + for item in tar: + tar.extract(item, extract_path) + if item.name.find(".tgz") != -1 or item.name.find(".tar") != -1: + _extract(item.name, "./" + item.name[:item.name.rfind('/')]) + +class TestCameCase: + + def setup_class(cls): + + current_file = os.path.abspath(__file__) + current_folder = os.path.dirname(current_file) + resource_folder = os.path.join(current_folder, 'resources') + input_file = os.path.join(resource_folder, 'app-test.yaml') + test_helm_chart = os.path.join(resource_folder, 'adminer-0.2.1.tgz') + + cls.OUTPUT_FOLDER = os.path.join(current_folder, "TEST_OUTPUT") + cls.helm_chart_folder = os.path.join(resource_folder, 'adminer') + cls.app_data = parse_yaml(input_file) + + if not os.path.exists(cls.OUTPUT_FOLDER): + os.makedirs(cls.OUTPUT_FOLDER, exist_ok=True) + + if not os.path.exists(cls.helm_chart_folder): + _extract(test_helm_chart, cls.helm_chart_folder) + + def teardown_class(cls): + + if os.path.exists(cls.OUTPUT_FOLDER): + shutil.rmtree(cls.OUTPUT_FOLDER) + + if os.path.exists(cls.helm_chart_folder): + shutil.rmtree(cls.helm_chart_folder) + + def test_generate_fluxcd_app(self): + + package_type = "fluxcd" + overwrite = True + package_only = False + no_package = False + + if not check_input_file(self.app_data): + assert False + + app = Application(self.app_data, package_type, self.OUTPUT_FOLDER) + + create_app_directories(app.output_folder, overwrite) + + generate_app(app, package_type, no_package, package_only) + + output_file = \ + os.path.join(self.OUTPUT_FOLDER, 'app-adminer', "FluxCD", 'app-adminer-1.0-1.tgz') + + file_exists = os.path.exists(output_file) + + assert file_exists == True \ No newline at end of file diff --git a/tests/unit/test_camel_case.py b/tests/unit/test_camel_case.py new file mode 100644 index 0000000..9375309 --- /dev/null +++ b/tests/unit/test_camel_case.py @@ -0,0 +1,18 @@ +from app_gen_tool.common import to_camel_case + + +class TestCameCase: + + def setup_class(cls): + pass + + def teardown_class(cls): + pass + + def test_camel_case(self): + + input_string = "ThisIsAString" + output_string = to_camel_case(input_string) + + assert output_string == input_string.lower() + diff --git a/tox.ini b/tox.ini index d0e6a1d..73bb12b 100644 --- a/tox.ini +++ b/tox.ini @@ -151,7 +151,7 @@ usedevelop = False skip_install = True deps = -r{toxinidir}/test-requirements.txt commands = - flake8 + flake8 ./app_gen_tool [testenv:venv] basepython = python3 @@ -159,12 +159,44 @@ commands = {posargs} [testenv:flake8] basepython = python3 -description = Dummy environment to allow flake8 to be run in subdir tox +description = Run Flake8 Linter +commands = + flake8 ./app_gen_tool [testenv:pylint] basepython = python3 -description = Dummy environment to allow pylint to be run in subdir tox +description = Run pylint linter +commands = + pylint ./app_gen_tool [testenv:bandit] basepython = python3 -description = Dummy environment to allow bandit to be run in subdir tox +description = Run bandit check +commands = + bandit -c ./bandit.yaml -r ./app_gen_tool + +[testenv:pytest] +basepython = python3 +description = Run bandit check +commands = + pytest tests/ + + +[testenv:coverage-run] +basepython = python3 +description = Run pytest with code coverage check +commands = + coverage run -m pytest -s tests/ + +[testenv:coverage-report] +basepython = python3 +description = Run pytest with code coverage report +commands = + coverage report --fail-under=55 + +[testenv:coverage] +basepython = python3 +description = Run coverage report +commands = + {[testenv:coverage-run]commands} + {[testenv:coverage-report]commands} \ No newline at end of file