From 421ced5174ddf9b3536a98ad0c17af8fb5f57fe6 Mon Sep 17 00:00:00 2001 From: Daniel Caires Date: Tue, 3 Oct 2023 14:33:58 -0300 Subject: [PATCH] Update app generator tool Update app-gen.py so the user can choose between Armada or FluxCD packaging, or both. - Create templates for FluxCD manifest - Create templates for Flux plugins - Update app-gen.py so it also supports FluxCD - Create template for the input file Story: 2010937 Task: 48913 Change-Id: I2521c3509556910841fd116f5a106e6577aad58a Signed-off-by: Daniel Caires --- README.md | 388 +++++++- app-gen.py | 906 +++++++++++++++--- app_manifest.yaml | 85 ++ .../armada-chart.template | 0 .../armada-chartgroup.template | 3 +- .../armada-manifest.template | 3 +- templates_flux/base/helmrepository.template | 7 + templates_flux/base/kustomization.template | 3 + templates_flux/base/namespace.template | 4 + .../fluxcd-manifest/helmrelease.template | 30 + .../fluxcd-manifest/kustomization.template | 12 + templates_flux/kustomization.template | 6 + templates_plugins/common.template | 7 + templates_plugins/helm.template | 30 + templates_plugins/kustomize.template | 18 + templates_plugins/lifecycle.template | 23 + 16 files changed, 1339 insertions(+), 186 deletions(-) create mode 100644 app_manifest.yaml rename {template => template_armada}/armada-chart.template (100%) rename {template => template_armada}/armada-chartgroup.template (84%) rename {template => template_armada}/armada-manifest.template (78%) create mode 100644 templates_flux/base/helmrepository.template create mode 100644 templates_flux/base/kustomization.template create mode 100644 templates_flux/base/namespace.template create mode 100644 templates_flux/fluxcd-manifest/helmrelease.template create mode 100644 templates_flux/fluxcd-manifest/kustomization.template create mode 100644 templates_flux/kustomization.template create mode 100644 templates_plugins/common.template create mode 100644 templates_plugins/helm.template create mode 100644 templates_plugins/kustomize.template create mode 100644 templates_plugins/lifecycle.template diff --git a/README.md b/README.md index 3766c45..f635a0f 100644 --- a/README.md +++ b/README.md @@ -1,60 +1,368 @@ # StarlingX Application Generation Tool -The purpose of this tool is to generate StarlingX user applications in an easy -way without stx build environment and armada manifest schema knowledge. +The purpose of this tool is to generate a StarlingX App from a workload/app +in an easy way without the complete StarlingX build environment. -## Pre-requisite +Below you will find the steps to deploy an application as a **StarlingX App**. -1. Helm2 installed -2. python3.5+ -3. pyyaml>=5.0.0 package +- [StarlingX Application Generation Tool](#starlingx-application-generation-tool) + - [Why deploy an application as a StarlingX application?](#why-deploy-an-application-as-a-starlingx-application) + - [Tools requirements](#tools-requirements) + - [Prerequisites](#prerequisites) + - [Generate the StarlingX Application package](#generate-the-starlingx-application-package) + - [App manifest configuration](#app-manifest-configuration) + - [Metadata File Configuration](#metadata-file-configuration) + - [App Setup configuration](#app-setup-configuration) + - [Run the StarlingX App Generator](#run-the-starlingx-app-generator) + - [Flux Packaging](#flux-packaging) + - [FluxCD Manifest](#fluxcd-manifest) + - [Plugins](#plugins) + - [Metadata](#metadata) + - [Tarballs](#tarballs) + - [Armada Packaging](#armada-packaging) + - [Armada Manifest](#armada-manifest) + - [Metadata](#metadata-1) + - [Tarballs](#tarballs-1) + - [Customizing the application](#customizing-the-application) + - [Flux Manifest](#flux-manifest) + - [FluxCD](#fluxcd) + - [Plugins](#plugins-1) + - [Other files](#other-files) + - [Armada Manifest](#armada-manifest-1) -`$ pip3 install pyyaml==5.1.2` +## Why deploy an application as a StarlingX application? -## 3 Steps to create a starlingx user app +It's important to understand that any user workload can be deployed in many +ways to the Kubernetes cluster(s) that StarlingX manages: -#### 1. Prepare a helm chart(s) +- with the most common Kubernetes package manager, [Helm](https://helm.sh/); +- with [Flux](https://fluxcd.io/), to enjoy all the benefits that come with it +; and finally +- as a StarlingX Application, which benefits from tight integration with the + [StarlingX system](https://opendev.org/starlingx/config). -##### What is helm and helm chart? + ```TODO Elaborate on the vantages of deploying an app as a StarlingX app``` -Helm is a Kubernetes package and operations manager. A Helm chart can contain -any number of Kubernetes objects, all of which are deployed as part of the -chart. +## Tools requirements -A list of official Helm Charts locates [here](https://github.com/helm/charts) +- Helm version 2+ +- Python version 3.8+ +- `pyyaml` version 6.0+ -##### How to develop a helm chart? +## Prerequisites -Refer to official [helm doc](https://helm.sh/docs/) +As the StarlingX Platform manages a distributed Kubernetes cluster, for an +application to be deployed as a StarlingX App it needs to be designed so it can +run on [Kubernetes](https://kubernetes.io/). -#### 2. Create an app manifest +Additionally, it needs to provide a [Helm Chart](https://helm.sh/) +which will be managed via [Flux](https://fluxcd.io/) by StarlingX itself. -A few essential fields needed to create the app, simplest one could be: +## Generate the StarlingX Application package + +Clone the app-generator repository. +```shell +git clone https://opendev.org/starlingx/tools.git +cd tools/app-gen-tool/ +``` + +This is what you'll find in the `app-gen-tool` folder of the repository: + +```shell +. +├── app-gen.py +├── app_manifest.yaml +├── bin +│   └── fetch_chart_info.sh +├── README.md +├── template +│   ├── armada-chartgroup.template +│   ├── armada-chart.template +│   └── armada-manifest.template +├── templates_flux +│   ├── base +│   │   ├── helmrepository.template +│   │   ├── kustomization.template +│   │   └── namespace.template +│   ├── fluxcd-manifest +│   │   ├── helmrelease.template +│   │   └── kustomization.template +│   └── kustomization.template +└── templates_plugins + ├── common.template + ├── helm.template + ├── kustomize.template + └── lifecycle.template ``` -appName: stx-app -namespace: stx-app -version: 1.0-1 -chart: -- name: chart1 - path: /path/to/chart1 -chartGroup: -- name: chartgroup1 - description: "This is the first chartgroup" - sequenced: true - chart_group: - - chart1 -manifest: - name: stx-app-manifest - releasePrefix: myprefix - chart_groups: - - chartgroup1 + +The `app_manifest.yaml` is the most important configuration step since it +specifies everything necessary to build the StarlingX application. + +The app manifest configuration can be divided into three parts, which will +have their own dedicated section below: + +- [App manifest configuration](#app-manifest-configuration) +- [Metadata file configuration](#metadata-file-configuration) +- [App Setup Configuration](#app-setup-configuration) + +### App manifest configuration + +In this stage the section **appManifestFile-config** from the +`app_manifest.yaml` will be configured. + +Below you will find a brief explanation of every one of the required fields +which will help you fill them out for you application: + +- **appName** field: desired StarlingX application name, referenced throughout + the whole system. +- **appVersion** field: the version of the application that the generated + package will represent. +- **namespace** field: desired StarlingX application namespace (note that this + namespace is not the same as the Kubernetes namespace). +- **chart** + - **name** field: your Helm chart name as it is in the chart metadata. + - **version** field: your chart version as it is in the chart metadata. + - **path** field: relative path to the Helm chart directory, Helm repo or + Helm package file. + > _NOTE_: Currently only Helm charts in directories have been tested. + - **chartGroup** field: chartgroup in which the helm-chart belong +- **chartGroup** + - **name**: Name of the chartgroup + - **description**: description of chart set + - **sequenced**: enables sequenced chart deployment in a group + - **chart_names**: a list of the names of the charts from your application. +- **manifest**: + - **name**: manifest name + - **releasePrefix**: appends to the front of all charts released by the manifest in order to manage releases throughout their lifecycle + +Note that the minimum required fields that will need to be filled in order +for the StarlingX App Generator to work properly will depend on the intended +type of packaging. + +>_NOTE_: The two other sections bellow ([Metadata file configuration](#metadata-file-configuration) +and [App Setup Configuration](#app-setup-configuration)) will only be necessary +if you intend to package your application utilizing FluxCD. +### Metadata File Configuration + +In this stage the section **metadataFile-config** from the +`app_manifest.yaml` will be configured. + +This section's objective is to configure the generation/creation of a +`metadata.yaml` file, which is the core metadata file for a StarlingX App +package. + +This `metadata.yaml` file is very flexible, hence the **metadataFile-config** +section is also very flexible. Other values may be passed in order to enable +advanced features within the StarlingX platform. For a better understanding of +each attribute in this section please refer to +[this link](https://wiki.openstack.org/wiki/StarlingX/Containers/StarlingXAppsInternals#metadata.yaml). + +### App Setup configuration + +In this stage the section **setupFile-config** from the `app_manifest.yaml` +will be configured. + +Below you will find a brief explanation of every one of the required fields +which will help you fill them out for you application: + +- **metadata** section: + - **author/author-email/url fields**: authorship information. + - **classifier** section: an array of additional information. + +This section is related to the `setup-cfg` file that will be generated. For +more advanced use cases you may want to refer to [the documentation](https://setuptools.pypa.io/en/latest/userguide/declarative_config.html). + +## Run the StarlingX App Generator + +```shell +python3 app-gen.py -i app_manifest.yaml -t ``` -For more details, please refer to example.yaml -#### 3. Run app-gen.py +With the command above, the StarlingX App Generator will create a set of files +and package everything in the chosed StarlingX format. -`$ python3 app-gen.py -i app_manifest.yaml [-o ./output] [--overwrite]` +The following sections explain in high-level the most important parts of the +package. -The application will be generated automatically along with the tarball located -in the folder of your application name. +### Flux Packaging +#### FluxCD Manifest + +The generator will first create the FluxCD Manifest following the structure below: + +```shell +fluxcd-manifests/ +├── base +│ ├── helmrepository.yaml +│ ├── kustomization.yaml +│ └── namespace.yaml +├── kustomization.yaml +└── CHART-NAME + ├── helmrelease.yaml + ├── kustomization.yaml + ├── CHART-NAME-static-overrides.yaml + └── CHART-NAME-system-overrides.yaml +``` + +For every Helm chart configured in the `app_manifest.yaml` file, a folder with +the name of the chart will be created. + +> **_NOTE_**: The `CHART-NAME-static-overrides.yaml` file will be empty. + +#### Plugins + +After the creation of the FluxCD Manifest, the generator will also create a set +of plugins with an empty implementation. + +The Structure of the plugins created will be: + +```shell +plugins/ +├── k8sapp_APP_NAME +│ ├── common +│ │ ├── __init__.py +│ │ └── constants.py +│ ├── helm +│ │ ├── __init__.py +│ │ └── CHART_NAME.py +│ ├── kustomize +│ │ ├── __init__.py +│ │ └── kustomize_APP_NAME.py +│ └── lifecycle +│ ├── __init__.py +│ └── lifecycle_APP_NAME.py +├── __init__.py +├── setup.cfg +└── setup.py +``` + +The `setup.cfg` file will be created according to the +[`setupFile-config`](#app-setup-configuration) section in the `app_manifest.yaml`. + +#### Metadata + +In the third step of the execution the `metadata.yaml` file will be generated +with the information given in [`metadataFile-config`](#metadata-file-configuration) +section in the `app_manifest.yaml`. + +#### Tarballs + +After the main files have been created, the generator will start packaging +everything. + +Firstly it will package every helm-chart, that was given in the +`app_manifest.yaml` file, into a `.tgz` file, saving these files into a folder +named `charts`. +>_NOTE_: For the Armada packaging, the creation of the helm-charts tarball will +be before the creation of the manifest. + +The generator, then, will package the plugins with the [wheel](https://peps.python.org/pep-0491/) +format. + +Lastly, creates a checksum sha256 signature file for the output tarball and +the output tarball itself, which will be called + +``` +-.tgz +``` + +The structure of the app inside the tarball will be the following: + +```shell + APPNAME-APPVERSION.tgz/ + ├── charts/ + ├── fluxcd-manifests/ + ├── plugins/ + ├── checksum.sha256 + └── metadata.yaml +``` + +> **Warning:** +> At this point, the generated package is a working StarlingX App, however it +> contains empty templates for some files. The following sections will describe +> how to further enhance your StarlingX App. +### Armada Packaging + +#### Armada Manifest + +TODO: Check about adding this link +https://opendev.org/airship/armada/src/commit/2b714888c490a9f7c5a11383eb18b7226d1b1dc8/docs/source/operations/guide-build-armada-yaml.rst +#### Metadata + +#TODO +#### Tarballs + +#TODO +## Customizing the application + +If you wish to add customization for the particularities of your application, +it is important to modify some of the generated files. + +In order to allow such customization, the generator provides additional +functions to modify specific files in the package. + +```shell +python3 app-gen.py -i app_manifest.yaml -t [-o ./output] [--overwrite] [--no-package]|[--package-only] +``` + +Where: + +- `-i/--input`: path to the `app_manifest.yaml` configuration file. +- `-t/--type`: type of packaging, needs to choose between armada, flux or both. +- `-o/--output`: output folder. Defaults to a new folder with the app name in + the current directory. +- `--overwrite`: deletes existing output folder before starting. +- `--no-package`: only creates the FluxCD manifest, plugins and the + metadata file, without compressing them in a tarball. +- `--package-only`: create the plugins wheels, sha256 file, helm-chart tarball + and package the entire application into a tarball. + +This means that, in order to be able to make additional configuration, one must: + +- first run the App Generator with `--no-package`; +- then do the changes (described in the following sections); +- finally, run the App Generator again with `--package-only`. + +### Flux Manifest + +#### FluxCD +> _NOTE_: this section needs improvement. + +Most of the generated manifest won't need any modification, but for every +Helm chart in the `app_manifest.yaml`, a static-overrides file will be created. +The static-overrides file contains all information that is not to be +overwritten inside the values.yaml of the Helm chart. + +#### Plugins + +The StarlingX App Generator will create 3 main plugins: the Helm, +the Kustomize and the Lifecycle plugins. + +- The `helm/APP_NAME.py` file is responsible for the overriding methods that will + be used to create the Helm overrides for the StarlingX App. + +- The `kustomize_APP_NAME.py` is a plugin that is used to make changes to the + top-level `kustomization` resource list based on the platform mode. + +- The `lifecycle_APP_NAME.py` is responsible for performing lifecycle actions on the + application using the lifecycle hooks of the StarlingX Platform. + +The files created by the generator will have an empty implementation and is up +to the developer to implement everything that is necessary for the application +to run as intended. + +The `sysinv` folder in the [StarlingX config repository](https://opendev.org/starlingx/config/src/branch/master/sysinv/sysinv/sysinv/sysinv) +contains a multitude of functions and variables that may be helpful in the +development of application plugins. + +#### Other files + +For the customization of the application the modifications above, in the FluxCD +and the plugins, should be enough for the application to run as expected in the +StarlingX platform. + +With that in mind, it is recommended to check if the `metadata` and the `setup.cfg` +have been created as they should. Particularly, the `setup.cfg` may need careful +attention if the modifications on the plugin file should be reflected in it. + +### Armada Manifest diff --git a/app-gen.py b/app-gen.py index a106526..d33125c 100644 --- a/app-gen.py +++ b/app-gen.py @@ -8,10 +8,22 @@ import re import shutil from urllib import request -SCHEMA_CHART_TEMPLATE = 'template/armada-chart.template' -SCHEMA_CHARTGROUP_TEMPLATE = 'template/armada-chartgroup.template' -SCHEMA_MANIFEST_TEMPLATE = 'template/armada-manifest.template' +## Variables for armada packaging +ARMADA_CHART_TEMPLATE = 'template_armada/armada-chart.template' +ARMADA_CHARTGROUP_TEMPLATE = 'template_armada/armada-chartgroup.template' +ARMADA_MANIFEST_TEMPLATE = 'template_armada/armada-manifest.template' BIN_FETCH_CHART_INFO = 'bin/fetch_chart_info.sh' + +## Variables for FluxCD packaging +FLUX_KUSTOMIZATION_TEMPLATE = 'templates_flux/kustomization.template' +FLUX_BASE_TEMPLATES = 'templates_flux/base/' +FLUX_MANIFEST_TEMPLATE = 'templates_flux/fluxcd-manifest' +FLUX_COMMON_TEMPLATE = 'templates_plugins/common.template' +FLUX_HELM_TEMPLATE = 'templates_plugins/helm.template' +FLUX_KUSTOMIZE_TEMPLATE = 'templates_plugins/kustomize.template' +FLUX_LIFECYCLE_TEMPLATE = 'templates_plugins/lifecycle.template' + + TEMP_USER_DIR = '/tmp/' + getpass.getuser() + '/' # Temp app work dir to hold git repo and upstream tarball # TEMP_APP_DIR = TEMP_USER_DIR/appName @@ -21,29 +33,48 @@ APP_GEN_PY_PATH = os.path.split(os.path.realpath(__file__))[0] def to_camel_case(s): return s[0].lower() + s.title().replace('_','')[1:] if s else s -class ArmadaApplication: +class Application: def __init__(self, app_data): # Initialize application config - self._armada_app = {} - # 'appName', 'namespace', 'version' are checked in check_manifest() - self._armada_app['appName'] = app_data['appName'] - self._armada_app['namespace'] = app_data['namespace'] - self._armada_app['version'] = app_data['version'] + self._app = {} + self._app = app_data['appManifestFile-config'] - # Initialize manifest - self._armada_manifest = app_data['manifest'] + self.APP_NAME = self._app['appName'] + self.APP_NAME_WITH_UNDERSCORE = self._app['appName'].replace('-', '_') + self.APP_NAME_CAMEL_CASE = self._app['appName'].replace('-', ' ').title().replace(' ', '') # Initialize chartgroup - self._armada_chartgroup = app_data['chartGroup'] + self._chartgroup = app_data['appManifestFile-config']['chartGroup'] + for i in range(len(self._chartgroup)): + self._chartgroup[i]['namespace'] = self._app['namespace'] + + # Create empty list that will contain all the charts + self._listcharts = {} + self._listcharts['chart_names'] = [] + self._listcharts['namespace'] = self._app['namespace'] # Initialize chart - self._armada_chart = app_data['chart'] - # add namespace and prefix to each chart - # 'namespace', 'releasePrefix' are checked in check_manifest() - for i in range(len(self._armada_chart)): - self._armada_chart[i]['namespace'] = self._armada_app['namespace'] - self._armada_chart[i]['releasePrefix'] = self._armada_manifest['releasePrefix'] + self._chart = app_data['appManifestFile-config']['chart'] + for i in range(len(self._chart)): + self._chart[i]['namespace'] = self._app['namespace'] + self._chart[i]['releasePrefix'] = self._app['manifest']['releasePrefix'] + self._listcharts['chart_names'].append(self._chart[i]['name']) + + # Initialize Armada manifest + self._manifest = app_data['appManifestFile-config']['manifest'] + self._manifest['chart_groups'] = [] + for i in range(len(self._chartgroup)): + self._manifest['chart_groups'].append(self._chartgroup[i]['name']) + + # Initialize setup data + self.plugin_setup = app_data['setupFile-config'] + + + # Initialize metadata + self.metadata = app_data['metadataFile-config'] + + # TODO: Validate values def _validate_app_values(self, app_data): @@ -62,21 +93,62 @@ class ArmadaApplication: return True def _validate_app_attributes(self): - if not self._validate_app_values(self._armada_app): + if not self._validate_app_values(self._app): return False - if not self._validate_manifest_values(self._armada_manifest): + if not self._validate_manifest_values(self._manifest): return False - if not self._validate_chartgroup_values(self._armada_chartgroup): + if not self._validate_chartgroup_values(self._chartgroup): return False - if not self._validate_chart_values(self._armada_chart): + if not self._validate_chart_values(self._chart): return False return True - def get_app_name(self): - return self._armada_app['appName'] - def _package_helm_chart(self, chart): + # Subprocess that check charts informations + def check_charts(self): + charts = self._chart + for chart in charts: + manifest_data = dict() + chart_file_data = dict() + manifest_data['name'], manifest_data['version'] = chart['name'], chart['version'] + if chart['_pathType'] == 'dir': + try: + chart_metadata_f = open(f'{chart["path"]}/Chart.yaml', 'r') + except Exception as e: + print(f'ERROR: {e}') + sys.exit(1) + chart_file_lines = chart_metadata_f.readlines() + chart_file_lines = [l for l in chart_file_lines if len(l) > 0 and l[0] != '#'] + chart_metadata_f.close() + for line in chart_file_lines: + line = line.rstrip('\n') + line_data = line.split() + if not line_data: + continue + if 'name:' in line_data[0]: + chart_file_data['name'] = line_data[-1] + elif 'version:' in line_data[0]: + chart_file_data['version'] = line_data[-1] + # To-do chart type different from dir + for key in manifest_data: + err_str = '' + if key not in chart_file_data: + err_str = f'"{key}" is present in app-manifest.yaml but not in {chart["path"]}/Chart.yaml' + raise KeyError(err_str) + if manifest_data[key] != chart_file_data[key]: + err_str = f'"{key}" has different values in app-manifest.yaml and {chart["path"]}/Chart.yaml' + raise ValueError(err_str) + + + def get_app_name(self): + return self._app['appName'] + + + # Sub-process of app generation + # generate application helm-charts tarball + # + def _package_helm_chart(self, chart, chart_dir): path = chart['path'] # lint helm chart cmd_lint = ['helm', 'lint', path] @@ -89,8 +161,8 @@ class ArmadaApplication: return False # package helm chart - cmd_package = ['helm', 'package', path, '--save=false', \ - '--destination=' + self._armada_app['outputChartDir']] + cmd_package = ['helm', 'package', path, \ + '--destination=' + chart_dir] subproc = subprocess.run(cmd_package, env=os.environ.copy(), \ stdout=subprocess.PIPE, stderr=subprocess.PIPE) if subproc.returncode == 0: @@ -105,6 +177,7 @@ class ArmadaApplication: return False return True + # pyyaml does not support writing yaml block with initial indent # add initial indent for yaml block substitution def _write_yaml_to_manifest(self, key, src, init_indent): @@ -112,14 +185,16 @@ class ArmadaApplication: # add heading key target[key] = src lines = yaml.safe_dump(target).split('\n') - # remove ending space + # remove ending space and first line lines.pop() + lines.pop(0) indents = ' ' * init_indent for i in range(len(lines)): lines[i] = indents + lines[i] # restore ending '\n' return '\n'.join(lines) + '\n' + def _substitute_values(self, in_line, dicts): out_line = in_line pattern = re.compile('\$.+?\$') @@ -143,6 +218,7 @@ class ArmadaApplication: else: return out_line, True + def _substitute_blocks(self, in_line, dicts): out_line = in_line result = re.search('@\S+\|\d+@',out_line) @@ -157,14 +233,15 @@ class ArmadaApplication: return out_line + # Fetch info from helm chart to fill # the values that needs to be substituted # Below info are needed: # - waitLabelKey # - chartArcname - # + def _fetch_info_from_chart(self, chart_idx): - a_chart = self._armada_chart[chart_idx] + a_chart = self._chart[chart_idx] bin_fetch_script = APP_GEN_PY_PATH + '/' + BIN_FETCH_CHART_INFO # waitLabelKey # search for the key of label which indicates '.Release.Name' @@ -195,16 +272,17 @@ class ArmadaApplication: return True + # Sub-process of app generation # lint and package helm chart # TODO: sub-chart dependency check # - def _gen_helm_chart_tarball(self, chart): + def _gen_helm_chart_tarball(self, chart, chart_dir): ret = False path = '' print('Processing chart %s...' % chart['name']) # check pathtype of the chart - if chart['_pathType'] is 'git': + if chart['_pathType'] == 'git': gitname = '' # download git if not os.path.exists(TEMP_APP_DIR): @@ -239,7 +317,7 @@ class ArmadaApplication: return False os.chdir(saved_pwd) path = TEMP_APP_DIR + chart['_gitname'] + '/' + chart['subpath'] - elif chart['_pathType'] is 'tarball': + elif chart['_pathType'] == 'tarball': if not os.path.exists(TEMP_APP_DIR): os.makedirs(TEMP_APP_DIR) try: @@ -267,42 +345,43 @@ class ArmadaApplication: print('Error: %s' % e) return False path = TEMP_APP_DIR + chart['_tarArcname'] + '/' + chart['subpath'] - elif chart['_pathType'] is 'dir': + elif chart['_pathType'] == 'dir': path = chart['path'] # update chart path # remove ending '/' chart['path'] = path.rstrip('/') # lint and package - ret = self._package_helm_chart(chart) + ret = self._package_helm_chart(chart, chart_dir) return ret + # Sub-process of app generation # generate application manifest file # def _gen_armada_manifest(self): # check manifest file existance - manifest_file = self._armada_app['outputDir'] + '/' + self._armada_app['appName'] + '.yaml' + manifest_file = self._app['outputArmadaDir'] + '/' + self._app['appName'] + '.yaml' if os.path.exists(manifest_file): os.remove(manifest_file) # update schema path to abspath - chart_template = APP_GEN_PY_PATH + '/' + SCHEMA_CHART_TEMPLATE - chartgroup_template = APP_GEN_PY_PATH + '/' + SCHEMA_CHARTGROUP_TEMPLATE - manifest_template = APP_GEN_PY_PATH + '/' + SCHEMA_MANIFEST_TEMPLATE + chart_template = APP_GEN_PY_PATH + '/' + ARMADA_CHART_TEMPLATE + chartgroup_template = APP_GEN_PY_PATH + '/' + ARMADA_CHARTGROUP_TEMPLATE + manifest_template = APP_GEN_PY_PATH + '/' + ARMADA_MANIFEST_TEMPLATE # generate chart schema try: with open(chart_template, 'r') as f: chart_schema = f.readlines() - except IOError: + except FileNotFoundError: print('File %s not found' % chart_template) return False with open(manifest_file, 'a') as f: # iterate each armada_chart - for idx in range(len(self._armada_chart)): - a_chart = self._armada_chart[idx] + for idx in range(len(self._chart)): + a_chart = self._chart[idx] # fetch chart specific info if not self._fetch_info_from_chart(idx): return False @@ -318,12 +397,12 @@ class ArmadaApplication: try: with open(chartgroup_template, 'r') as f: chartgroup_schema = f.readlines() - except IOError: + except FileNotFoundError: print('File %s not found' % chartgroup_template) return False with open(manifest_file, 'a') as f: # iterate each chartgroup - for chartgroup in self._armada_chartgroup: + for chartgroup in self._chartgroup: for line in chartgroup_schema: # substitute template values to chartgroup values out_line, substituted = self._substitute_values(line, chartgroup) @@ -336,12 +415,12 @@ class ArmadaApplication: try: with open(manifest_template, 'r') as f: manifest_schema = f.readlines() - except IOError: + except FileNotFoundError: print('File %s not found' % manifest_template) return False with open(manifest_file, 'a') as f: # only one manifest in an application - manifest = self._armada_manifest + manifest = self._manifest # substitute values for line in manifest_schema: # substitute template values to manifest values @@ -353,47 +432,414 @@ class ArmadaApplication: return True + # Sub-process of app generation - # generate application metadata + # generate application fluxcd manifest files # - def _gen_metadata(self): - # check metadata file existance - metadata_file = self._armada_app['outputDir'] + '/metadata.yaml' - if os.path.exists(metadata_file): - os.remove(metadata_file) - with open(metadata_file, 'a') as f: - f.write('app_name: ' + self._armada_app['appName'] + '\n') - f.write('app_version: ' + self._armada_app['version'] + '\n') + def _gen_fluxcd_manifest(self): + # check manifest file existance + flux_dir = self._app['outputManifestDir'] + + # update schema path to abspath + kustomization_template = APP_GEN_PY_PATH + '/' + FLUX_KUSTOMIZATION_TEMPLATE + + base_helmrepo_template = APP_GEN_PY_PATH + '/' + FLUX_BASE_TEMPLATES + '/helmrepository.template' + base_kustom_template = APP_GEN_PY_PATH + '/' + FLUX_BASE_TEMPLATES + '/kustomization.template' + base_namespace_template = APP_GEN_PY_PATH + '/' + FLUX_BASE_TEMPLATES + '/namespace.template' + + manifest_helmrelease_template = APP_GEN_PY_PATH + '/' + FLUX_MANIFEST_TEMPLATE + '/helmrelease.template' + manifest_kustomization_template = APP_GEN_PY_PATH + '/' + FLUX_MANIFEST_TEMPLATE + '/kustomization.template' + + manifest = self._app + chartgroup = self._listcharts + chart = self._chart + + # generate kustomization file + try: + with open(kustomization_template, 'r') as f: + kustomization_schema = f.readlines() + except FileNotFoundError: + print('File %s not found' % kustomization_template) + return False + kustom_file = flux_dir + 'kustomization.yaml' + with open(kustom_file, 'a') as f: + # substitute values + for line in kustomization_schema: + # substitute template values to manifest values + out_line, substituted = self._substitute_values(line, chartgroup) + if not substituted: + # substitute template blocks to manifest blocks + out_line = self._substitute_blocks(line, chartgroup) + f.write(out_line) + + # generate base/namespace file + try: + with open(base_namespace_template, 'r') as f: + base_namespace_schema = f.readlines() + except FileNotFoundError: + print('File %s not found' % base_namespace_template) + return False + base_namespace_file = flux_dir + 'base/namespace.yaml' + with open(base_namespace_file, 'a') as f: + # substitute values + for line in base_namespace_schema: + # substitute template values to manifest values + out_line, substituted = self._substitute_values(line, manifest) + if not substituted: + # substitute template blocks to manifest blocks + out_line = self._substitute_blocks(line, manifest) + f.write(out_line) + + # generate base/kustomization file + # generate base/helmrepository file + # Both yaml files don't need to add informations from the input file + try: + with open(base_kustom_template, 'r') as f: + base_kustom_schema = f.readlines() + except FileNotFoundError: + print('File %s not found' % base_kustom_template) + return False + base_kustom_file = flux_dir + 'base/kustomization.yaml' + with open(base_kustom_file, 'a') as f: + for line in base_kustom_schema: + out_line = line + f.write(out_line) + + try: + with open(base_helmrepo_template, 'r') as f: + base_helmrepo_schema = f.readlines() + except FileNotFoundError: + print('File %s not found' % base_helmrepo_template) + return False + base_helmrepo_file = flux_dir + 'base/helmrepository.yaml' + with open(base_helmrepo_file, 'a') as f: + for line in base_helmrepo_schema: + out_line = line + f.write(out_line) + + # iterate each fluxcd_chart for the generation of its fluxcd manifests + for idx in range(len(chart)): + a_chart = chart[idx] + + # generate manifest/helmrelease file + try: + with open(manifest_helmrelease_template, 'r') as f: + manifest_helmrelease_schema = f.readlines() + except FileNotFoundError: + print('File %s not found' % manifest_helmrelease_template) + return False + manifest_helmrelease_file = flux_dir + a_chart['name'] + '/helmrelease.yaml' + with open(manifest_helmrelease_file, 'a') as f: + # fetch chart specific info + for line in manifest_helmrelease_schema: + # substitute template values to chart values + out_line, substituted = self._substitute_values(line, a_chart) + if not substituted: + # substitute template blocks to chart blocks + out_line = self._substitute_blocks(line, a_chart) + f.write(out_line) + + # generate manifest/kustomizaion file + try: + with open(manifest_kustomization_template, 'r') as f: + manifest_kustomization_schema = f.readlines() + except FileNotFoundError: + print('File %s not found' % manifest_kustomization_template) + return False + manifest_kustomization_file = flux_dir + a_chart['name'] + '/kustomization.yaml' + with open(manifest_kustomization_file, 'a') as f: + # fetch chart specific info + for line in manifest_kustomization_schema: + # substitute template values to chart values + out_line, substituted = self._substitute_values(line, a_chart) + if not substituted: + # substitute template blocks to chart blocks + out_line = self._substitute_blocks(line, a_chart) + f.write(out_line) + + # generate an empty manifest/system-overrides file + system_override_file = flux_dir + '/' + a_chart['name'] + '/' + a_chart['name'] + '-system-overrides.yaml' + open(system_override_file, 'w').close() + + # generate a manifest/static-overrides file + static_override_file = flux_dir + '/' + a_chart['name'] + '/' + a_chart['name'] + '-static-overrides.yaml' + open(static_override_file, 'w').close() + return True - def _gen_md5(self, in_file): + # Sub-process of app generation + # generate application plugin files + # + def _gen_plugins(self): + + plugin_dir = self._app['outputPluginDir'] + + common_template = APP_GEN_PY_PATH + '/' + FLUX_COMMON_TEMPLATE + helm_template = APP_GEN_PY_PATH + '/' + FLUX_HELM_TEMPLATE + kustomize_template = APP_GEN_PY_PATH + '/' + FLUX_KUSTOMIZE_TEMPLATE + lifecycle_template = APP_GEN_PY_PATH + '/' + FLUX_LIFECYCLE_TEMPLATE + + appname = 'k8sapp_' + self.APP_NAME_WITH_UNDERSCORE + namespace = self._app['namespace'] + chart = self._chart + name = self._chart[0]['name'] + + # generate Common files + try: + with open(common_template, 'r') as f: + common_schema = f.read() + except FileNotFoundError: + print('File %s not found' % common_template) + return False + common_file = plugin_dir + '/' + appname + '/common/constants.py' + output = common_schema.format(appname=appname, name=name, namespace=namespace) + + with open(common_file, "w") as f: + f.write(output) + + self.create_init_file(self._app['outputCommonDir']) + + # Generate Helm files + try: + with open(helm_template, 'r') as f: + helm_schema = f.read() + except FileNotFoundError: + print('File %s not found' % helm_template) + return False + + for idx in range(len(chart)): + a_chart = chart[idx] + + helm_file = plugin_dir + '/' + appname + '/helm/' + a_chart['name'].replace(" ", "_").replace("-", "_") + '.py' + + name = a_chart['name'].replace('-', ' ').title().replace(' ','') + namespace = a_chart['namespace'] + + output = helm_schema.format(appname=appname, name=name) + + with open(helm_file, "w") as f: + f.write(output) + + self.create_init_file(self._app['outputHelmDir']) + + # Generate Kustomize files + try: + with open(kustomize_template, 'r') as f: + kustomize_schema = f.read() + except FileNotFoundError: + print('File %s not found' % kustomize_template) + return False + kustomize_file = plugin_dir + '/' + appname + '/kustomize/kustomize_' + self.APP_NAME_WITH_UNDERSCORE + '.py' + output = kustomize_schema.format(appname=appname, appnameStriped=self.APP_NAME_CAMEL_CASE) + + with open(kustomize_file, "w") as f: + f.write(output) + + self.create_init_file(self._app['outputKustomizeDir']) + + # Generate Lifecycle files + try: + with open(lifecycle_template, 'r') as f: + lifecycle_schema = f.read() + except FileNotFoundError: + print('File %s not found' % lifecycle_template) + return False + lifecycle_file = plugin_dir + '/' + appname + '/lifecycle/lifecycle_' + self.APP_NAME_WITH_UNDERSCORE + '.py' + output = lifecycle_schema.format(appnameStriped=self.APP_NAME_CAMEL_CASE) + + with open(lifecycle_file, "w") as f: + f.write(output) + + self.create_init_file(self._app['outputLifecycleDir']) + + # Generate setup.py + setupPy_file = plugin_dir + '/setup.py' + file = f"""import setuptools\n\nsetuptools.setup(\n setup_requires=['pbr>=2.0.0'],\n pbr=True)""" + + with open(setupPy_file, 'w') as f: + f.write(file) + + # Generate setup.cfg file + self.write_app_setup() + + + self.create_init_file(plugin_dir) + + dir = plugin_dir + '/' + appname + self.create_init_file(dir) + + return True + + + # Subprocess that creates __init__.py file + def create_init_file(self, path): + init_file = path + '/__init__.py' + open(init_file, 'w').close() + + + #Subprocess that writes the setup.cfg file + def write_app_setup(self): + def split_and_format_value(value) -> str: + if type(value) == str: + return ''.join([f'\t{lin}\n' for lin in value.split('\n')]) + else: + return ''.join([f'\t{lin}\n' for lin in value]) + + def expected_order(tup: tuple) -> int: + if tup[0] == 'name': + return 0 + elif tup[0] == 'summary': + return 1 + return 2 + + yml_data = self.plugin_setup + yml_data['metadata']['name'] = f'k8sapp-{self.APP_NAME}' + yml_data['metadata']['summary'] = f'StarlingX sysinv extensions for {self.APP_NAME}' + yml_data['metadata'] = dict(sorted(yml_data['metadata'].items(), key=expected_order)) + out = '' + for label in yml_data: + out += f'[{label}]\n' + for key, val in yml_data[label].items(): + if label == 'metadata' and val is None: + raise ValueError(f'You should\'ve written a value for: {key}') + elif type(val) != list: + out += f'{key} = {val}\n' + else: + out += f'{key} =\n' + out += split_and_format_value(val) + out += '\n' + charts_data = self._chart + plugins_names = [] + for dic in charts_data: + plugins_names.append(dic['name']) + out += f'[files]\npackages =\n\tk8sapp_{self.APP_NAME_WITH_UNDERSCORE}\n\n' + out += '[global]\nsetup-hooks =\n\tpbr.hooks.setup_hook\n\n' + out += '[entry_points]\nsystemconfig.helm_applications =\n\t' \ + f'{self.APP_NAME} = systemconfig.helm_plugins.{self.APP_NAME_WITH_UNDERSCORE}\n\n' \ + f'systemconfig.helm_plugins.{self.APP_NAME_WITH_UNDERSCORE} =\n' + for i, plug in enumerate(plugins_names): + out += f'\t{i+1:03d}_{plug} = k8sapp_{self.APP_NAME_WITH_UNDERSCORE}.helm.{plug.replace("-","_")}' + out += f':{plug.replace("-", " ").title().replace(" ", "")}Helm\n' + out += '\n' + out += 'systemconfig.fluxcd.kustomize_ops =\n' \ + f'\t{self.APP_NAME} = k8sapp_{self.APP_NAME_WITH_UNDERSCORE}.kustomize.kustomize_' \ + f'{self.APP_NAME_WITH_UNDERSCORE}:{self.APP_NAME_CAMEL_CASE}FluxCDKustomizeOperator\n\n' \ + 'systemconfig.app_lifecycle =\n' \ + f'\t{self.APP_NAME} = k8sapp_{self.APP_NAME_WITH_UNDERSCORE}.lifecycle.lifecycle_' \ + f'{self.APP_NAME_WITH_UNDERSCORE}:{self.APP_NAME_CAMEL_CASE}AppLifecycleOperator\n\n' + out += '[bdist_wheel]\nuniversal = 1' + with open(f'{self._app["outputPluginDir"]}/setup.cfg', 'w+') as f: + f.write(out) + + + # Sub-process of app generation + # generate application metadata + # + def _gen_metadata(self, package_type, output): + """ + gets the keys and values defined in the input yaml and writes the metadata.yaml app file. + """ + + yml_data = self.metadata + app_name, app_version = self._app['appName'], self._app['appVersion'] + file = output + '/metadata.yaml' + try: + with open(file, 'w') as f: + f.write(f'app_name: {app_name}\napp_version: {app_version}') + + if package_type == 'flux': + with open(file, 'a') as f: + f.write('\nhelm_repo: stx-platform\n') + if yml_data is not None: + yaml.safe_dump(yml_data, f) + except: + return False + + return True + + + # Sub-process of app generation + # generate application sha256 file + # + def _gen_sha256(self, in_file): with open(in_file, 'rb') as f: - out_md5 = hashlib.md5(f.read()).hexdigest() - return out_md5 + out_sha256 = hashlib.sha256(f.read()).hexdigest() + return out_sha256 + + + # Sub-process of app generation + # generate plugin wheels + # + def _gen_plugin_wheels(self): + dirplugins = self._app['outputPluginDir'] + + store_cwd = os.getcwd() + os.makedirs(dirplugins, exist_ok=True) + os.chdir(dirplugins) + + command = [ + "python3", + "setup.py", + "bdist_wheel", + "--universal", + "-d", + dirplugins] + + try: + subprocess.call(command, stderr=subprocess.STDOUT) + except: + return False + + files = [ + f'{dirplugins}/ChangeLog', + f'{dirplugins}/AUTHORS'] + for file in files: + if os.path.exists(file): + os.remove(file) + + dirs = [ + f'{dirplugins}/build/', + f'{dirplugins}/k8sapp_{self.APP_NAME_WITH_UNDERSCORE}.egg-info/'] + for dir in dirs: + if os.path.exists(dir): + shutil.rmtree(dir) + + os.chdir(store_cwd) + + return True + # Sub-process of app generation # generate application checksum file and tarball # - def _gen_checksum_and_app_tarball(self): + def _gen_checksum_and_app_tarball(self, output): store_cwd = os.getcwd() - os.chdir(self._armada_app['outputDir']) + os.chdir(output) # gen checksum # check checksum file existance - checksum_file = 'checksum.md5' + checksum_file = 'checksum.sha256' if os.path.exists(checksum_file): os.remove(checksum_file) app_files = [] - for parent, dirnames, filenames in os.walk('./'): - for filename in filenames: - app_files.append(os.path.join(parent, filename)) - with open(checksum_file, 'a') as f: - for target_file in app_files: - f.write(self._gen_md5(target_file) + ' ' + target_file + '\n') + try: + for parent, dirnames, filenames in os.walk('./'): + for filename in filenames: + if filename[-3:] != '.py' and filename[-4:] != '.cfg': + app_files.append(os.path.join(parent, filename)) + except Exception as e: + print('Error: %s' % e) + try: + with open(checksum_file, 'a') as f: + for target_file in sorted(app_files): + f.write(self._gen_sha256(target_file) + ' *' + target_file + '\n') + except Exception as e: + print('Error: %s' % e) + app_files.append('./' + checksum_file) # gen application tarball - tarname = self._armada_app['appName'] + '-' + self._armada_app['version'] + '.tgz' + tarname = f"{self._app['appName']}-{self._app['appVersion']}.tgz" t = tarfile.open(tarname, 'w:gz') for target_file in app_files: t.add(target_file) @@ -401,126 +847,264 @@ class ArmadaApplication: os.chdir(store_cwd) return tarname + + def _create_flux_dir(self, output_dir): + + if not os.path.exists(self._app['outputFluxChartDir']): + os.makedirs(self._app['outputFluxChartDir']) + if not os.path.exists(self._app['outputManifestDir']): + os.makedirs(self._app['outputFluxBaseDir']) + for idx in range(len(self._chart)): + chart = self._chart[idx] + self._app['outputFluxManifestDir'] = output_dir + '/FluxCD/fluxcd-manifests/' + chart['name'] + os.makedirs(self._app['outputFluxManifestDir']) + + + def _create_plugins_dir(self): + + if not os.path.exists(self._app['outputPluginDir']): + os.makedirs(self._app['outputPluginDir']) + if not os.path.exists(self._app['outputHelmDir']): + os.makedirs(self._app['outputHelmDir']) + if not os.path.exists(self._app['outputCommonDir']): + os.makedirs(self._app['outputCommonDir']) + if not os.path.exists(self._app['outputKustomizeDir']): + os.makedirs(self._app['outputKustomizeDir']) + if not os.path.exists(self._app['outputLifecycleDir']): + os.makedirs(self._app['outputLifecycleDir']) + + + def _create_armada_dir(self): + + if not os.path.exists(self._app['outputArmadaDir']): + os.makedirs(self._app['outputArmadaDir']) + if not os.path.exists(self._app['outputArmadaChartDir']): + os.makedirs(self._app['outputArmadaChartDir']) + + # Generate armada application, including: - # 1. helm chart tarballs - # 2. armada manifest - # 3. metadata file - # 4. checksum file - # 5. application tarball + # 1. Check chart values + # 2. Create Armada directory + # 3. Generate helm chart tarballs + # 4. Generate armada manifest + # 5. Generate metadata file + # 6. Generage checksum file + # 7. Package Armada application # - def gen_app(self, output_dir, overwrite): + def gen_armada_app(self, output_dir, no_package, package_only): ret = False if not self._validate_app_attributes(): print('Error: Some of the app attributes are not valid!') return ret - self._armada_app['outputDir'] = output_dir - self._armada_app['outputChartDir'] = output_dir + '/charts' - if not os.path.exists(self._armada_app['outputDir']): - os.makedirs(self._armada_app['outputDir']) - elif overwrite: - shutil.rmtree(self._armada_app['outputDir']) - else: - print('Output folder %s exists, please remove it or use --overwrite.' % self._armada_app['outputDir']) - return ret - if not os.path.exists(self._armada_app['outputChartDir']): - os.makedirs(self._armada_app['outputChartDir']) - # 1. Generating helm chart tarball - for chart in self._armada_chart: - ret = self._gen_helm_chart_tarball(chart) + + self._app['outputDir'] = output_dir + self._app['outputArmadaDir'] = output_dir + '/Armada' + self._app['outputArmadaChartDir'] = output_dir + '/Armada/charts' + + # 1 - Validate input file and helm chart data + self.check_charts() + + if not package_only: + # 2. Generate armada directories + self._create_armada_dir() + + # 3. Generating helm chart tarball + for chart in self._chart: + ret = self._gen_helm_chart_tarball( + chart, self._app['outputArmadaChartDir']) + if ret: + print('Helm chart %s tarball generated!' % chart['name']) + print('') + else: + print('Generating tarball for helm chart: %s error!' % chart['name']) + return ret + + # 4. Generating armada manifest + ret = self._gen_armada_manifest() if ret: - print('Helm chart %s tarball generated!' % chart['name']) - print('') + print('Armada manifest generated!') else: - print('Generating tarball for helm chart: %s error!' % chart['name']) + print('Armada manifest generation failed!') + return ret + + # 5. Generating metadata file + ret = self._gen_metadata('armada', self._app['outputArmadaDir']) + if ret: + print('Metadata generated!') + else: + print('Armada Metadata generation failed!') + return ret + + if not no_package: + # 6&7. Generating checksum file and tarball + ret = self._gen_checksum_and_app_tarball(self._app['outputArmadaDir']) + if ret: + print('Checksum generated!') + print('Armada App tarball generated at %s/%s' % (self._app['outputArmadaDir'], ret)) + print('') + else: + print('Checksum and App tarball generation failed!') return ret - # 2. Generating armada manifest - ret = self._gen_armada_manifest() - if ret: - print('Armada manifest generated!') - else: - print('Armada manifest generation failed!') return ret - # 3. Generating metadata file - ret = self._gen_metadata() - if ret: - print('Metadata generated!') - else: - print('Metadata generation failed!') - return ret - # 4&5. Generating checksum file and tarball - ret = self._gen_checksum_and_app_tarball() - if ret: - print('Checksum generated!') - print('App tarball generated at %s/%s' % (self._armada_app['outputDir'], ret)) - print('') - else: - print('Checksum and App tarball generation failed!') - return ret + # Function to call all process fot the creation of the FluxCd app tarball + # 1 - Validate input file and helm chart data + # 2 - Create application directories + # 3 - Generate FluxCD Manifests + # 4 - Generate application plugins + # 5 - Generate application metadata + # 6 - Package helm-charts + # 7 - Package plugins in wheel format + # 8 - Generate checksum + # 9 - Package entire application + def gen_flux_app(self, output_dir, no_package, package_only): + + self._app['outputDir'] = output_dir + self._app['outputFluxCDDir'] = output_dir + '/FluxCD' + self._app['outputFluxChartDir'] = output_dir + '/FluxCD/charts/' + self._app['outputManifestDir'] = output_dir + '/FluxCD/fluxcd-manifests/' + self._app['outputFluxBaseDir'] = output_dir + '/FluxCD/fluxcd-manifests/base/' + + self._app['outputPluginDir'] = output_dir + '/FluxCD/plugins' + self._app['outputHelmDir'] = output_dir + '/FluxCD/plugins/k8sapp_' + self._app['appName'].replace(" ", "_").replace("-", "_") + '/helm/' + self._app['outputCommonDir'] = output_dir + '/FluxCD/plugins/k8sapp_' + self._app['appName'].replace(" ", "_").replace("-", "_") + '/common/' + self._app['outputKustomizeDir'] = output_dir + '/FluxCD/plugins/k8sapp_' + self._app['appName'].replace(" ", "_").replace("-", "_") + '/kustomize/' + self._app['outputLifecycleDir'] = output_dir + '/FluxCD/plugins/k8sapp_' + self._app['appName'].replace(" ", "_").replace("-", "_") + '/lifecycle/' + + # 1 - Validate input file and helm chart data + self.check_charts() + + if not package_only: + + # 2 - Create application directories + self._create_flux_dir(output_dir) + self._create_plugins_dir() + + # 3 - Generate FluxCD Manifests + ret = self._gen_fluxcd_manifest() + if ret: + print('FluxCD manifest generated!') + else: + print('FluxCCD manifest generation failed!') + return ret + + # 4 - Generate application plugins + ret = self._gen_plugins() + if ret: + print('FluxCD Plugins generated!') + else: + print('FluxCD Plugins generation failed!') + return ret + + # 5 - Generate application metadata + ret = self._gen_metadata('flux', self._app['outputFluxCDDir']) + if ret: + print('FluxCD Metadata generated!') + else: + print('FluxCD Metadata generation failed!') + return ret + + if not no_package: + + # 6 - Package helm-charts + for chart in self._chart: + ret = self._gen_helm_chart_tarball( + chart, self._app['outputFluxChartDir']) + if ret: + print('Helm chart %s tarball generated!' % chart['name']) + print('') + else: + print('Generating tarball for helm chart: %s error!' % chart['name']) + return ret + + # 7 - Package plugins in wheel format + ret = self._gen_plugin_wheels() + if ret: + print('Plugin wheels generated!') + else: + print('Plugin wheels generation failed!') + return ret + + # 8 - Generate checksum && + # 9 - Package entire application + ret = self._gen_checksum_and_app_tarball(self._app['outputFluxCDDir']) + if ret: + print('Checksum generated!') + print('FluxCD App tarball generated at %s/%s' % (self._app['outputDir'], ret)) + print('') + else: + print('Checksum and App tarball generation failed!') + return ret - return ret # For debug def print_app_data(self): - print(self._armada_app) - print(self._armada_manifest) - print(self._armada_chartgroup) - print(self._armada_chart) + print(self._app) + print(self._manifest) + print(self._chartgroup) + print(self._chart) + def parse_yaml(yaml_in): yaml_data='' try: with open(yaml_in) as f: yaml_data = yaml.safe_load(f) - except IOError: + except FileNotFoundError: print('Error: %s no found' % yaml_in ) except Exception as e: print('Error: Invalid yaml file') return yaml_data + def check_manifest(manifest_data): # TODO: check more mandatory key/values in manifest yaml # check app values - if 'appName' not in manifest_data: + if 'appName' not in manifest_data['appManifestFile-config']: print('Error: \'appName\' is missing.') return False - if 'namespace' not in manifest_data: + if 'namespace' not in manifest_data['appManifestFile-config']: print('Error: \'namespace\' is missing.') return False - if 'version' not in manifest_data: - print('Error: \'version\' is missing.') + if 'appVersion' not in manifest_data['appManifestFile-config']: + print('Error: \'appVersion\' is missing.') return False - # check manifest values - if 'manifest' not in manifest_data: - print('Error: \'manifest\'is missing.') - return False + # # check manifest values + # if 'manifest' not in manifest_data['appManifestFile-config']: + # print('Error: \'manifest\'is missing.') + # return False - if 'releasePrefix' not in manifest_data['manifest']: - print('Error: Manifest attribute \'releasePrefix\' is missing.') - return False + # if 'releasePrefix' not in manifest_data['manifest']: + # print('Error: Manifest attribute \'releasePrefix\' is missing.') + # return False # check chartGroup values - if 'chartGroup' not in manifest_data: + if 'chartGroup' not in manifest_data['appManifestFile-config']: print('Error: \'chartGroup\' is missing.') return False # check chart values - if 'chart' not in manifest_data: + if 'chart' not in manifest_data['appManifestFile-config']: print('Error: \'chart\' is missing.') return False - for chart in manifest_data['chart']: + for chart in manifest_data['appManifestFile-config']['chart']: # check chart name if 'name' not in chart: print('Error: Chart attribute \'name\' is missing.') return False + # check chart version + if 'version' not in chart: + print('Error: Chart attribute \'version\' is missing.') + return False + # check chart path, supporting: dir, git, tarball if 'path' not in chart: print('Error: Chart attribute \'path\' is missing in chart %s.' % chart['name']) @@ -560,7 +1144,8 @@ def check_manifest(manifest_data): return True -def generate_app(file_in, out_folder, overwrite): + +def generate_app(file_in, out_folder, package_type, overwrite, no_package, package_only): global TEMP_APP_DIR app_data = parse_yaml(file_in) if not app_data: @@ -569,20 +1154,40 @@ def generate_app(file_in, out_folder, overwrite): if not check_manifest(app_data): print('Application manifest is not valid') return - armada_app = ArmadaApplication(app_data) - TEMP_APP_DIR = TEMP_USER_DIR + armada_app.get_app_name() + '/' - app_out = out_folder + '/' + armada_app.get_app_name() - armada_app.gen_app(app_out, overwrite) + app = Application(app_data) + TEMP_APP_DIR = TEMP_USER_DIR + app.get_app_name() + '/' + app_out = out_folder + '/' + app.get_app_name() + + if not os.path.exists(app_out): + os.makedirs(app_out) + elif overwrite: + shutil.rmtree(app_out) + elif package_only: + pass + else: + print('Output folder %s exists, please remove it or use --overwrite.' % app_out) + sys.exit() + + if package_type == 'armada' or package_type == 'both': + app.gen_armada_app(app_out, no_package, package_only) + + if package_type == 'flux' or package_type == 'both': + app.gen_flux_app(app_out, no_package, package_only) + def main(argv): input_file = '' output_folder = '.' + package_type = '' overwrite = False + package_only = False + no_package = False try: - options, args = getopt.getopt(argv, 'hi:o:', \ - ['help', 'input==', 'output==', 'overwrite']) + options, args = getopt.getopt(argv, 'hi:o:t:', \ + ['help', 'input=', 'output=', 'type=', 'overwrite', 'no-package', 'package-only']) except getopt.GetoptError: - sys.exit() + print('Error: Invalid argument') + sys.exit(1) for option, value in options: if option in ('-h', '--help'): print('StarlingX User Application Generator') @@ -593,7 +1198,10 @@ def main(argv): print('Options:') print(' -i, --input yaml_file generate app from yaml_file') print(' -o, --output folder generate app to output folder') + print(' -t, --type package select Armada,Flux or Both packaging') print(' --overwrite overwrite the output dir') + print(' --no-package does not create app tarball') + print(' --package-only only creates tarball from dir') print(' -h, --help this help') if option in ('--overwrite'): overwrite = True @@ -601,12 +1209,22 @@ def main(argv): input_file = value if option in ('-o', '--output'): output_folder = value + if option in ('-t', '--type'): + package_type = value.lower() + if option in ('--no-package'): + no_package = True + if option in ('--package-only'): + package_only = True + if not package_type: + print('Error: Select type of packaging (armada/flux/both)') + sys.exit(1) if not os.path.isfile(os.path.abspath(input_file)): print('Error: input file not found') - sys.exit() + sys.exit(1) if input_file: - generate_app(os.path.abspath(input_file), os.path.abspath(output_folder), overwrite) + generate_app(os.path.abspath(input_file), os.path.abspath(output_folder), package_type, overwrite, no_package, package_only) + if __name__ == '__main__': - main(sys.argv[1:]) + main(sys.argv[1:]) \ No newline at end of file diff --git a/app_manifest.yaml b/app_manifest.yaml new file mode 100644 index 0000000..b863c35 --- /dev/null +++ b/app_manifest.yaml @@ -0,0 +1,85 @@ +--- +## App Manifest Configuration +appManifestFile-config: + appName: # required + appVersion: # required + namespace: # required + chart: + - name: # required + version: # required + path: # required + chartGroup: # required + # add more if you have more than one chart + chartGroup: + - name: # required + description: # required for Armada + sequenced: # required for Armada + chart_names: + - # required + - # optional + # add more if you have more than one chartgroup for your Armada app + manifest: + name: # required for Armada + releasePrefix: # required for Armada + +## For Armada packaging the sections bellow are not necessary. + +################################################# +## App Metadata Configuration +# for further details about possible configurations on this file, please +# visit the link: https://wiki.openstack.org/wiki/StarlingX/Containers/StarlingXAppsInternals#metadata.yaml +metadataFile-config: +# the following configurations are optional +# uncomment and configure properly the ones you need for your application metadata + # upgrades: + # auto_update: + # update_failure_no_rollback: + # from_versions: + # - + # - + # # ... + # supported_k8s_version: + # minimum: + # maximum: + # supported_releases: + # : + # - + # - + # # ... + # repo: + # disabled_charts: + # - + # - + # # ... + # maintain_attributes: + # maintain_user_overrides: + # behavior: + # platform_managed_app: + # desired_state: + # evaluate_reapply: + # after: + # - + # - + # - ... + # triggers: + # - type: + # - type: + # - ... + # filters: + # - : + # - : + # - ... + # filter_field: + # apply_progress_adjust: <0/1/2/.../100> + +################################################# +## App Setup Configuration +# if you wish to see a setup.cfg example, please see the link +# https://opendev.org/starlingx/app-dell-storage/src/branch/master/python3-k8sapp-dell-storage/k8sapp_dell_storage/setup.cfg +setupFile-config: + metadata: + author: # required + author-email: # required + url: # required + classifier: # required + - # required diff --git a/template/armada-chart.template b/template_armada/armada-chart.template similarity index 100% rename from template/armada-chart.template rename to template_armada/armada-chart.template diff --git a/template/armada-chartgroup.template b/template_armada/armada-chartgroup.template similarity index 84% rename from template/armada-chartgroup.template rename to template_armada/armada-chartgroup.template index e878d02..95755ae 100644 --- a/template/armada-chartgroup.template +++ b/template_armada/armada-chartgroup.template @@ -6,5 +6,6 @@ metadata: data: description: $DESCRIPTION%This is a chartgroup$ sequenced: $SEQUENCED%false$ - @CHART_GROUP|2@ + chart_group: + @CHART_NAMES|4@ diff --git a/template/armada-manifest.template b/template_armada/armada-manifest.template similarity index 78% rename from template/armada-manifest.template rename to template_armada/armada-manifest.template index 8293d63..2da5220 100644 --- a/template/armada-manifest.template +++ b/template_armada/armada-manifest.template @@ -5,4 +5,5 @@ metadata: name: $NAME$ data: release_prefix: $RELEASE_PREFIX$ - @CHART_GROUPS|2@ + chart_groups: + @CHART_GROUPS|4@ diff --git a/templates_flux/base/helmrepository.template b/templates_flux/base/helmrepository.template new file mode 100644 index 0000000..5e1a1f3 --- /dev/null +++ b/templates_flux/base/helmrepository.template @@ -0,0 +1,7 @@ +apiVersion: source.toolkit.fluxcd.io/v1beta1 +kind: HelmRepository +metadata: + name: stx-platform +spec: + url: http://192.168.206.1:8080/helm_charts/stx-platform + interval: 60m \ No newline at end of file diff --git a/templates_flux/base/kustomization.template b/templates_flux/base/kustomization.template new file mode 100644 index 0000000..dd3987a --- /dev/null +++ b/templates_flux/base/kustomization.template @@ -0,0 +1,3 @@ +resources: + - helmrepository.yaml + - namespace.yaml \ No newline at end of file diff --git a/templates_flux/base/namespace.template b/templates_flux/base/namespace.template new file mode 100644 index 0000000..e278bd5 --- /dev/null +++ b/templates_flux/base/namespace.template @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: $NAMESPACE$ \ No newline at end of file diff --git a/templates_flux/fluxcd-manifest/helmrelease.template b/templates_flux/fluxcd-manifest/helmrelease.template new file mode 100644 index 0000000..4efeaf5 --- /dev/null +++ b/templates_flux/fluxcd-manifest/helmrelease.template @@ -0,0 +1,30 @@ +apiVersion: "helm.toolkit.fluxcd.io/v2beta1" +kind: HelmRelease +metadata: + name: $NAME$ + labels: + chart_group: $CHART_GROUP$ +spec: + releaseName: $NAME$ + chart: + spec: + chart: $NAME$ + version: $VERSION$ + sourceRef: + kind: HelmRepository + name: stx-platform + interval: 5m + timeout: 30m + test: + enable: false + install: + disableHooks: false + upgrade: + disableHooks: false + valuesFrom: + - kind: Secret + name: $NAME$-static-overrides + valuesKey: $NAME$-static-overrides.yaml + - kind: Secret + name: $NAME$-system-overrides + valuesKey: $NAME$-system-overrides.yaml \ No newline at end of file diff --git a/templates_flux/fluxcd-manifest/kustomization.template b/templates_flux/fluxcd-manifest/kustomization.template new file mode 100644 index 0000000..5793d8f --- /dev/null +++ b/templates_flux/fluxcd-manifest/kustomization.template @@ -0,0 +1,12 @@ +namespace: $NAMESPACE$ +resources: +- helmrelease.yaml +secretGenerator: + - name: $NAME$-static-overrides + files: + - $NAME$-static-overrides.yaml + - name: $NAME$-system-overrides + files: + - $NAME$-system-overrides.yaml +generatorOptions: + disableNameSuffixHash: true \ No newline at end of file diff --git a/templates_flux/kustomization.template b/templates_flux/kustomization.template new file mode 100644 index 0000000..552eed4 --- /dev/null +++ b/templates_flux/kustomization.template @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: $NAMESPACE$ +resources: + - base + - @CHART_NAMES|2@ \ No newline at end of file diff --git a/templates_plugins/common.template b/templates_plugins/common.template new file mode 100644 index 0000000..ec1d5f4 --- /dev/null +++ b/templates_plugins/common.template @@ -0,0 +1,7 @@ +HELM_NS = "{namespace}" + +HELM_APP = "{name}" + +HELM_CHART = "{name}" + +FLUXCD_HELMRELEASE = "{name}" \ No newline at end of file diff --git a/templates_plugins/helm.template b/templates_plugins/helm.template new file mode 100644 index 0000000..4580dd1 --- /dev/null +++ b/templates_plugins/helm.template @@ -0,0 +1,30 @@ +from sysinv.helm import base +from sysinv.common import exception + +from {appname}.common import constants + +class {name}Helm(base.FluxCDBaseHelm): + """Class to encapsulate helm operations for the app chart""" + + SUPPORTED_NAMESPACES = base.BaseHelm.SUPPORTED_NAMESPACES + \ + [constants.HELM_NS] + SUPPORTED_APP_NAMESPACES = {{constants.HELM_APP: SUPPORTED_NAMESPACES, + }} + + CHART = constants.HELM_CHART + HELM_RELEASE = constants.FLUXCD_HELMRELEASE + + def get_namespaces(self): + return self.SUPPORTED_NAMESPACES + + def get_overrides(self, namespace=None): + + overrides = {{}} + + if namespace in self.SUPPORTED_NAMESPACES: + return overrides[namespace] + elif namespace: + raise exception.InvalidHelmNamespace(chart=self.CHART, + namespace=namespace) + else: + return overrides \ No newline at end of file diff --git a/templates_plugins/kustomize.template b/templates_plugins/kustomize.template new file mode 100644 index 0000000..f854741 --- /dev/null +++ b/templates_plugins/kustomize.template @@ -0,0 +1,18 @@ +from {appname}.common import constants +from sysinv.helm import kustomize_base as base + + +class {appnameStriped}FluxCDKustomizeOperator(base.FluxCDKustomizeOperator): + + APP = constants.HELM_APP + + def platform_mode_kustomize_updates(self, dbapi, mode): + """ Update the top-level kustomization resource list + + Make changes to the top-level kustomization resource list based on the + platform mode + + :param dbapi: DB api object + :param mode: mode to control when to update the resource list + """ + pass \ No newline at end of file diff --git a/templates_plugins/lifecycle.template b/templates_plugins/lifecycle.template new file mode 100644 index 0000000..e3ed84d --- /dev/null +++ b/templates_plugins/lifecycle.template @@ -0,0 +1,23 @@ +""" System inventory App lifecycle operator.""" + +from oslo_log import log as logging +from sysinv.common import constants +from sysinv.helm import lifecycle_base as base +from sysinv.helm import lifecycle_utils as lifecycle_utils + +LOG = logging.getLogger(__name__) + +class {appnameStriped}AppLifecycleOperator(base.AppLifecycleOperator): + + def app_lifecycle_actions(self, context, conductor_obj, app_op, app, hook_info): + """ Perform lifecycle actions for an operation + + :param context: request context + :param conductor_obj: conductor object + :param app_op: AppOperator object + :param app: AppOperator.Application object + :param hook_info: LifecycleHookInfo object + + """ + + super({appnameStriped}AppLifecycleOperator, self).app_lifecycle_actions(context, conductor_obj, app_op, app, hook_info)