starlingx/config
Code Issues Proposed changes

Add runtime reconfiguration of kubelet

Browse Source 
This adds the CLI command 'system kube-config-kubelet'. This invokes
puppet runtime manifests to reconfigure kubelet-config ConfigMap
with new parameters, and to upgrade kubernetes nodes with new
parameters, and restart kubelet. This gives the ability to update
kubelet parameters with a software patch.

The specific kubelet-config parameters are provided within the puppet
manifests and its supporting parameters script. The specific settings
values and engineering are described in the puppet component.
Identical settings are also configured at install time in
ansible-playbooks.

TESTING:
PASS - manually fill /var/lib/docker to exceed imageGC and
       verify GC operates
PASS - AIO-DX fresh install gets updated kubelet config
PASS - AIO-DX apply/remove designer patch with updated kubelet config
PASS - 'system kube-config-kubelet' updates K8S nodes kubelet config
PASS - AIO-DX reinstall controller-1 has updated kubelet config
PASS - AIO-DX install new worker node gets updated kubelet config
PASS - build and view REST documentation

Partial-Bug: 1977754
Depends-On: https://review.opendev.org/c/starlingx/stx-puppet/+/844298
Depends-On: https://review.opendev.org/c/starlingx/ansible-playbooks/+/844305

Signed-off-by: Jim Gauld <james.gauld@windriver.com>
Change-Id: Iad32a724d3f681bc9854fa663299f8539f70fd2a
This commit is contained in:
Jim Gauld 2022-06-01 11:24:14 -04:00
parent 15bb4dc070
commit d57d3a07b8
10 changed files with 244 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
api-ref/source/api-ref-sysinv-v1-config.rst
View File

@ -655,6 +655,16 @@ itemNotFound (404)
"href": "http://10.10.10.3:6385/kube_upgrade/",
"rel": "bookmark"
}
],
"kube_config_kubelet": [
{
"href": "http://10.10.10.3:6385/v1/kube_config_kubelet/",
"rel": "self"
},
{
"href": "http://10.10.10.3:6385/kube_config_kubelet/",
"rel": "bookmark"
}
]
}
@ -12515,3 +12525,43 @@ forbidden (403), badMethod (405), overLimit (413)
}
This operation does not accept a request body.
-------------------------
Kubernetes config kubelet
-------------------------
These APIs allow the reconfiguration of kubelet-config parameters and restart of kubelet on each node.
******************************************************************************
Apply kubelet-config parameters reconfiguration and restart kubelet procedure
******************************************************************************
.. rest_method:: POST /v1/kube_config_kubelet/apply
**Normal response codes**
200
**Error response codes**
computeFault (400, 500, ...), serviceUnavailable (503), badRequest (400),
unauthorized (401), forbidden (403), badMethod (405), overLimit (413)
**Request parameters**
**Response parameters**
.. csv-table::
:header: "Parameter", "Style", "Type", "Description"
:widths: 20, 20, 20, 60
"success", "plain", "xsd:string", "The success message indicating start of the kube-config-kubelet apply"
"error", "plain", "xsd:string", "The error message in case something wrong happen on the API execution"
::
{
"success": "kube-config-kubelet applied.",
"error": ""
}

3
sysinv/cgts-client/cgts-client/cgtsclient/v1/client.py
View File

@ -55,6 +55,7 @@ from cgtsclient.v1 import isystem
from cgtsclient.v1 import iuser
from cgtsclient.v1 import kube_cluster
from cgtsclient.v1 import kube_cmd_version
from cgtsclient.v1 import kube_config_kubelet
from cgtsclient.v1 import kube_host_upgrade
from cgtsclient.v1 import kube_rootca_update
from cgtsclient.v1 import kube_upgrade
@ -181,3 +182,5 @@ class Client(object):
self.device_label = device_label.DeviceLabelManager(self.http_client)
self.restore = restore.RestoreManager(self.http_client)
self.kube_rootca_update = kube_rootca_update.KubeRootCAUpdateManager(self.http_client)
self.kube_config_kubelet = \
kube_config_kubelet.KubeConfigKubeletManager(self.http_client)

26
sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_config_kubelet.py Normal file
View File

@ -0,0 +1,26 @@
#
# Copyright (c) 2022 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
from cgtsclient.common import base
class KubeConfigKubelet(base.Resource):
def __repr__(self):
return "<kube_config_kubelet %s>" % self._info
class KubeConfigKubeletManager(base.Manager):
resource_class = KubeConfigKubelet
@staticmethod
def _path(id=None):
return '/v1/kube_config_kubelet/%s' % id if id \
else '/v1/kube_config_kubelet'
def apply(self):
path = self._path("apply")
_, body = self.api.json_request('POST', path)
return body

29
sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_config_kubelet_shell.py Normal file
View File

@ -0,0 +1,29 @@
#
# Copyright (c) 2022 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# All Rights Reserved.
#
from cgtsclient import exc
def do_kube_config_kubelet(cc, args):
"""Apply the kubelet config."""
try:
response = cc.kube_config_kubelet.apply()
except exc.HTTPNotFound:
raise exc.CommandError('Failed to apply kubelet config. No response.')
except Exception as e:
raise exc.CommandError('Failed to apply kubelet config: %s' % (e))
else:
success = response.get('success')
error = response.get('error')
if success:
print("Success: " + success)
if error:
print("Error: " + error)

4
sysinv/cgts-client/cgts-client/cgtsclient/v1/shell.py
View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2013-2021 Wind River Systems, Inc.
# Copyright (c) 2013-2022 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -42,6 +42,7 @@ from cgtsclient.v1 import isystem_shell
from cgtsclient.v1 import iuser_shell
from cgtsclient.v1 import kube_cluster_shell
from cgtsclient.v1 import kube_config_kubelet_shell
from cgtsclient.v1 import kube_rootca_update_shell
from cgtsclient.v1 import kube_upgrade_shell
from cgtsclient.v1 import kube_version_shell
@ -128,6 +129,7 @@ COMMAND_MODULES = [
app_shell,
host_fs_shell,
kube_cluster_shell,
kube_config_kubelet_shell,
kube_version_shell,
kube_upgrade_shell,
kube_rootca_update_shell,

13
sysinv/sysinv/sysinv/sysinv/api/controllers/v1/__init__.py
View File

@ -47,6 +47,7 @@ from sysinv.api.controllers.v1 import kube_rootca_update
from sysinv.api.controllers.v1 import kube_upgrade
from sysinv.api.controllers.v1 import kube_version
from sysinv.api.controllers.v1 import kube_cmd_version
from sysinv.api.controllers.v1 import kube_config_kubelet
from sysinv.api.controllers.v1 import label
from sysinv.api.controllers.v1 import interface
from sysinv.api.controllers.v1 import interface_network
@ -281,6 +282,9 @@ class V1(base.APIBase):
kube_host_upgrades = [link.Link]
"Links to the kube_host_upgrade resource"
kube_config_kubelet = [link.Link]
"Links to the kube_config_kubelet resource "
device_images = [link.Link]
"Links to the device images resource"
@ -870,6 +874,14 @@ class V1(base.APIBase):
'kube_host_upgrades', '',
bookmark=True)]
v1.kube_config_kubelet = [
link.Link.make_link('self', pecan.request.host_url,
'kube_config_kubelet', ''),
link.Link.make_link('bookmark', pecan.request.host_url,
'kube_config_kubelet', '',
bookmark=True)
]
v1.device_images = [link.Link.make_link('self', pecan.request.host_url,
'device_images', ''),
link.Link.make_link('bookmark',
@ -974,6 +986,7 @@ class Controller(rest.RestController):
kube_upgrade = kube_upgrade.KubeUpgradeController()
kube_rootca_update = kube_rootca_update.KubeRootCAUpdateController()
kube_host_upgrades = kube_host_upgrade.KubeHostUpgradeController()
kube_config_kubelet = kube_config_kubelet.KubeConfigKubeletController()
device_images = device_image.DeviceImageController()
device_image_state = device_image_state.DeviceImageStateController()
device_labels = device_label.DeviceLabelController()

36
sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_config_kubelet.py Normal file
View File

@ -0,0 +1,36 @@
########################################################################
#
# Copyright (c) 2022 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
########################################################################
import pecan
from pecan import expose
from pecan import rest
from sysinv.common import utils as cutils
from sysinv.openstack.common.rpc.common import RemoteError
LOCK_NAME = 'KubeConfigKubeletController'
class KubeConfigKubeletController(rest.RestController):
"""REST controller for kube_config_kubelet."""
_custom_actions = {
'apply': ['POST'],
}
@expose('json')
@cutils.synchronized(LOCK_NAME)
def apply(self):
try:
pecan.request.rpcapi.kube_config_kubelet(pecan.request.context)
except RemoteError as e:
return dict(success="", error=e.value)
except Exception as ex:
return dict(success="", error=str(ex))
return dict(success="kube-config-kubelet applied.", error="")

77
sysinv/sysinv/sysinv/sysinv/conductor/manager.py
View File

@ -1703,6 +1703,30 @@ class ConductorManager(service.PeriodicService):
return False
def kube_config_kubelet(self, context):
"""Update kubernetes nodes kubelet configuration ConfigMap.
This method updates kubelet parameters in configmaps/kubelet-config.
This leverages puppet report status so we can wait for completion
of the runtime manifest and trigger subsequent per-node configuration.
:param context: request context
"""
active_controller = utils.HostHelper.get_active_controller(self.dbapi)
personalities = [constants.CONTROLLER]
config_uuid = self._config_update_hosts(context, personalities,
[active_controller.uuid])
config_dict = {
"personalities": personalities,
"host_uuids": [active_controller.uuid],
"classes": [
'platform::kubernetes::master::update_kubelet_params::runtime'],
puppet_common.REPORT_STATUS_CFG:
puppet_common.REPORT_KUBE_UPDATE_KUBELET_PARAMS
}
self._config_apply_runtime_manifest(
context, config_uuid=config_uuid, config_dict=config_dict)
def update_keystone_password(self, context):
"""This method calls a puppet class
'openstack::keystone::password::runtime'
@ -8488,6 +8512,26 @@ class ConductorManager(service.PeriodicService):
self.report_sysparam_http_update_success, [],
self.report_sysparam_http_update_failure, [error]
)
# Kubernetes kubelet parameters update and per node configuration
elif reported_cfg == puppet_common.REPORT_KUBE_UPDATE_KUBELET_PARAMS:
# The agent is reporting the runtime update_kubelet_params has been applied.
host_uuid = iconfig['host_uuid']
if status == puppet_common.REPORT_SUCCESS:
# Update action was successful.
# Invoke per-node kubelet upgrade runtime configuration.
success = True
self.handle_kube_update_params_success(context, host_uuid)
elif status == puppet_common.REPORT_FAILURE:
# Update action has failed
args = {'cfg': reported_cfg, 'status': status, 'iconfig': iconfig}
LOG.error("config runtime failure, "
"reported_cfg: %(cfg)s status: %(status)s "
"iconfig: %(iconfig)s" % args)
else:
args = {'cfg': reported_cfg, 'status': status, 'iconfig': iconfig}
LOG.error("No match for sysinv-agent manifest application reported! "
"reported_cfg: %(cfg)s status: %(status)s "
"iconfig: %(iconfig)s" % args)
else:
LOG.error("Reported configuration '%(cfg)s' is not handled by"
" report_config_status! iconfig: %(iconfig)s" %
@ -9266,6 +9310,37 @@ class ConductorManager(service.PeriodicService):
upgrade.uuid,
{'state': constants.UPGRADE_ACTIVATION_FAILED})
def handle_kube_update_params_success(self, context, host_uuid):
"""
Callback for Sysinv Agent on kube update params success.
This is invoked after kubelet-config ConfigMap is updated,
and does per-node kubernetes configuration.
This will download the current kubelet-config ConfigMap,
regenerate the configuration file /var/lib/kubelet/config.yaml,
and restart kubelet per node.
:param context: request context
:param host_uuid: host unique id
"""
LOG.info("Kube update params phase succeeded on host: %s"
% (host_uuid))
personalities = [constants.CONTROLLER, constants.WORKER]
hosts = self.dbapi.ihost_get_list()
host_uuids = [x.uuid for x in hosts if x.personality in personalities]
config_uuid = self._config_update_hosts(context, personalities,
host_uuids=host_uuids)
config_dict = {
"personalities": personalities,
"host_uuids": host_uuids,
"classes": [
'platform::kubernetes::update_kubelet_config::runtime']
}
self._config_apply_runtime_manifest(
context, config_uuid=config_uuid, config_dict=config_dict)
def report_kube_rootca_update_success(self, host_uuid, reported_cfg):
"""
Callback for Sysinv Agent on kube root CA update success
@ -11165,7 +11240,7 @@ class ConductorManager(service.PeriodicService):
config_dict=config_dict)
if filter_classes and config_dict['classes'] in filter_classes:
LOG.info("config runtime filter_clasess add %s %s" %
LOG.info("config runtime filter_classes add %s %s" %
(filter_classes, config_dict))
self._add_runtime_class_apply_in_progress(filter_classes,
host_uuids=config_dict.get('host_uuids', None))

7
sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py
View File

@ -2088,6 +2088,13 @@ class ConductorAPI(sysinv.openstack.common.rpc.proxy.RpcProxy):
return self.cast(context, self.make_msg('kube_upgrade_networking',
kube_version=kube_version))
def kube_config_kubelet(self, context):
"""Sychronously, have the conductor configure kubelet.
:param context: request context.
"""
return self.call(context, self.make_msg('kube_config_kubelet'))
def store_bitstream_file(self, context, filename):
"""Asynchronously, have the conductor store the device image
on this host.

1
sysinv/sysinv/sysinv/sysinv/puppet/common.py
View File

@ -51,6 +51,7 @@ REPORT_KUBE_CERT_UPDATE_PODS_TRUSTBOTHCAS = \
'pods_' + constants.KUBE_CERT_UPDATE_TRUSTBOTHCAS
REPORT_KUBE_CERT_UPDATE_PODS_TRUSTNEWCA = \
'pods_' + constants.KUBE_CERT_UPDATE_TRUSTNEWCA
REPORT_KUBE_UPDATE_KUBELET_PARAMS = 'update_kubelet_params'
REPORT_HTTP_CONFIG = 'http_config'