docker images management

Currently, the images pulled from public registries are used
by helm charts for pods launching. This commit updates this
behavior to pull images from the local docker registry to
launch pods. In order to pull images from local registry,
images for each supported chart are updated in sysinv overrides
to point to the local docker registry. For the non stx images
or images not included in sysinv overrides, they are handled
when application apply by getting them from charts and adding
the converted image tags to the override files.

A list of docker images required for stx-openstack app installation
is maintained in a file. When applying stx-openstack app (before
launching pods), if the required images do not exist in the local
registry, images will be pulled from the public registries and
pushed to the local registry.

To be able to use the local docker registry, the default disk
size for docker-distribution is increased from 1G to 16G.

Validated on AIO-SX/AIO-DX/2+2+2:
 - config_controller with kubernetes
 - During the initial stx-openstack app apply, images pulled from
   public registry and pushed to local registry. Images are pulled
   from local registry for pods launching on each node
 - Cleanup docker cache for each node, all images pulled from local
   registry during the subsequent application apply
 - Cleanup the images in local registry, images pulled from public
   and pushed to the local during the application apply

Story: 2004520
Task: 28526
Depends-On: https://review.openstack.org/#/c/626394/
Change-Id: I2001fff237cc2ccf4be1a8b15ab346d730bb32e9
Signed-off-by: Angie Wang <angie.wang@windriver.com>
This commit is contained in:
Angie Wang 2018-12-18 20:54:31 -05:00
parent 634e513d62
commit f91b02c205
20 changed files with 177 additions and 83 deletions

View File

@ -20,7 +20,7 @@ labels:
images:
tags:
nova_api_proxy: 128.224.186.231:9001/abailey/stx-nova-api-proxy:latest
nova_api_proxy: docker.io/starlingx/stx-nova-api-proxy:dev-centos-pike-latest
ks_endpoints: docker.io/openstackhelm/heat:pike
dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1
pullPolicy: IfNotPresent

View File

@ -283,18 +283,19 @@ DEFAULT_SMALL_DISK_SIZE = 240
# DEFAULT_EXTENSION_STOR_SIZE 1
# DEFAULT_GNOCCHI_STOR_SIZE 5
# KUBERNETES_DOCKER_STOR_SIZE (--kubernetes) 16
# DEFAULT_DOCKER_DISTRIBUTION_STOR_SIZE (--kubernetes) 1
# DOCKER_DISTRIBUTION_STOR_SIZE (--kubernetes) 16
# ETCD_STOR_SIZE (--kubernetes) 5
# buffer inside VG for LV creation 1
# root partition (created in kickstarts) 20
# boot partition (created in kickstarts) 1
# buffer for partition creation 1
# -------------------------------------------------------
# 152
MINIMUM_DISK_SIZE = 152
# 167
MINIMUM_DISK_SIZE = 167
# Docker lv size when Kubernetes is configured
KUBERNETES_DOCKER_STOR_SIZE = 16
DOCKER_DISTRIBUTION_STOR_SIZE = 16
ETCD_STOR_SIZE = 5
# Openstack Interface names

View File

@ -289,7 +289,10 @@ class AppOperator(object):
def _get_image_tags_by_path(self, path):
""" Mine the image tags from values.yaml files in the chart directory,
intended for custom apps. """
intended for custom apps.
TODO(awang): Support custom apps to pull images from local registry
"""
image_tags = []
ids = []
@ -304,35 +307,80 @@ class AppOperator(object):
return list(set(image_tags))
def _get_image_tags_by_charts(self, app_path, charts):
""" Mine the image tags from both the chart path and the overrides,
intended for system app. """
""" Mine the image tags from the chart paths. Add the converted
image tags to the overrides if the image tags from the chart
paths do not exist. Intended for system app.
The image tagging conversion(local docker registry address prepended):
${LOCAL_DOCKER_REGISTRY_IP}:${REGISTRY_PORT}/<image-name>
(ie..192.168.204.2:9001/docker.io/mariadb:10.2.13)
"""
local_docker_registry_ip = self._dbapi.address_get_by_name(
cutils.format_address_name(constants.CONTROLLER_HOSTNAME,
constants.NETWORK_TYPE_MGMT)
).address
image_tags = []
for chart in charts:
tags = []
images_charts = {}
images_overrides = {}
overrides = chart.namespace + '-' + chart.name + '.yaml'
overrides_file = os.path.join(common.HELM_OVERRIDES_PATH,
overrides)
chart_path = os.path.join(app_path, chart.name)
chart_name = os.path.join(app_path, chart.name)
chart_path = os.path.join(chart_name, 'values.yaml')
# Get the image tags from the chart path
if os.path.exists(chart_path):
with open(chart_path, 'r') as file:
try:
doc = yaml.load(file)
images_charts = doc["images"]["tags"]
except (TypeError, KeyError):
pass
# Get the image tags from the overrides file
if os.path.exists(overrides_file):
with open(overrides_file, 'r') as file:
try:
y = yaml.load(file)
tags = y["data"]["values"]["images"]["tags"].values()
images_overrides = y["data"]["values"]["images"]["tags"]
except (TypeError, KeyError):
LOG.info("Overrides file %s has no img tags" %
overrides_file)
if tags:
image_tags.extend(tags)
continue
pass
# Either this chart does not have overrides file or image tags are
# not in its overrides file, walk the chart path to find image tags
chart_path = os.path.join(app_path, chart.name)
if os.path.exists(chart_path):
tags = self._get_image_tags_by_path(chart_path)
if tags:
image_tags.extend(tags)
# Add the converted image tags to the overrides if the images from
# the chart path do not exist in the overrides
tags_updated = False
for key, image_tag in images_charts.items():
if (key not in images_overrides and
not image_tag.startswith(local_docker_registry_ip)):
images_overrides.update(
{key: '{}:{}/{}'.format(local_docker_registry_ip, common.REGISTRY_PORT, image_tag)})
tags_updated = True
if tags_updated:
with open(overrides_file, 'w') as file:
try:
if "images" not in y["data"]["values"]:
file.seek(0)
file.truncate()
y["data"]["values"]["images"] = {"tags": images_overrides}
else:
y["data"]["values"]["images"]["tags"] = images_overrides
yaml.safe_dump(y, file, explicit_start=True,
default_flow_style=False)
LOG.info("Overrides file %s updated with new image tags" %
overrides_file)
except (TypeError, KeyError):
LOG.error("Overrides file %s fails to update" %
overrides_file)
if images_overrides:
image_tags.extend(images_overrides.values())
return list(set(image_tags))
@ -361,10 +409,9 @@ class AppOperator(object):
self._helm.generate_helm_application_overrides(
app.name, cnamespace=None, armada_format=True, combined=True)
app.charts = self._get_list_of_charts(app.armada_mfile_abs)
# Grab the image tags from the overrides. If they don't exist
# then mine them from the chart paths.
images_to_download = self._get_image_tags_by_charts(app.charts_dir,
app.charts)
# Get the list of images from the updated images overrides
images_to_download = self._get_image_tags_by_charts(
app.charts_dir, app.charts)
else:
# For custom apps, mine image tags from application path
images_to_download = self._get_image_tags_by_path(app.path)
@ -1150,18 +1197,34 @@ class DockerHelper(object):
(request, manifest_file, e))
return rc
def download_an_image(self, img_tag):
def download_an_image(self, loc_img_tag):
rc = True
start = time.time()
try:
LOG.info("Image %s download started" % img_tag)
client = docker.from_env(timeout=INSTALLATION_TIMEOUT)
client.images.pull(img_tag)
# Pull image from local docker registry
LOG.info("Image %s download started from local registry" % loc_img_tag)
client = docker.APIClient(timeout=INSTALLATION_TIMEOUT)
client.pull(loc_img_tag)
except docker.errors.NotFound:
try:
# Image is not available in local docker registry, get the image
# from the public registry and push to the local registry
LOG.info("Image %s is not available in local registry, "
"download started from public registry" % loc_img_tag)
pub_img_tag = loc_img_tag[1 + loc_img_tag.find('/'):]
client.pull(pub_img_tag)
client.tag(pub_img_tag, loc_img_tag)
client.push(loc_img_tag)
except Exception as e:
rc = False
LOG.error("Image %s download failed from public registry: %s" % (pub_img_tag, e))
except Exception as e:
rc = False
LOG.error("Image %s download failed: %s" % (img_tag, e))
LOG.error("Image %s download failed from local registry: %s" % (loc_img_tag, e))
elapsed_time = time.time() - start
LOG.info("Image %s download succeeded in %d seconds" %
(img_tag, elapsed_time))
return img_tag, rc
if rc:
LOG.info("Image %s download succeeded in %d seconds" %
(loc_img_tag, elapsed_time))
return loc_img_tag, rc

View File

@ -6542,9 +6542,9 @@ class ConductorManager(service.PeriodicService):
# Defaults: 500G root disk
#
# Min size of the cgts-vg PV is:
# 169.0 G - PV for cgts-vg (specified in the kickstart)
# 184.0 G - PV for cgts-vg (specified in the kickstart)
# or
# 177.0 G - (for DCSC non-AIO)
# 192.0 G - (for DCSC non-AIO)
# 8 G - /var/log (reserved in kickstart)
# 8 G - /scratch (reserved in kickstart)
# 2 G - cgcs_lv (DRBD bootstrap manifest)
@ -6569,22 +6569,22 @@ class ConductorManager(service.PeriodicService):
# 5 G - /opt/gnocchi
# 1 G - anchor_lv
# 16 G - /var/lib/docker (--kubernetes)
# 1 G - /var/lib/docker-distribution (--kubernetes)
# 16 G - /var/lib/docker-distribution (--kubernetes)
# 5 G - /opt/etcd (--kubernetes)
# 8 G - /opt/patch-vault (DRBD ctlr manifest for
# Distributed Cloud System Controller non-AIO only)
# -----
# 177 G (for DCSC non-AIO) or 169 G
# 192 G (for DCSC non-AIO) or 184 G
#
# The absolute minimum disk size for these default settings:
# 0.5 G - /boot
# 20.0 G - /
# 169.0 G - cgts-vg PV
# or 177.0 G - (DCSC non-AIO)
# 184.0 G - cgts-vg PV
# or 192.0 G - (DCSC non-AIO)
# -------
# 189.5 G => ~190G min size disk
# 204.5 G => ~205G min size disk
# or
# 197.5 G => ~198G min size disk
# 212.5 G => ~213G min size disk
#
# If required disk is size 500G:
# 1) Standard controller - will use all free space for the PV
@ -6595,8 +6595,8 @@ class ConductorManager(service.PeriodicService):
# 2) AIO - will leave unused space for further partitioning
# 0.5 G - /boot
# 20.0 G - /
# 169.0 G - cgts-vg PV
# 310.5 G - unpartitioned free space
# 184.0 G - cgts-vg PV
# 295.5 G - unpartitioned free space
#
database_storage = constants.DEFAULT_DATABASE_STOR_SIZE
if glance_local:
@ -6619,9 +6619,9 @@ class ConductorManager(service.PeriodicService):
# Small disk: under 240G root disk
#
# Min size of the cgts-vg PV is:
# 129.0 G - PV for cgts-vg (specified in the kickstart)
# 144.0 G - PV for cgts-vg (specified in the kickstart)
# or
# 137.0 G - (for DCSC non-AIO)
# 152.0 G - (for DCSC non-AIO)
# 8 G - /var/log (reserved in kickstart)
# 8 G - /scratch (reserved in kickstart)
# 2 G - cgcs_lv (DRBD bootstrap manifest)
@ -6646,22 +6646,22 @@ class ConductorManager(service.PeriodicService):
# 5 G - /opt/gnocchi
# 1 G - anchor_lv
# 16 G - /var/lib/docker (--kubernetes)
# 1 G - /var/lib/docker-distribution (--kubernetes)
# 16 G - /var/lib/docker-distribution (--kubernetes)
# 5 G - /opt/etcd (--kubernetes)
# 8 G - /opt/patch-vault (DRBD ctlr manifest for DCSC non-AIO only)
# -----
# 137 G (for DCSC non-AIO) or 129 G
# 152 G (for DCSC non-AIO) or 144 G
#
# The absolute minimum disk size for these default settings:
# 0.5 G - /boot
# 20.0 G - /
# 129.0 G - cgts-vg PV
# 144.0 G - cgts-vg PV
# or
# 137.0 G - (for DCSC non-AIO)
# 152.0 G - (for DCSC non-AIO)
# -------
# 149.5 G => ~150G min size disk
# 164.5 G => ~165G min size disk
# or
# 157.5 G => ~158G min size disk
# 172.5 G => ~173G min size disk
#
# If required disk is size 240G:
# 1) Standard controller - will use all free space for the PV
@ -6671,8 +6671,8 @@ class ConductorManager(service.PeriodicService):
# 2) AIO - will leave unused space for further partitioning
# 0.5 G - /boot
# 20.0 G - /
# 129.0 G - cgts-vg PV
# 90.5 G - unpartitioned free space
# 144.0 G - cgts-vg PV
# 75.5 G - unpartitioned free space
#
database_storage = \
constants.DEFAULT_SMALL_DATABASE_STOR_SIZE
@ -6797,7 +6797,7 @@ class ConductorManager(service.PeriodicService):
data = {
'name': constants.FILESYSTEM_NAME_DOCKER_DISTRIBUTION,
'size': constants.DEFAULT_DOCKER_DISTRIBUTION_STOR_SIZE,
'size': constants.DOCKER_DISTRIBUTION_STOR_SIZE,
'logical_volume': constants.FILESYSTEM_LV_DICT[
constants.FILESYSTEM_NAME_DOCKER_DISTRIBUTION],
'replicated': True,

View File

@ -26,7 +26,7 @@ class AodhHelm(openstack.OpenstackBaseHelm):
@property
def docker_repo_source(self):
return common.DOCKER_SRC_STX
return common.DOCKER_SRC_LOC
def get_namespaces(self):
return self.SUPPORTED_NAMESPACES

View File

@ -49,10 +49,16 @@ class BaseHelm(object):
@property
def docker_image(self):
return "{}/{}{}:{}".format(
common.DOCKER_SRCS[self.docker_repo_source][common.IMG_BASE_KEY],
common.DOCKER_SRCS[self.docker_repo_source][common.IMG_PREFIX_KEY],
self.SERVICE_NAME, self.docker_repo_tag)
if self.docker_repo_source == common.DOCKER_SRC_LOC:
return "{}:{}/{}/{}{}:{}".format(
self._get_management_address(), common.REGISTRY_PORT, common.REPO_LOC,
common.DOCKER_SRCS[self.docker_repo_source][common.IMG_PREFIX_KEY],
self.SERVICE_NAME, self.docker_repo_tag)
else:
return "{}/{}{}:{}".format(
common.DOCKER_SRCS[self.docker_repo_source][common.IMG_BASE_KEY],
common.DOCKER_SRCS[self.docker_repo_source][common.IMG_PREFIX_KEY],
self.SERVICE_NAME, self.docker_repo_tag)
@staticmethod
def quoted_str(value):

View File

@ -27,7 +27,7 @@ class CeilometerHelm(openstack.OpenstackBaseHelm):
@property
def docker_repo_source(self):
return common.DOCKER_SRC_STX
return common.DOCKER_SRC_LOC
def get_namespaces(self):
return self.SUPPORTED_NAMESPACES
@ -58,11 +58,15 @@ class CeilometerHelm(openstack.OpenstackBaseHelm):
'tags': {
'ks_service': heat_image,
'ks_user': heat_image,
'ks_endpoints': heat_image,
'db_init': self.docker_image,
'ceilometer_db_sync': self.docker_image,
'ceilometer_central': self.docker_image,
'ceilometer_compute': self.docker_image,
'ceilometer_ipmi': self.docker_image,
'ceilometer_notification': self.docker_image
'ceilometer_notification': self.docker_image,
'ceilometer_collector': self.docker_image,
'ceilometer_api': self.docker_image
}
}

View File

@ -29,7 +29,7 @@ class CinderHelm(openstack.OpenstackBaseHelm):
@property
def docker_repo_source(self):
return common.DOCKER_SRC_STX
return common.DOCKER_SRC_LOC
@property
def docker_repo_tag(self):
@ -202,8 +202,8 @@ class CinderHelm(openstack.OpenstackBaseHelm):
# TODO: Remove after ceph upgrade
# Format the name of the stx specific ceph config helper
ceph_config_helper_image = "{}/{}{}:{}".format(
common.DOCKER_SRCS[self.docker_repo_source][common.IMG_BASE_KEY],
ceph_config_helper_image = "{}:{}/{}/{}{}:{}".format(
self._get_management_address(), common.REGISTRY_PORT, common.REPO_LOC,
common.DOCKER_SRCS[self.docker_repo_source][common.IMG_PREFIX_KEY],
'ceph-config-helper', self.docker_repo_tag)

View File

@ -54,13 +54,14 @@ DOCKER_SRC_LOC = 'controller'
# TODO (rchurch): These values and settings are currently provided for early
# integration scenarios. As we formalize delivery mechanisms, these will need to
# be adjusted accordingly.
REGISTRY_PORT = '9001'
REGISTRY_OSH = 'docker.io'
REGISTRY_STX = '128.224.186.231:9001'
REGISTRY_LOC = '192.168.204.2:9001'
REGISTRY_STX = '128.224.186.231:%s' % REGISTRY_PORT
REGISTRY_LOC = '192.168.204.2:%s' % REGISTRY_PORT
REPO_OSH = 'openstackhelm'
REPO_STX = 'abailey'
REPO_LOC = 'stx'
REPO_LOC = 'starlingx'
IMG_PREFIX_KEY = 'prefix'
IMG_BASE_KEY = 'base'
@ -89,6 +90,6 @@ DOCKER_SRCS = {
DOCKER_SRC_LOC: {
IMG_BASE_KEY: '{}/{}'.format(REGISTRY_LOC, REPO_LOC),
IMG_PREFIX_KEY: IMG_PREFIX_LOC,
IMG_TAG_KEY: TAGS_LATEST
IMG_TAG_KEY: TAGS_STX_LATEST
}
}

View File

@ -31,7 +31,7 @@ class GlanceHelm(openstack.OpenstackBaseHelm):
@property
def docker_repo_source(self):
return common.DOCKER_SRC_STX
return common.DOCKER_SRC_LOC
@property
def docker_repo_tag(self):

View File

@ -26,7 +26,7 @@ class GnocchiHelm(openstack.OpenstackBaseHelm):
@property
def docker_repo_source(self):
return common.DOCKER_SRC_STX
return common.DOCKER_SRC_LOC
@property
def docker_repo_tag(self):
@ -67,6 +67,7 @@ class GnocchiHelm(openstack.OpenstackBaseHelm):
'gnocchi_api': self.docker_image,
'gnocchi_metricd': self.docker_image,
'gnocchi_resources_cleaner': self.docker_image,
'gnocchi_statsd': self.docker_image,
'ks_endpoints': heat_image,
'ks_service': heat_image,
'ks_user': heat_image,

View File

@ -26,7 +26,7 @@ class HeatHelm(openstack.OpenstackBaseHelm):
@property
def docker_repo_source(self):
return common.DOCKER_SRC_STX
return common.DOCKER_SRC_LOC
@property
def docker_repo_tag(self):

View File

@ -25,7 +25,7 @@ class HorizonHelm(openstack.OpenstackBaseHelm):
@property
def docker_repo_source(self):
return common.DOCKER_SRC_STX
return common.DOCKER_SRC_LOC
def get_namespaces(self):
return self.SUPPORTED_NAMESPACES

View File

@ -34,7 +34,7 @@ class KeystoneHelm(openstack.OpenstackBaseHelm):
@property
def docker_repo_source(self):
return common.DOCKER_SRC_STX
return common.DOCKER_SRC_LOC
@property
def docker_repo_tag(self):

View File

@ -25,7 +25,7 @@ class LibvirtHelm(openstack.OpenstackBaseHelm):
@property
def docker_repo_source(self):
return common.DOCKER_SRC_STX
return common.DOCKER_SRC_LOC
@property
def docker_repo_tag(self):

View File

@ -32,7 +32,7 @@ class NeutronHelm(openstack.OpenstackBaseHelm):
@property
def docker_repo_source(self):
return common.DOCKER_SRC_STX
return common.DOCKER_SRC_LOC
@property
def docker_repo_tag(self):

View File

@ -48,7 +48,7 @@ class NovaHelm(openstack.OpenstackBaseHelm):
@property
def docker_repo_source(self):
return common.DOCKER_SRC_STX
return common.DOCKER_SRC_LOC
@property
def docker_repo_tag(self):

View File

@ -24,6 +24,10 @@ class NovaApiProxyHelm(openstack.OpenstackBaseHelm):
SERVICE_NAME = 'nova'
AUTH_USERS = ['nova']
@property
def docker_repo_source(self):
return common.DOCKER_SRC_LOC
def get_namespaces(self):
return self.SUPPORTED_NAMESPACES
@ -45,6 +49,7 @@ class NovaApiProxyHelm(openstack.OpenstackBaseHelm):
},
}
},
'images': self._get_images_overrides(),
'endpoints': self._get_endpoints_overrides(),
}
}
@ -57,6 +62,22 @@ class NovaApiProxyHelm(openstack.OpenstackBaseHelm):
else:
return overrides
def _get_images_overrides(self):
nova_api_proxy_image = "{}:{}/{}/{}{}:{}".format(
self._get_management_address(), common.REGISTRY_PORT, common.REPO_LOC,
common.DOCKER_SRCS[self.docker_repo_source][common.IMG_PREFIX_KEY],
'nova-api-proxy', self.docker_repo_tag)
heat_image = self._operator.chart_operators[
constants.HELM_CHART_HEAT].docker_image
return {
'tags': {
'nova_api_proxy': nova_api_proxy_image,
'ks_endpoints': heat_image
}
}
def _get_endpoints_overrides(self):
return {
'identity': {

View File

@ -26,7 +26,7 @@ class PankoHelm(openstack.OpenstackBaseHelm):
@property
def docker_repo_source(self):
return common.DOCKER_SRC_STX
return common.DOCKER_SRC_LOC
def get_namespaces(self):
return self.SUPPORTED_NAMESPACES

View File

@ -28,7 +28,7 @@ class RbdProvisionerHelm(base.BaseHelm):
@property
def docker_repo_source(self):
return common.DOCKER_SRC_STX
return common.DOCKER_SRC_LOC
@property
def docker_repo_tag(self):
@ -130,17 +130,14 @@ class RbdProvisionerHelm(base.BaseHelm):
def _get_images_overrides(self):
# TODO: Remove after ceph upgrade
# Format the name of the stx specific ceph config helper
ceph_config_helper_image = "{}/{}{}:{}".format(
common.DOCKER_SRCS[self.docker_repo_source][common.IMG_BASE_KEY],
local_docker_registry_ip = self._get_management_address()
ceph_config_helper_image = "{}:{}/{}/{}{}:{}".format(
local_docker_registry_ip, common.REGISTRY_PORT, common.REPO_LOC,
common.DOCKER_SRCS[self.docker_repo_source][common.IMG_PREFIX_KEY],
'ceph-config-helper', self.docker_repo_tag)
rbd_provisioner_image = \
'quay.io/external_storage/rbd-provisioner:latest'
return {
'tags': {
'rbd_provisioner': rbd_provisioner_image,
'rbd_provisioner_storage_init': ceph_config_helper_image,
}
}