Support multiple Cinder backend types

This is the first step in supporting multiple Cinder backend types at
once.  It initially converts the existing hard-coded multi-lvm support
to a new cinder_backends driver form.  Eventually the cinder_plugins
will be converted to this form so they can be enabled more than just
one at a time using CINDER_ENABLED_BACKENDS.

The default configuration should be identical to the previous defaults,
including for both True and False values of CINDER_MULTI_LVM_BACKEND.

The existing cinder_plugins are expected to be removed when this is
complete. They should continue to work until they have been converted.

Add wait for c-api to ensure it is started before continuing.

Change-Id: I93b8ef32832269d730c76a6dc24ddb4f20c6d9df
This commit is contained in:
Dean Troyer 2014-07-03 10:46:57 -05:00 committed by Sébastien Han
parent 0ae54daa4a
commit 097183356e
5 changed files with 331 additions and 141 deletions

View File

@ -28,6 +28,7 @@ set +o xtrace
# set up default driver
CINDER_DRIVER=${CINDER_DRIVER:-default}
CINDER_PLUGINS=$TOP_DIR/lib/cinder_plugins
CINDER_BACKENDS=$TOP_DIR/lib/cinder_backends
# grab plugin config if specified via cinder_driver
if [[ -r $CINDER_PLUGINS/$CINDER_DRIVER ]]; then
@ -57,9 +58,24 @@ else
CINDER_BIN_DIR=$(get_python_exec_prefix)
fi
# Maintain this here for backward-compatibility with the old configuration
# DEPRECATED: Use CINDER_ENABLED_BACKENDS instead
# Support for multi lvm backend configuration (default is no support)
CINDER_MULTI_LVM_BACKEND=$(trueorfalse False $CINDER_MULTI_LVM_BACKEND)
# Default backends
# The backend format is type:name where type is one of the supported backend
# types (lvm, nfs, etc) and name is the identifier used in the Cinder
# configuration and for the volume type name. Multiple backends are
# comma-separated.
if [[ $CINDER_MULTI_LVM_BACKEND == "False" ]]; then
CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1}
else
CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1,lvm:lvmdriver-2}
fi
# Should cinder perform secure deletion of volumes?
# Defaults to true, can be set to False to avoid this bug when testing:
# https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1023755
@ -73,22 +89,22 @@ CINDER_SECURE_DELETE=`trueorfalse True $CINDER_SECURE_DELETE`
# https://bugs.launchpad.net/cinder/+bug/1180976
CINDER_PERIODIC_INTERVAL=${CINDER_PERIODIC_INTERVAL:-60}
# Name of the lvm volume groups to use/create for iscsi volumes
VOLUME_GROUP=${VOLUME_GROUP:-stack-volumes}
VOLUME_BACKING_FILE=${VOLUME_BACKING_FILE:-$DATA_DIR/${VOLUME_GROUP}-backing-file}
VOLUME_BACKING_DEVICE=${VOLUME_BACKING_DEVICE:-}
# VOLUME_GROUP2 is used only if CINDER_MULTI_LVM_BACKEND = True
VOLUME_GROUP2=${VOLUME_GROUP2:-stack-volumes2}
VOLUME_BACKING_FILE2=${VOLUME_BACKING_FILE2:-$DATA_DIR/${VOLUME_GROUP2}-backing-file}
VOLUME_BACKING_DEVICE2=${VOLUME_BACKING_DEVICE2:-}
VOLUME_NAME_PREFIX=${VOLUME_NAME_PREFIX:-volume-}
# Tell Tempest this project is present
TEMPEST_SERVICES+=,cinder
# Source the enabled backends
if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
for be in ${CINDER_ENABLED_BACKENDS//,/ }; do
BE_TYPE=${be%%:*}
BE_NAME=${be##*:}
if [[ -r $CINDER_BACKENDS/${BE_TYPE} ]]; then
source $CINDER_BACKENDS/${BE_TYPE}
fi
done
fi
# Functions
# ---------
@ -99,41 +115,6 @@ function is_cinder_enabled {
return 1
}
# _clean_lvm_lv removes all cinder LVM volumes
#
# Usage: _clean_lvm_lv $VOLUME_GROUP $VOLUME_NAME_PREFIX
function _clean_lvm_lv {
local vg=$1
local lv_prefix=$2
# Clean out existing volumes
for lv in `sudo lvs --noheadings -o lv_name $vg`; do
# lv_prefix prefixes the LVs we want
if [[ "${lv#$lv_prefix}" != "$lv" ]]; then
sudo lvremove -f $vg/$lv
fi
done
}
# _clean_lvm_backing_file() removes the backing file of the
# volume group used by cinder
#
# Usage: _clean_lvm_backing_file() $VOLUME_GROUP
function _clean_lvm_backing_file {
local vg=$1
# if there is no logical volume left, it's safe to attempt a cleanup
# of the backing file
if [ -z "`sudo lvs --noheadings -o lv_name $vg`" ]; then
# if the backing physical device is a loop device, it was probably setup by devstack
if [[ -n "$VG_DEV" ]] && [[ -e "$VG_DEV" ]]; then
VG_DEV=$(sudo losetup -j $DATA_DIR/${vg}-backing-file | awk -F':' '/backing-file/ { print $1}')
sudo losetup -d $VG_DEV
rm -f $DATA_DIR/${vg}-backing-file
fi
fi
}
# cleanup_cinder() - Remove residual data files, anything left over from previous
# runs that a clean run would need to clean up
function cleanup_cinder {
@ -160,23 +141,20 @@ function cleanup_cinder {
done
fi
if is_service_enabled cinder; then
sudo rm -rf $CINDER_STATE_PATH/volumes/*
fi
if is_ubuntu; then
stop_service tgt
else
stop_service tgtd
fi
# Campsite rule: leave behind a volume group at least as clean as we found it
_clean_lvm_lv $VOLUME_GROUP $VOLUME_NAME_PREFIX
_clean_lvm_backing_file $VOLUME_GROUP
if [ "$CINDER_MULTI_LVM_BACKEND" = "True" ]; then
_clean_lvm_lv $VOLUME_GROUP2 $VOLUME_NAME_PREFIX
_clean_lvm_backing_file $VOLUME_GROUP2
if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
for be in ${CINDER_ENABLED_BACKENDS//,/ }; do
BE_TYPE=${be%%:*}
BE_NAME=${be##*:}
if type cleanup_cinder_backend_${BE_TYPE} >/dev/null 2>&1; then
cleanup_cinder_backend_${BE_TYPE} ${BE_NAME}
fi
done
fi
}
@ -243,23 +221,7 @@ function configure_cinder {
iniset $CINDER_CONF DEFAULT auth_strategy keystone
iniset $CINDER_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
iniset $CINDER_CONF DEFAULT verbose True
if [ "$CINDER_MULTI_LVM_BACKEND" = "True" ]; then
iniset $CINDER_CONF DEFAULT enabled_backends lvmdriver-1,lvmdriver-2
iniset $CINDER_CONF lvmdriver-1 volume_group $VOLUME_GROUP
iniset $CINDER_CONF lvmdriver-1 volume_driver cinder.volume.drivers.lvm.LVMISCSIDriver
iniset $CINDER_CONF lvmdriver-1 volume_backend_name LVM_iSCSI
iniset $CINDER_CONF lvmdriver-2 volume_group $VOLUME_GROUP2
iniset $CINDER_CONF lvmdriver-2 volume_driver cinder.volume.drivers.lvm.LVMISCSIDriver
iniset $CINDER_CONF lvmdriver-2 volume_backend_name LVM_iSCSI_2
# NOTE(mriedem): Work around Cinder "wishlist" bug 1255593
if [[ "$CINDER_SECURE_DELETE" == "False" ]]; then
iniset $CINDER_CONF lvmdriver-1 volume_clear none
iniset $CINDER_CONF lvmdriver-2 volume_clear none
fi
else
iniset $CINDER_CONF DEFAULT volume_group $VOLUME_GROUP
iniset $CINDER_CONF DEFAULT volume_name_template ${VOLUME_NAME_PREFIX}%s
fi
iniset $CINDER_CONF DEFAULT my_ip "$CINDER_SERVICE_HOST"
iniset $CINDER_CONF DEFAULT iscsi_helper tgtadm
iniset $CINDER_CONF DEFAULT sql_connection `database_connection_url cinder`
@ -274,6 +236,26 @@ function configure_cinder {
# supported.
iniset $CINDER_CONF DEFAULT enable_v1_api true
if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
enabled_backends=""
default_type=""
for be in ${CINDER_ENABLED_BACKENDS//,/ }; do
BE_TYPE=${be%%:*}
BE_NAME=${be##*:}
if type configure_cinder_backend_${BE_TYPE} >/dev/null 2>&1; then
configure_cinder_backend_${BE_TYPE} ${BE_NAME}
fi
if [[ -z "$default_type" ]]; then
default_type=$BE_TYPE}
fi
enabled_backends+=$BE_NAME,
done
iniset $CINDER_CONF DEFAULT enabled_backends ${enabled_backends%,*}
if [[ -n "$default_type" ]]; then
iniset $CINDER_CONF DEFAULT default_volume_type ${enabled_backends%,*}
fi
fi
if is_service_enabled swift; then
iniset $CINDER_CONF DEFAULT backup_swift_url "http://$SERVICE_HOST:8080/v1/AUTH_"
fi
@ -371,53 +353,6 @@ function create_cinder_cache_dir {
rm -f $CINDER_AUTH_CACHE_DIR/*
}
function create_cinder_volume_group {
# According to the ``CINDER_MULTI_LVM_BACKEND`` value, configure one or two default volumes
# group called ``stack-volumes`` (and ``stack-volumes2``) for the volume
# service if it (they) does (do) not yet exist. If you don't wish to use a
# file backed volume group, create your own volume group called ``stack-volumes``
# and ``stack-volumes2`` before invoking ``stack.sh``.
#
# The two backing files are ``VOLUME_BACKING_FILE_SIZE`` in size, and they are stored in
# the ``DATA_DIR``.
if ! sudo vgs $VOLUME_GROUP; then
if [ -z "$VOLUME_BACKING_DEVICE" ]; then
# Only create if the file doesn't already exists
[[ -f $VOLUME_BACKING_FILE ]] || truncate -s $VOLUME_BACKING_FILE_SIZE $VOLUME_BACKING_FILE
DEV=`sudo losetup -f --show $VOLUME_BACKING_FILE`
# Only create if the loopback device doesn't contain $VOLUME_GROUP
if ! sudo vgs $VOLUME_GROUP; then
sudo vgcreate $VOLUME_GROUP $DEV
fi
else
sudo vgcreate $VOLUME_GROUP $VOLUME_BACKING_DEVICE
fi
fi
if [ "$CINDER_MULTI_LVM_BACKEND" = "True" ]; then
#set up the second volume if CINDER_MULTI_LVM_BACKEND is enabled
if ! sudo vgs $VOLUME_GROUP2; then
if [ -z "$VOLUME_BACKING_DEVICE2" ]; then
# Only create if the file doesn't already exists
[[ -f $VOLUME_BACKING_FILE2 ]] || truncate -s $VOLUME_BACKING_FILE_SIZE $VOLUME_BACKING_FILE2
DEV=`sudo losetup -f --show $VOLUME_BACKING_FILE2`
# Only create if the loopback device doesn't contain $VOLUME_GROUP
if ! sudo vgs $VOLUME_GROUP2; then
sudo vgcreate $VOLUME_GROUP2 $DEV
fi
else
sudo vgcreate $VOLUME_GROUP2 $VOLUME_BACKING_DEVICE2
fi
fi
fi
mkdir -p $CINDER_STATE_PATH/volumes
}
# init_cinder() - Initialize database and volume group
function init_cinder {
# Force nova volumes off
@ -431,26 +366,17 @@ function init_cinder {
$CINDER_BIN_DIR/cinder-manage db sync
fi
if is_service_enabled c-vol; then
create_cinder_volume_group
if sudo vgs $VOLUME_GROUP; then
if is_fedora || is_suse; then
# service is not started by default
start_service tgtd
if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
for be in ${CINDER_ENABLED_BACKENDS//,/ }; do
BE_TYPE=${be%%:*}
BE_NAME=${be##*:}
if type init_cinder_backend_${BE_TYPE} >/dev/null 2>&1; then
init_cinder_backend_${BE_TYPE} ${BE_NAME}
fi
# Remove iscsi targets
sudo tgtadm --op show --mode target | grep $VOLUME_NAME_PREFIX | grep Target | cut -f3 -d ' ' | sudo xargs -n1 tgt-admin --delete || true
# Start with a clean volume group
_clean_lvm_lv $VOLUME_GROUP $VOLUME_NAME_PREFIX
if [ "$CINDER_MULTI_LVM_BACKEND" = "True" ]; then
_clean_lvm_lv $VOLUME_GROUP2 $VOLUME_NAME_PREFIX
fi
fi
done
fi
mkdir -p $CINDER_STATE_PATH/volumes
create_cinder_cache_dir
}
@ -502,6 +428,11 @@ function start_cinder {
fi
screen_it c-api "cd $CINDER_DIR && $CINDER_BIN_DIR/cinder-api --config-file $CINDER_CONF"
echo "Waiting for Cinder API to start..."
if ! wait_for_service $SERVICE_TIMEOUT $CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT; then
die $LINENO "c-api did not start"
fi
screen_it c-sch "cd $CINDER_DIR && $CINDER_BIN_DIR/cinder-scheduler --config-file $CINDER_CONF"
screen_it c-bak "cd $CINDER_DIR && $CINDER_BIN_DIR/cinder-backup --config-file $CINDER_CONF"
screen_it c-vol "cd $CINDER_DIR && $CINDER_BIN_DIR/cinder-volume --config-file $CINDER_CONF"
@ -532,6 +463,30 @@ function stop_cinder {
fi
}
# create_volume_types() - Create Cinder's configured volume types
function create_volume_types {
# Create volume types
if is_service_enabled c-api && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then
for be in ${CINDER_ENABLED_BACKENDS//,/ }; do
BE_TYPE=${be%%:*}
BE_NAME=${be##*:}
if type configure_cinder_backend_${BE_TYPE} >/dev/null 2>&1; then
# openstack volume type create --property volume_backend_name="${BE_TYPE}" ${BE_NAME}
cinder type-create ${BE_NAME} && \
cinder type-key ${BE_NAME} set volume_backend_name="${BE_NAME}"
fi
done
fi
}
# Compatibility for Grenade
function create_cinder_volume_group {
# During a transition period Grenade needs to have this function defined
# It is effectively a no-op in the Grenade 'target' use case
:
}
# Restore xtrace
$XTRACE

179
lib/cinder_backends/lvm Normal file
View File

@ -0,0 +1,179 @@
# lib/cinder_backends/lvm
# Configure the LVM backend
# Enable with:
#
# CINDER_ENABLED_BACKENDS+=,lvm:lvmname
# Dependencies:
#
# - ``functions`` file
# - ``cinder`` configurations
# CINDER_CONF
# DATA_DIR
# clean_cinder_backend_lvm - called from clean_cinder()
# configure_cinder_backend_lvm - called from configure_cinder()
# init_cinder_backend_lvm - called from init_cinder()
# Save trace setting
MY_XTRACE=$(set +o | grep xtrace)
set +o xtrace
# Defaults
# --------
# Name of the lvm volume groups to use/create for iscsi volumes
# This monkey-motion is for compatibility with icehouse-generation Grenade
# If ``VOLUME_GROUP`` is set, use it, otherwise we'll build a VG name based
# on ``VOLUME_GROUP_NAME`` that includes the backend name
# Grenade doesn't use ``VOLUME_GROUP2`` so it is left out
VOLUME_GROUP_NAME=${VOLUME_GROUP:-${VOLUME_GROUP_NAME:-stack-volumes}}
# TODO: resurrect backing device...need to know how to set values
#VOLUME_BACKING_DEVICE=${VOLUME_BACKING_DEVICE:-}
VOLUME_NAME_PREFIX=${VOLUME_NAME_PREFIX:-volume-}
# Entry Points
# ------------
# Compatibility for getting a volume group name from either ``VOLUME_GROUP``
# or from ``VOLUME_GROUP_NAME`` plus the backend name
function get_volume_group_name {
local be_name=$1
# Again with the icehouse-generation compatibility
local volume_group_name=$VOLUME_GROUP_NAME
if [[ -z $VOLUME_GROUP ]]; then
volume_group_name+="-$be_name"
fi
echo $volume_group_name
}
function cleanup_cinder_backend_lvm {
local be_name=$1
# Again with the icehouse-generation compatibility
local volume_group_name=$(get_volume_group_name $be_name)
# Campsite rule: leave behind a volume group at least as clean as we found it
_clean_lvm_lv ${volume_group_name} $VOLUME_NAME_PREFIX
_clean_lvm_backing_file ${volume_group_name} $DATA_DIR/${volume_group_name}-backing-file
}
# configure_cinder_backend_lvm - Set config files, create data dirs, etc
# configure_cinder_backend_lvm $name
function configure_cinder_backend_lvm {
local be_name=$1
# Again with the icehouse-generation compatibility
local volume_group_name=$(get_volume_group_name $be_name)
iniset $CINDER_CONF $be_name volume_backend_name $be_name
iniset $CINDER_CONF $be_name volume_driver "cinder.volume.drivers.lvm.LVMISCSIDriver"
iniset $CINDER_CONF $be_name volume_group $volume_group_name
if [[ "$CINDER_SECURE_DELETE" == "False" ]]; then
iniset $CINDER_CONF $be_name volume_clear none
fi
}
function init_cinder_backend_lvm {
local be_name=$1
# Again with the icehouse-generation compatibility
local volume_group_name=$(get_volume_group_name $be_name)
# Start with a clean volume group
_create_cinder_volume_group ${volume_group_name} $DATA_DIR/${volume_group_name}-backing-file
if is_fedora || is_suse; then
# service is not started by default
start_service tgtd
fi
# Remove iscsi targets
sudo tgtadm --op show --mode target | grep $VOLUME_NAME_PREFIX | grep Target | cut -f3 -d ' ' | sudo xargs -n1 tgt-admin --delete || true
_clean_lvm_lv ${volume_group_name} $VOLUME_NAME_PREFIX
}
# _clean_lvm_lv removes all cinder LVM volumes
#
# Usage: _clean_lvm_lv volume-group-name $VOLUME_NAME_PREFIX
function _clean_lvm_lv {
local vg=$1
local lv_prefix=$2
# Clean out existing volumes
for lv in $(sudo lvs --noheadings -o lv_name $vg 2>/dev/null); do
# lv_prefix prefixes the LVs we want
if [[ "${lv#$lv_prefix}" != "$lv" ]]; then
sudo lvremove -f $vg/$lv
fi
done
}
# _clean_lvm_backing_file() removes the backing file of the
# volume group used by cinder
#
# Usage: _clean_lvm_backing_file() volume-group-name backing-file-name
function _clean_lvm_backing_file {
local vg=$1
local backing_file=$2
# if there is no logical volume left, it's safe to attempt a cleanup
# of the backing file
if [[ -z "$(sudo lvs --noheadings -o lv_name $vg 2>/dev/null)" ]]; then
# if the backing physical device is a loop device, it was probably setup by devstack
VG_DEV=$(sudo losetup -j $backing_file | awk -F':' '/backing-file/ { print $1}')
if [[ -n "$VG_DEV" ]] && [[ -e "$VG_DEV" ]]; then
sudo losetup -d $VG_DEV
rm -f $backing_file
fi
fi
}
# _create_cinder_volume_group volume-group-name backing-file-name
function _create_cinder_volume_group {
# According to the ``CINDER_MULTI_LVM_BACKEND`` value, configure one or two default volumes
# group called ``stack-volumes`` (and ``stack-volumes2``) for the volume
# service if it (they) does (do) not yet exist. If you don't wish to use a
# file backed volume group, create your own volume group called ``stack-volumes``
# and ``stack-volumes2`` before invoking ``stack.sh``.
#
# The two backing files are ``VOLUME_BACKING_FILE_SIZE`` in size, and they are stored in
# the ``DATA_DIR``.
local vg_name=$1
local backing_file=$2
if ! sudo vgs $vg_name; then
# TODO: fix device handling
if [ -z "$VOLUME_BACKING_DEVICE" ]; then
# Only create if the file doesn't already exists
[[ -f $backing_file ]] || truncate -s $VOLUME_BACKING_FILE_SIZE $backing_file
DEV=`sudo losetup -f --show $backing_file`
# Only create if the loopback device doesn't contain $VOLUME_GROUP
if ! sudo vgs $vg_name; then
sudo vgcreate $vg_name $DEV
fi
else
sudo vgcreate $vg_name $VOLUME_BACKING_DEVICE
fi
fi
}
# Restore xtrace
$MY_XTRACE
# mode: shell-script
# End:

43
lib/cinder_backends/nfs Normal file
View File

@ -0,0 +1,43 @@
# lib/cinder_backends/nfs
# Configure the nfs backend
# Enable with:
#
# CINDER_ENABLED_BACKENDS+=,nfs:<volume-type-name>
# Dependencies:
#
# - ``functions`` file
# - ``cinder`` configurations
# CINDER_CONF
# CINDER_CONF_DIR
# CINDER_NFS_SERVERPATH - contents of nfs shares config file
# configure_cinder_backend_nfs - Configure Cinder for NFS backends
# Save trace setting
NFS_XTRACE=$(set +o | grep xtrace)
set +o xtrace
# Entry Points
# ------------
# configure_cinder_backend_nfs - Set config files, create data dirs, etc
function configure_cinder_backend_nfs {
local be_name=$1
iniset $CINDER_CONF $be_name volume_backend_name $be_name
iniset $CINDER_CONF $be_name volume_driver "cinder.volume.drivers.nfs.NfsDriver"
iniset $CINDER_CONF $be_name nfs_shares_config "$CINDER_CONF_DIR/nfs-shares-$be_name.conf"
echo "$CINDER_NFS_SERVERPATH" | tee "$CINDER_CONF_DIR/nfs-shares-$be_name.conf"
}
# Restore xtrace
$NFS_XTRACE
# Local variables:
# mode: shell-script
# End:

View File

@ -1245,6 +1245,7 @@ fi
if is_service_enabled cinder; then
echo_summary "Starting Cinder"
start_cinder
create_volume_types
fi
if is_service_enabled ceilometer; then
echo_summary "Starting Ceilometer"
@ -1501,6 +1502,19 @@ if [[ -n "$Q_SRV_EXTRA_DEFAULT_OPTS" ]]; then
done
fi
# TODO(dtroyer): Remove CINDER_MULTI_LVM_BACKEND after stable/juno branch is cut
if [[ "$CINDER_MULTI_LVM_BACKEND" = "True" ]]; then
echo ""
echo_summary "WARNING: CINDER_MULTI_LVM_BACKEND is used"
echo "You are using CINDER_MULTI_LVM_BACKEND to configure Cinder's multiple LVM backends"
echo "Please convert that configuration in local.conf to use CINDER_ENABLED_BACKENDS."
echo "CINDER_ENABLED_BACKENDS will be removed early in the 'K' development cycle"
echo "
[[local|localrc]]
CINDER_ENABLED_BACKENDS=lvm:lvmdriver-1,lvm:lvmdriver-2
"
fi
# Indicate how long this took to run (bash maintained variable ``SECONDS``)
echo_summary "stack.sh completed in $SECONDS seconds."

View File

@ -407,8 +407,7 @@ fi
# 10Gb default volume backing file size
VOLUME_BACKING_FILE_SIZE=${VOLUME_BACKING_FILE_SIZE:-10250M}
# Name of the LVM volume group to use/create for iscsi volumes
VOLUME_GROUP=${VOLUME_GROUP:-stack-volumes}
# Prefixes for volume and instance names
VOLUME_NAME_PREFIX=${VOLUME_NAME_PREFIX:-volume-}
INSTANCE_NAME_PREFIX=${INSTANCE_NAME_PREFIX:-instance-}