From d470867f08a2be3c30c66d9519a95eb5801cfbc9 Mon Sep 17 00:00:00 2001 From: Daniel Genin Date: Fri, 31 Oct 2014 15:01:29 -0400 Subject: [PATCH] Adds support for LVM ephemeral storage in Nova DevStack currently lacks support for LVM ephemeral storage in Nova. This support is important for testing of Nova's LVM backend. The proposed change adds a default volume group, to be shared by Cinder and Nova. It also adds a configuration option NOVA_BACKEND, which must be LVM if it is set, that determines whether Nova should be configured to use LVM ephemeral storage. Change-Id: I4eb9afff3536fbcd563939f2d325efbb845081bb --- HACKING.rst | 4 +- lib/cinder | 4 +- lib/cinder_backends/lvm | 130 +++------------------------------------- lib/lvm | 124 ++++++++++++++++++++++++++++++++++++++ lib/nova | 5 ++ stack.sh | 5 ++ unstack.sh | 3 + 7 files changed, 150 insertions(+), 125 deletions(-) create mode 100644 lib/lvm diff --git a/HACKING.rst b/HACKING.rst index 3ffe1e2103..b7d9a49458 100644 --- a/HACKING.rst +++ b/HACKING.rst @@ -110,8 +110,8 @@ follows: * Global configuration that may be referenced in ``local.conf``, i.e. ``DEST``, ``DATA_DIR`` * Global service configuration like ``ENABLED_SERVICES`` * Variables used by multiple services that do not have a clear owner, i.e. - ``VOLUME_BACKING_FILE_SIZE`` (nova-volumes and cinder) or ``PUBLIC_NETWORK_NAME`` - (nova-network and neutron) + ``VOLUME_BACKING_FILE_SIZE`` (nova-compute, nova-volumes and cinder) or + ``PUBLIC_NETWORK_NAME`` (nova-network and neutron) * Variables that can not be cleanly declared in a project file due to dependency ordering, i.e. the order of sourcing the project files can not be changed for other reasons but the earlier file needs to dereference a diff --git a/lib/cinder b/lib/cinder index 7fc694957b..f1a5de7e05 100644 --- a/lib/cinder +++ b/lib/cinder @@ -76,9 +76,9 @@ CINDER_MULTI_LVM_BACKEND=$(trueorfalse False CINDER_MULTI_LVM_BACKEND) # configuration and for the volume type name. Multiple backends are # comma-separated. if [[ $CINDER_MULTI_LVM_BACKEND == "False" ]]; then - CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1} + CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:${DEFAULT_VOLUME_GROUP_NAME##*-}} else - CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1,lvm:lvmdriver-2} + CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:${DEFAULT_VOLUME_GROUP_NAME##*-},lvm:cinder} fi diff --git a/lib/cinder_backends/lvm b/lib/cinder_backends/lvm index 280baf7b3e..43e13e4dac 100644 --- a/lib/cinder_backends/lvm +++ b/lib/cinder_backends/lvm @@ -14,6 +14,7 @@ # CINDER_CONF # DATA_DIR +# VOLUME_GROUP_NAME # clean_cinder_backend_lvm - called from clean_cinder() # configure_cinder_backend_lvm - called from configure_cinder() @@ -25,157 +26,44 @@ MY_XTRACE=$(set +o | grep xtrace) set +o xtrace -# Defaults -# -------- - -# Name of the lvm volume groups to use/create for iscsi volumes -# This monkey-motion is for compatibility with icehouse-generation Grenade -# If ``VOLUME_GROUP`` is set, use it, otherwise we'll build a VG name based -# on ``VOLUME_GROUP_NAME`` that includes the backend name -# Grenade doesn't use ``VOLUME_GROUP2`` so it is left out -VOLUME_GROUP=${VOLUME_GROUP:-} -VOLUME_GROUP_NAME=${VOLUME_GROUP:-${VOLUME_GROUP_NAME:-stack-volumes}} - # TODO: resurrect backing device...need to know how to set values #VOLUME_BACKING_DEVICE=${VOLUME_BACKING_DEVICE:-} -VOLUME_NAME_PREFIX=${VOLUME_NAME_PREFIX:-volume-} - - # Entry Points # ------------ -# Compatibility for getting a volume group name from either ``VOLUME_GROUP`` -# or from ``VOLUME_GROUP_NAME`` plus the backend name -function get_volume_group_name { - local be_name=$1 - - # Again with the icehouse-generation compatibility - local volume_group_name=$VOLUME_GROUP_NAME - if [[ -z $VOLUME_GROUP ]]; then - volume_group_name+="-$be_name" - fi - echo $volume_group_name -} - +# cleanup_cinder_backend_lvm - Delete volume group and remove backing file +# cleanup_cinder_backend_lvm $be_name function cleanup_cinder_backend_lvm { local be_name=$1 - # Again with the icehouse-generation compatibility - local volume_group_name=$(get_volume_group_name $be_name) - # Campsite rule: leave behind a volume group at least as clean as we found it - _clean_lvm_lv ${volume_group_name} $VOLUME_NAME_PREFIX - _clean_lvm_backing_file ${volume_group_name} $DATA_DIR/${volume_group_name}-backing-file + clean_lvm_volume_group $VOLUME_GROUP_NAME-$be_name } # configure_cinder_backend_lvm - Set config files, create data dirs, etc -# configure_cinder_backend_lvm $name +# configure_cinder_backend_lvm $be_name function configure_cinder_backend_lvm { local be_name=$1 - # Again with the icehouse-generation compatibility - local volume_group_name=$(get_volume_group_name $be_name) - iniset $CINDER_CONF $be_name volume_backend_name $be_name iniset $CINDER_CONF $be_name volume_driver "cinder.volume.drivers.lvm.LVMISCSIDriver" - iniset $CINDER_CONF $be_name volume_group $volume_group_name + iniset $CINDER_CONF $be_name volume_group $VOLUME_GROUP_NAME-$be_name if [[ "$CINDER_SECURE_DELETE" == "False" ]]; then iniset $CINDER_CONF $be_name volume_clear none fi } - +# init_cinder_backend_lvm - Initialize volume group +# init_cinder_backend_lvm $be_name function init_cinder_backend_lvm { local be_name=$1 - # Again with the icehouse-generation compatibility - local volume_group_name=$(get_volume_group_name $be_name) - # Start with a clean volume group - _create_cinder_volume_group ${volume_group_name} $DATA_DIR/${volume_group_name}-backing-file - - if is_fedora || is_suse; then - # service is not started by default - start_service tgtd - fi - - # Remove iscsi targets - sudo tgtadm --op show --mode target | grep $VOLUME_NAME_PREFIX | grep Target | cut -f3 -d ' ' | sudo xargs -n1 tgt-admin --delete || true - _clean_lvm_lv ${volume_group_name} $VOLUME_NAME_PREFIX + init_lvm_volume_group $VOLUME_GROUP_NAME-$be_name $VOLUME_BACKING_FILE_SIZE } - -# _clean_lvm_lv removes all cinder LVM volumes -# -# Usage: _clean_lvm_lv volume-group-name $VOLUME_NAME_PREFIX -function _clean_lvm_lv { - local vg=$1 - local lv_prefix=$2 - - # Clean out existing volumes - local lv - for lv in $(sudo lvs --noheadings -o lv_name $vg 2>/dev/null); do - # lv_prefix prefixes the LVs we want - if [[ "${lv#$lv_prefix}" != "$lv" ]]; then - sudo lvremove -f $vg/$lv - fi - done -} - -# _clean_lvm_backing_file() removes the backing file of the -# volume group used by cinder -# -# Usage: _clean_lvm_backing_file() volume-group-name backing-file-name -function _clean_lvm_backing_file { - local vg=$1 - local backing_file=$2 - - # if there is no logical volume left, it's safe to attempt a cleanup - # of the backing file - if [[ -z "$(sudo lvs --noheadings -o lv_name $vg 2>/dev/null)" ]]; then - # if the backing physical device is a loop device, it was probably setup by devstack - local vg_dev=$(sudo losetup -j $backing_file | awk -F':' '/backing-file/ { print $1}') - if [[ -n "$vg_dev" ]] && [[ -e "$vg_dev" ]]; then - sudo losetup -d $vg_dev - rm -f $backing_file - fi - fi -} - -# _create_cinder_volume_group volume-group-name backing-file-name -function _create_cinder_volume_group { - # According to the ``CINDER_MULTI_LVM_BACKEND`` value, configure one or two default volumes - # group called ``stack-volumes`` (and ``stack-volumes2``) for the volume - # service if it (they) does (do) not yet exist. If you don't wish to use a - # file backed volume group, create your own volume group called ``stack-volumes`` - # and ``stack-volumes2`` before invoking ``stack.sh``. - # - # The two backing files are ``VOLUME_BACKING_FILE_SIZE`` in size, and they are stored in - # the ``DATA_DIR``. - - local vg_name=$1 - local backing_file=$2 - - if ! sudo vgs $vg_name; then - # TODO: fix device handling - if [ -z "$VOLUME_BACKING_DEVICE" ]; then - # Only create if the file doesn't already exists - [[ -f $backing_file ]] || truncate -s $VOLUME_BACKING_FILE_SIZE $backing_file - local vg_dev=`sudo losetup -f --show $backing_file` - - # Only create if the loopback device doesn't contain $VOLUME_GROUP - if ! sudo vgs $vg_name; then - sudo vgcreate $vg_name $vg_dev - fi - else - sudo vgcreate $vg_name $VOLUME_BACKING_DEVICE - fi - fi -} - - # Restore xtrace $MY_XTRACE diff --git a/lib/lvm b/lib/lvm new file mode 100644 index 0000000000..4ef260d593 --- /dev/null +++ b/lib/lvm @@ -0,0 +1,124 @@ +# lib/lvm +# Configure the default LVM volume group used by Cinder and Nova + +# Dependencies: +# +# - ``functions`` file +# - ``cinder`` configurations + +# DATA_DIR + +# clean_default_volume_group - called from clean() +# configure_default_volume_group - called from configure() +# init_default_volume_group - called from init() + + +# Save trace setting +MY_XTRACE=$(set +o | grep xtrace) +set +o xtrace + + +# Defaults +# -------- +# Name of the lvm volume groups to use/create for iscsi volumes +# This monkey-motion is for compatibility with icehouse-generation Grenade +# If ``VOLUME_GROUP`` is set, use it, otherwise we'll build a VG name based +# on ``VOLUME_GROUP_NAME`` that includes the backend name +# Grenade doesn't use ``VOLUME_GROUP2`` so it is left out +VOLUME_GROUP_NAME=${VOLUME_GROUP:-${VOLUME_GROUP_NAME:-stack-volumes}} +DEFAULT_VOLUME_GROUP_NAME=$VOLUME_GROUP_NAME-default + +# Backing file name is of the form $VOLUME_GROUP$BACKING_FILE_SUFFIX +BACKING_FILE_SUFFIX=-backing-file + + +# Entry Points +# ------------ + +# _clean_lvm_volume_group removes all default LVM volumes +# +# Usage: clean_lvm_volume_group $vg +function _clean_lvm_volume_group { + local vg=$1 + + # Clean out existing volumes + sudo lvremove -f $vg +} + +# _clean_lvm_backing_file() removes the backing file of the +# volume group +# +# Usage: _clean_lvm_backing_file() $backing_file +function _clean_lvm_backing_file { + local backing_file=$1 + + # if the backing physical device is a loop device, it was probably setup by devstack + if [[ -n "$backing_file" ]] && [[ -e "$backing_file" ]]; then + local vg_dev=$(sudo losetup -j $backing_file | awk -F':' '/'$BACKING_FILE_SUFFIX'/ { print $1}') + sudo losetup -d $vg_dev + rm -f $backing_file + fi +} + +# clean_lvm_volume_group() cleans up the volume group and removes the +# backing file +# +# Usage: clean_lvm_volume_group $vg +function clean_lvm_volume_group { + local vg=$1 + + _clean_lvm_volume_group $vg + # if there is no logical volume left, it's safe to attempt a cleanup + # of the backing file + if [[ -z "$(sudo lvs --noheadings -o lv_name $vg 2>/dev/null)" ]]; then + _clean_lvm_backing_file $DATA_DIR/$vg$BACKING_FILE_SUFFIX + fi +} + + +# _create_volume_group creates default volume group +# +# Usage: _create_lvm_volume_group() $vg $size +function _create_lvm_volume_group { + local vg=$1 + local size=$2 + + local backing_file=$DATA_DIR/$vg$BACKING_FILE_SUFFIX + if ! sudo vgs $vg; then + # Only create if the file doesn't already exists + [[ -f $DATA_DIR/$backing_file ]] || truncate -s $size $backing_file + local vg_dev=`sudo losetup -f --show $backing_file` + + # Only create volume group if it doesn't already exist + if ! sudo vgs $vg; then + sudo vgcreate $vg $vg_dev + fi + fi +} + +# init_lvm_volume_group() initializes the volume group creating the backing +# file if necessary +# +# Usage: init_lvm_volume_group() $vg +function init_lvm_volume_group { + local vg=$1 + local size=$2 + # Start with a clean volume group + _create_lvm_volume_group $vg $size + + if is_fedora || is_suse; then + # service is not started by default + start_service tgtd + fi + + # Remove iscsi targets + sudo tgtadm --op show --mode target | grep Target | cut -f3 -d ' ' | sudo xargs -n1 tgt-admin --delete || true + + _clean_lvm_volume_group $vg +} + +# Restore xtrace +$MY_XTRACE + +# mode: shell-script +# End: diff --git a/lib/nova b/lib/nova index 0e9282bb1b..61216ff9d6 100644 --- a/lib/nova +++ b/lib/nova @@ -547,6 +547,11 @@ function create_nova_conf { iniset $NOVA_CONF DEFAULT ec2_workers "$API_WORKERS" iniset $NOVA_CONF DEFAULT metadata_workers "$API_WORKERS" + if [[ "$NOVA_BACKEND" == "LVM" ]]; then + iniset $NOVA_CONF libvirt images_type "lvm" + iniset $NOVA_CONF libvirt images_volume_group $DEFAULT_VOLUME_GROUP_NAME + fi + if is_ssl_enabled_service glance || is_service_enabled tls-proxy; then iniset $NOVA_CONF DEFAULT glance_protocol https fi diff --git a/stack.sh b/stack.sh index 585d1ceaf6..45113b50d2 100755 --- a/stack.sh +++ b/stack.sh @@ -523,6 +523,7 @@ source $TOP_DIR/lib/tls source $TOP_DIR/lib/infra source $TOP_DIR/lib/oslo source $TOP_DIR/lib/stackforge +source $TOP_DIR/lib/lvm source $TOP_DIR/lib/horizon source $TOP_DIR/lib/keystone source $TOP_DIR/lib/glance @@ -939,6 +940,10 @@ init_service_check # A better kind of sysstat, with the top process per time slice start_dstat +# Initialize default LVM volume group +# ----------------------------------- +init_lvm_volume_group $DEFAULT_VOLUME_GROUP_NAME $VOLUME_BACKING_FILE_SIZE + # Start Services # ============== diff --git a/unstack.sh b/unstack.sh index bff01d1ca4..b8b7f4a130 100755 --- a/unstack.sh +++ b/unstack.sh @@ -55,6 +55,7 @@ source $TOP_DIR/lib/tls source $TOP_DIR/lib/infra source $TOP_DIR/lib/oslo source $TOP_DIR/lib/stackforge +source $TOP_DIR/lib/lvm source $TOP_DIR/lib/horizon source $TOP_DIR/lib/keystone source $TOP_DIR/lib/glance @@ -177,3 +178,5 @@ if [[ -n "$SCREEN" ]]; then screen -X -S $SESSION quit fi fi + +clean_lvm_volume_group $DEFAULT_VOLUME_GROUP_NAME