devstack/lib/ceph
Matt Riedemann e446fc3f59 Set ATTACH_ENCRYPTED_VOLUME_AVAILABLE=False if testing with Ceph
The encrypted Cinder volume tests in Tempest don't actually work
properly for a Ceph backend in cinder since the volume encryption
support is not in Nova for RBD volume types.

This is needed for Cinder change
I03f8cae05cc117e14f7482115de685fc9f3fa54a which tells Nova that the rbd
volume connection is on an encrypted volume type.

Related-Bug: #1463525

Change-Id: I8548d41095513b9e669f773e3f35353e9228ead9
2015-07-08 12:10:15 -07:00

365 lines
14 KiB
Bash

#!/bin/bash
#
# lib/ceph
# Functions to control the configuration and operation of the **Ceph** storage service
# Dependencies:
#
# - ``functions`` file
# - ``CEPH_DATA_DIR`` or ``DATA_DIR`` must be defined
# ``stack.sh`` calls the entry points in this order (via ``extras.d/60-ceph.sh``):
#
# - install_ceph
# - configure_ceph
# - init_ceph
# - start_ceph
# - stop_ceph
# - cleanup_ceph
# Save trace setting
XTRACE=$(set +o | grep xtrace)
set +o xtrace
# Defaults
# --------
# Set ``CEPH_DATA_DIR`` to the location of Ceph drives and objects.
# Default is the common DevStack data directory.
CEPH_DATA_DIR=${CEPH_DATA_DIR:-/var/lib/ceph}
CEPH_DISK_IMAGE=${CEPH_DATA_DIR}/drives/images/ceph.img
# Set ``CEPH_CONF_DIR`` to the location of the configuration files.
# Default is ``/etc/ceph``.
CEPH_CONF_DIR=${CEPH_CONF_DIR:-/etc/ceph}
# DevStack will create a loop-back disk formatted as XFS to store the
# Ceph data. Set ``CEPH_LOOPBACK_DISK_SIZE`` to the disk size in
# kilobytes.
# Default is 1 gigabyte.
CEPH_LOOPBACK_DISK_SIZE_DEFAULT=4G
CEPH_LOOPBACK_DISK_SIZE=${CEPH_LOOPBACK_DISK_SIZE:-$CEPH_LOOPBACK_DISK_SIZE_DEFAULT}
# Common
CEPH_FSID=$(uuidgen)
CEPH_CONF_FILE=${CEPH_CONF_DIR}/ceph.conf
# Glance
GLANCE_CEPH_USER=${GLANCE_CEPH_USER:-glance}
GLANCE_CEPH_POOL=${GLANCE_CEPH_POOL:-images}
GLANCE_CEPH_POOL_PG=${GLANCE_CEPH_POOL_PG:-8}
GLANCE_CEPH_POOL_PGP=${GLANCE_CEPH_POOL_PGP:-8}
# Nova
NOVA_CEPH_POOL=${NOVA_CEPH_POOL:-vms}
NOVA_CEPH_POOL_PG=${NOVA_CEPH_POOL_PG:-8}
NOVA_CEPH_POOL_PGP=${NOVA_CEPH_POOL_PGP:-8}
# Cinder
CINDER_CEPH_POOL=${CINDER_CEPH_POOL:-volumes}
CINDER_CEPH_POOL_PG=${CINDER_CEPH_POOL_PG:-8}
CINDER_CEPH_POOL_PGP=${CINDER_CEPH_POOL_PGP:-8}
CINDER_CEPH_USER=${CINDER_CEPH_USER:-cinder}
CINDER_CEPH_UUID=${CINDER_CEPH_UUID:-$(uuidgen)}
# Set ``CEPH_REPLICAS`` to configure how many replicas are to be
# configured for your Ceph cluster. By default we are configuring
# only one replica since this is way less CPU and memory intensive. If
# you are planning to test Ceph replication feel free to increase this value
CEPH_REPLICAS=${CEPH_REPLICAS:-1}
CEPH_REPLICAS_SEQ=$(seq ${CEPH_REPLICAS})
# Connect to an existing Ceph cluster
REMOTE_CEPH=$(trueorfalse False REMOTE_CEPH)
REMOTE_CEPH_ADMIN_KEY_PATH=${REMOTE_CEPH_ADMIN_KEY_PATH:-$CEPH_CONF_DIR/ceph.client.admin.keyring}
# Cinder encrypted volume tests are not supported with a Ceph backend due to
# bug 1463525.
ATTACH_ENCRYPTED_VOLUME_AVAILABLE=False
# Functions
# ------------
function get_ceph_version {
local ceph_version_str=$(sudo ceph daemon mon.$(hostname) version | cut -d '"' -f 4 | cut -f 1,2 -d '.')
echo $ceph_version_str
}
# import_libvirt_secret_ceph() - Imports Cinder user key into libvirt
# so it can connect to the Ceph cluster while attaching a Cinder block device
function import_libvirt_secret_ceph {
cat > secret.xml <<EOF
<secret ephemeral='no' private='no'>
<uuid>${CINDER_CEPH_UUID}</uuid>
<usage type='ceph'>
<name>client.${CINDER_CEPH_USER} secret</name>
</usage>
</secret>
EOF
sudo virsh secret-define --file secret.xml
sudo virsh secret-set-value --secret ${CINDER_CEPH_UUID} --base64 $(sudo ceph -c ${CEPH_CONF_FILE} auth get-key client.${CINDER_CEPH_USER})
sudo rm -f secret.xml
}
# undefine_virsh_secret() - Undefine Cinder key secret from libvirt
function undefine_virsh_secret {
if is_service_enabled cinder || is_service_enabled nova; then
local virsh_uuid=$(sudo virsh secret-list | awk '/^ ?[0-9a-z]/ { print $1 }')
sudo virsh secret-undefine ${virsh_uuid} >/dev/null 2>&1
fi
}
# check_os_support_ceph() - Check if the operating system provides a decent version of Ceph
function check_os_support_ceph {
if [[ ! ${DISTRO} =~ (trusty|f21|f22) ]]; then
echo "WARNING: your distro $DISTRO does not provide (at least) the Firefly release. Please use Ubuntu Trusty or Fedora 20 (and higher)"
if [[ "$FORCE_CEPH_INSTALL" != "yes" ]]; then
die $LINENO "If you wish to install Ceph on this distribution anyway run with FORCE_CEPH_INSTALL=yes"
fi
NO_UPDATE_REPOS=False
fi
}
# cleanup_ceph() - Remove residual data files, anything left over from previous
# runs that a clean run would need to clean up
function cleanup_ceph_remote {
# do a proper cleanup from here to avoid leftover on the remote Ceph cluster
if is_service_enabled glance; then
sudo ceph osd pool delete $GLANCE_CEPH_POOL $GLANCE_CEPH_POOL --yes-i-really-really-mean-it > /dev/null 2>&1
sudo ceph auth del client.$GLANCE_CEPH_USER > /dev/null 2>&1
fi
if is_service_enabled cinder; then
sudo ceph osd pool delete $CINDER_CEPH_POOL $CINDER_CEPH_POOL --yes-i-really-really-mean-it > /dev/null 2>&1
sudo ceph auth del client.$CINDER_CEPH_USER > /dev/null 2>&1
fi
if is_service_enabled c-bak; then
sudo ceph osd pool delete $CINDER_BAK_CEPH_POOL $CINDER_BAK_CEPH_POOL --yes-i-really-really-mean-it > /dev/null 2>&1
sudo ceph auth del client.$CINDER_BAK_CEPH_USER > /dev/null 2>&1
fi
if is_service_enabled nova; then
iniset $NOVA_CONF libvirt rbd_secret_uuid ""
sudo ceph osd pool delete $NOVA_CEPH_POOL $NOVA_CEPH_POOL --yes-i-really-really-mean-it > /dev/null 2>&1
fi
}
function cleanup_ceph_embedded {
sudo killall -w -9 ceph-mon
sudo killall -w -9 ceph-osd
sudo rm -rf ${CEPH_DATA_DIR}/*/*
if egrep -q ${CEPH_DATA_DIR} /proc/mounts; then
sudo umount ${CEPH_DATA_DIR}
fi
if [[ -e ${CEPH_DISK_IMAGE} ]]; then
sudo rm -f ${CEPH_DISK_IMAGE}
fi
# purge ceph config file and keys
sudo rm -rf ${CEPH_CONF_DIR}/*
}
function cleanup_ceph_general {
undefine_virsh_secret
uninstall_package ceph ceph-common python-ceph libcephfs1 > /dev/null 2>&1
}
# configure_ceph() - Set config files, create data dirs, etc
function configure_ceph {
local count=0
# create a backing file disk
create_disk ${CEPH_DISK_IMAGE} ${CEPH_DATA_DIR} ${CEPH_LOOPBACK_DISK_SIZE}
# populate ceph directory
sudo mkdir -p ${CEPH_DATA_DIR}/{bootstrap-mds,bootstrap-osd,mds,mon,osd,tmp}
# create ceph monitor initial key and directory
sudo ceph-authtool /var/lib/ceph/tmp/keyring.mon.$(hostname) --create-keyring --name=mon. --add-key=$(ceph-authtool --gen-print-key) --cap mon 'allow *'
sudo mkdir /var/lib/ceph/mon/ceph-$(hostname)
# create a default ceph configuration file
sudo tee ${CEPH_CONF_FILE} > /dev/null <<EOF
[global]
fsid = ${CEPH_FSID}
mon_initial_members = $(hostname)
mon_host = ${SERVICE_HOST}
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
filestore_xattr_use_omap = true
osd crush chooseleaf type = 0
osd journal size = 100
EOF
# bootstrap the ceph monitor
sudo ceph-mon -c ${CEPH_CONF_FILE} --mkfs -i $(hostname) --keyring /var/lib/ceph/tmp/keyring.mon.$(hostname)
if is_ubuntu; then
sudo touch /var/lib/ceph/mon/ceph-$(hostname)/upstart
sudo initctl emit ceph-mon id=$(hostname)
else
sudo touch /var/lib/ceph/mon/ceph-$(hostname)/sysvinit
sudo service ceph start mon.$(hostname)
fi
# wait for the admin key to come up otherwise we will not be able to do the actions below
until [ -f ${CEPH_CONF_DIR}/ceph.client.admin.keyring ]; do
echo_summary "Waiting for the Ceph admin key to be ready..."
count=$(($count + 1))
if [ $count -eq 3 ]; then
die $LINENO "Maximum of 3 retries reached"
fi
sleep 5
done
# pools data and metadata were removed in the Giant release so depending on the version we apply different commands
local ceph_version=$(get_ceph_version)
# change pool replica size according to the CEPH_REPLICAS set by the user
if [[ ${ceph_version%%.*} -eq 0 ]] && [[ ${ceph_version##*.} -lt 87 ]]; then
sudo ceph -c ${CEPH_CONF_FILE} osd pool set rbd size ${CEPH_REPLICAS}
sudo ceph -c ${CEPH_CONF_FILE} osd pool set data size ${CEPH_REPLICAS}
sudo ceph -c ${CEPH_CONF_FILE} osd pool set metadata size ${CEPH_REPLICAS}
else
sudo ceph -c ${CEPH_CONF_FILE} osd pool set rbd size ${CEPH_REPLICAS}
fi
# create a simple rule to take OSDs instead of host with CRUSH
# then apply this rules to the default pool
if [[ $CEPH_REPLICAS -ne 1 ]]; then
sudo ceph -c ${CEPH_CONF_FILE} osd crush rule create-simple devstack default osd
RULE_ID=$(sudo ceph -c ${CEPH_CONF_FILE} osd crush rule dump devstack | awk '/rule_id/ {print $3}' | cut -d ',' -f1)
sudo ceph -c ${CEPH_CONF_FILE} osd pool set rbd crush_ruleset ${RULE_ID}
sudo ceph -c ${CEPH_CONF_FILE} osd pool set data crush_ruleset ${RULE_ID}
sudo ceph -c ${CEPH_CONF_FILE} osd pool set metadata crush_ruleset ${RULE_ID}
fi
# create the OSD(s)
for rep in ${CEPH_REPLICAS_SEQ}; do
OSD_ID=$(sudo ceph -c ${CEPH_CONF_FILE} osd create)
sudo mkdir -p ${CEPH_DATA_DIR}/osd/ceph-${OSD_ID}
sudo ceph-osd -c ${CEPH_CONF_FILE} -i ${OSD_ID} --mkfs
sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create osd.${OSD_ID} mon 'allow profile osd ' osd 'allow *' | sudo tee ${CEPH_DATA_DIR}/osd/ceph-${OSD_ID}/keyring
# ceph's init script is parsing ${CEPH_DATA_DIR}/osd/ceph-${OSD_ID}/ and looking for a file
# 'upstart' or 'sysinitv', thanks to these 'touches' we are able to control OSDs daemons
# from the init script.
if is_ubuntu; then
sudo touch ${CEPH_DATA_DIR}/osd/ceph-${OSD_ID}/upstart
else
sudo touch ${CEPH_DATA_DIR}/osd/ceph-${OSD_ID}/sysvinit
fi
done
}
function configure_ceph_embedded_glance {
# configure Glance service options, ceph pool, ceph user and ceph key
sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${GLANCE_CEPH_POOL} size ${CEPH_REPLICAS}
if [[ $CEPH_REPLICAS -ne 1 ]]; then
sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${GLANCE_CEPH_POOL} crush_ruleset ${RULE_ID}
fi
}
# configure_ceph_glance() - Glance config needs to come after Glance is set up
function configure_ceph_glance {
sudo ceph -c ${CEPH_CONF_FILE} osd pool create ${GLANCE_CEPH_POOL} ${GLANCE_CEPH_POOL_PG} ${GLANCE_CEPH_POOL_PGP}
sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create client.${GLANCE_CEPH_USER} mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool=${GLANCE_CEPH_POOL}" | sudo tee ${CEPH_CONF_DIR}/ceph.client.${GLANCE_CEPH_USER}.keyring
sudo chown ${STACK_USER}:$(id -g -n $whoami) ${CEPH_CONF_DIR}/ceph.client.${GLANCE_CEPH_USER}.keyring
iniset $GLANCE_API_CONF glance_store default_store rbd
iniset $GLANCE_API_CONF glance_store stores "file, http, rbd"
iniset $GLANCE_API_CONF glance_store rbd_store_ceph_conf $CEPH_CONF_FILE
iniset $GLANCE_API_CONF glance_store rbd_store_user $GLANCE_CEPH_USER
iniset $GLANCE_API_CONF glance_store rbd_store_pool $GLANCE_CEPH_POOL
}
function configure_ceph_embedded_nova {
# configure Nova service options, ceph pool, ceph user and ceph key
sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${NOVA_CEPH_POOL} size ${CEPH_REPLICAS}
if [[ $CEPH_REPLICAS -ne 1 ]]; then
sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${NOVA_CEPH_POOL} crush_ruleset ${RULE_ID}
fi
}
# configure_ceph_nova() - Nova config needs to come after Nova is set up
function configure_ceph_nova {
sudo ceph -c ${CEPH_CONF_FILE} osd pool create ${NOVA_CEPH_POOL} ${NOVA_CEPH_POOL_PG} ${NOVA_CEPH_POOL_PGP}
iniset $NOVA_CONF libvirt rbd_user ${CINDER_CEPH_USER}
iniset $NOVA_CONF libvirt rbd_secret_uuid ${CINDER_CEPH_UUID}
iniset $NOVA_CONF libvirt inject_key false
iniset $NOVA_CONF libvirt inject_partition -2
iniset $NOVA_CONF libvirt disk_cachemodes "network=writeback"
iniset $NOVA_CONF libvirt images_type rbd
iniset $NOVA_CONF libvirt images_rbd_pool ${NOVA_CEPH_POOL}
iniset $NOVA_CONF libvirt images_rbd_ceph_conf ${CEPH_CONF_FILE}
if ! is_service_enabled cinder; then
sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create client.${CINDER_CEPH_USER} mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool=${CINDER_CEPH_POOL}, allow rwx pool=${NOVA_CEPH_POOL},allow rx pool=${GLANCE_CEPH_POOL}" | sudo tee ${CEPH_CONF_DIR}/ceph.client.${CINDER_CEPH_USER}.keyring > /dev/null
sudo chown ${STACK_USER}:$(id -g -n $whoami) ${CEPH_CONF_DIR}/ceph.client.${CINDER_CEPH_USER}.keyring
fi
}
function configure_ceph_embedded_cinder {
# Configure Cinder service options, ceph pool, ceph user and ceph key
sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CINDER_CEPH_POOL} size ${CEPH_REPLICAS}
if [[ $CEPH_REPLICAS -ne 1 ]]; then
sudo ceph -c ${CEPH_CONF_FILE} osd pool set ${CINDER_CEPH_POOL} crush_ruleset ${RULE_ID}
fi
}
# configure_ceph_cinder() - Cinder config needs to come after Cinder is set up
function configure_ceph_cinder {
sudo ceph -c ${CEPH_CONF_FILE} osd pool create ${CINDER_CEPH_POOL} ${CINDER_CEPH_POOL_PG} ${CINDER_CEPH_POOL_PGP}
sudo ceph -c ${CEPH_CONF_FILE} auth get-or-create client.${CINDER_CEPH_USER} mon "allow r" osd "allow class-read object_prefix rbd_children, allow rwx pool=${CINDER_CEPH_POOL}, allow rwx pool=${NOVA_CEPH_POOL},allow rx pool=${GLANCE_CEPH_POOL}" | sudo tee ${CEPH_CONF_DIR}/ceph.client.${CINDER_CEPH_USER}.keyring
sudo chown ${STACK_USER}:$(id -g -n $whoami) ${CEPH_CONF_DIR}/ceph.client.${CINDER_CEPH_USER}.keyring
}
# init_ceph() - Initialize databases, etc.
function init_ceph {
# clean up from previous (possibly aborted) runs
# make sure to kill all ceph processes first
sudo pkill -f ceph-mon || true
sudo pkill -f ceph-osd || true
}
# install_ceph() - Collect source and prepare
function install_ceph_remote {
install_package ceph-common
}
function install_ceph {
install_package ceph
}
# start_ceph() - Start running processes, including screen
function start_ceph {
if is_ubuntu; then
sudo initctl emit ceph-mon id=$(hostname)
for id in $(sudo ceph -c ${CEPH_CONF_FILE} osd ls); do
sudo start ceph-osd id=${id}
done
else
sudo service ceph start
fi
}
# stop_ceph() - Stop running processes (non-screen)
function stop_ceph {
if is_ubuntu; then
sudo service ceph-mon-all stop > /dev/null 2>&1
sudo service ceph-osd-all stop > /dev/null 2>&1
else
sudo service ceph stop > /dev/null 2>&1
fi
}
# Restore xtrace
$XTRACE
## Local variables:
## mode: shell-script
## End: