Standalone nfs-ganesha with cephadm deployment

Manila supports using a standalone NFS-Ganesha server
as well as a ceph orchestrator deployed NFS-Ganesha cluster
("ceph nfs service"). We've only ever allowed using
ceph orch deployed NFS with ceph orch deployed clusters
through this devstack plugin. With this change,
the plugin can optionally deploy a standalone
NFS-Ganesha service with a ceph orch deployed
ceph cluster. This will greatly simplify testing when we sunset
the package based installation/deployment of ceph.

Depends-On: I2198eee3892b2bb0eb835ec66e21b708152b33a9
Change-Id: If983bb5d5a5fc0c16c1cead84b5fa30ea961d21b
Implements: bp/cephadm-deploy
Signed-off-by: Goutham Pacha Ravi <gouthampravi@gmail.com>
This commit is contained in:
Goutham Pacha Ravi 2024-04-01 17:11:05 -07:00
parent 13f94aaaf2
commit ca2486efb4
5 changed files with 212 additions and 156 deletions

View File

@ -88,10 +88,11 @@
parent: manila-tempest-plugin-cephfs-native-cephadm
- job:
name: devstack-plugin-ceph-cephfs-nfs
name: devstack-plugin-ceph-cephfs-nfs-standalone
description: |
Runs manila tempest plugin tests with CephFS via NFS-Ganesha as a manila
back end (DHSS=False)
back end (DHSS=False). The Ceph cluster is created with cephadm
while nfs-ganesha is installed "standalone" via a package.
parent: manila-tempest-plugin-cephfs-nfs
- job:
@ -167,7 +168,7 @@
- devstack-plugin-ceph-cephfs-native:
irrelevant-files: *irrelevant-files
voting: false
- devstack-plugin-ceph-cephfs-nfs:
- devstack-plugin-ceph-cephfs-nfs-standalone:
irrelevant-files: *irrelevant-files
voting: false
# - devstack-plugin-ceph-master-tempest:

View File

@ -21,6 +21,7 @@
# Save trace setting
XTRACE=$(set +o | grep xtrace)
set +o xtrace
source $CEPH_PLUGIN_DIR/lib/common
# Defaults
@ -30,20 +31,6 @@ TEST_MASTER=$(trueorfalse False TEST_MASTER)
CEPH_RELEASE=${CEPH_RELEASE:-quincy}
GANESHA_RELEASE=${GANESHA_RELEASE:-'unspecified'}
# Remove "v" and "-stable" prefix/suffix tags
GANESHA_RELEASE=$(echo $GANESHA_RELEASE | sed -e "s/^v//" -e "s/-stable$//")
if [[ "$MANILA_CEPH_DRIVER" == "cephfsnfs" && "$GANESHA_RELEASE" == "unspecified" ]]; then
# default ganesha release based on ceph release
case $CEPH_RELEASE in
pacific)
GANESHA_RELEASE='3.5' ;;
*)
GANESHA_RELEASE='4.0' ;;
esac
fi
# Deploy a Ceph demo container instead of a non-containerized version
CEPH_CONTAINERIZED=$(trueorfalse False CEPH_CONTAINERIZED)
@ -111,10 +98,6 @@ CEPHFS_DATA_POOL=${CEPHFS_DATA_POOL:-cephfs_data}
MANILA_CEPH_DRIVER=${MANILA_CEPH_DRIVER:-cephfsnative}
MANILA_CEPH_USER=${MANILA_CEPH_USER:-manila}
# Allows driver to store NFS-Ganesha exports and export counter as
# RADOS objects in CephFS's data pool. This needs NFS-Ganesha v2.5.4 or later,
# Ceph v12.2.2 or later, and OpenStack Queens or later.
MANILA_CEPH_GANESHA_RADOS_STORE=${MANILA_CEPH_GANESHA_RADOS_STORE:-True}
# Set ``CEPH_REPLICAS`` to configure how many replicas are to be
# configured for your Ceph cluster. By default we are configuring
@ -757,67 +740,13 @@ function configure_ceph_manila {
if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
configure_nfs_ganesha
# NFS-Ganesha server cannot run alongwith with other kernel NFS server.
sudo systemctl stop nfs-server || true
sudo systemctl disable nfs-server || true
sudo systemctl enable nfs-ganesha
sudo systemctl start nfs-ganesha || (
echo "Ganesha didn't start. Let's debug..." >&2
sudo systemctl status nfs-ganesha || true
echo "**Ganesha conf file**" >&2
sudo cat /etc/ganesha/ganesha.conf || true
echo "**Ganesha log file**" >&2
sudo cat /var/log/ganesha/ganesha.log || true
echo "**Exiting**" >&2
exit 1
)
echo "Ganesha started successfully!" >&2
start_nfs_ganesha
fi
# RESTART DOCKER CONTAINER
}
function configure_nfs_ganesha {
# Configure NFS-Ganesha to work with Manila's CephFS driver
sudo mkdir -p /etc/ganesha/export.d
if [ $MANILA_CEPH_GANESHA_RADOS_STORE == 'True' ]; then
# Create an empty placeholder ganesha export index object
echo | sudo rados -p ${CEPHFS_DATA_POOL} put ganesha-export-index -
cat <<EOF | sudo tee /etc/ganesha/ganesha.conf>/dev/null
RADOS_URLS {
ceph_conf = ${CEPH_CONF_FILE};
userid = admin;
}
CACHEINODE {
Dir_Max = 1;
Dir_Chunk = 0;
Cache_FDs = false;
NParts = 1;
Cache_Size = 1;
}
EXPORT_DEFAULTS {
Attr_Expiration_Time = 0;
}
%url rados://${CEPHFS_DATA_POOL}/ganesha-export-index
EOF
else
sudo touch /etc/ganesha/export.d/INDEX.conf
echo "%include /etc/ganesha/export.d/INDEX.conf" | sudo tee /etc/ganesha/ganesha.conf
fi
}
function cleanup_nfs_ganesha {
sudo systemctl stop nfs-ganesha
sudo systemctl disable nfs-ganesha
sudo uninstall_package nfs-ganesha nfs-ganesha-ceph libntirpc3 nfs-ganesha-rados-urls nfs-ganesha-vfs
}
function configure_ceph_embedded_manila {
if [[ $CEPH_REPLICAS -ne 1 ]]; then
sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool set ${CEPHFS_DATA_POOL} \
@ -958,19 +887,6 @@ EOF
sudo dnf config-manager --add-repo ceph.repo
}
function dnf_add_repository_nfsganesha {
local repo=""
case $ganesha_release in
3.*)
repo="centos-release-nfs-ganesha30" ;;
*)
repo="centos-release-nfs-ganesha4" ;;
esac
sudo dnf -y install ${repo}
}
# configure_repo_ceph() - Configure Ceph repositories
# Usage: configure_repo_ceph <package_release>
# - package_release: to override the os_RELEASE variable
@ -1009,44 +925,11 @@ function cleanup_repo_ceph {
fi
}
# configure_repo_nfsganesha() - Configure NFS Ganesha repositories
function configure_repo_nfsganesha {
if is_ubuntu; then
# NOTE(gouthamr): Ubuntu PPAs contain the latest build from each major
# version; we can't use a build microversion unlike el8/el9 builds
if [[ $GANESHA_RELEASE =~ 3 ]]; then
sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-3.0
sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-3.0
elif [[ $GANESHA_RELEASE =~ 4 ]]; then
sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-4
sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-4
else
die $LINENO "NFS-Ganesha $GANESHA_RELEASE is not supported by the Ceph plugin for Devstack"
fi
sudo apt-get -y update
elif is_fedora; then
dnf_add_repository_nfsganesha
fi
}
# cleanup_repo_nfsganesha() - Remove NFS Ganesha repositories
# Usage: cleanup_repo_nfsganesha
function cleanup_repo_nfsganesha {
if is_ubuntu; then
sudo rm -rf "/etc/apt/sources.list.d/nfs-ganesha-ubuntu*"
elif is_fedora; then
sudo rm -rf /etc/yum.repos.d/nfs-ganesha.repo
fi
}
function setup_packages_for_manila_on_ubuntu {
CEPH_PACKAGES="${CEPH_PACKAGES} ceph-mds libcephfs2"
if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
configure_repo_nfsganesha
LIBNTIRPC_PACKAGE="libntirpc${GANESHA_RELEASE:0:1}"
CEPH_PACKAGES="${CEPH_PACKAGES} $LIBNTIRPC_PACKAGE \
nfs-ganesha nfs-ganesha-ceph nfs-ganesha-rados-urls nfs-ganesha-vfs"
install_nfs_ganesha
fi
if python3_enabled; then
@ -1056,9 +939,7 @@ function setup_packages_for_manila_on_ubuntu {
function setup_packages_for_manila_on_fedora_family {
if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
configure_repo_nfsganesha
CEPH_PACKAGES="${CEPH_PACKAGES} nfs-ganesha nfs-ganesha-ceph \
nfs-ganesha-rados-urls nfs-ganesha-vfs"
install_nfs_ganesha
fi
}
@ -1175,8 +1056,7 @@ function stop_ceph {
fi
if is_ceph_enabled_for_service manila; then
if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
sudo systemctl stop nfs-ganesha
sudo systemctl disable nfs-ganesha
stop_nfs_ganesha
fi
sudo systemctl stop ceph-mds@${MDS_ID}
sudo systemctl disable ceph-mds@${MDS_ID}

View File

@ -65,7 +65,11 @@ RBD_CLIENT_LOG=/var/log/ceph/qemu-guest-\$pid.log
# MANILA DEFAULTS
MANILA_CEPH_USER=${MANILA_CEPH_USER:-manila}
# NFS OPTIONS
# NFS OPTIONS: Only apply when ENABLE_CEPH_MANILA=True
# Whether or not cephadm should deploy/manage NFS-Ganesha? If set to False,
# we'll deploy a "standalone" NFS Ganesha instead, not managed by cephadm.
CEPHADM_DEPLOY_NFS=${CEPHADM_DEPLOY_NFS:-True}
# Clustered NFS Options
FSNAME=${FSNAME:-'cephfs'}
NFS_PORT=2049
CEPHFS_CLIENT=0
@ -124,7 +128,7 @@ function export_spec {
# Pre-install ceph: install required dependencies
function install_deps {
if [[ "$REMOTE_CEPH" == "False" ]]; then
if [[ "$REMOTE_CEPH" = "False" ]]; then
install_package python3-cephfs python3-prettytable python3-rados python3-rbd python3-requests
fi
}
@ -189,16 +193,16 @@ function start_ceph {
--skip-mon-network \
--mon-ip "$HOST_IP"
test -e $CEPH_CONFIG
test -e $CEPH_KEYRING
test -e $CEPH_CONFIG
test -e $CEPH_KEYRING
if [ "$CEPHADM_DEV_OSD" == 'True' ]; then
create_osd_dev
fi
# Wait cephadm backend to be operational
# and add osds via drivegroups
sleep "$SLEEP"
add_osds
if [ "$CEPHADM_DEV_OSD" == 'True' ]; then
create_osd_dev
fi
# Wait cephadm backend to be operational
# and add osds via drivegroups
sleep "$SLEEP"
add_osds
fi
}
@ -242,7 +246,7 @@ function add_osds {
while [ "$ATTEMPTS" -ne 0 ]; do
num_osds=$($SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \
--keyring $CEPH_KEYRING -- ceph -s -f json | jq '.osdmap | .num_up_osds')
--keyring $CEPH_KEYRING -- ceph -s -f json | jq '.osdmap | .num_up_osds')
if [ "$num_osds" -ge "$MIN_OSDS" ]; then
break;
fi
@ -313,8 +317,8 @@ function create_keys {
# Install ceph: add MDS
function cephfs_config {
# Two pools are generated by this action
# - $FSNAME.FSNAME.data
# - $FSNAME.FSNAME.meta
# - cephfs.$FSNAME.data
# - cephfs.$FSNAME.meta
# and the mds daemon is deployed
$SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \
--keyring $CEPH_KEYRING -- ceph fs volume create "$FSNAME"
@ -330,8 +334,7 @@ function _get_ceph_version {
echo $ceph_version_str
}
# Install ceph: add NFS
function ceph_nfs_config {
function _install_and_configure_clustered_nfs {
local ceph_version
ceph_version=$(_get_ceph_version)
@ -350,6 +353,23 @@ function ceph_nfs_config {
fi
}
function _install_and_configure_standalone_nfs {
source $CEPH_PLUGIN_DIR/lib/common
install_nfs_ganesha
configure_nfs_ganesha
start_nfs_ganesha
}
# Install ceph: add NFS
function ceph_nfs_config {
if [[ "$CEPHADM_DEPLOY_NFS" == "True" ]]; then
_install_and_configure_clustered_nfs
else
_install_and_configure_standalone_nfs
fi
}
function _create_swift_endpoint {
local swift_service
@ -453,17 +473,17 @@ function configure_ceph_manila {
function enable_services {
for item in "${SERVICES[@]}"; do
case "$item" in
cephfs|CEPHFS)
cephfs|CEPHFS)
echo "[CEPHADM] Config cephfs volume on node $HOSTNAME"
cephfs_config
CEPHFS_CLIENT=1
;;
nfs|NFS)
nfs|NFS)
echo "[CEPHADM] Deploying NFS on node $HOSTNAME"
ceph_nfs_config
CEPHFS_CLIENT=1
;;
rgw|RGW)
rgw|RGW)
echo "[CEPHADM] Deploying RGW on node $HOSTNAME"
rgw
;;
@ -706,8 +726,14 @@ function cleanup_ceph {
# purge ceph config file and keys
$SUDO rm -f ${CEPH_CONF_DIR}/*
if is_ceph_enabled_for_service nova; then
_undefine_virsh_secret
_undefine_virsh_secret
fi
if [[ "$CEPHADM_DEPLOY_NFS" != "True" ]]; then
stop_nfs_ganesha
cleanup_nfs_ganesha
cleanup_repo_nfs_ganesha
fi
}
function disable_cephadm {

149
devstack/lib/common Executable file
View File

@ -0,0 +1,149 @@
#!/bin/bash
# Allows driver to store NFS-Ganesha exports and export counter as
# RADOS objects in CephFS's data pool. This needs NFS-Ganesha v2.5.4 or later,
# Ceph v12.2.2 or later, and OpenStack Queens or later.
MANILA_CEPH_GANESHA_RADOS_STORE=${MANILA_CEPH_GANESHA_RADOS_STORE:-True}
GANESHA_RELEASE=${GANESHA_RELEASE:-'unspecified'}
# Remove "v" and "-stable" prefix/suffix tags
GANESHA_RELEASE=$(echo $GANESHA_RELEASE | sed -e "s/^v//" -e "s/-stable$//")
if [[ "$CEPHADM_DEPLOY" = "True" ]]; then
FSNAME=${FSNAME:-'cephfs'}
CEPHFS_DATA_POOL="cephfs.$FSNAME.data"
else
CEPHFS_DATA_POOL=${CEPHFS_DATA_POOL:-cephfs_data}
fi
if [[ "$MANILA_CEPH_DRIVER" == "cephfsnfs" && "$GANESHA_RELEASE" == "unspecified" ]]; then
# default ganesha release based on ceph release
case $CEPH_RELEASE in
pacific)
GANESHA_RELEASE='3.5'
;;
*)
GANESHA_RELEASE='5.0'
;;
esac
fi
# configure_repo_nfsganesha - Configure NFS Ganesha repositories
function configure_repo_nfsganesha {
if is_ubuntu; then
# NOTE(gouthamr): Ubuntu PPAs contain the latest build from each major
# version; we can't use a build microversion unlike el8/el9 builds
case $GANESHA_RELEASE in
3.*)
sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-3.0
sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-3.0
;;
*)
GANESHA_PPA_VERSION="${GANESHA_RELEASE:0:1}"
sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-"$GANESHA_PPA_VERSION"
sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-"$GANESHA_PPA_VERSION"
;;
esac
sudo apt-get -y update
elif is_fedora; then
local repo=""
case $GANESHA_RELEASE in
3.*)
repo="centos-release-nfs-ganesha30"
;;
*)
repo="centos-release-nfs-ganesha5"
;;
esac
sudo dnf -y install ${repo}
fi
}
function install_nfs_ganesha {
configure_repo_nfsganesha
NFS_GANESHA_PACKAGES="nfs-ganesha nfs-ganesha-ceph \
nfs-ganesha-rados-urls nfs-ganesha-vfs"
if is_ubuntu; then
LIBNTIRPC_PACKAGE="libntirpc${GANESHA_RELEASE:0:1}"
NFS_GANESHA_PACKAGES="${LIBNTIRPC_PACKAGE} ${NFS_GANESHA_PACKAGES}"
fi
install_package $NFS_GANESHA_PACKAGES
}
function configure_nfs_ganesha {
# Configure NFS-Ganesha to work with Manila's CephFS driver
rados_cmd="sudo rados -p ${CEPHFS_DATA_POOL}"
if [[ "$CEPHADM_DEPLOY" = "True" ]]; then
CEPHADM=${TARGET_BIN}/cephadm
rados_cmd="sudo $CEPHADM shell rados -p ${CEPHFS_DATA_POOL}"
fi
sudo mkdir -p /etc/ganesha/export.d
if [ $MANILA_CEPH_GANESHA_RADOS_STORE == 'True' ]; then
# Create an empty placeholder ganesha export index object
echo | $rados_cmd put ganesha-export-index -
cat <<EOF | sudo tee /etc/ganesha/ganesha.conf >/dev/null
RADOS_URLS {
ceph_conf = ${CEPH_CONF_FILE};
userid = admin;
}
CACHEINODE {
Dir_Max = 1;
Dir_Chunk = 0;
Cache_FDs = false;
NParts = 1;
Cache_Size = 1;
}
EXPORT_DEFAULTS {
Attr_Expiration_Time = 0;
}
%url rados://${CEPHFS_DATA_POOL}/ganesha-export-index
EOF
else
sudo touch /etc/ganesha/export.d/INDEX.conf
echo "%include /etc/ganesha/export.d/INDEX.conf" | sudo tee /etc/ganesha/ganesha.conf
fi
}
function start_nfs_ganesha {
# NFS-Ganesha server cannot run alongwith with other kernel NFS server.
sudo systemctl stop nfs-server || true
sudo systemctl disable nfs-server || true
sudo systemctl enable nfs-ganesha
sudo systemctl start nfs-ganesha || (
echo "Ganesha didn't start. Let's debug..." >&2
sudo systemctl status nfs-ganesha || true
echo "**Ganesha conf file**" >&2
sudo cat /etc/ganesha/ganesha.conf || true
echo "**Ganesha log file**" >&2
sudo cat /var/log/ganesha/ganesha.log || true
echo "**Exiting**" >&2
exit 1
)
echo "Standalone NFS-Ganesha started successfully!" >&2
}
function stop_nfs_ganesha {
sudo systemctl stop nfs-ganesha
sudo systemctl disable nfs-ganesha
}
function cleanup_nfs_ganesha {
sudo systemctl stop nfs-ganesha
sudo systemctl disable nfs-ganesha
sudo uninstall_package nfs-ganesha nfs-ganesha-ceph libntirpc3 nfs-ganesha-rados-urls nfs-ganesha-vfs
}
# cleanup_repo_nfsganesha() - Remove NFS Ganesha repositories
# Usage: cleanup_repo_nfsganesha
function cleanup_repo_nfsganesha {
if is_ubuntu; then
sudo rm -rf "/etc/apt/sources.list.d/nfs-ganesha-ubuntu*"
elif is_fedora; then
sudo rm -rf /etc/yum.repos.d/nfs-ganesha.repo
fi
}

View File

@ -62,16 +62,16 @@ if (is_ceph_enabled_for_service manila); then
MANILA_OPTGROUP_cephfsnfs1_cephfs_conf_path=${CEPH_CONF_FILE}
MANILA_OPTGROUP_cephfsnfs1_cephfs_auth_id=${MANILA_CEPH_USER}
MANILA_OPTGROUP_cephfsnfs1_cephfs_protocol_helper_type=NFS
MANILA_OPTGROUP_cephfsnfs1_cephfs_ganesha_server_ip=$HOST_IP
MANILA_CEPH_GANESHA_RADOS_STORE=$(trueorfalse False MANILA_CEPH_GANESHA_RADOS_STORE)
if [ "$MANILA_CEPH_GANESHA_RADOS_STORE" = "True" ]; then
MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_enable=${MANILA_CEPH_GANESHA_RADOS_STORE}
MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_pool_name=${CEPHFS_DATA_POOL}
fi
if [ "$CEPHADM_DEPLOY" = "True" ]; then
if [[ $CEPHADM_DEPLOY_NFS == "True" ]]; then
MANILA_OPTGROUP_cephfsnfs1_cephfs_nfs_cluster_id=${FSNAME}
else
MANILA_OPTGROUP_cephfsnfs1_cephfs_ganesha_server_ip=$HOST_IP
MANILA_CEPH_GANESHA_RADOS_STORE=$(trueorfalse False MANILA_CEPH_GANESHA_RADOS_STORE)
if [ "$MANILA_CEPH_GANESHA_RADOS_STORE" = "True" ]; then
MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_enable=${MANILA_CEPH_GANESHA_RADOS_STORE}
MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_pool_name=${CEPHFS_DATA_POOL}
fi
fi
fi
fi