Merge "Standalone nfs-ganesha with cephadm deployment"

This commit is contained in:
Zuul 2024-08-08 20:00:20 +00:00 committed by Gerrit Code Review
commit 62c5b084cb
5 changed files with 212 additions and 156 deletions

View File

@ -90,10 +90,11 @@
parent: manila-tempest-plugin-cephfs-native-cephadm
- job:
name: devstack-plugin-ceph-cephfs-nfs
name: devstack-plugin-ceph-cephfs-nfs-standalone
description: |
Runs manila tempest plugin tests with CephFS via NFS-Ganesha as a manila
back end (DHSS=False)
back end (DHSS=False). The Ceph cluster is created with cephadm
while nfs-ganesha is installed "standalone" via a package.
parent: manila-tempest-plugin-cephfs-nfs
- job:
@ -174,7 +175,7 @@
- devstack-plugin-ceph-cephfs-native:
irrelevant-files: *irrelevant-files
voting: false
- devstack-plugin-ceph-cephfs-nfs:
- devstack-plugin-ceph-cephfs-nfs-standalone:
irrelevant-files: *irrelevant-files
voting: false
# - devstack-plugin-ceph-master-tempest:

View File

@ -21,6 +21,7 @@
# Save trace setting
XTRACE=$(set +o | grep xtrace)
set +o xtrace
source $CEPH_PLUGIN_DIR/lib/common
# Defaults
@ -30,20 +31,6 @@ TEST_MASTER=$(trueorfalse False TEST_MASTER)
CEPH_RELEASE=${CEPH_RELEASE:-quincy}
GANESHA_RELEASE=${GANESHA_RELEASE:-'unspecified'}
# Remove "v" and "-stable" prefix/suffix tags
GANESHA_RELEASE=$(echo $GANESHA_RELEASE | sed -e "s/^v//" -e "s/-stable$//")
if [[ "$MANILA_CEPH_DRIVER" == "cephfsnfs" && "$GANESHA_RELEASE" == "unspecified" ]]; then
# default ganesha release based on ceph release
case $CEPH_RELEASE in
pacific)
GANESHA_RELEASE='3.5' ;;
*)
GANESHA_RELEASE='4.0' ;;
esac
fi
# Deploy a Ceph demo container instead of a non-containerized version
CEPH_CONTAINERIZED=$(trueorfalse False CEPH_CONTAINERIZED)
@ -111,10 +98,6 @@ CEPHFS_DATA_POOL=${CEPHFS_DATA_POOL:-cephfs_data}
MANILA_CEPH_DRIVER=${MANILA_CEPH_DRIVER:-cephfsnative}
MANILA_CEPH_USER=${MANILA_CEPH_USER:-manila}
# Allows driver to store NFS-Ganesha exports and export counter as
# RADOS objects in CephFS's data pool. This needs NFS-Ganesha v2.5.4 or later,
# Ceph v12.2.2 or later, and OpenStack Queens or later.
MANILA_CEPH_GANESHA_RADOS_STORE=${MANILA_CEPH_GANESHA_RADOS_STORE:-True}
# Set ``CEPH_REPLICAS`` to configure how many replicas are to be
# configured for your Ceph cluster. By default we are configuring
@ -757,67 +740,13 @@ function configure_ceph_manila {
if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
configure_nfs_ganesha
# NFS-Ganesha server cannot run alongwith with other kernel NFS server.
sudo systemctl stop nfs-server || true
sudo systemctl disable nfs-server || true
sudo systemctl enable nfs-ganesha
sudo systemctl start nfs-ganesha || (
echo "Ganesha didn't start. Let's debug..." >&2
sudo systemctl status nfs-ganesha || true
echo "**Ganesha conf file**" >&2
sudo cat /etc/ganesha/ganesha.conf || true
echo "**Ganesha log file**" >&2
sudo cat /var/log/ganesha/ganesha.log || true
echo "**Exiting**" >&2
exit 1
)
echo "Ganesha started successfully!" >&2
start_nfs_ganesha
fi
# RESTART DOCKER CONTAINER
}
function configure_nfs_ganesha {
# Configure NFS-Ganesha to work with Manila's CephFS driver
sudo mkdir -p /etc/ganesha/export.d
if [ $MANILA_CEPH_GANESHA_RADOS_STORE == 'True' ]; then
# Create an empty placeholder ganesha export index object
echo | sudo rados -p ${CEPHFS_DATA_POOL} put ganesha-export-index -
cat <<EOF | sudo tee /etc/ganesha/ganesha.conf>/dev/null
RADOS_URLS {
ceph_conf = ${CEPH_CONF_FILE};
userid = admin;
}
CACHEINODE {
Dir_Max = 1;
Dir_Chunk = 0;
Cache_FDs = false;
NParts = 1;
Cache_Size = 1;
}
EXPORT_DEFAULTS {
Attr_Expiration_Time = 0;
}
%url rados://${CEPHFS_DATA_POOL}/ganesha-export-index
EOF
else
sudo touch /etc/ganesha/export.d/INDEX.conf
echo "%include /etc/ganesha/export.d/INDEX.conf" | sudo tee /etc/ganesha/ganesha.conf
fi
}
function cleanup_nfs_ganesha {
sudo systemctl stop nfs-ganesha
sudo systemctl disable nfs-ganesha
sudo uninstall_package nfs-ganesha nfs-ganesha-ceph libntirpc3 nfs-ganesha-rados-urls nfs-ganesha-vfs
}
function configure_ceph_embedded_manila {
if [[ $CEPH_REPLICAS -ne 1 ]]; then
sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool set ${CEPHFS_DATA_POOL} \
@ -958,19 +887,6 @@ EOF
sudo dnf config-manager --add-repo ceph.repo
}
function dnf_add_repository_nfsganesha {
local repo=""
case $ganesha_release in
3.*)
repo="centos-release-nfs-ganesha30" ;;
*)
repo="centos-release-nfs-ganesha4" ;;
esac
sudo dnf -y install ${repo}
}
# configure_repo_ceph() - Configure Ceph repositories
# Usage: configure_repo_ceph <package_release>
# - package_release: to override the os_RELEASE variable
@ -1009,44 +925,11 @@ function cleanup_repo_ceph {
fi
}
# configure_repo_nfsganesha() - Configure NFS Ganesha repositories
function configure_repo_nfsganesha {
if is_ubuntu; then
# NOTE(gouthamr): Ubuntu PPAs contain the latest build from each major
# version; we can't use a build microversion unlike el8/el9 builds
if [[ $GANESHA_RELEASE =~ 3 ]]; then
sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-3.0
sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-3.0
elif [[ $GANESHA_RELEASE =~ 4 ]]; then
sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-4
sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-4
else
die $LINENO "NFS-Ganesha $GANESHA_RELEASE is not supported by the Ceph plugin for Devstack"
fi
sudo apt-get -y update
elif is_fedora; then
dnf_add_repository_nfsganesha
fi
}
# cleanup_repo_nfsganesha() - Remove NFS Ganesha repositories
# Usage: cleanup_repo_nfsganesha
function cleanup_repo_nfsganesha {
if is_ubuntu; then
sudo rm -rf "/etc/apt/sources.list.d/nfs-ganesha-ubuntu*"
elif is_fedora; then
sudo rm -rf /etc/yum.repos.d/nfs-ganesha.repo
fi
}
function setup_packages_for_manila_on_ubuntu {
CEPH_PACKAGES="${CEPH_PACKAGES} ceph-mds libcephfs2"
if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
configure_repo_nfsganesha
LIBNTIRPC_PACKAGE="libntirpc${GANESHA_RELEASE:0:1}"
CEPH_PACKAGES="${CEPH_PACKAGES} $LIBNTIRPC_PACKAGE \
nfs-ganesha nfs-ganesha-ceph nfs-ganesha-rados-urls nfs-ganesha-vfs"
install_nfs_ganesha
fi
if python3_enabled; then
@ -1056,9 +939,7 @@ function setup_packages_for_manila_on_ubuntu {
function setup_packages_for_manila_on_fedora_family {
if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
configure_repo_nfsganesha
CEPH_PACKAGES="${CEPH_PACKAGES} nfs-ganesha nfs-ganesha-ceph \
nfs-ganesha-rados-urls nfs-ganesha-vfs"
install_nfs_ganesha
fi
}
@ -1175,8 +1056,7 @@ function stop_ceph {
fi
if is_ceph_enabled_for_service manila; then
if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
sudo systemctl stop nfs-ganesha
sudo systemctl disable nfs-ganesha
stop_nfs_ganesha
fi
sudo systemctl stop ceph-mds@${MDS_ID}
sudo systemctl disable ceph-mds@${MDS_ID}

View File

@ -67,7 +67,11 @@ MDS_LOGS=${MDS_LOGS:-"False"}
# MANILA DEFAULTS
MANILA_CEPH_USER=${MANILA_CEPH_USER:-manila}
# NFS OPTIONS
# NFS OPTIONS: Only apply when ENABLE_CEPH_MANILA=True
# Whether or not cephadm should deploy/manage NFS-Ganesha? If set to False,
# we'll deploy a "standalone" NFS Ganesha instead, not managed by cephadm.
CEPHADM_DEPLOY_NFS=${CEPHADM_DEPLOY_NFS:-True}
# Clustered NFS Options
FSNAME=${FSNAME:-'cephfs'}
NFS_PORT=2049
CEPHFS_CLIENT=0
@ -153,7 +157,7 @@ function export_spec {
# Pre-install ceph: install required dependencies
function install_deps {
if [[ "$REMOTE_CEPH" == "False" ]]; then
if [[ "$REMOTE_CEPH" = "False" ]]; then
install_package python3-cephfs python3-prettytable python3-rados python3-rbd python3-requests
fi
}
@ -218,16 +222,16 @@ function start_ceph {
--skip-mon-network \
--mon-ip "$HOST_IP"
test -e $CEPH_CONFIG
test -e $CEPH_KEYRING
test -e $CEPH_CONFIG
test -e $CEPH_KEYRING
if [ "$CEPHADM_DEV_OSD" == 'True' ]; then
create_osd_dev
fi
# Wait cephadm backend to be operational
# and add osds via drivegroups
sleep "$SLEEP"
add_osds
if [ "$CEPHADM_DEV_OSD" == 'True' ]; then
create_osd_dev
fi
# Wait cephadm backend to be operational
# and add osds via drivegroups
sleep "$SLEEP"
add_osds
fi
}
@ -271,7 +275,7 @@ function add_osds {
while [ "$ATTEMPTS" -ne 0 ]; do
num_osds=$($SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \
--keyring $CEPH_KEYRING -- ceph -s -f json | jq '.osdmap | .num_up_osds')
--keyring $CEPH_KEYRING -- ceph -s -f json | jq '.osdmap | .num_up_osds')
if [ "$num_osds" -ge "$MIN_OSDS" ]; then
break;
fi
@ -342,8 +346,8 @@ function create_keys {
# Install ceph: add MDS
function cephfs_config {
# Two pools are generated by this action
# - $FSNAME.FSNAME.data
# - $FSNAME.FSNAME.meta
# - cephfs.$FSNAME.data
# - cephfs.$FSNAME.meta
# and the mds daemon is deployed
$SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \
--keyring $CEPH_KEYRING -- ceph fs volume create "$FSNAME"
@ -359,8 +363,7 @@ function _get_ceph_version {
echo $ceph_version_str
}
# Install ceph: add NFS
function ceph_nfs_config {
function _install_and_configure_clustered_nfs {
local ceph_version
ceph_version=$(_get_ceph_version)
@ -379,6 +382,23 @@ function ceph_nfs_config {
fi
}
function _install_and_configure_standalone_nfs {
source $CEPH_PLUGIN_DIR/lib/common
install_nfs_ganesha
configure_nfs_ganesha
start_nfs_ganesha
}
# Install ceph: add NFS
function ceph_nfs_config {
if [[ "$CEPHADM_DEPLOY_NFS" == "True" ]]; then
_install_and_configure_clustered_nfs
else
_install_and_configure_standalone_nfs
fi
}
function _create_swift_endpoint {
local swift_service
@ -482,17 +502,17 @@ function configure_ceph_manila {
function enable_services {
for item in "${SERVICES[@]}"; do
case "$item" in
cephfs|CEPHFS)
cephfs|CEPHFS)
echo "[CEPHADM] Config cephfs volume on node $HOSTNAME"
cephfs_config
CEPHFS_CLIENT=1
;;
nfs|NFS)
nfs|NFS)
echo "[CEPHADM] Deploying NFS on node $HOSTNAME"
ceph_nfs_config
CEPHFS_CLIENT=1
;;
rgw|RGW)
rgw|RGW)
echo "[CEPHADM] Deploying RGW on node $HOSTNAME"
rgw
;;
@ -735,8 +755,14 @@ function cleanup_ceph {
# purge ceph config file and keys
$SUDO rm -f ${CEPH_CONF_DIR}/*
if is_ceph_enabled_for_service nova; then
_undefine_virsh_secret
_undefine_virsh_secret
fi
if [[ "$CEPHADM_DEPLOY_NFS" != "True" ]]; then
stop_nfs_ganesha
cleanup_nfs_ganesha
cleanup_repo_nfs_ganesha
fi
}
function disable_cephadm {

149
devstack/lib/common Executable file
View File

@ -0,0 +1,149 @@
#!/bin/bash
# Allows driver to store NFS-Ganesha exports and export counter as
# RADOS objects in CephFS's data pool. This needs NFS-Ganesha v2.5.4 or later,
# Ceph v12.2.2 or later, and OpenStack Queens or later.
MANILA_CEPH_GANESHA_RADOS_STORE=${MANILA_CEPH_GANESHA_RADOS_STORE:-True}
GANESHA_RELEASE=${GANESHA_RELEASE:-'unspecified'}
# Remove "v" and "-stable" prefix/suffix tags
GANESHA_RELEASE=$(echo $GANESHA_RELEASE | sed -e "s/^v//" -e "s/-stable$//")
if [[ "$CEPHADM_DEPLOY" = "True" ]]; then
FSNAME=${FSNAME:-'cephfs'}
CEPHFS_DATA_POOL="cephfs.$FSNAME.data"
else
CEPHFS_DATA_POOL=${CEPHFS_DATA_POOL:-cephfs_data}
fi
if [[ "$MANILA_CEPH_DRIVER" == "cephfsnfs" && "$GANESHA_RELEASE" == "unspecified" ]]; then
# default ganesha release based on ceph release
case $CEPH_RELEASE in
pacific)
GANESHA_RELEASE='3.5'
;;
*)
GANESHA_RELEASE='5.0'
;;
esac
fi
# configure_repo_nfsganesha - Configure NFS Ganesha repositories
function configure_repo_nfsganesha {
if is_ubuntu; then
# NOTE(gouthamr): Ubuntu PPAs contain the latest build from each major
# version; we can't use a build microversion unlike el8/el9 builds
case $GANESHA_RELEASE in
3.*)
sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-3.0
sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-3.0
;;
*)
GANESHA_PPA_VERSION="${GANESHA_RELEASE:0:1}"
sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-"$GANESHA_PPA_VERSION"
sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-"$GANESHA_PPA_VERSION"
;;
esac
sudo apt-get -y update
elif is_fedora; then
local repo=""
case $GANESHA_RELEASE in
3.*)
repo="centos-release-nfs-ganesha30"
;;
*)
repo="centos-release-nfs-ganesha5"
;;
esac
sudo dnf -y install ${repo}
fi
}
function install_nfs_ganesha {
configure_repo_nfsganesha
NFS_GANESHA_PACKAGES="nfs-ganesha nfs-ganesha-ceph \
nfs-ganesha-rados-urls nfs-ganesha-vfs"
if is_ubuntu; then
LIBNTIRPC_PACKAGE="libntirpc${GANESHA_RELEASE:0:1}"
NFS_GANESHA_PACKAGES="${LIBNTIRPC_PACKAGE} ${NFS_GANESHA_PACKAGES}"
fi
install_package $NFS_GANESHA_PACKAGES
}
function configure_nfs_ganesha {
# Configure NFS-Ganesha to work with Manila's CephFS driver
rados_cmd="sudo rados -p ${CEPHFS_DATA_POOL}"
if [[ "$CEPHADM_DEPLOY" = "True" ]]; then
CEPHADM=${TARGET_BIN}/cephadm
rados_cmd="sudo $CEPHADM shell rados -p ${CEPHFS_DATA_POOL}"
fi
sudo mkdir -p /etc/ganesha/export.d
if [ $MANILA_CEPH_GANESHA_RADOS_STORE == 'True' ]; then
# Create an empty placeholder ganesha export index object
echo | $rados_cmd put ganesha-export-index -
cat <<EOF | sudo tee /etc/ganesha/ganesha.conf >/dev/null
RADOS_URLS {
ceph_conf = ${CEPH_CONF_FILE};
userid = admin;
}
CACHEINODE {
Dir_Max = 1;
Dir_Chunk = 0;
Cache_FDs = false;
NParts = 1;
Cache_Size = 1;
}
EXPORT_DEFAULTS {
Attr_Expiration_Time = 0;
}
%url rados://${CEPHFS_DATA_POOL}/ganesha-export-index
EOF
else
sudo touch /etc/ganesha/export.d/INDEX.conf
echo "%include /etc/ganesha/export.d/INDEX.conf" | sudo tee /etc/ganesha/ganesha.conf
fi
}
function start_nfs_ganesha {
# NFS-Ganesha server cannot run alongwith with other kernel NFS server.
sudo systemctl stop nfs-server || true
sudo systemctl disable nfs-server || true
sudo systemctl enable nfs-ganesha
sudo systemctl start nfs-ganesha || (
echo "Ganesha didn't start. Let's debug..." >&2
sudo systemctl status nfs-ganesha || true
echo "**Ganesha conf file**" >&2
sudo cat /etc/ganesha/ganesha.conf || true
echo "**Ganesha log file**" >&2
sudo cat /var/log/ganesha/ganesha.log || true
echo "**Exiting**" >&2
exit 1
)
echo "Standalone NFS-Ganesha started successfully!" >&2
}
function stop_nfs_ganesha {
sudo systemctl stop nfs-ganesha
sudo systemctl disable nfs-ganesha
}
function cleanup_nfs_ganesha {
sudo systemctl stop nfs-ganesha
sudo systemctl disable nfs-ganesha
sudo uninstall_package nfs-ganesha nfs-ganesha-ceph libntirpc3 nfs-ganesha-rados-urls nfs-ganesha-vfs
}
# cleanup_repo_nfsganesha() - Remove NFS Ganesha repositories
# Usage: cleanup_repo_nfsganesha
function cleanup_repo_nfsganesha {
if is_ubuntu; then
sudo rm -rf "/etc/apt/sources.list.d/nfs-ganesha-ubuntu*"
elif is_fedora; then
sudo rm -rf /etc/yum.repos.d/nfs-ganesha.repo
fi
}

View File

@ -62,16 +62,16 @@ if (is_ceph_enabled_for_service manila); then
MANILA_OPTGROUP_cephfsnfs1_cephfs_conf_path=${CEPH_CONF_FILE}
MANILA_OPTGROUP_cephfsnfs1_cephfs_auth_id=${MANILA_CEPH_USER}
MANILA_OPTGROUP_cephfsnfs1_cephfs_protocol_helper_type=NFS
MANILA_OPTGROUP_cephfsnfs1_cephfs_ganesha_server_ip=$HOST_IP
MANILA_CEPH_GANESHA_RADOS_STORE=$(trueorfalse False MANILA_CEPH_GANESHA_RADOS_STORE)
if [ "$MANILA_CEPH_GANESHA_RADOS_STORE" = "True" ]; then
MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_enable=${MANILA_CEPH_GANESHA_RADOS_STORE}
MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_pool_name=${CEPHFS_DATA_POOL}
fi
if [ "$CEPHADM_DEPLOY" = "True" ]; then
if [[ $CEPHADM_DEPLOY_NFS == "True" ]]; then
MANILA_OPTGROUP_cephfsnfs1_cephfs_nfs_cluster_id=${FSNAME}
else
MANILA_OPTGROUP_cephfsnfs1_cephfs_ganesha_server_ip=$HOST_IP
MANILA_CEPH_GANESHA_RADOS_STORE=$(trueorfalse False MANILA_CEPH_GANESHA_RADOS_STORE)
if [ "$MANILA_CEPH_GANESHA_RADOS_STORE" = "True" ]; then
MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_enable=${MANILA_CEPH_GANESHA_RADOS_STORE}
MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_pool_name=${CEPHFS_DATA_POOL}
fi
fi
fi
fi