openstack/devstack-plugin-ceph
Code Issues Proposed changes

Merge "Standalone nfs-ganesha with cephadm deployment"

Browse Source
This commit is contained in:
Zuul 2024-08-08 20:00:20 +00:00 committed by Gerrit Code Review
commit 62c5b084cb
5 changed files with 212 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.zuul.yaml
View File

@ -90,10 +90,11 @@
parent: manila-tempest-plugin-cephfs-native-cephadm parent: manila-tempest-plugin-cephfs-native-cephadm
- job: - job:
name: devstack-plugin-ceph-cephfs-nfs name: devstack-plugin-ceph-cephfs-nfs-standalone
description: | description: |
Runs manila tempest plugin tests with CephFS via NFS-Ganesha as a manila Runs manila tempest plugin tests with CephFS via NFS-Ganesha as a manila
back end (DHSS=False) back end (DHSS=False). The Ceph cluster is created with cephadm
while nfs-ganesha is installed "standalone" via a package.
parent: manila-tempest-plugin-cephfs-nfs parent: manila-tempest-plugin-cephfs-nfs
- job: - job:
@ -174,7 +175,7 @@
- devstack-plugin-ceph-cephfs-native: - devstack-plugin-ceph-cephfs-native:
irrelevant-files: *irrelevant-files irrelevant-files: *irrelevant-files
voting: false voting: false
- devstack-plugin-ceph-cephfs-nfs: - devstack-plugin-ceph-cephfs-nfs-standalone:
irrelevant-files: *irrelevant-files irrelevant-files: *irrelevant-files
voting: false voting: false
# - devstack-plugin-ceph-master-tempest: # - devstack-plugin-ceph-master-tempest:

130
devstack/lib/ceph
View File

@ -21,6 +21,7 @@
# Save trace setting # Save trace setting
XTRACE=$(set +o | grep xtrace) XTRACE=$(set +o | grep xtrace)
set +o xtrace set +o xtrace
source $CEPH_PLUGIN_DIR/lib/common
# Defaults # Defaults
@ -30,20 +31,6 @@ TEST_MASTER=$(trueorfalse False TEST_MASTER)
CEPH_RELEASE=${CEPH_RELEASE:-quincy} CEPH_RELEASE=${CEPH_RELEASE:-quincy}
GANESHA_RELEASE=${GANESHA_RELEASE:-'unspecified'}
# Remove "v" and "-stable" prefix/suffix tags
GANESHA_RELEASE=$(echo $GANESHA_RELEASE | sed -e "s/^v//" -e "s/-stable$//")
if [[ "$MANILA_CEPH_DRIVER" == "cephfsnfs" && "$GANESHA_RELEASE" == "unspecified" ]]; then
# default ganesha release based on ceph release
case $CEPH_RELEASE in
pacific)
GANESHA_RELEASE='3.5' ;;
*)
GANESHA_RELEASE='4.0' ;;
esac
fi
# Deploy a Ceph demo container instead of a non-containerized version # Deploy a Ceph demo container instead of a non-containerized version
CEPH_CONTAINERIZED=$(trueorfalse False CEPH_CONTAINERIZED) CEPH_CONTAINERIZED=$(trueorfalse False CEPH_CONTAINERIZED)
@ -111,10 +98,6 @@ CEPHFS_DATA_POOL=${CEPHFS_DATA_POOL:-cephfs_data}
MANILA_CEPH_DRIVER=${MANILA_CEPH_DRIVER:-cephfsnative} MANILA_CEPH_DRIVER=${MANILA_CEPH_DRIVER:-cephfsnative}
MANILA_CEPH_USER=${MANILA_CEPH_USER:-manila} MANILA_CEPH_USER=${MANILA_CEPH_USER:-manila}
# Allows driver to store NFS-Ganesha exports and export counter as
# RADOS objects in CephFS's data pool. This needs NFS-Ganesha v2.5.4 or later,
# Ceph v12.2.2 or later, and OpenStack Queens or later.
MANILA_CEPH_GANESHA_RADOS_STORE=${MANILA_CEPH_GANESHA_RADOS_STORE:-True}
# Set ``CEPH_REPLICAS`` to configure how many replicas are to be # Set ``CEPH_REPLICAS`` to configure how many replicas are to be
# configured for your Ceph cluster. By default we are configuring # configured for your Ceph cluster. By default we are configuring
@ -757,67 +740,13 @@ function configure_ceph_manila {
if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
configure_nfs_ganesha configure_nfs_ganesha
# NFS-Ganesha server cannot run alongwith with other kernel NFS server. start_nfs_ganesha
sudo systemctl stop nfs-server || true
sudo systemctl disable nfs-server || true
sudo systemctl enable nfs-ganesha
sudo systemctl start nfs-ganesha || (
echo "Ganesha didn't start. Let's debug..." >&2
sudo systemctl status nfs-ganesha || true
echo "**Ganesha conf file**" >&2
sudo cat /etc/ganesha/ganesha.conf || true
echo "**Ganesha log file**" >&2
sudo cat /var/log/ganesha/ganesha.log || true
echo "**Exiting**" >&2
exit 1
)
echo "Ganesha started successfully!" >&2
fi fi
# RESTART DOCKER CONTAINER # RESTART DOCKER CONTAINER
} }
function configure_nfs_ganesha {
# Configure NFS-Ganesha to work with Manila's CephFS driver
sudo mkdir -p /etc/ganesha/export.d
if [ $MANILA_CEPH_GANESHA_RADOS_STORE == 'True' ]; then
# Create an empty placeholder ganesha export index object
echo | sudo rados -p ${CEPHFS_DATA_POOL} put ganesha-export-index -
cat <<EOF | sudo tee /etc/ganesha/ganesha.conf>/dev/null
RADOS_URLS {
ceph_conf = ${CEPH_CONF_FILE};
userid = admin;
}
CACHEINODE {
Dir_Max = 1;
Dir_Chunk = 0;
Cache_FDs = false;
NParts = 1;
Cache_Size = 1;
}
EXPORT_DEFAULTS {
Attr_Expiration_Time = 0;
}
%url rados://${CEPHFS_DATA_POOL}/ganesha-export-index
EOF
else
sudo touch /etc/ganesha/export.d/INDEX.conf
echo "%include /etc/ganesha/export.d/INDEX.conf" | sudo tee /etc/ganesha/ganesha.conf
fi
}
function cleanup_nfs_ganesha {
sudo systemctl stop nfs-ganesha
sudo systemctl disable nfs-ganesha
sudo uninstall_package nfs-ganesha nfs-ganesha-ceph libntirpc3 nfs-ganesha-rados-urls nfs-ganesha-vfs
}
function configure_ceph_embedded_manila { function configure_ceph_embedded_manila {
if [[ $CEPH_REPLICAS -ne 1 ]]; then if [[ $CEPH_REPLICAS -ne 1 ]]; then
sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool set ${CEPHFS_DATA_POOL} \ sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool set ${CEPHFS_DATA_POOL} \
@ -958,19 +887,6 @@ EOF
sudo dnf config-manager --add-repo ceph.repo sudo dnf config-manager --add-repo ceph.repo
} }
function dnf_add_repository_nfsganesha {
local repo=""
case $ganesha_release in
3.*)
repo="centos-release-nfs-ganesha30" ;;
*)
repo="centos-release-nfs-ganesha4" ;;
esac
sudo dnf -y install ${repo}
}
# configure_repo_ceph() - Configure Ceph repositories # configure_repo_ceph() - Configure Ceph repositories
# Usage: configure_repo_ceph <package_release> # Usage: configure_repo_ceph <package_release>
# - package_release: to override the os_RELEASE variable # - package_release: to override the os_RELEASE variable
@ -1009,44 +925,11 @@ function cleanup_repo_ceph {
fi fi
} }
# configure_repo_nfsganesha() - Configure NFS Ganesha repositories
function configure_repo_nfsganesha {
if is_ubuntu; then
# NOTE(gouthamr): Ubuntu PPAs contain the latest build from each major
# version; we can't use a build microversion unlike el8/el9 builds
if [[ $GANESHA_RELEASE =~ 3 ]]; then
sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-3.0
sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-3.0
elif [[ $GANESHA_RELEASE =~ 4 ]]; then
sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-4
sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-4
else
die $LINENO "NFS-Ganesha $GANESHA_RELEASE is not supported by the Ceph plugin for Devstack"
fi
sudo apt-get -y update
elif is_fedora; then
dnf_add_repository_nfsganesha
fi
}
# cleanup_repo_nfsganesha() - Remove NFS Ganesha repositories
# Usage: cleanup_repo_nfsganesha
function cleanup_repo_nfsganesha {
if is_ubuntu; then
sudo rm -rf "/etc/apt/sources.list.d/nfs-ganesha-ubuntu*"
elif is_fedora; then
sudo rm -rf /etc/yum.repos.d/nfs-ganesha.repo
fi
}
function setup_packages_for_manila_on_ubuntu { function setup_packages_for_manila_on_ubuntu {
CEPH_PACKAGES="${CEPH_PACKAGES} ceph-mds libcephfs2" CEPH_PACKAGES="${CEPH_PACKAGES} ceph-mds libcephfs2"
if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
configure_repo_nfsganesha install_nfs_ganesha
LIBNTIRPC_PACKAGE="libntirpc${GANESHA_RELEASE:0:1}"
CEPH_PACKAGES="${CEPH_PACKAGES} $LIBNTIRPC_PACKAGE \
nfs-ganesha nfs-ganesha-ceph nfs-ganesha-rados-urls nfs-ganesha-vfs"
fi fi
if python3_enabled; then if python3_enabled; then
@ -1056,9 +939,7 @@ function setup_packages_for_manila_on_ubuntu {
function setup_packages_for_manila_on_fedora_family { function setup_packages_for_manila_on_fedora_family {
if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
configure_repo_nfsganesha install_nfs_ganesha
CEPH_PACKAGES="${CEPH_PACKAGES} nfs-ganesha nfs-ganesha-ceph \
nfs-ganesha-rados-urls nfs-ganesha-vfs"
fi fi
} }
@ -1175,8 +1056,7 @@ function stop_ceph {
fi fi
if is_ceph_enabled_for_service manila; then if is_ceph_enabled_for_service manila; then
if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
sudo systemctl stop nfs-ganesha stop_nfs_ganesha
sudo systemctl disable nfs-ganesha
fi fi
sudo systemctl stop ceph-mds@${MDS_ID} sudo systemctl stop ceph-mds@${MDS_ID}
sudo systemctl disable ceph-mds@${MDS_ID} sudo systemctl disable ceph-mds@${MDS_ID}

38
devstack/lib/cephadm
View File

@ -67,7 +67,11 @@ MDS_LOGS=${MDS_LOGS:-"False"}
# MANILA DEFAULTS # MANILA DEFAULTS
MANILA_CEPH_USER=${MANILA_CEPH_USER:-manila} MANILA_CEPH_USER=${MANILA_CEPH_USER:-manila}
# NFS OPTIONS # NFS OPTIONS: Only apply when ENABLE_CEPH_MANILA=True
# Whether or not cephadm should deploy/manage NFS-Ganesha? If set to False,
# we'll deploy a "standalone" NFS Ganesha instead, not managed by cephadm.
CEPHADM_DEPLOY_NFS=${CEPHADM_DEPLOY_NFS:-True}
# Clustered NFS Options
FSNAME=${FSNAME:-'cephfs'} FSNAME=${FSNAME:-'cephfs'}
NFS_PORT=2049 NFS_PORT=2049
CEPHFS_CLIENT=0 CEPHFS_CLIENT=0
@ -153,7 +157,7 @@ function export_spec {
# Pre-install ceph: install required dependencies # Pre-install ceph: install required dependencies
function install_deps { function install_deps {
if [[ "$REMOTE_CEPH" == "False" ]]; then if [[ "$REMOTE_CEPH" = "False" ]]; then
install_package python3-cephfs python3-prettytable python3-rados python3-rbd python3-requests install_package python3-cephfs python3-prettytable python3-rados python3-rbd python3-requests
fi fi
} }
@ -342,8 +346,8 @@ function create_keys {
# Install ceph: add MDS # Install ceph: add MDS
function cephfs_config { function cephfs_config {
# Two pools are generated by this action # Two pools are generated by this action
# - $FSNAME.FSNAME.data # - cephfs.$FSNAME.data
# - $FSNAME.FSNAME.meta # - cephfs.$FSNAME.meta
# and the mds daemon is deployed # and the mds daemon is deployed
$SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \ $SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \
--keyring $CEPH_KEYRING -- ceph fs volume create "$FSNAME" --keyring $CEPH_KEYRING -- ceph fs volume create "$FSNAME"
@ -359,8 +363,7 @@ function _get_ceph_version {
echo $ceph_version_str echo $ceph_version_str
} }
# Install ceph: add NFS function _install_and_configure_clustered_nfs {
function ceph_nfs_config {
local ceph_version local ceph_version
ceph_version=$(_get_ceph_version) ceph_version=$(_get_ceph_version)
@ -379,6 +382,23 @@ function ceph_nfs_config {
fi fi
} }
function _install_and_configure_standalone_nfs {
source $CEPH_PLUGIN_DIR/lib/common
install_nfs_ganesha
configure_nfs_ganesha
start_nfs_ganesha
}
# Install ceph: add NFS
function ceph_nfs_config {
if [[ "$CEPHADM_DEPLOY_NFS" == "True" ]]; then
_install_and_configure_clustered_nfs
else
_install_and_configure_standalone_nfs
fi
}
function _create_swift_endpoint { function _create_swift_endpoint {
local swift_service local swift_service
@ -737,6 +757,12 @@ function cleanup_ceph {
if is_ceph_enabled_for_service nova; then if is_ceph_enabled_for_service nova; then
_undefine_virsh_secret _undefine_virsh_secret
fi fi
if [[ "$CEPHADM_DEPLOY_NFS" != "True" ]]; then
stop_nfs_ganesha
cleanup_nfs_ganesha
cleanup_repo_nfs_ganesha
fi
} }
function disable_cephadm { function disable_cephadm {

149
devstack/lib/common Executable file
View File

@ -0,0 +1,149 @@
#!/bin/bash
# Allows driver to store NFS-Ganesha exports and export counter as
# RADOS objects in CephFS's data pool. This needs NFS-Ganesha v2.5.4 or later,
# Ceph v12.2.2 or later, and OpenStack Queens or later.
MANILA_CEPH_GANESHA_RADOS_STORE=${MANILA_CEPH_GANESHA_RADOS_STORE:-True}
GANESHA_RELEASE=${GANESHA_RELEASE:-'unspecified'}
# Remove "v" and "-stable" prefix/suffix tags
GANESHA_RELEASE=$(echo $GANESHA_RELEASE | sed -e "s/^v//" -e "s/-stable$//")
if [[ "$CEPHADM_DEPLOY" = "True" ]]; then
FSNAME=${FSNAME:-'cephfs'}
CEPHFS_DATA_POOL="cephfs.$FSNAME.data"
else
CEPHFS_DATA_POOL=${CEPHFS_DATA_POOL:-cephfs_data}
fi
if [[ "$MANILA_CEPH_DRIVER" == "cephfsnfs" && "$GANESHA_RELEASE" == "unspecified" ]]; then
# default ganesha release based on ceph release
case $CEPH_RELEASE in
pacific)
GANESHA_RELEASE='3.5'
;;
*)
GANESHA_RELEASE='5.0'
;;
esac
fi
# configure_repo_nfsganesha - Configure NFS Ganesha repositories
function configure_repo_nfsganesha {
if is_ubuntu; then
# NOTE(gouthamr): Ubuntu PPAs contain the latest build from each major
# version; we can't use a build microversion unlike el8/el9 builds
case $GANESHA_RELEASE in
3.*)
sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-3.0
sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-3.0
;;
*)
GANESHA_PPA_VERSION="${GANESHA_RELEASE:0:1}"
sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-"$GANESHA_PPA_VERSION"
sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-"$GANESHA_PPA_VERSION"
;;
esac
sudo apt-get -y update
elif is_fedora; then
local repo=""
case $GANESHA_RELEASE in
3.*)
repo="centos-release-nfs-ganesha30"
;;
*)
repo="centos-release-nfs-ganesha5"
;;
esac
sudo dnf -y install ${repo}
fi
}
function install_nfs_ganesha {
configure_repo_nfsganesha
NFS_GANESHA_PACKAGES="nfs-ganesha nfs-ganesha-ceph \
nfs-ganesha-rados-urls nfs-ganesha-vfs"
if is_ubuntu; then
LIBNTIRPC_PACKAGE="libntirpc${GANESHA_RELEASE:0:1}"
NFS_GANESHA_PACKAGES="${LIBNTIRPC_PACKAGE} ${NFS_GANESHA_PACKAGES}"
fi
install_package $NFS_GANESHA_PACKAGES
}
function configure_nfs_ganesha {
# Configure NFS-Ganesha to work with Manila's CephFS driver
rados_cmd="sudo rados -p ${CEPHFS_DATA_POOL}"
if [[ "$CEPHADM_DEPLOY" = "True" ]]; then
CEPHADM=${TARGET_BIN}/cephadm
rados_cmd="sudo $CEPHADM shell rados -p ${CEPHFS_DATA_POOL}"
fi
sudo mkdir -p /etc/ganesha/export.d
if [ $MANILA_CEPH_GANESHA_RADOS_STORE == 'True' ]; then
# Create an empty placeholder ganesha export index object
echo | $rados_cmd put ganesha-export-index -
cat <<EOF | sudo tee /etc/ganesha/ganesha.conf >/dev/null
RADOS_URLS {
ceph_conf = ${CEPH_CONF_FILE};
userid = admin;
}
CACHEINODE {
Dir_Max = 1;
Dir_Chunk = 0;
Cache_FDs = false;
NParts = 1;
Cache_Size = 1;
}
EXPORT_DEFAULTS {
Attr_Expiration_Time = 0;
}
%url rados://${CEPHFS_DATA_POOL}/ganesha-export-index
EOF
else
sudo touch /etc/ganesha/export.d/INDEX.conf
echo "%include /etc/ganesha/export.d/INDEX.conf" | sudo tee /etc/ganesha/ganesha.conf
fi
}
function start_nfs_ganesha {
# NFS-Ganesha server cannot run alongwith with other kernel NFS server.
sudo systemctl stop nfs-server || true
sudo systemctl disable nfs-server || true
sudo systemctl enable nfs-ganesha
sudo systemctl start nfs-ganesha || (
echo "Ganesha didn't start. Let's debug..." >&2
sudo systemctl status nfs-ganesha || true
echo "**Ganesha conf file**" >&2
sudo cat /etc/ganesha/ganesha.conf || true
echo "**Ganesha log file**" >&2
sudo cat /var/log/ganesha/ganesha.log || true
echo "**Exiting**" >&2
exit 1
)
echo "Standalone NFS-Ganesha started successfully!" >&2
}
function stop_nfs_ganesha {
sudo systemctl stop nfs-ganesha
sudo systemctl disable nfs-ganesha
}
function cleanup_nfs_ganesha {
sudo systemctl stop nfs-ganesha
sudo systemctl disable nfs-ganesha
sudo uninstall_package nfs-ganesha nfs-ganesha-ceph libntirpc3 nfs-ganesha-rados-urls nfs-ganesha-vfs
}
# cleanup_repo_nfsganesha() - Remove NFS Ganesha repositories
# Usage: cleanup_repo_nfsganesha
function cleanup_repo_nfsganesha {
if is_ubuntu; then
sudo rm -rf "/etc/apt/sources.list.d/nfs-ganesha-ubuntu*"
elif is_fedora; then
sudo rm -rf /etc/yum.repos.d/nfs-ganesha.repo
fi
}

8
devstack/settings
View File

@ -62,16 +62,16 @@ if (is_ceph_enabled_for_service manila); then
MANILA_OPTGROUP_cephfsnfs1_cephfs_conf_path=${CEPH_CONF_FILE} MANILA_OPTGROUP_cephfsnfs1_cephfs_conf_path=${CEPH_CONF_FILE}
MANILA_OPTGROUP_cephfsnfs1_cephfs_auth_id=${MANILA_CEPH_USER} MANILA_OPTGROUP_cephfsnfs1_cephfs_auth_id=${MANILA_CEPH_USER}
MANILA_OPTGROUP_cephfsnfs1_cephfs_protocol_helper_type=NFS MANILA_OPTGROUP_cephfsnfs1_cephfs_protocol_helper_type=NFS
MANILA_OPTGROUP_cephfsnfs1_cephfs_ganesha_server_ip=$HOST_IP
if [[ $CEPHADM_DEPLOY_NFS == "True" ]]; then
MANILA_OPTGROUP_cephfsnfs1_cephfs_nfs_cluster_id=${FSNAME}
else
MANILA_OPTGROUP_cephfsnfs1_cephfs_ganesha_server_ip=$HOST_IP
MANILA_CEPH_GANESHA_RADOS_STORE=$(trueorfalse False MANILA_CEPH_GANESHA_RADOS_STORE) MANILA_CEPH_GANESHA_RADOS_STORE=$(trueorfalse False MANILA_CEPH_GANESHA_RADOS_STORE)
if [ "$MANILA_CEPH_GANESHA_RADOS_STORE" = "True" ]; then if [ "$MANILA_CEPH_GANESHA_RADOS_STORE" = "True" ]; then
MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_enable=${MANILA_CEPH_GANESHA_RADOS_STORE} MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_enable=${MANILA_CEPH_GANESHA_RADOS_STORE}
MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_pool_name=${CEPHFS_DATA_POOL} MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_pool_name=${CEPHFS_DATA_POOL}
fi fi
if [ "$CEPHADM_DEPLOY" = "True" ]; then
MANILA_OPTGROUP_cephfsnfs1_cephfs_nfs_cluster_id=${FSNAME}
fi fi
fi fi
fi fi