Merge "Standalone nfs-ganesha with cephadm deployment"

.zuul.yaml

@@ -90,10 +90,11 @@
     parent: manila-tempest-plugin-cephfs-native-cephadm
 
 - job:
-    name: devstack-plugin-ceph-cephfs-nfs
+    name: devstack-plugin-ceph-cephfs-nfs-standalone
     description: |
       Runs manila tempest plugin tests with CephFS via NFS-Ganesha as a manila
-      back end (DHSS=False)
+      back end (DHSS=False). The Ceph cluster is created with cephadm
+      while nfs-ganesha is installed "standalone" via a package.
     parent: manila-tempest-plugin-cephfs-nfs
 
 - job:
@@ -174,7 +175,7 @@
         - devstack-plugin-ceph-cephfs-native:
             irrelevant-files: *irrelevant-files
             voting: false
-        - devstack-plugin-ceph-cephfs-nfs:
+        - devstack-plugin-ceph-cephfs-nfs-standalone:
             irrelevant-files: *irrelevant-files
             voting: false
         # - devstack-plugin-ceph-master-tempest:

devstack/lib/ceph

@@ -21,6 +21,7 @@
 # Save trace setting
 XTRACE=$(set +o | grep xtrace)
 set +o xtrace
+source $CEPH_PLUGIN_DIR/lib/common
 
 
 # Defaults
@@ -30,20 +31,6 @@ TEST_MASTER=$(trueorfalse False TEST_MASTER)
 
 CEPH_RELEASE=${CEPH_RELEASE:-quincy}
 
-GANESHA_RELEASE=${GANESHA_RELEASE:-'unspecified'}
-# Remove "v" and "-stable" prefix/suffix tags
-GANESHA_RELEASE=$(echo $GANESHA_RELEASE | sed -e "s/^v//" -e "s/-stable$//")
-
-if [[ "$MANILA_CEPH_DRIVER" == "cephfsnfs" && "$GANESHA_RELEASE" == "unspecified" ]]; then
-    # default ganesha release based on ceph release
-    case $CEPH_RELEASE in
-        pacific)
-            GANESHA_RELEASE='3.5' ;;
-        *)
-            GANESHA_RELEASE='4.0' ;;
-    esac
-fi
-
 # Deploy a Ceph demo container instead of a non-containerized version
 CEPH_CONTAINERIZED=$(trueorfalse False CEPH_CONTAINERIZED)
 
@@ -111,10 +98,6 @@ CEPHFS_DATA_POOL=${CEPHFS_DATA_POOL:-cephfs_data}
 
 MANILA_CEPH_DRIVER=${MANILA_CEPH_DRIVER:-cephfsnative}
 MANILA_CEPH_USER=${MANILA_CEPH_USER:-manila}
-# Allows driver to store NFS-Ganesha exports and export counter as
-# RADOS objects in CephFS's data pool. This needs NFS-Ganesha v2.5.4 or later,
-# Ceph v12.2.2 or later, and OpenStack Queens or later.
-MANILA_CEPH_GANESHA_RADOS_STORE=${MANILA_CEPH_GANESHA_RADOS_STORE:-True}
 
 # Set ``CEPH_REPLICAS`` to configure how many replicas are to be
 # configured for your Ceph cluster. By default we are configuring
@@ -757,67 +740,13 @@ function configure_ceph_manila {
 
     if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
         configure_nfs_ganesha
-        # NFS-Ganesha server cannot run alongwith with other kernel NFS server.
+        start_nfs_ganesha
-        sudo systemctl stop nfs-server || true
-        sudo systemctl disable nfs-server || true
-        sudo systemctl enable nfs-ganesha
-        sudo systemctl start nfs-ganesha || (
-            echo "Ganesha didn't start. Let's debug..." >&2
-            sudo systemctl status nfs-ganesha || true
-            echo "**Ganesha conf file**" >&2
-            sudo cat /etc/ganesha/ganesha.conf || true
-            echo "**Ganesha log file**" >&2
-            sudo cat /var/log/ganesha/ganesha.log || true
-            echo "**Exiting**" >&2
-            exit 1
-        )
-        echo "Ganesha started successfully!" >&2
     fi
 
 # RESTART DOCKER CONTAINER
 
 }
 
-function configure_nfs_ganesha {
-    # Configure NFS-Ganesha to work with Manila's CephFS driver
-    sudo mkdir -p /etc/ganesha/export.d
-    if [ $MANILA_CEPH_GANESHA_RADOS_STORE == 'True' ]; then
-        # Create an empty placeholder ganesha export index object
-        echo | sudo rados -p ${CEPHFS_DATA_POOL} put ganesha-export-index -
-        cat <<EOF | sudo tee /etc/ganesha/ganesha.conf>/dev/null
-RADOS_URLS {
-    ceph_conf = ${CEPH_CONF_FILE};
-    userid = admin;
-}
-
-CACHEINODE {
-        Dir_Max = 1;
-        Dir_Chunk = 0;
-
-        Cache_FDs = false;
-
-        NParts = 1;
-        Cache_Size = 1;
-}
-
-EXPORT_DEFAULTS {
-        Attr_Expiration_Time = 0;
-}
-
-%url rados://${CEPHFS_DATA_POOL}/ganesha-export-index
-EOF
-    else
-        sudo touch /etc/ganesha/export.d/INDEX.conf
-        echo "%include /etc/ganesha/export.d/INDEX.conf" | sudo tee /etc/ganesha/ganesha.conf
-    fi
-}
-
-function cleanup_nfs_ganesha {
-    sudo systemctl stop nfs-ganesha
-    sudo systemctl disable nfs-ganesha
-    sudo uninstall_package nfs-ganesha nfs-ganesha-ceph libntirpc3 nfs-ganesha-rados-urls nfs-ganesha-vfs
-}
-
 function configure_ceph_embedded_manila {
     if [[ $CEPH_REPLICAS -ne 1 ]]; then
         sudo $DOCKER_EXEC ceph -c ${CEPH_CONF_FILE} osd pool set ${CEPHFS_DATA_POOL} \
@@ -958,19 +887,6 @@ EOF
     sudo dnf config-manager --add-repo ceph.repo
 }
 
-function dnf_add_repository_nfsganesha {
-    local repo=""
-
-    case $ganesha_release in
-      3.*)
-        repo="centos-release-nfs-ganesha30" ;;
-      *)
-        repo="centos-release-nfs-ganesha4" ;;
-    esac
-
-    sudo dnf -y install ${repo}
-}
-
 # configure_repo_ceph() - Configure Ceph repositories
 # Usage: configure_repo_ceph <package_release>
 # - package_release: to override the os_RELEASE variable
@@ -1009,44 +925,11 @@ function cleanup_repo_ceph {
     fi
 }
 
-# configure_repo_nfsganesha() - Configure NFS Ganesha repositories
-function configure_repo_nfsganesha {
-    if is_ubuntu; then
-        # NOTE(gouthamr): Ubuntu PPAs contain the latest build from each major
-        # version; we can't use a build microversion unlike el8/el9 builds
-        if [[ $GANESHA_RELEASE =~ 3 ]]; then
-            sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-3.0
-            sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-3.0
-        elif [[ $GANESHA_RELEASE =~ 4 ]]; then
-            sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-4
-            sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-4
-        else
-            die $LINENO "NFS-Ganesha $GANESHA_RELEASE is not supported by the Ceph plugin for Devstack"
-        fi
-        sudo apt-get -y update
-    elif is_fedora; then
-        dnf_add_repository_nfsganesha
-    fi
-}
-
-# cleanup_repo_nfsganesha() - Remove NFS Ganesha repositories
-# Usage: cleanup_repo_nfsganesha
-function cleanup_repo_nfsganesha {
-    if is_ubuntu; then
-        sudo rm -rf "/etc/apt/sources.list.d/nfs-ganesha-ubuntu*"
-    elif is_fedora; then
-        sudo rm -rf /etc/yum.repos.d/nfs-ganesha.repo
-    fi
-}
-
 function setup_packages_for_manila_on_ubuntu {
     CEPH_PACKAGES="${CEPH_PACKAGES} ceph-mds libcephfs2"
 
     if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
-        configure_repo_nfsganesha
+        install_nfs_ganesha
-        LIBNTIRPC_PACKAGE="libntirpc${GANESHA_RELEASE:0:1}"
-        CEPH_PACKAGES="${CEPH_PACKAGES} $LIBNTIRPC_PACKAGE \
-          nfs-ganesha nfs-ganesha-ceph nfs-ganesha-rados-urls nfs-ganesha-vfs"
     fi
 
     if python3_enabled; then
@@ -1056,9 +939,7 @@ function setup_packages_for_manila_on_ubuntu {
 
 function setup_packages_for_manila_on_fedora_family {
     if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
-        configure_repo_nfsganesha
+        install_nfs_ganesha
-        CEPH_PACKAGES="${CEPH_PACKAGES} nfs-ganesha nfs-ganesha-ceph \
-           nfs-ganesha-rados-urls nfs-ganesha-vfs"
     fi
 }
 
@@ -1175,8 +1056,7 @@ function stop_ceph {
     fi
     if is_ceph_enabled_for_service manila; then
         if [ $MANILA_CEPH_DRIVER == 'cephfsnfs' ]; then
-            sudo systemctl stop nfs-ganesha
+            stop_nfs_ganesha
-            sudo systemctl disable nfs-ganesha
         fi
         sudo systemctl stop ceph-mds@${MDS_ID}
         sudo systemctl disable ceph-mds@${MDS_ID}

devstack/lib/cephadm

@@ -67,7 +67,11 @@ MDS_LOGS=${MDS_LOGS:-"False"}
 # MANILA DEFAULTS
 MANILA_CEPH_USER=${MANILA_CEPH_USER:-manila}
 
-# NFS OPTIONS
+# NFS OPTIONS: Only apply when ENABLE_CEPH_MANILA=True
+# Whether or not cephadm should deploy/manage NFS-Ganesha? If set to False,
+# we'll deploy a "standalone" NFS Ganesha instead, not managed by cephadm.
+CEPHADM_DEPLOY_NFS=${CEPHADM_DEPLOY_NFS:-True}
+# Clustered NFS Options
 FSNAME=${FSNAME:-'cephfs'}
 NFS_PORT=2049
 CEPHFS_CLIENT=0
@@ -153,7 +157,7 @@ function export_spec {
 
 # Pre-install ceph: install required dependencies
 function install_deps {
-    if [[ "$REMOTE_CEPH" == "False" ]]; then
+    if [[ "$REMOTE_CEPH" = "False" ]]; then
         install_package python3-cephfs python3-prettytable python3-rados python3-rbd python3-requests
     fi
 }
@@ -218,16 +222,16 @@ function start_ceph {
             --skip-mon-network \
             --mon-ip "$HOST_IP"
 
-    test -e $CEPH_CONFIG
+        test -e $CEPH_CONFIG
-    test -e $CEPH_KEYRING
+        test -e $CEPH_KEYRING
 
-    if [ "$CEPHADM_DEV_OSD" == 'True' ]; then
+        if [ "$CEPHADM_DEV_OSD" == 'True' ]; then
-        create_osd_dev
+            create_osd_dev
-    fi
+        fi
-    # Wait cephadm backend to be operational
+        # Wait cephadm backend to be operational
-    # and add osds via drivegroups
+        # and add osds via drivegroups
-    sleep "$SLEEP"
+        sleep "$SLEEP"
-    add_osds
+        add_osds
     fi
 }
 
@@ -271,7 +275,7 @@ function add_osds {
 
     while [ "$ATTEMPTS" -ne 0 ]; do
         num_osds=$($SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \
-        --keyring $CEPH_KEYRING -- ceph -s -f json | jq '.osdmap | .num_up_osds')
+            --keyring $CEPH_KEYRING -- ceph -s -f json | jq '.osdmap | .num_up_osds')
         if [ "$num_osds" -ge "$MIN_OSDS" ]; then
             break;
         fi
@@ -342,8 +346,8 @@ function create_keys {
 # Install ceph: add MDS
 function cephfs_config {
     # Two pools are generated by this action
-    # - $FSNAME.FSNAME.data
+    # - cephfs.$FSNAME.data
-    # - $FSNAME.FSNAME.meta
+    # - cephfs.$FSNAME.meta
     # and the mds daemon is deployed
     $SUDO "$CEPHADM" shell --fsid $FSID --config $CEPH_CONFIG \
         --keyring $CEPH_KEYRING -- ceph fs volume create "$FSNAME"
@@ -359,8 +363,7 @@ function _get_ceph_version {
     echo $ceph_version_str
 }
 
-# Install ceph: add NFS
+function _install_and_configure_clustered_nfs {
-function ceph_nfs_config {
     local ceph_version
     ceph_version=$(_get_ceph_version)
 
@@ -379,6 +382,23 @@ function ceph_nfs_config {
     fi
 }
 
+function _install_and_configure_standalone_nfs {
+    source $CEPH_PLUGIN_DIR/lib/common
+    install_nfs_ganesha
+    configure_nfs_ganesha
+    start_nfs_ganesha
+}
+
+# Install ceph: add NFS
+function ceph_nfs_config {
+    if [[ "$CEPHADM_DEPLOY_NFS" == "True" ]]; then
+        _install_and_configure_clustered_nfs
+    else
+        _install_and_configure_standalone_nfs
+    fi
+
+}
+
 function _create_swift_endpoint {
 
     local swift_service
@@ -482,17 +502,17 @@ function configure_ceph_manila {
 function enable_services {
     for item in "${SERVICES[@]}"; do
         case "$item" in
-            cephfs|CEPHFS)
+        cephfs|CEPHFS)
             echo "[CEPHADM] Config cephfs volume on node $HOSTNAME"
             cephfs_config
             CEPHFS_CLIENT=1
             ;;
-            nfs|NFS)
+        nfs|NFS)
             echo "[CEPHADM] Deploying NFS on node $HOSTNAME"
             ceph_nfs_config
             CEPHFS_CLIENT=1
             ;;
-            rgw|RGW)
+        rgw|RGW)
             echo "[CEPHADM] Deploying RGW on node $HOSTNAME"
             rgw
             ;;
@@ -735,8 +755,14 @@ function cleanup_ceph {
     # purge ceph config file and keys
     $SUDO rm -f ${CEPH_CONF_DIR}/*
     if is_ceph_enabled_for_service nova; then
-    _undefine_virsh_secret
+        _undefine_virsh_secret
     fi
+    if [[ "$CEPHADM_DEPLOY_NFS" != "True" ]]; then
+        stop_nfs_ganesha
+        cleanup_nfs_ganesha
+        cleanup_repo_nfs_ganesha
+    fi
+
 }
 
 function disable_cephadm {

devstack/lib/common

@@ -0,0 +1,149 @@
+#!/bin/bash
+
+# Allows driver to store NFS-Ganesha exports and export counter as
+# RADOS objects in CephFS's data pool. This needs NFS-Ganesha v2.5.4 or later,
+# Ceph v12.2.2 or later, and OpenStack Queens or later.
+MANILA_CEPH_GANESHA_RADOS_STORE=${MANILA_CEPH_GANESHA_RADOS_STORE:-True}
+GANESHA_RELEASE=${GANESHA_RELEASE:-'unspecified'}
+# Remove "v" and "-stable" prefix/suffix tags
+GANESHA_RELEASE=$(echo $GANESHA_RELEASE | sed -e "s/^v//" -e "s/-stable$//")
+if [[ "$CEPHADM_DEPLOY" = "True" ]]; then
+    FSNAME=${FSNAME:-'cephfs'}
+    CEPHFS_DATA_POOL="cephfs.$FSNAME.data"
+else
+    CEPHFS_DATA_POOL=${CEPHFS_DATA_POOL:-cephfs_data}
+fi
+
+if [[ "$MANILA_CEPH_DRIVER" == "cephfsnfs" && "$GANESHA_RELEASE" == "unspecified" ]]; then
+    # default ganesha release based on ceph release
+    case $CEPH_RELEASE in
+    pacific)
+        GANESHA_RELEASE='3.5'
+        ;;
+    *)
+        GANESHA_RELEASE='5.0'
+        ;;
+    esac
+fi
+
+# configure_repo_nfsganesha - Configure NFS Ganesha repositories
+function configure_repo_nfsganesha {
+    if is_ubuntu; then
+        # NOTE(gouthamr): Ubuntu PPAs contain the latest build from each major
+        # version; we can't use a build microversion unlike el8/el9 builds
+        case $GANESHA_RELEASE in
+        3.*)
+            sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-3.0
+            sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-3.0
+            ;;
+        *)
+            GANESHA_PPA_VERSION="${GANESHA_RELEASE:0:1}"
+            sudo add-apt-repository -y ppa:nfs-ganesha/libntirpc-"$GANESHA_PPA_VERSION"
+            sudo add-apt-repository -y ppa:nfs-ganesha/nfs-ganesha-"$GANESHA_PPA_VERSION"
+            ;;
+        esac
+        sudo apt-get -y update
+    elif is_fedora; then
+        local repo=""
+        case $GANESHA_RELEASE in
+        3.*)
+            repo="centos-release-nfs-ganesha30"
+            ;;
+        *)
+            repo="centos-release-nfs-ganesha5"
+            ;;
+        esac
+        sudo dnf -y install ${repo}
+    fi
+}
+
+function install_nfs_ganesha {
+    configure_repo_nfsganesha
+    NFS_GANESHA_PACKAGES="nfs-ganesha nfs-ganesha-ceph \
+        nfs-ganesha-rados-urls nfs-ganesha-vfs"
+    if is_ubuntu; then
+        LIBNTIRPC_PACKAGE="libntirpc${GANESHA_RELEASE:0:1}"
+        NFS_GANESHA_PACKAGES="${LIBNTIRPC_PACKAGE} ${NFS_GANESHA_PACKAGES}"
+    fi
+    install_package $NFS_GANESHA_PACKAGES
+}
+
+function configure_nfs_ganesha {
+    # Configure NFS-Ganesha to work with Manila's CephFS driver
+    rados_cmd="sudo rados -p ${CEPHFS_DATA_POOL}"
+    if [[ "$CEPHADM_DEPLOY" = "True" ]]; then
+        CEPHADM=${TARGET_BIN}/cephadm
+        rados_cmd="sudo $CEPHADM shell rados -p ${CEPHFS_DATA_POOL}"
+    fi
+
+
+    sudo mkdir -p /etc/ganesha/export.d
+    if [ $MANILA_CEPH_GANESHA_RADOS_STORE == 'True' ]; then
+        # Create an empty placeholder ganesha export index object
+        echo | $rados_cmd put ganesha-export-index -
+        cat <<EOF | sudo tee /etc/ganesha/ganesha.conf >/dev/null
+    RADOS_URLS {
+        ceph_conf = ${CEPH_CONF_FILE};
+        userid = admin;
+}
+
+CACHEINODE {
+        Dir_Max = 1;
+        Dir_Chunk = 0;
+
+        Cache_FDs = false;
+
+        NParts = 1;
+        Cache_Size = 1;
+}
+
+EXPORT_DEFAULTS {
+        Attr_Expiration_Time = 0;
+}
+
+%url rados://${CEPHFS_DATA_POOL}/ganesha-export-index
+EOF
+    else
+        sudo touch /etc/ganesha/export.d/INDEX.conf
+        echo "%include /etc/ganesha/export.d/INDEX.conf" | sudo tee /etc/ganesha/ganesha.conf
+    fi
+}
+
+function start_nfs_ganesha {
+    # NFS-Ganesha server cannot run alongwith with other kernel NFS server.
+    sudo systemctl stop nfs-server || true
+    sudo systemctl disable nfs-server || true
+    sudo systemctl enable nfs-ganesha
+    sudo systemctl start nfs-ganesha || (
+        echo "Ganesha didn't start. Let's debug..." >&2
+        sudo systemctl status nfs-ganesha || true
+        echo "**Ganesha conf file**" >&2
+        sudo cat /etc/ganesha/ganesha.conf || true
+        echo "**Ganesha log file**" >&2
+        sudo cat /var/log/ganesha/ganesha.log || true
+        echo "**Exiting**" >&2
+        exit 1
+    )
+    echo "Standalone NFS-Ganesha started successfully!" >&2
+}
+
+function stop_nfs_ganesha {
+    sudo systemctl stop nfs-ganesha
+    sudo systemctl disable nfs-ganesha
+}
+
+function cleanup_nfs_ganesha {
+    sudo systemctl stop nfs-ganesha
+    sudo systemctl disable nfs-ganesha
+    sudo uninstall_package nfs-ganesha nfs-ganesha-ceph libntirpc3 nfs-ganesha-rados-urls nfs-ganesha-vfs
+}
+
+# cleanup_repo_nfsganesha() - Remove NFS Ganesha repositories
+# Usage: cleanup_repo_nfsganesha
+function cleanup_repo_nfsganesha {
+    if is_ubuntu; then
+        sudo rm -rf "/etc/apt/sources.list.d/nfs-ganesha-ubuntu*"
+    elif is_fedora; then
+        sudo rm -rf /etc/yum.repos.d/nfs-ganesha.repo
+    fi
+}

devstack/settings

@@ -62,16 +62,16 @@ if (is_ceph_enabled_for_service manila); then
         MANILA_OPTGROUP_cephfsnfs1_cephfs_conf_path=${CEPH_CONF_FILE}
         MANILA_OPTGROUP_cephfsnfs1_cephfs_auth_id=${MANILA_CEPH_USER}
         MANILA_OPTGROUP_cephfsnfs1_cephfs_protocol_helper_type=NFS
-        MANILA_OPTGROUP_cephfsnfs1_cephfs_ganesha_server_ip=$HOST_IP
 
-        MANILA_CEPH_GANESHA_RADOS_STORE=$(trueorfalse False MANILA_CEPH_GANESHA_RADOS_STORE)
+        if [[ $CEPHADM_DEPLOY_NFS == "True" ]]; then
-        if [ "$MANILA_CEPH_GANESHA_RADOS_STORE" = "True" ]; then
-            MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_enable=${MANILA_CEPH_GANESHA_RADOS_STORE}
-            MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_pool_name=${CEPHFS_DATA_POOL}
-        fi
-
-        if [ "$CEPHADM_DEPLOY" = "True" ]; then
             MANILA_OPTGROUP_cephfsnfs1_cephfs_nfs_cluster_id=${FSNAME}
+        else
+            MANILA_OPTGROUP_cephfsnfs1_cephfs_ganesha_server_ip=$HOST_IP
+            MANILA_CEPH_GANESHA_RADOS_STORE=$(trueorfalse False MANILA_CEPH_GANESHA_RADOS_STORE)
+            if [ "$MANILA_CEPH_GANESHA_RADOS_STORE" = "True" ]; then
+                MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_enable=${MANILA_CEPH_GANESHA_RADOS_STORE}
+                MANILA_OPTGROUP_cephfsnfs1_ganesha_rados_store_pool_name=${CEPHFS_DATA_POOL}
+            fi
         fi
     fi
 fi