remove non RabbitMQ messaging

Part of what was decided at summit is devstack needs to return to a
more opinionated stance, the following removes support for non
RabbitMQ messaging. RabbitMQ is used by over 95% of our community
(statistically all of it), so it's a pretty clear line to draw that
this shouldn't be in tree.

iniset_rpc_backend will be our stable hook for other projects that
want to implement this out of tree. The burden on creating those out
of tree plugins will be on those that wish to support those
alternative stacks.

Change-Id: I8073a895c03ec927a2598eff6c2f01e5c82606fc

README.md

@@ -115,22 +115,6 @@ following in the `localrc` section:
 
 `mysql` is the default database.
 
-# RPC Backend
-
-Multiple RPC backends are available. Currently, this
-includes RabbitMQ (default), Qpid, and ZeroMQ. Your backend of
-choice may be selected via the `localrc` section.
-
-Note that selecting more than one RPC backend will result in a failure.
-
-Example (ZeroMQ):
-
-    ENABLED_SERVICES="$ENABLED_SERVICES,-rabbit,-qpid,zeromq"
-
-Example (Qpid):
-
-    ENABLED_SERVICES="$ENABLED_SERVICES,-rabbit,-zeromq,qpid"
-
 # Apache Frontend
 
 Apache web server can be enabled for wsgi services that support being deployed

doc/source/faq.rst

@@ -88,7 +88,7 @@ Q: Is enabling a service that defaults to off done with the reverse of the above
 
     ::
 
-        enable_service qpid
+        enable_service q-svc
 
 Q: How do I run a specific OpenStack milestone?
    A: OpenStack milestones have tags set in the git repo. Set the

files/debs/neutron

@@ -9,11 +9,9 @@ sudo
 postgresql-server-dev-all
 python-mysqldb
 python-mysql.connector
-python-qpid # NOPRIME
 dnsmasq-base
 dnsmasq-utils # for dhcp_release only available in dist:precise
 rabbitmq-server # NOPRIME
-qpidd # NOPRIME
 sqlite3
 vlan
 radvd # NOPRIME

files/debs/nova

@@ -24,10 +24,8 @@ vlan
 curl
 genisoimage # required for config_drive
 rabbitmq-server # NOPRIME
-qpidd # NOPRIME
 socat # used by ajaxterm
 python-libvirt # NOPRIME
 python-libxml2
 python-numpy # used by websockify for spice console
 python-m2crypto
-python-qpid # NOPRIME

files/debs/qpid

@@ -1 +0,0 @@
-sasl2-bin # NOPRIME

files/rpms-suse/neutron

@@ -11,6 +11,3 @@ sqlite3
 sudo
 vlan
 radvd # NOPRIME
-
-# FIXME: qpid is not part of openSUSE, those names are tentative
-qpidd # NOPRIME

files/rpms-suse/nova

@@ -22,7 +22,3 @@ socat
 sqlite3
 sudo
 vlan
-
-# FIXME: qpid is not part of openSUSE, those names are tentative
-python-qpid # NOPRIME
-qpidd # NOPRIME

files/rpms/neutron

@@ -11,7 +11,6 @@ mysql-server # NOPRIME
 openvswitch # NOPRIME
 postgresql-devel
 rabbitmq-server # NOPRIME
-qpid-cpp-server        # NOPRIME
 sqlite
 sudo
 radvd # NOPRIME

files/rpms/nova

@@ -22,6 +22,5 @@ mysql-server # NOPRIME
 parted
 polkit
 rabbitmq-server # NOPRIME
-qpid-cpp-server # NOPRIME
 sqlite
 sudo

files/rpms/qpid

@@ -1,3 +0,0 @@
-qpid-proton-c-devel # NOPRIME
-cyrus-sasl-lib # NOPRIME
-cyrus-sasl-plain # NOPRIME

functions-common

@@ -1671,7 +1671,7 @@ function disable_service {
 # ``ENABLED_SERVICES`` list, if they are not already present.
 #
 # For example:
-#   enable_service qpid
+#   enable_service q-svc
 #
 # This function does not know about the special cases
 # for nova, glance, and neutron built into is_service_enabled().

lib/glance

@@ -112,9 +112,7 @@ function configure_glance {
     iniset $GLANCE_REGISTRY_CONF DEFAULT workers "$API_WORKERS"
     iniset $GLANCE_REGISTRY_CONF paste_deploy flavor keystone
     configure_auth_token_middleware $GLANCE_REGISTRY_CONF glance $GLANCE_AUTH_CACHE_DIR/registry
-    if is_service_enabled qpid || [ -n "$RABBIT_HOST" ] && [ -n "$RABBIT_PASSWORD" ]; then
     iniset $GLANCE_REGISTRY_CONF DEFAULT notification_driver messaging
-    fi
     iniset_rpc_backend glance $GLANCE_REGISTRY_CONF
 
     cp $GLANCE_DIR/etc/glance-api.conf $GLANCE_API_CONF
@@ -125,9 +123,7 @@ function configure_glance {
     iniset $GLANCE_API_CONF DEFAULT image_cache_dir $GLANCE_CACHE_DIR/
     iniset $GLANCE_API_CONF paste_deploy flavor keystone+cachemanagement
     configure_auth_token_middleware $GLANCE_API_CONF glance $GLANCE_AUTH_CACHE_DIR/api
-    if is_service_enabled qpid || [ -n "$RABBIT_HOST" ] && [ -n "$RABBIT_PASSWORD" ]; then
     iniset $GLANCE_API_CONF DEFAULT notification_driver messaging
-    fi
     iniset_rpc_backend glance $GLANCE_API_CONF
     if [ "$VIRT_DRIVER" = 'xenserver' ]; then
         iniset $GLANCE_API_CONF DEFAULT container_formats "ami,ari,aki,bare,ovf,tgz"

lib/rpc_backend

@@ -1,72 +1,32 @@
 #!/bin/bash
 #
 # lib/rpc_backend
-# Interface for interactig with different RPC backends
+# Interface for installing RabbitMQ on the system
 
 # Dependencies:
 #
 # - ``functions`` file
 # - ``RABBIT_{HOST|PASSWORD|USERID}`` must be defined when RabbitMQ is used
-# - ``RPC_MESSAGING_PROTOCOL`` option for configuring the messaging protocol
 
 # ``stack.sh`` calls the entry points in this order:
 #
 # - check_rpc_backend
 # - install_rpc_backend
 # - restart_rpc_backend
-# - iniset_rpc_backend
+# - iniset_rpc_backend (stable interface)
+#
+# Note: if implementing an out of tree plugin for an RPC backend, you
+# should install all services through normal plugin methods, then
+# redefine ``iniset_rpc_backend`` in your code. That's the one portion
+# of this file which is a standard interface.
 
 # Save trace setting
 XTRACE=$(set +o | grep xtrace)
 set +o xtrace
 
-RPC_MESSAGING_PROTOCOL=${RPC_MESSAGING_PROTOCOL:-0.9}
-
-# TODO(sdague): RPC backend selection is super wonky because we treat
-# messaging server as a service, which it really isn't for multi host
-QPID_HOST=${QPID_HOST:-}
-
-
 # Functions
 # ---------
 
-# Make sure we only have one rpc backend enabled.
-# Also check the specified rpc backend is available on your platform.
-function check_rpc_backend {
-    local c svc
-
-    local rpc_needed=1
-    # We rely on the fact that filenames in lib/* match the service names
-    # that can be passed as arguments to is_service_enabled.
-    # We check for a call to iniset_rpc_backend in these files, meaning
-    # the service needs a backend.
-    rpc_candidates=$(grep -rl iniset_rpc_backend $TOP_DIR/lib/ | awk -F/ '{print $NF}')
-    for c in ${rpc_candidates}; do
-        if is_service_enabled $c; then
-            rpc_needed=0
-            break
-        fi
-    done
-    local rpc_backend_cnt=0
-    for svc in qpid zeromq rabbit; do
-        is_service_enabled $svc &&
-        (( rpc_backend_cnt++ )) || true
-    done
-    if [ "$rpc_backend_cnt" -gt 1 ]; then
-        echo "ERROR: only one rpc backend may be enabled,"
-        echo "       set only one of 'rabbit', 'qpid', 'zeromq'"
-        echo "       via ENABLED_SERVICES."
-    elif [ "$rpc_backend_cnt" == 0 ] && [ "$rpc_needed" == 0 ]; then
-        echo "ERROR: at least one rpc backend must be enabled,"
-        echo "       set one of 'rabbit', 'qpid', 'zeromq'"
-        echo "       via ENABLED_SERVICES."
-    fi
-
-    if is_service_enabled qpid && ! qpid_is_supported; then
-        die $LINENO "Qpid support is not available for this version of your distribution."
-    fi
-}
-
 # clean up after rpc backend - eradicate all traces so changing backends
 # produces a clean switch
 function cleanup_rpc_backend {
@@ -79,110 +39,14 @@ function cleanup_rpc_backend {
             # And the Erlang runtime too
             apt_get purge -y erlang*
         fi
-    elif is_service_enabled qpid; then
-        if is_fedora; then
-            uninstall_package qpid-cpp-server
-        elif is_ubuntu; then
-            uninstall_package qpidd
-        else
-            exit_distro_not_supported "qpid installation"
-        fi
-    elif is_service_enabled zeromq; then
-        if is_fedora; then
-            uninstall_package zeromq python-zmq
-            if [ "$ZEROMQ_MATCHMAKER" == "redis" ]; then
-                uninstall_package redis python-redis
-            fi
-        elif is_ubuntu; then
-            uninstall_package libzmq1 python-zmq
-            if [ "$ZEROMQ_MATCHMAKER" == "redis" ]; then
-                uninstall_package redis-server python-redis
-            fi
-        elif is_suse; then
-            uninstall_package libzmq1 python-pyzmq
-            if [ "$ZEROMQ_MATCHMAKER" == "redis" ]; then
-                uninstall_package redis python-redis
-            fi
-        else
-            exit_distro_not_supported "zeromq installation"
-        fi
-    fi
-
-    # Remove the AMQP 1.0 messaging libraries
-    if [ "$RPC_MESSAGING_PROTOCOL" == "AMQP1" ]; then
-        if is_fedora; then
-            uninstall_package qpid-proton-c-devel
-            uninstall_package python-qpid-proton
-        fi
-        # TODO(kgiusti) ubuntu cleanup
     fi
 }
 
 # install rpc backend
 function install_rpc_backend {
-    # Regardless of the broker used, if AMQP 1.0 is configured load
-    # the necessary messaging client libraries for oslo.messaging
-    if [ "$RPC_MESSAGING_PROTOCOL" == "AMQP1" ]; then
-        if is_fedora; then
-            install_package qpid-proton-c-devel
-            install_package python-qpid-proton
-        elif is_ubuntu; then
-            # TODO(kgiusti) The QPID AMQP 1.0 protocol libraries
-            # are not yet in the ubuntu repos. Enable these installs
-            # once they are present:
-            #install_package libqpid-proton2-dev
-            #install_package python-qpid-proton
-            # Also add 'uninstall' directives in cleanup_rpc_backend()!
-            exit_distro_not_supported "QPID AMQP 1.0 Proton libraries"
-        else
-            exit_distro_not_supported "QPID AMQP 1.0 Proton libraries"
-        fi
-        # Install pyngus client API
-        # TODO(kgiusti) can remove once python qpid bindings are
-        # available on all supported platforms _and_ pyngus is added
-        # to the requirements.txt file in oslo.messaging
-        pip_install_gr pyngus
-    fi
-
     if is_service_enabled rabbit; then
         # Install rabbitmq-server
         install_package rabbitmq-server
-    elif is_service_enabled qpid; then
-        if is_fedora; then
-            install_package qpid-cpp-server
-        elif is_ubuntu; then
-            install_package qpidd
-        else
-            exit_distro_not_supported "qpid installation"
-        fi
-        _configure_qpid
-    elif is_service_enabled zeromq; then
-        if is_fedora; then
-            install_package zeromq python-zmq
-            if [ "$ZEROMQ_MATCHMAKER" == "redis" ]; then
-                install_package redis python-redis
-            fi
-        elif is_ubuntu; then
-            install_package libzmq1 python-zmq
-            if [ "$ZEROMQ_MATCHMAKER" == "redis" ]; then
-                install_package redis-server python-redis
-            fi
-        elif is_suse; then
-            install_package libzmq1 python-pyzmq
-            if [ "$ZEROMQ_MATCHMAKER" == "redis" ]; then
-                install_package redis python-redis
-            fi
-        else
-            exit_distro_not_supported "zeromq installation"
-        fi
-        # Necessary directory for socket location.
-        sudo mkdir -p /var/run/openstack
-        sudo chown $STACK_USER /var/run/openstack
-    fi
-
-    # If using the QPID broker, install the QPID python client API
-    if is_service_enabled qpid || [ -n "$QPID_HOST" ]; then
-        install_package python-qpid
     fi
 }
 
@@ -232,17 +96,12 @@ function restart_rpc_backend {
                 sudo rabbitmqctl set_permissions -p child_cell $RABBIT_USERID ".*" ".*" ".*"
             fi
         fi
-    elif is_service_enabled qpid; then
-        echo_summary "Starting qpid"
-        restart_service qpidd
     fi
 }
 
 # builds transport url string
 function get_transport_url {
-    if is_service_enabled qpid || [ -n "$QPID_HOST" ]; then
+    if is_service_enabled rabbit || { [ -n "$RABBIT_HOST" ] && [ -n "$RABBIT_PASSWORD" ]; }; then
-        echo "qpid://$QPID_USERNAME:$QPID_PASSWORD@$QPID_HOST:5672/"
-    elif is_service_enabled rabbit || { [ -n "$RABBIT_HOST" ] && [ -n "$RABBIT_PASSWORD" ]; }; then
         echo "rabbit://$RABBIT_USERID:$RABBIT_PASSWORD@$RABBIT_HOST:5672/"
     fi
 }
@@ -252,29 +111,7 @@ function iniset_rpc_backend {
     local package=$1
     local file=$2
     local section=${3:-DEFAULT}
-    if is_service_enabled zeromq; then
+    if is_service_enabled rabbit || { [ -n "$RABBIT_HOST" ] && [ -n "$RABBIT_PASSWORD" ]; }; then
-        iniset $file $section rpc_backend "zmq"
-        iniset $file $section rpc_zmq_host `hostname`
-        if [ "$ZEROMQ_MATCHMAKER" == "redis" ]; then
-            iniset $file $section rpc_zmq_matchmaker "redis"
-            MATCHMAKER_REDIS_HOST=${MATCHMAKER_REDIS_HOST:-127.0.0.1}
-            iniset $file matchmaker_redis host $MATCHMAKER_REDIS_HOST
-        else
-            die $LINENO "Other matchmaker drivers not supported"
-        fi
-    elif is_service_enabled qpid || [ -n "$QPID_HOST" ]; then
-        # For Qpid use the 'amqp' oslo.messaging transport when AMQP 1.0 is used
-        if [ "$RPC_MESSAGING_PROTOCOL" == "AMQP1" ]; then
-            iniset $file $section rpc_backend "amqp"
-        else
-            iniset $file $section rpc_backend "qpid"
-        fi
-        iniset $file $section qpid_hostname ${QPID_HOST:-$SERVICE_HOST}
-        if [ -n "$QPID_USERNAME" ]; then
-            iniset $file $section qpid_username $QPID_USERNAME
-            iniset $file $section qpid_password $QPID_PASSWORD
-        fi
-    elif is_service_enabled rabbit || { [ -n "$RABBIT_HOST" ] && [ -n "$RABBIT_PASSWORD" ]; }; then
         iniset $file $section rpc_backend "rabbit"
         iniset $file oslo_messaging_rabbit rabbit_hosts $RABBIT_HOST
         iniset $file oslo_messaging_rabbit rabbit_password $RABBIT_PASSWORD
@@ -288,17 +125,6 @@ function iniset_rpc_backend {
     fi
 }
 
-# Check if qpid can be used on the current distro.
-# qpid_is_supported
-function qpid_is_supported {
-    if [[ -z "$DISTRO" ]]; then
-        GetDistro
-    fi
-
-    # Qpid is not in openSUSE
-    ( ! is_suse )
-}
-
 function rabbit_setuser {
     local user="$1" pass="$2" found="" out=""
     out=$(sudo rabbitmqctl list_users) ||
@@ -314,85 +140,6 @@ function rabbit_setuser {
     sudo rabbitmqctl set_permissions "$user" ".*" ".*" ".*"
 }
 
-# Set up the various configuration files used by the qpidd broker
-function _configure_qpid {
-
-    # the location of the configuration files have changed since qpidd 0.14
-    local qpid_conf_file
-    if [ -e /etc/qpid/qpidd.conf ]; then
-        qpid_conf_file=/etc/qpid/qpidd.conf
-    elif [ -e /etc/qpidd.conf ]; then
-        qpid_conf_file=/etc/qpidd.conf
-    else
-        exit_distro_not_supported "qpidd.conf file not found!"
-    fi
-
-    # force the ACL file to a known location
-    local qpid_acl_file=/etc/qpid/qpidd.acl
-    if [ ! -e $qpid_acl_file ]; then
-        sudo mkdir -p -m 755 `dirname $qpid_acl_file`
-        sudo touch $qpid_acl_file
-        sudo chmod o+r $qpid_acl_file
-    fi
-    sudo sed -i.bak '/^acl-file=/d' $qpid_conf_file
-    echo "acl-file=$qpid_acl_file" | sudo tee --append $qpid_conf_file
-
-    sudo sed -i '/^auth=/d' $qpid_conf_file
-    if [ -z "$QPID_USERNAME" ]; then
-        # no QPID user configured, so disable authentication
-        # and access control
-        echo "auth=no" | sudo tee --append $qpid_conf_file
-        cat <<EOF | sudo tee $qpid_acl_file
-acl allow all all
-EOF
-    else
-        # Configure qpidd to use PLAIN authentication, and add
-        # QPID_USERNAME to the ACL:
-        echo "auth=yes" | sudo tee --append $qpid_conf_file
-        if [ -z "$QPID_PASSWORD" ]; then
-            read_password QPID_PASSWORD "ENTER A PASSWORD FOR QPID USER $QPID_USERNAME"
-        fi
-        # Create ACL to allow $QPID_USERNAME full access
-        cat <<EOF | sudo tee $qpid_acl_file
-group admin ${QPID_USERNAME}@QPID
-acl allow admin all
-acl deny all all
-EOF
-        # Add user to SASL database
-        if is_ubuntu; then
-            install_package sasl2-bin
-        elif is_fedora; then
-            install_package cyrus-sasl-lib
-            install_package cyrus-sasl-plain
-        fi
-        local sasl_conf_file=/etc/sasl2/qpidd.conf
-        sudo sed -i.bak '/PLAIN/!s/mech_list: /mech_list: PLAIN /' $sasl_conf_file
-        local sasl_db=`sudo grep sasldb_path $sasl_conf_file | cut -f 2 -d ":" | tr -d [:blank:]`
-        if [ ! -e $sasl_db ]; then
-            sudo mkdir -p -m 755 `dirname $sasl_db`
-        fi
-        echo $QPID_PASSWORD | sudo saslpasswd2 -c -p -f $sasl_db -u QPID $QPID_USERNAME
-        sudo chmod o+r $sasl_db
-    fi
-
-    # If AMQP 1.0 is specified, ensure that the version of the
-    # broker can support AMQP 1.0 and configure the queue and
-    # topic address patterns used by oslo.messaging.
-    if [ "$RPC_MESSAGING_PROTOCOL" == "AMQP1" ]; then
-        QPIDD=$(type -p qpidd)
-        if ! $QPIDD --help | grep -q "queue-patterns"; then
-            exit_distro_not_supported "qpidd with AMQP 1.0 support"
-        fi
-        if ! grep -q "queue-patterns=exclusive" $qpid_conf_file; then
-            cat <<EOF | sudo tee --append $qpid_conf_file
-queue-patterns=exclusive
-queue-patterns=unicast
-topic-patterns=broadcast
-EOF
-        fi
-    fi
-}
-
 # Restore xtrace
 $XTRACE
 

lib/zaqar

@@ -128,10 +128,9 @@ function configure_zaqar {
         configure_redis
     fi
 
-    if is_service_enabled qpid || [ -n "$RABBIT_HOST" ] && [ -n "$RABBIT_PASSWORD" ]; then
     iniset $ZAQAR_CONF DEFAULT notification_driver messaging
     iniset $ZAQAR_CONF DEFAULT control_exchange zaqar
-    fi
+
     iniset_rpc_backend zaqar $ZAQAR_CONF
 
     cleanup_zaqar

stack.sh

@@ -500,10 +500,6 @@ rm -f $SSL_BUNDLE_FILE
 source $TOP_DIR/lib/database
 source $TOP_DIR/lib/rpc_backend
 
-# Make sure we only have one rpc backend enabled,
-# and the specified rpc backend is available on your platform.
-check_rpc_backend
-
 # Service to enable with SSL if ``USE_SSL`` is True
 SSL_ENABLED_SERVICES="key,nova,cinder,glance,s-proxy,neutron,sahara"
 
@@ -1014,15 +1010,6 @@ if is_service_enabled keystone; then
     export OS_REGION_NAME=$REGION_NAME
 fi
 
-
-# ZeroMQ
-# ------
-if is_service_enabled zeromq; then
-    echo_summary "Starting zeromq receiver"
-    run_process zeromq "$OSLO_BIN_DIR/oslo-messaging-zmq-receiver"
-fi
-
-
 # Horizon
 # -------