Merge "Move Sahara into in-tree plugin"

MAINTAINERS.rst

@@ -63,11 +63,6 @@ OpenFlow Agent (ofagent)
 * YAMAMOTO Takashi <yamamoto@valinux.co.jp>
 * Fumihiko Kakuma <kakuma@valinux.co.jp>
 
-Sahara
-~~~~~~
-
-* Sergey Lukjanov <slukjanov@mirantis.com>
-
 Swift
 ~~~~~
 

doc/source/index.rst

@@ -170,7 +170,6 @@ Scripts
 * `lib/nova <lib/nova.html>`__
 * `lib/oslo <lib/oslo.html>`__
 * `lib/rpc\_backend <lib/rpc_backend.html>`__
-* `lib/sahara <lib/sahara.html>`__
 * `lib/swift <lib/swift.html>`__
 * `lib/tempest <lib/tempest.html>`__
 * `lib/tls <lib/tls.html>`__
@@ -181,7 +180,6 @@ Scripts
 
 * `extras.d/50-ironic.sh <extras.d/50-ironic.sh.html>`__
 * `extras.d/60-ceph.sh <extras.d/60-ceph.sh.html>`__
-* `extras.d/70-sahara.sh <extras.d/70-sahara.sh.html>`__
 * `extras.d/70-tuskar.sh <extras.d/70-tuskar.sh.html>`__
 * `extras.d/70-zaqar.sh <extras.d/70-zaqar.sh.html>`__
 * `extras.d/80-tempest.sh <extras.d/80-tempest.sh.html>`__
@@ -238,7 +236,6 @@ Exercises
 * `exercises/floating\_ips.sh <exercises/floating_ips.sh.html>`__
 * `exercises/horizon.sh <exercises/horizon.sh.html>`__
 * `exercises/neutron-adv-test.sh <exercises/neutron-adv-test.sh.html>`__
-* `exercises/sahara.sh <exercises/sahara.sh.html>`__
 * `exercises/sec\_groups.sh <exercises/sec_groups.sh.html>`__
 * `exercises/swift.sh <exercises/swift.sh.html>`__
 * `exercises/volumes.sh <exercises/volumes.sh.html>`__

doc/source/plugin-registry.rst

@@ -22,6 +22,8 @@ The following are plugins that exist for official OpenStack projects.
 +--------------------+-------------------------------------------+--------------------+
 |magnum              |git://git.openstack.org/openstack/magnum   |                    |
 +--------------------+-------------------------------------------+--------------------+
+|sahara              |git://git.openstack.org/openstack/sahara   |                    |
++--------------------+-------------------------------------------+--------------------+
 |trove               |git://git.openstack.org/openstack/trove    |                    |
 +--------------------+-------------------------------------------+--------------------+
 |zaqar               |git://git.openstack.org/openstack/zarar    |                    |

exercises/sahara.sh

@@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-
-# **sahara.sh**
-
-# Sanity check that Sahara started if enabled
-
-echo "*********************************************************************"
-echo "Begin DevStack Exercise: $0"
-echo "*********************************************************************"
-
-# This script exits on an error so that errors don't compound and you see
-# only the first error that occurred.
-set -o errexit
-
-# Print the commands being run so that we can see the command that triggers
-# an error.  It is also useful for following allowing as the install occurs.
-set -o xtrace
-
-
-# Settings
-# ========
-
-# Keep track of the current directory
-EXERCISE_DIR=$(cd $(dirname "$0") && pwd)
-TOP_DIR=$(cd $EXERCISE_DIR/..; pwd)
-
-# Import common functions
-source $TOP_DIR/functions
-
-# Import configuration
-source $TOP_DIR/openrc
-
-# Import exercise configuration
-source $TOP_DIR/exerciserc
-
-is_service_enabled sahara || exit 55
-
-if is_ssl_enabled_service "sahara" || is_service_enabled tls-proxy; then
-    SAHARA_SERVICE_PROTOCOL="https"
-fi
-
-SAHARA_SERVICE_PROTOCOL=${SAHARA_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
-
-$CURL_GET $SAHARA_SERVICE_PROTOCOL://$SERVICE_HOST:8386/ 2>/dev/null | grep -q 'Auth' || die $LINENO "Sahara API isn't functioning!"
-
-set +o xtrace
-echo "*********************************************************************"
-echo "SUCCESS: End DevStack Exercise: $0"
-echo "*********************************************************************"

extras.d/70-sahara.sh

@@ -1,29 +0,0 @@
-# sahara.sh - DevStack extras script to install Sahara
-
-if is_service_enabled sahara; then
-    if [[ "$1" == "source" ]]; then
-        # Initial source
-        source $TOP_DIR/lib/sahara
-    elif [[ "$1" == "stack" && "$2" == "install" ]]; then
-        echo_summary "Installing sahara"
-        install_sahara
-        install_python_saharaclient
-        cleanup_sahara
-    elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
-        echo_summary "Configuring sahara"
-        configure_sahara
-        create_sahara_accounts
-    elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
-        echo_summary "Initializing sahara"
-        sahara_register_images
-        start_sahara
-    fi
-
-    if [[ "$1" == "unstack" ]]; then
-        stop_sahara
-    fi
-
-    if [[ "$1" == "clean" ]]; then
-        cleanup_sahara
-    fi
-fi

lib/sahara

@@ -1,259 +0,0 @@
-#!/bin/bash
-#
-# lib/sahara
-
-# Dependencies:
-# ``functions`` file
-# ``DEST``, ``DATA_DIR``, ``STACK_USER`` must be defined
-
-# ``stack.sh`` calls the entry points in this order:
-#
-# install_sahara
-# install_python_saharaclient
-# configure_sahara
-# sahara_register_images
-# start_sahara
-# stop_sahara
-# cleanup_sahara
-
-# Save trace setting
-XTRACE=$(set +o | grep xtrace)
-set +o xtrace
-
-
-# Defaults
-# --------
-
-# Set up default repos
-
-# Set up default directories
-GITDIR["python-saharaclient"]=$DEST/python-saharaclient
-SAHARA_DIR=$DEST/sahara
-
-SAHARA_CONF_DIR=${SAHARA_CONF_DIR:-/etc/sahara}
-SAHARA_CONF_FILE=${SAHARA_CONF_DIR}/sahara.conf
-
-if is_ssl_enabled_service "sahara" || is_service_enabled tls-proxy; then
-    SAHARA_SERVICE_PROTOCOL="https"
-fi
-SAHARA_SERVICE_HOST=${SAHARA_SERVICE_HOST:-$SERVICE_HOST}
-SAHARA_SERVICE_PORT=${SAHARA_SERVICE_PORT:-8386}
-SAHARA_SERVICE_PORT_INT=${SAHARA_SERVICE_PORT_INT:-18386}
-SAHARA_SERVICE_PROTOCOL=${SAHARA_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}
-
-SAHARA_AUTH_CACHE_DIR=${SAHARA_AUTH_CACHE_DIR:-/var/cache/sahara}
-
-SAHARA_ENABLED_PLUGINS=${SAHARA_ENABLED_PLUGINS:-vanilla,hdp,cdh,spark,fake}
-
-# Support entry points installation of console scripts
-if [[ -d $SAHARA_DIR/bin ]]; then
-    SAHARA_BIN_DIR=$SAHARA_DIR/bin
-else
-    SAHARA_BIN_DIR=$(get_python_exec_prefix)
-fi
-
-# Tell Tempest this project is present
-TEMPEST_SERVICES+=,sahara
-
-# Functions
-# ---------
-
-# create_sahara_accounts() - Set up common required sahara accounts
-#
-# Tenant      User       Roles
-# ------------------------------
-# service     sahara    admin
-function create_sahara_accounts {
-
-    create_service_user "sahara"
-
-    if [[ "$KEYSTONE_CATALOG_BACKEND" = 'sql' ]]; then
-
-        # TODO: remove "data_processing" service when #1356053 will be fixed
-        local sahara_service_old=$(openstack service create \
-            "data_processing" \
-            --name "sahara" \
-            --description "Sahara Data Processing" \
-            -f value -c id
-        )
-        local sahara_service_new=$(openstack service create \
-            "data-processing" \
-            --name "sahara" \
-            --description "Sahara Data Processing" \
-            -f value -c id
-        )
-        get_or_create_endpoint $sahara_service_old \
-            "$REGION_NAME" \
-            "$SAHARA_SERVICE_PROTOCOL://$SAHARA_SERVICE_HOST:$SAHARA_SERVICE_PORT/v1.1/\$(tenant_id)s" \
-            "$SAHARA_SERVICE_PROTOCOL://$SAHARA_SERVICE_HOST:$SAHARA_SERVICE_PORT/v1.1/\$(tenant_id)s" \
-            "$SAHARA_SERVICE_PROTOCOL://$SAHARA_SERVICE_HOST:$SAHARA_SERVICE_PORT/v1.1/\$(tenant_id)s"
-        get_or_create_endpoint $sahara_service_new \
-            "$REGION_NAME" \
-            "$SAHARA_SERVICE_PROTOCOL://$SAHARA_SERVICE_HOST:$SAHARA_SERVICE_PORT/v1.1/\$(tenant_id)s" \
-            "$SAHARA_SERVICE_PROTOCOL://$SAHARA_SERVICE_HOST:$SAHARA_SERVICE_PORT/v1.1/\$(tenant_id)s" \
-            "$SAHARA_SERVICE_PROTOCOL://$SAHARA_SERVICE_HOST:$SAHARA_SERVICE_PORT/v1.1/\$(tenant_id)s"
-    fi
-}
-
-# cleanup_sahara() - Remove residual data files, anything left over from
-# previous runs that would need to clean up.
-function cleanup_sahara {
-
-    # Cleanup auth cache dir
-    sudo rm -rf $SAHARA_AUTH_CACHE_DIR
-}
-
-# configure_sahara() - Set config files, create data dirs, etc
-function configure_sahara {
-    sudo install -d -o $STACK_USER $SAHARA_CONF_DIR
-
-    if [[ -f $SAHARA_DIR/etc/sahara/policy.json ]]; then
-        cp -p $SAHARA_DIR/etc/sahara/policy.json $SAHARA_CONF_DIR
-    fi
-
-    # Create auth cache dir
-    sudo install -d -o $STACK_USER -m 700 $SAHARA_AUTH_CACHE_DIR
-    rm -rf $SAHARA_AUTH_CACHE_DIR/*
-
-    configure_auth_token_middleware $SAHARA_CONF_FILE sahara $SAHARA_AUTH_CACHE_DIR
-
-    iniset_rpc_backend sahara $SAHARA_CONF_FILE DEFAULT
-
-    # Set configuration to send notifications
-
-    if is_service_enabled ceilometer; then
-        iniset $SAHARA_CONF_FILE DEFAULT enable_notifications "true"
-        iniset $SAHARA_CONF_FILE DEFAULT notification_driver "messaging"
-    fi
-
-    iniset $SAHARA_CONF_FILE DEFAULT verbose True
-    iniset $SAHARA_CONF_FILE DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL
-
-    iniset $SAHARA_CONF_FILE DEFAULT plugins $SAHARA_ENABLED_PLUGINS
-
-    iniset $SAHARA_CONF_FILE database connection `database_connection_url sahara`
-
-    if is_service_enabled neutron; then
-        iniset $SAHARA_CONF_FILE DEFAULT use_neutron true
-
-        if is_ssl_enabled_service "neutron" || is_service_enabled tls-proxy; then
-            iniset $SAHARA_CONF_FILE neutron ca_file $SSL_BUNDLE_FILE
-        fi
-    else
-        iniset $SAHARA_CONF_FILE DEFAULT use_neutron false
-    fi
-
-    if is_service_enabled heat; then
-        iniset $SAHARA_CONF_FILE DEFAULT infrastructure_engine heat
-
-        if is_ssl_enabled_service "heat" || is_service_enabled tls-proxy; then
-            iniset $SAHARA_CONF_FILE heat ca_file $SSL_BUNDLE_FILE
-        fi
-    else
-        iniset $SAHARA_CONF_FILE DEFAULT infrastructure_engine direct
-    fi
-
-    if is_ssl_enabled_service "cinder" || is_service_enabled tls-proxy; then
-        iniset $SAHARA_CONF_FILE cinder ca_file $SSL_BUNDLE_FILE
-    fi
-
-    if is_ssl_enabled_service "nova" || is_service_enabled tls-proxy; then
-        iniset $SAHARA_CONF_FILE nova ca_file $SSL_BUNDLE_FILE
-    fi
-
-    if is_ssl_enabled_service "swift" || is_service_enabled tls-proxy; then
-        iniset $SAHARA_CONF_FILE swift ca_file $SSL_BUNDLE_FILE
-    fi
-
-    if is_ssl_enabled_service "key" || is_service_enabled tls-proxy; then
-        iniset $SAHARA_CONF_FILE keystone ca_file $SSL_BUNDLE_FILE
-    fi
-
-    # Register SSL certificates if provided
-    if is_ssl_enabled_service sahara; then
-        ensure_certificates SAHARA
-
-        iniset $SAHARA_CONF_FILE ssl cert_file "$SAHARA_SSL_CERT"
-        iniset $SAHARA_CONF_FILE ssl key_file "$SAHARA_SSL_KEY"
-    fi
-
-    iniset $SAHARA_CONF_FILE DEFAULT use_syslog $SYSLOG
-
-    # Format logging
-    if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ]; then
-        setup_colorized_logging $SAHARA_CONF_FILE DEFAULT
-    fi
-
-    if is_service_enabled tls-proxy; then
-        # Set the service port for a proxy to take the original
-        iniset $SAHARA_CONF_FILE DEFAULT port $SAHARA_SERVICE_PORT_INT
-    fi
-
-    recreate_database sahara
-    $SAHARA_BIN_DIR/sahara-db-manage --config-file $SAHARA_CONF_FILE upgrade head
-}
-
-# install_sahara() - Collect source and prepare
-function install_sahara {
-    git_clone $SAHARA_REPO $SAHARA_DIR $SAHARA_BRANCH
-    setup_develop $SAHARA_DIR
-}
-
-# install_python_saharaclient() - Collect source and prepare
-function install_python_saharaclient {
-    if use_library_from_git "python-saharaclient"; then
-        git_clone_by_name "python-saharaclient"
-        setup_dev_lib "python-saharaclient"
-    fi
-}
-
-# sahara_register_images() - Registers images in sahara image registry
-function sahara_register_images {
-    if is_service_enabled heat && [[ ! -z "$HEAT_CFN_IMAGE_URL" ]]; then
-        # Register heat image for Fake plugin
-        local fake_plugin_properties="--property _sahara_tag_0.1=True"
-        fake_plugin_properties+=" --property _sahara_tag_fake=True"
-        fake_plugin_properties+=" --property _sahara_username=fedora"
-        openstack --os-url $GLANCE_SERVICE_PROTOCOL://$GLANCE_HOSTPORT image set $(basename "$HEAT_CFN_IMAGE_URL" ".qcow2") $fake_plugin_properties
-    fi
-}
-
-# start_sahara() - Start running processes, including screen
-function start_sahara {
-    local service_port=$SAHARA_SERVICE_PORT
-    local service_protocol=$SAHARA_SERVICE_PROTOCOL
-    if is_service_enabled tls-proxy; then
-        service_port=$SAHARA_SERVICE_PORT_INT
-        service_protocol="http"
-    fi
-
-    run_process sahara "$SAHARA_BIN_DIR/sahara-all --config-file $SAHARA_CONF_FILE"
-    run_process sahara-api "$SAHARA_BIN_DIR/sahara-api --config-file $SAHARA_CONF_FILE"
-    run_process sahara-eng "$SAHARA_BIN_DIR/sahara-engine --config-file $SAHARA_CONF_FILE"
-
-    echo "Waiting for Sahara to start..."
-    if ! wait_for_service $SERVICE_TIMEOUT $service_protocol://$SAHARA_SERVICE_HOST:$service_port; then
-        die $LINENO "Sahara did not start"
-    fi
-
-    # Start proxies if enabled
-    if is_service_enabled tls-proxy; then
-        start_tls_proxy '*' $SAHARA_SERVICE_PORT $SAHARA_SERVICE_HOST $SAHARA_SERVICE_PORT_INT &
-    fi
-}
-
-# stop_sahara() - Stop running processes
-function stop_sahara {
-    # Kill the Sahara screen windows
-    stop_process sahara
-    stop_process sahara-api
-    stop_process sahara-eng
-}
-
-
-# Restore xtrace
-$XTRACE
-
-# Local variables:
-# mode: shell-script
-# End:

stack.sh

@@ -505,7 +505,7 @@ source $TOP_DIR/lib/rpc_backend
 check_rpc_backend
 
 # Service to enable with SSL if ``USE_SSL`` is True
-SSL_ENABLED_SERVICES="key,nova,cinder,glance,s-proxy,neutron,sahara"
+SSL_ENABLED_SERVICES="key,nova,cinder,glance,s-proxy,neutron"
 
 if is_service_enabled tls-proxy && [ "$USE_SSL" == "True" ]; then
     die $LINENO "tls-proxy and SSL are mutually exclusive"

stackrc

@@ -231,10 +231,6 @@ NEUTRON_VPNAAS_BRANCH=${NEUTRON_VPNAAS_BRANCH:-master}
 NOVA_REPO=${NOVA_REPO:-${GIT_BASE}/openstack/nova.git}
 NOVA_BRANCH=${NOVA_BRANCH:-master}
 
-# data processing service
-SAHARA_REPO=${SAHARA_REPO:-${GIT_BASE}/openstack/sahara.git}
-SAHARA_BRANCH=${SAHARA_BRANCH:-master}
-
 # object storage service
 SWIFT_REPO=${SWIFT_REPO:-${GIT_BASE}/openstack/swift.git}
 SWIFT_BRANCH=${SWIFT_BRANCH:-master}
@@ -296,10 +292,6 @@ GITBRANCH["python-neutronclient"]=${NEUTRONCLIENT_BRANCH:-master}
 GITREPO["python-novaclient"]=${NOVACLIENT_REPO:-${GIT_BASE}/openstack/python-novaclient.git}
 GITBRANCH["python-novaclient"]=${NOVACLIENT_BRANCH:-master}
 
-# python saharaclient
-GITREPO["python-saharaclient"]=${SAHARACLIENT_REPO:-${GIT_BASE}/openstack/python-saharaclient.git}
-GITBRANCH["python-saharaclient"]=${SAHARACLIENT_BRANCH:-master}
-
 # python swift client library
 GITREPO["python-swiftclient"]=${SWIFTCLIENT_REPO:-${GIT_BASE}/openstack/python-swiftclient.git}
 GITBRANCH["python-swiftclient"]=${SWIFTCLIENT_BRANCH:-master}

tests/test_libs_from_pypi.sh

@@ -35,7 +35,7 @@ ALL_LIBS+=" python-glanceclient python-ironicclient tempest-lib"
 ALL_LIBS+=" oslo.messaging oslo.log cliff python-heatclient stevedore"
 ALL_LIBS+=" python-cinderclient glance_store oslo.concurrency oslo.db"
 ALL_LIBS+=" oslo.versionedobjects oslo.vmware keystonemiddleware"
-ALL_LIBS+=" oslo.serialization python-saharaclient django_openstack_auth"
+ALL_LIBS+=" oslo.serialization django_openstack_auth"
 ALL_LIBS+=" python-openstackclient oslo.rootwrap oslo.i18n"
 ALL_LIBS+=" python-ceilometerclient oslo.utils python-swiftclient"
 ALL_LIBS+=" python-neutronclient tooz ceilometermiddleware oslo.policy"