initial devstack + essex

* update horizon apache config
* updates apts for horizon
* better logging for horizon
* keystone conf settings
* only add swift endpoint if it is enabled
* new nova paste

Change-Id: I1edacbe5d8adc4bd5265d36abcaf01ce5490aefd

exercises/euca.sh

@@ -22,7 +22,7 @@ source ./openrc
 popd
 
 # find a machine image to boot
-IMAGE=`euca-describe-images | grep machine | cut -f2`
+IMAGE=`euca-describe-images | grep machine | cut -f2 | head -n1`
 
 # launch it
 INSTANCE=`euca-run-instances $IMAGE | grep INSTANCE | cut -f2`

files/000-default.template

@@ -6,7 +6,7 @@
     WSGIProcessGroup horizon
 
     DocumentRoot %HORIZON_DIR%/.blackhole/
-    Alias /media %HORIZON_DIR%/openstack-dashboard/media
+    Alias /media %HORIZON_DIR%/openstack-dashboard/dashboard/static
     Alias /vpn /opt/stack/vpn
 
     <Directory />

files/apts/horizon

@@ -13,3 +13,8 @@ pylint
 pep8
 python-eventlet
 python-nose
+python-sphinx
+python-mox
+python-kombu
+python-coverage
+python-cherrypy3 # why?

files/horizon_settings.py

@@ -12,23 +12,13 @@ DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
         'NAME': os.path.join(LOCAL_PATH, 'dashboard_openstack.sqlite3'),
+        'TEST_NAME': os.path.join(LOCAL_PATH, 'test.sqlite3'),
     },
 }
 
+# The default values for these two settings seem to cause issues with apache
 CACHE_BACKEND = 'dummy://'
-
+SESSION_ENGINE = 'django.contrib.sessions.backends.cached_db'
-# Add apps to horizon installation.
-INSTALLED_APPS = (
-    'dashboard',
-    'django.contrib.contenttypes',
-    'django.contrib.sessions',
-    'django.contrib.messages',
-    'django.contrib.staticfiles',
-    'django_openstack',
-    'django_openstack.templatetags',
-    'mailer',
-)
-
 
 # Send email to the console by default
 EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
@@ -44,32 +34,40 @@ MAILER_EMAIL_BACKEND = EMAIL_BACKEND
 # EMAIL_HOST_USER = 'djangomail'
 # EMAIL_HOST_PASSWORD = 'top-secret!'
 
-# FIXME: This needs to be changed to allow for multi-node setup.
+HORIZON_CONFIG = {
-OPENSTACK_KEYSTONE_URL = "http://localhost:5000/v2.0/"
+    'dashboards': ('nova', 'syspanel', 'settings',),
-OPENSTACK_KEYSTONE_ADMIN_URL = "http://localhost:35357/v2.0"
+    'default_dashboard': 'nova',
+    'user_home': 'dashboard.views.user_home',
+}
+
+OPENSTACK_HOST = "127.0.0.1"
+OPENSTACK_KEYSTONE_URL = "http://%s:5000/v2.0" % OPENSTACK_HOST
+# FIXME: this is only needed until keystone fixes its GET /tenants call
+# so that it doesn't return everything for admins
+OPENSTACK_KEYSTONE_ADMIN_URL = "http://%s:35357/v2.0" % OPENSTACK_HOST
 OPENSTACK_KEYSTONE_DEFAULT_ROLE = "Member"
 
-# NOTE(tres): Available services should come from the service
+SWIFT_PAGINATE_LIMIT = 100
-#             catalog in Keystone.
-SWIFT_ENABLED = False
 
 # Configure quantum connection details for networking
 QUANTUM_ENABLED = False
-QUANTUM_URL = '127.0.0.1'
+QUANTUM_URL = '%s'  % OPENSTACK_HOST
 QUANTUM_PORT = '9696'
 QUANTUM_TENANT = '1234'
 QUANTUM_CLIENT_VERSION='0.1'
 
-# No monitoring links currently
+# If you have external monitoring links, eg:
-EXTERNAL_MONITORING = []
+# EXTERNAL_MONITORING = [
+#     ['Nagios','http://foo.com'],
+#     ['Ganglia','http://bar.com'],
+# ]
 
-# Uncomment the following segment to silence most logging
-# django.db and boto DEBUG logging is extremely verbose.
 #LOGGING = {
 #        'version': 1,
-#        # set to True will disable all logging except that specified, unless
+#        # When set to True this will disable all logging except
-#        # nothing is specified except that django.db.backends will still log,
+#        # for loggers specified in this configuration dictionary. Note that
-#        # even when set to True, so disable explicitly
+#        # if nothing is specified here and disable_existing_loggers is True,
+#        # django.db.backends will still log unless it is disabled explicitly.
 #        'disable_existing_loggers': False,
 #        'handlers': {
 #            'null': {
@@ -77,20 +75,34 @@ EXTERNAL_MONITORING = []
 #                'class': 'django.utils.log.NullHandler',
 #                },
 #            'console': {
-#                'level': 'DEBUG',
+#                # Set the level to "DEBUG" for verbose output logging.
+#                'level': 'INFO',
 #                'class': 'logging.StreamHandler',
 #                },
 #            },
 #        'loggers': {
-#            # Comment or Uncomment these to turn on/off logging output
+#            # Logging from django.db.backends is VERY verbose, send to null
+#            # by default.
 #            'django.db.backends': {
 #                'handlers': ['null'],
 #                'propagate': False,
 #                },
-#            'django_openstack': {
+#            'horizon': {
-#                'handlers': ['null'],
+#                'handlers': ['console'],
 #                'propagate': False,
 #            },
+#            'novaclient': {
+#                'handlers': ['console'],
+#                'propagate': False,
+#            },
+#            'keystoneclient': {
+#                'handlers': ['console'],
+#                'propagate': False,
+#            },
+#            'nose.plugins.manager': {
+#                'handlers': ['console'],
+#                'propagate': False,
+#            }
 #        }
 #}
 

files/keystone.conf

@@ -25,6 +25,9 @@ service-header-mappings = {
 	'swift' : 'X-Storage-Url',
 	'cdn' : 'X-CDN-Management-Url'}
 
+#List of extensions currently supported
+extensions= osksadm,oskscatalog
+
 # Address to bind the API server
 # TODO Properties defined within app not available via pipeline.
 service_host = 0.0.0.0
@@ -32,23 +35,47 @@ service_host = 0.0.0.0
 # Port the bind the API server to
 service_port = 5000
 
+# SSL for API server
+service_ssl = False
+
 # Address to bind the Admin API server
 admin_host = 0.0.0.0
 
 # Port the bind the Admin API server to
 admin_port = 35357
 
+# SSL for API Admin server
+admin_ssl = False
+
+# Keystone certificate file (modify as needed)
+# Only required if *_ssl is set to True
+certfile = /etc/keystone/ssl/certs/keystone.pem
+
+# Keystone private key file (modify as needed)
+# Only required if *_ssl is set to True
+keyfile = /etc/keystone/ssl/private/keystonekey.pem
+
+# Keystone trusted CA certificates  (modify as needed)
+# Only required if *_ssl is set to True
+ca_certs = /etc/keystone/ssl/certs/ca.pem
+
+# Client certificate required
+# Only relevant if *_ssl is set to True
+cert_required = True
+
 #Role that allows to perform admin operations.
-keystone-admin-role = KeystoneAdmin
+keystone-admin-role = Admin
 
 #Role that allows to perform service admin operations.
 keystone-service-admin-role = KeystoneServiceAdmin
 
+#Tells whether password user need to be hashed in the backend
+hash-password = True
+
 [keystone.backends.sqlalchemy]
 # SQLAlchemy connection string for the reference implementation registry
 # server. Any valid SQLAlchemy connection string is fine.
 # See: http://bit.ly/ideIpI
-#sql_connection = sqlite:///keystone.db
 sql_connection = %SQL_CONN%
 backend_entities = ['UserRoleAssociation', 'Endpoints', 'Role', 'Tenant',
                     'User', 'Credentials', 'EndpointTemplates', 'Token',
@@ -84,3 +111,6 @@ paste.filter_factory = keystone.frontends.legacy_token_auth:filter_factory
 
 [filter:RAX-KEY-extension]
 paste.filter_factory = keystone.contrib.extensions.service.raxkey.frontend:filter_factory
+
+[filter:debug]
+paste.filter_factory = keystone.common.wsgi:debug_filter_factory

files/keystone_data.sh

@@ -30,13 +30,17 @@ $BIN_DIR/keystone-manage $* role grant KeystoneServiceAdmin admin
 $BIN_DIR/keystone-manage $* service add nova compute "Nova Compute Service"
 $BIN_DIR/keystone-manage $* service add glance image "Glance Image Service"
 $BIN_DIR/keystone-manage $* service add keystone identity "Keystone Identity Service"
+if [[ "$ENABLED_SERVICES" =~ "swift" ]]; then
     $BIN_DIR/keystone-manage $* service add swift object-store "Swift Service"
+fi
 
 #endpointTemplates
 $BIN_DIR/keystone-manage $* endpointTemplates add RegionOne nova http://%HOST_IP%:8774/v1.1/%tenant_id% http://%HOST_IP%:8774/v1.1/%tenant_id%  http://%HOST_IP%:8774/v1.1/%tenant_id% 1 1
 $BIN_DIR/keystone-manage $* endpointTemplates add RegionOne glance http://%HOST_IP%:9292/v1.1/%tenant_id% http://%HOST_IP%:9292/v1.1/%tenant_id% http://%HOST_IP%:9292/v1.1/%tenant_id% 1 1
 $BIN_DIR/keystone-manage $* endpointTemplates add RegionOne keystone http://%HOST_IP%:5000/v2.0 http://%HOST_IP%:35357/v2.0 http://%HOST_IP%:5000/v2.0 1 1
+if [[ "$ENABLED_SERVICES" =~ "swift" ]]; then
     $BIN_DIR/keystone-manage $* endpointTemplates add RegionOne swift http://%HOST_IP%:8080/v1/AUTH_%tenant_id% http://%HOST_IP%:8080/ http://%HOST_IP%:8080/v1/AUTH_%tenant_id% 1 1
+fi
 
 # Tokens
 $BIN_DIR/keystone-manage $* token add %SERVICE_TOKEN% admin admin 2015-02-05T00:00

files/nova-api-paste.ini

@@ -1,34 +1,54 @@
+############
+# Metadata #
+############
+[composite:metadata]
+use = egg:Paste#urlmap
+/: metaversions
+/latest: meta
+/2007-01-19: meta
+/2007-03-01: meta
+/2007-08-29: meta
+/2007-10-10: meta
+/2007-12-15: meta
+/2008-02-01: meta
+/2008-09-01: meta
+/2009-04-04: meta
+
+[pipeline:metaversions]
+pipeline = ec2faultwrap logrequest metaverapp
+
+[pipeline:meta]
+pipeline = ec2faultwrap logrequest metaapp
+
+[app:metaverapp]
+paste.app_factory = nova.api.metadata.handler:Versions.factory
+
+[app:metaapp]
+paste.app_factory = nova.api.metadata.handler:MetadataRequestHandler.factory
+
 #######
 # EC2 #
 #######
 
 [composite:ec2]
 use = egg:Paste#urlmap
-/: ec2versions
 /services/Cloud: ec2cloud
 /services/Admin: ec2admin
-/latest: ec2metadata
-/2007-01-19: ec2metadata
-/2007-03-01: ec2metadata
-/2007-08-29: ec2metadata
-/2007-10-10: ec2metadata
-/2007-12-15: ec2metadata
-/2008-02-01: ec2metadata
-/2008-09-01: ec2metadata
-/2009-04-04: ec2metadata
-/1.0: ec2metadata
 
 [pipeline:ec2cloud]
-pipeline = logrequest totoken authtoken keystonecontext cloudrequest authorizer ec2executor
+pipeline = ec2faultwrap logrequest totoken authtoken keystonecontext cloudrequest authorizer ec2executor
 
 [pipeline:ec2admin]
-pipeline = logrequest totoken authtoken keystonecontext adminrequest authorizer ec2executor
+pipeline = ec2faultwrap logrequest totoken authtoken keystonecontext adminrequest authorizer ec2executor
 
 [pipeline:ec2metadata]
-pipeline = logrequest ec2md
+pipeline = ec2faultwrap logrequest ec2md
 
 [pipeline:ec2versions]
-pipeline = logrequest ec2ver
+pipeline = ec2faultwrap logrequest ec2ver
+
+[filter:ec2faultwrap]
+paste.filter_factory = nova.api.ec2:FaultWrapper.factory
 
 [filter:logrequest]
 paste.filter_factory = nova.api.ec2:RequestLogging.factory
@@ -59,54 +79,45 @@ paste.filter_factory = nova.api.ec2:Authorizer.factory
 [app:ec2executor]
 paste.app_factory = nova.api.ec2:Executor.factory
 
-[app:ec2ver]
-paste.app_factory = nova.api.ec2:Versions.factory
-
-[app:ec2md]
-paste.app_factory = nova.api.ec2.metadatarequesthandler:MetadataRequestHandler.factory
-
 #############
 # Openstack #
 #############
 
 [composite:osapi]
-use = egg:Paste#urlmap
+use = call:nova.api.openstack.v2.urlmap:urlmap_factory
 /: osversions
-/v1.0: openstackapi10
+/v1.1: openstack_api_v2
-/v1.1: openstackapi11
+/v2: openstack_api_v2
 
-[pipeline:openstackapi10]
+[pipeline:openstack_api_v2]
-pipeline = faultwrap authtoken keystonecontext ratelimit osapiapp10
+pipeline = faultwrap authtoken keystonecontext ratelimit serialize extensions osapi_app_v2
-
-[pipeline:openstackapi11]
-pipeline = faultwrap authtoken keystonecontext ratelimit extensions osapiapp11
 
 [filter:faultwrap]
-paste.filter_factory = nova.api.openstack:FaultWrapper.factory
+paste.filter_factory = nova.api.openstack.v2:FaultWrapper.factory
 
 [filter:auth]
-paste.filter_factory = nova.api.openstack.auth:AuthMiddleware.factory
+paste.filter_factory = nova.api.openstack.v2.auth:AuthMiddleware.factory
 
 [filter:noauth]
-paste.filter_factory = nova.api.openstack.auth:NoAuthMiddleware.factory
+paste.filter_factory = nova.api.openstack.v2.auth:NoAuthMiddleware.factory
 
 [filter:ratelimit]
-paste.filter_factory = nova.api.openstack.limits:RateLimitingMiddleware.factory
+paste.filter_factory = nova.api.openstack.v2.limits:RateLimitingMiddleware.factory
+
+[filter:serialize]
+paste.filter_factory = nova.api.openstack.wsgi:LazySerializationMiddleware.factory
 
 [filter:extensions]
-paste.filter_factory = nova.api.openstack.extensions:ExtensionMiddleware.factory
+paste.filter_factory = nova.api.openstack.v2.extensions:ExtensionMiddleware.factory
 
-[app:osapiapp10]
+[app:osapi_app_v2]
-paste.app_factory = nova.api.openstack:APIRouterV10.factory
+paste.app_factory = nova.api.openstack.v2:APIRouter.factory
-
-[app:osapiapp11]
-paste.app_factory = nova.api.openstack:APIRouterV11.factory
 
 [pipeline:osversions]
 pipeline = faultwrap osversionapp
 
 [app:osversionapp]
-paste.app_factory = nova.api.openstack.versions:Versions.factory
+paste.app_factory = nova.api.openstack.v2.versions:Versions.factory
 
 ##########
 # Shared #

files/pips/horizon

@@ -1,9 +1,15 @@
 Django==1.3
-django-nose==0.1.2
 django-mailer
+django-nose==0.1.2
+django-nose-selenium
 django-registration==0.7
-python-cloudfiles
+glance==2011.3
+pycrypto==2.3
+quantum
 sqlalchemy-migrate
+python-cloudfiles
 
+-e git+https://github.com/cloudbuilders/openstackx.git#egg=openstackx
 -e git+https://github.com/jacobian/openstack.compute.git#egg=openstack
-
+-e git+https://github.com/rackspace/python-novaclient.git#egg=python-novaclient
+-e git+https://github.com/4P/python-keystoneclient.git#egg=python-keystoneclient

stack.sh

@@ -537,7 +537,7 @@ if [[ "$ENABLED_SERVICES" =~ "openstackx" ]]; then
     cd $OPENSTACKX_DIR; sudo python setup.py develop
 fi
 if [[ "$ENABLED_SERVICES" =~ "horizon" ]]; then
-    cd $HORIZON_DIR/django-openstack; sudo python setup.py develop
+    cd $HORIZON_DIR/horizon; sudo python setup.py develop
     cd $HORIZON_DIR/openstack-dashboard; sudo python setup.py develop
 fi
 if [[ "$ENABLED_SERVICES" =~ "quantum" ]]; then
@@ -618,7 +618,13 @@ if [[ "$ENABLED_SERVICES" =~ "horizon" ]]; then
 
 
     # ``local_settings.py`` is used to override horizon default settings.
-    cp $FILES/horizon_settings.py $HORIZON_DIR/openstack-dashboard/local/local_settings.py
+    local_settings=$HORIZON_DIR/openstack-dashboard/local/local_settings.py
+    cp $FILES/horizon_settings.py $local_settings
+
+    # Enable quantum in dashboard, if requested
+    if [[ "$ENABLED_SERVICES" =~ "quantum" ]]; then
+        sudo sed -e "s,QUANTUM_ENABLED = False,QUANTUM_ENABLED = True,g" -i $local_settings
+    fi
 
     # Initialize the horizon database (it stores sessions and notices shown to
     # users).  The user system is external (keystone).
@@ -934,10 +940,12 @@ add_nova_flag "--vlan_interface=$VLAN_INTERFACE"
 add_nova_flag "--sql_connection=$BASE_SQL_CONN/nova"
 add_nova_flag "--libvirt_type=$LIBVIRT_TYPE"
 if [[ "$ENABLED_SERVICES" =~ "openstackx" ]]; then
-    add_nova_flag "--osapi_extensions_path=$OPENSTACKX_DIR/extensions"
+    add_nova_flag "--osapi_extension=nova.api.openstack.v2.contrib.standard_extensions"
+    add_nova_flag "--osapi_extension=extensions.admin.Admin"
 fi
 if [[ "$ENABLED_SERVICES" =~ "n-vnc" ]]; then
-    add_nova_flag "--vncproxy_url=http://$HOST_IP:6080"
+    VNCPROXY_URL=${VNCPROXY_URL:-"http://$HOST_IP:6080"}
+    add_nova_flag "--vncproxy_url=$VNCPROXY_URL"
     add_nova_flag "--vncproxy_wwwroot=$NOVNC_DIR/"
 fi
 add_nova_flag "--api_paste_config=$NOVA_DIR/bin/nova-api-paste.ini"
@@ -1008,7 +1016,7 @@ if [[ "$ENABLED_SERVICES" =~ "key" ]]; then
     mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'DROP DATABASE IF EXISTS keystone;'
     mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'CREATE DATABASE keystone;'
 
-    # FIXME (anthony) keystone should use keystone.conf.example
+    # Configure keystone.conf
     KEYSTONE_CONF=$KEYSTONE_DIR/etc/keystone.conf
     cp $FILES/keystone.conf $KEYSTONE_CONF
     sudo sed -e "s,%SQL_CONN%,$BASE_SQL_CONN/keystone,g" -i $KEYSTONE_CONF
@@ -1021,7 +1029,7 @@ if [[ "$ENABLED_SERVICES" =~ "key" ]]; then
     sudo sed -e "s,%SERVICE_TOKEN%,$SERVICE_TOKEN,g" -i $KEYSTONE_DATA
     sudo sed -e "s,%ADMIN_PASSWORD%,$ADMIN_PASSWORD,g" -i $KEYSTONE_DATA
     # initialize keystone with default users/endpoints
-    BIN_DIR=$KEYSTONE_DIR/bin bash $KEYSTONE_DATA
+    ENABLED_SERVICES=$ENABLED_SERVICES BIN_DIR=$KEYSTONE_DIR/bin bash $KEYSTONE_DATA
 fi
 
 

stackrc

@@ -1,10 +1,10 @@
 # compute service
 NOVA_REPO=https://github.com/openstack/nova.git
-NOVA_BRANCH=stable/diablo
+NOVA_BRANCH=master
 
 # storage service
 SWIFT_REPO=https://github.com/openstack/swift.git
-SWIFT_BRANCH=stable/diablo
+SWIFT_BRANCH=master
 
 # swift and keystone integration
 SWIFT_KEYSTONE_REPO=https://github.com/cloudbuilders/swift-keystone2.git
@@ -12,7 +12,7 @@ SWIFT_KEYSTONE_BRANCH=master
 
 # image catalog service
 GLANCE_REPO=https://github.com/openstack/glance.git
-GLANCE_BRANCH=stable/diablo
+GLANCE_BRANCH=master
 
 # unified auth system (manages accounts/tokens)
 KEYSTONE_REPO=https://github.com/openstack/keystone.git
@@ -20,11 +20,11 @@ KEYSTONE_BRANCH=stable/diablo
 
 # a websockets/html5 or flash powered VNC console for vm instances
 NOVNC_REPO=https://github.com/cloudbuilders/noVNC.git
-NOVNC_BRANCH=diablo
+NOVNC_BRANCH=master
 
 # django powered web control panel for openstack
 HORIZON_REPO=https://github.com/openstack/horizon.git
-HORIZON_BRANCH=stable/diablo
+HORIZON_BRANCH=master
 
 # python client library to nova that horizon (and others) use
 NOVACLIENT_REPO=https://github.com/openstack/python-novaclient.git
@@ -33,11 +33,11 @@ NOVACLIENT_BRANCH=master
 # openstackx is a collection of extensions to openstack.compute & nova
 # that is *deprecated*.  The code is being moved into python-novaclient & nova.
 OPENSTACKX_REPO=https://github.com/cloudbuilders/openstackx.git
-OPENSTACKX_BRANCH=diablo
+OPENSTACKX_BRANCH=master
 
 # quantum service
 QUANTUM_REPO=https://github.com/openstack/quantum
-QUANTUM_BRANCH=stable/diablo
+QUANTUM_BRANCH=master
 
 # CI test suite
 CITEST_REPO=https://github.com/openstack/openstack-integration-tests.git

tools/get_uec_image.sh

@@ -10,6 +10,7 @@ TOP_DIR=`cd $TOOLS_DIR/..; pwd`
 
 # exit on error to stop unexpected errors
 set -o errexit
+set -o xtrace
 
 usage() {
     echo "Usage: $0 - Fetch and prepare Ubuntu images"