From 6b87e4026f0dada4095e2e713520450009b6d645 Mon Sep 17 00:00:00 2001 From: Emilien Macchi Date: Thu, 5 Nov 2015 12:02:15 -0500 Subject: [PATCH] Implement all-in-one.sh script This script will help newcomers to run this script and deploy a simple scenario of an OpenStack Cloud with Puppet OpenStack modules. Change-Id: I559e5e0a0d21a02d04b3d999b77dee23bb38c341 --- README.md | 52 +++-- all-in-one.sh | 81 ++++++++ fixtures/scenario003.pp | 418 ++++++++++++++++++++++++++++++++++++++++ functions | 15 ++ run_tests.sh | 13 +- 5 files changed, 554 insertions(+), 25 deletions(-) create mode 100755 all-in-one.sh create mode 100644 fixtures/scenario003.pp diff --git a/README.md b/README.md index 1b12185ff..e365cd69e 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,8 @@ puppet-openstack-integration 1. [Overview - What is Puppet OpenStack Integration?](#overview) 2. [Description - What does the project do?](#description) 3. [Development - Guide for contributing](#development) -4. [Contributors - Those with commits](#contributors) +4. [All-in-one - How to deploy a cloud with Puppet](#All-In-One) +5. [Contributors - Those with commits](#contributors) Overview @@ -27,20 +28,22 @@ OpenStack Infrastructure is deploying two jobs per supported Operating System OpenStack services are balanced between two scenarios because OpenStack Infastructure Jenkins slaves can not afford the load of running all on the same node. +One manifest (scenario003) is used for people who want to [run a simple All-In-One +scenario](#All-In-One). -| - | scenario001 | scenario002 | -|:----------:|:-----------:|:-----------:| -| keystone | X | X | -| glance | X | X | -| nova | X | X | -| neutron | X | X | -| cinder | X | | -| ceilometer | X | | -| heat | | X | -| swift | | X | -| sahara | | X | -| trove | X | | -| horizon | | X | +| - | scenario001 | scenario002 | scenario003 | +|:----------:|:-----------:|:-----------:|:-----------:| +| keystone | X | X | X | +| glance | X | X | X | +| nova | X | X | X | +| neutron | X | X | X | +| cinder | X | | X | +| ceilometer | X | | | +| heat | | X | | +| swift | | X | | +| sahara | | X | | +| trove | X | | | +| horizon | | X | X | When the Jenkins slave is created, the *run_tests.sh* script will executed. This script will execute *install_modules.sh* that prepare /etc/puppet/modules @@ -65,6 +68,27 @@ Developer documentation for the entire Puppet OpenStack project: * https://wiki.openstack.org/wiki/Puppet +All-In-One +---------- + +If you're new in Puppet OpenStack and you want to deploy an All-In-One setup of +an OpenStack Cloud with the Puppet modules, please follow the steps: + +```bash +git clone git://git.openstack.org/openstack/puppet-openstack-integration +cd puppet-openstack-integration +./all-in-one.sh +``` +or + +```bash +curl -sL http://git.openstack.org/cgit/openstack/puppet-openstack-integration/plain/all-in-one.sh | bash +``` + +Look at [Description](#description) to see which services it will install +(scenario003). + + Contributors ------------ diff --git a/all-in-one.sh b/all-in-one.sh new file mode 100755 index 000000000..c00df3586 --- /dev/null +++ b/all-in-one.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Deploy Puppet OpenStack modules, deploy OpenStack and test the setup. +# Software requirements: +# * Ubuntu 14.04 LTS or CentOS7 fresh install +# * 'git' installed +# +# Hardware requirements: +# * At least 4GB of memory, but 8GB is recommended +# * At least 10GB of storage +# +# Usage: +# $ git clone git://git.openstack.org/openstack/puppet-openstack-integration +# $ cd puppet-openstack-integration +# $ ./all-in-one.sh +# +# or +# +# $ curl -sL http://git.openstack.org/cgit/openstack/puppet-openstack-integration/plain/all-in-one.sh | bash +# + +set -e + +if [ -n "$DEBUG" ]; then + set -x +fi + +# Prepare puppet-openstack-integration repository +rm -rf /tmp/puppet-openstack-integration +git clone git://git.openstack.org/openstack/puppet-openstack-integration /tmp/puppet-openstack-integration +cd /tmp/puppet-openstack-integration + +SCENARIO=scenario003 +export SCRIPT_DIR=$(cd `dirname $0` && pwd -P) +source $SCRIPT_DIR/functions + +if is_fedora; then + sudo yum -y remove facter puppet rdo-release + sudo yum -y install libxml2-devel libxslt-devel ruby-devel rubygems + sudo yum -y groupinstall "Development Tools" + DASHBOARD="dashboard" +elif uses_debs; then + sudo apt-get remove -y --purge facter puppet puppet-common + sudo apt-get update + sudo apt-get install -y libxml2-dev libxslt-dev zlib1g-dev ruby + DASHBOARD="horizon" +fi + +mkdir -p .bundled_gems +export GEM_HOME=`pwd`/.bundled_gems +gem install bundler --no-rdoc --no-ri --verbose + +set -e +./run_tests.sh +RESULT=$? +set +e +if [ $RESULT -ne 0 ]; then + echo "Deployment failed to finish." + exit 1 +fi + +cat <<-EOF + +OpenStack Dashboard available: http://127.0.0.1/${DASHBOARD} +To access through Horizon, use the following user/password: + admin / a_big_secret + +EOF diff --git a/fixtures/scenario003.pp b/fixtures/scenario003.pp new file mode 100644 index 000000000..acca1b6d2 --- /dev/null +++ b/fixtures/scenario003.pp @@ -0,0 +1,418 @@ +# +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +Exec { logoutput => 'on_failure' } + +# Common resources +case $::osfamily { + 'Debian': { + include ::apt + class { '::openstack_extras::repo::debian::ubuntu': + release => 'liberty', + repo => 'proposed', + package_require => true, + } + $package_provider = 'apt' + } + 'RedHat': { + class { '::openstack_extras::repo::redhat::redhat': + manage_rdo => false, + repo_hash => { + 'openstack-common-testing' => { + 'baseurl' => 'http://cbs.centos.org/repos/cloud7-openstack-common-testing/x86_64/os/', + 'descr' => 'openstack-common-testing', + 'gpgcheck' => 'no', + }, + 'openstack-liberty-testing' => { + 'baseurl' => 'http://cbs.centos.org/repos/cloud7-openstack-liberty-testing/x86_64/os/', + 'descr' => 'openstack-liberty-testing', + 'gpgcheck' => 'no', + }, + 'openstack-liberty-trunk' => { + 'baseurl' => 'http://trunk.rdoproject.org/centos7-liberty/current-passed-ci/', + 'descr' => 'openstack-liberty-trunk', + 'gpgcheck' => 'no', + }, + }, + } + package { 'openstack-selinux': ensure => 'latest' } + $package_provider = 'yum' + } + default: { + fail("Unsupported osfamily (${::osfamily})") + } +} + +# Deploy MySQL Server +class { '::mysql::server': } + +# Deploy RabbitMQ +class { '::rabbitmq': + delete_guest_user => true, + package_provider => $package_provider, +} +rabbitmq_vhost { '/': + provider => 'rabbitmqctl', + require => Class['rabbitmq'], +} +rabbitmq_user { ['neutron', 'nova', 'cinder', 'glance']: + admin => true, + password => 'an_even_bigger_secret', + provider => 'rabbitmqctl', + require => Class['rabbitmq'], +} +rabbitmq_user_permissions { ['neutron@/', 'nova@/', 'cinder@/', 'glance@/']: + configure_permission => '.*', + write_permission => '.*', + read_permission => '.*', + provider => 'rabbitmqctl', + require => Class['rabbitmq'], +} + +# Deploy Keystone +class { '::keystone::client': } +class { '::keystone::cron::token_flush': } +class { '::keystone::db::mysql': + password => 'keystone', +} +class { '::keystone': + verbose => true, + debug => true, + database_connection => 'mysql://keystone:keystone@127.0.0.1/keystone', + admin_token => 'admin_token', + enabled => true, + service_name => 'httpd', +} +include ::apache +class { '::keystone::wsgi::apache': + ssl => false, + workers => 2, +} +class { '::keystone::roles::admin': + email => 'test@example.tld', + password => 'a_big_secret', +} +class { '::keystone::endpoint': + default_domain => 'admin', +} + +# Deploy Glance +class { '::glance::db::mysql': + password => 'glance', +} +include ::glance +include ::glance::backend::file +include ::glance::client +class { '::glance::keystone::auth': + password => 'a_big_secret', +} +class { '::glance::api': + debug => true, + verbose => true, + database_connection => 'mysql://glance:glance@127.0.0.1/glance?charset=utf8', + keystone_password => 'a_big_secret', + workers => 2, +} +class { '::glance::registry': + debug => true, + verbose => true, + database_connection => 'mysql://glance:glance@127.0.0.1/glance?charset=utf8', + keystone_password => 'a_big_secret', + workers => 2, +} +class { '::glance::notify::rabbitmq': + rabbit_userid => 'glance', + rabbit_password => 'an_even_bigger_secret', + rabbit_host => '127.0.0.1', + notification_driver => 'messagingv2', +} + +# Deploy Neutron +class { '::neutron::db::mysql': + password => 'neutron', +} +class { '::neutron::keystone::auth': + password => 'a_big_secret', +} +class { '::neutron': + rabbit_user => 'neutron', + rabbit_password => 'an_even_bigger_secret', + rabbit_host => '127.0.0.1', + allow_overlapping_ips => true, + core_plugin => 'ml2', + service_plugins => ['router', 'metering'], + debug => true, + verbose => true, +} +class { '::neutron::client': } +class { '::neutron::server': + database_connection => 'mysql://neutron:neutron@127.0.0.1/neutron?charset=utf8', + auth_password => 'a_big_secret', + identity_uri => 'http://127.0.0.1:35357/', + sync_db => true, + api_workers => 4, +} +class { '::neutron::plugins::ml2': + type_drivers => ['vxlan'], + tenant_network_types => ['vxlan'], + mechanism_drivers => ['openvswitch'], +} +class { '::neutron::agents::ml2::ovs': + enable_tunneling => true, + local_ip => '127.0.0.1', + tunnel_types => ['vxlan'], +} +class { '::neutron::agents::metadata': + debug => true, + auth_password => 'a_big_secret', + shared_secret => 'a_big_secret', + metadata_workers => 2, +} +class { '::neutron::agents::lbaas': + debug => true, +} +class { '::neutron::agents::l3': + debug => true, +} +class { '::neutron::agents::dhcp': + debug => true, +} +class { '::neutron::agents::metering': + debug => true, +} +class { '::neutron::server::notifications': + nova_admin_password => 'a_big_secret', +} + +# Deploy Nova +class { '::nova::db::mysql': + password => 'nova', +} +class { '::nova::keystone::auth': + password => 'a_big_secret', +} +class { '::nova': + database_connection => 'mysql://nova:nova@127.0.0.1/nova?charset=utf8', + rabbit_host => '127.0.0.1', + rabbit_userid => 'nova', + rabbit_password => 'an_even_bigger_secret', + glance_api_servers => 'localhost:9292', + verbose => true, + debug => true, + notification_driver => 'messagingv2', + notify_on_state_change => 'vm_and_task_state', +} +class { '::nova::api': + admin_password => 'a_big_secret', + identity_uri => 'http://127.0.0.1:35357/', + osapi_v3 => true, + neutron_metadata_proxy_shared_secret => 'a_big_secret', + osapi_compute_workers => 2, + ec2_workers => 2, + metadata_workers => 2, + default_floating_pool => 'public', +} +class { '::nova::cert': } +class { '::nova::client': } +class { '::nova::conductor': } +class { '::nova::consoleauth': } +class { '::nova::cron::archive_deleted_rows': } +class { '::nova::compute': + vnc_enabled => true, + instance_usage_audit => true, + instance_usage_audit_period => 'hour', +} +class { '::nova::compute::libvirt': + libvirt_virt_type => 'qemu', + migration_support => true, + vncserver_listen => '0.0.0.0', +} +class { '::nova::scheduler': } +class { '::nova::vncproxy': } +class { '::nova::network::neutron': + neutron_admin_password => 'a_big_secret', + neutron_admin_auth_url => 'http://127.0.0.1:35357/v2.0', +} + +# Deploy Cinder +class { '::cinder::db::mysql': + password => 'cinder', +} +class { '::cinder::keystone::auth': + password => 'a_big_secret', +} +class { '::cinder': + database_connection => 'mysql://cinder:cinder@127.0.0.1/cinder?charset=utf8', + rabbit_host => '127.0.0.1', + rabbit_userid => 'cinder', + rabbit_password => 'an_even_bigger_secret', + verbose => true, + debug => true, +} +class { '::cinder::api': + keystone_password => 'a_big_secret', + identity_uri => 'http://127.0.0.1:35357/', + default_volume_type => 'BACKEND_1', + service_workers => 2, +} +class { '::cinder::quota': } +class { '::cinder::scheduler': } +class { '::cinder::scheduler::filter': } +class { '::cinder::volume': } +class { '::cinder::cron::db_purge': } +class { '::cinder::glance': + glance_api_servers => 'localhost:9292', +} +class { '::cinder::setup_test_volume': + size => '15G', +} +cinder::backend::iscsi { 'BACKEND_1': + iscsi_ip_address => '127.0.0.1', +} +class { '::cinder::backends': + enabled_backends => ['BACKEND_1'], +} +Cinder::Type { + os_password => 'a_big_secret', + os_tenant_name => 'services', + os_username => 'cinder', + os_auth_url => 'http://127.0.0.1:5000/v2.0', +} +cinder::type { 'BACKEND_1': + set_key => 'volume_backend_name', + set_value => 'BACKEND_1', + notify => Service['cinder-volume'], + require => Service['cinder-api'], +} + +# Deploy Horizon +$vhost_params = { add_listen => false } +class { '::horizon': + secret_key => 'big_secret', + vhost_extra_params => $vhost_params, + servername => $::hostname, + allowed_hosts => $::hostname, + # need to disable offline compression due to + # https://bugs.launchpad.net/ubuntu/+source/horizon/+bug/1424042 + compress_offline => false, +} + +# Configure Tempest and the resources +$os_auth_options = '--os-username admin --os-password a_big_secret --os-tenant-name openstack --os-auth-url http://127.0.0.1:5000/v2.0' + +exec { 'manage_m1.nano_nova_flavor': + path => '/usr/bin:/bin:/usr/sbin:/sbin', + provider => shell, + command => "nova ${os_auth_options} flavor-create m1.nano 42 128 0 1", + unless => "nova ${os_auth_options} flavor-list | grep m1.nano", +} +Keystone_user_role['admin@openstack'] -> Exec['manage_m1.nano_nova_flavor'] + +exec { 'manage_m1.micro_nova_flavor': + path => '/usr/bin:/bin:/usr/sbin:/sbin', + provider => shell, + command => "nova ${os_auth_options} flavor-create m1.micro 84 128 0 1", + unless => "nova ${os_auth_options} flavor-list | grep m1.micro", +} +Keystone_user_role['admin@openstack'] -> Exec['manage_m1.micro_nova_flavor'] + +neutron_network { 'public': + tenant_name => 'openstack', + router_external => true, +} +Keystone_user_role['admin@openstack'] -> Neutron_network<||> + +neutron_subnet { 'public-subnet': + cidr => '172.24.5.0/24', + ip_version => '4', + allocation_pools => ['start=172.24.5.10,end=172.24.5.200'], + gateway_ip => '172.24.5.1', + enable_dhcp => false, + network_name => 'public', + tenant_name => 'openstack', +} + +include ::vswitch::ovs +vs_bridge { 'br-ex': + ensure => present, + notify => Exec['create_br-ex_vif'], +} + +# creates br-ex virtual interface to reach floating-ip network +exec { 'create_br-ex_vif': + path => '/usr/bin:/bin:/usr/sbin:/sbin', + provider => shell, + command => 'ip addr add 172.24.5.1/24 dev br-ex; ip link set br-ex up', + refreshonly => true, +} + +glance_image { 'cirros': + ensure => present, + container_format => 'bare', + disk_format => 'qcow2', + is_public => 'yes', + # TODO(emilien) optimization by 1/ using Hiera to configure Glance image source + # and 2/ if running in the gate, use /home/jenkins/cache/files/ cirros image. + # source => '/home/jenkins/cache/files/cirros-0.3.4-x86_64-disk.img', + source => 'http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-disk.img', +} +glance_image { 'cirros_alt': + ensure => present, + container_format => 'bare', + disk_format => 'qcow2', + is_public => 'yes', + # TODO(emilien) optimization by 1/ using Hiera to configure Glance image source + # and 2/ if running in the gate, use /home/jenkins/cache/files/ cirros image. + # source => '/home/jenkins/cache/files/cirros-0.3.4-x86_64-disk.img', + source => 'http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-disk.img', +} + +class { '::tempest': + debug => true, + use_stderr => false, + log_file => 'tempest.log', + git_clone => true, + tempest_clone_path => '/tmp/tempest', + tempest_clone_owner => $::id, + lock_path => '/tmp/tempest', + tempest_config_file => '/tmp/tempest/etc/tempest.conf', + configure_images => true, + configure_networks => true, + identity_uri => 'http://127.0.0.1:5000/v2.0', + identity_uri_v3 => 'http://127.0.0.1:5000/v3', + admin_username => 'admin', + admin_tenant_name => 'openstack', + admin_password => 'a_big_secret', + admin_domain_name => 'Default', + auth_version => 'v3', + image_name => 'cirros', + image_name_alt => 'cirros_alt', + cinder_available => true, + glance_available => true, + horizon_available => true, + nova_available => true, + neutron_available => true, + public_network_name => 'public', + flavor_ref => '42', + flavor_ref_alt => '84', + image_ssh_user => 'cirros', + image_alt_ssh_user => 'cirros', + img_file => 'cirros-0.3.4-x86_64-disk.img', + # TODO(emilien) optimization by 1/ using Hiera to configure Glance image source + # and 2/ if running in the gate, use /home/jenkins/cache/files/ cirros image. + # img_dir => '/home/jenkins/cache/files', + img_dir => '/tmp/tempest', +} diff --git a/functions b/functions index 476763670..556c1a205 100644 --- a/functions +++ b/functions @@ -68,3 +68,18 @@ install_modules() { install_all fi } + +is_fedora() { + if [ -f /etc/os-release ]; then + source /etc/os-release + test "$ID" = "fedora" -o "$ID" = "centos" + else + return 1 + fi +} + +uses_debs() { + # check if apt-get is installed, valid for debian based + type "apt-get" 2>/dev/null +} + diff --git a/run_tests.sh b/run_tests.sh index e03c1e411..48eaaf748 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -14,6 +14,8 @@ # under the License. export SCENARIO=${SCENARIO:-scenario001} +export SCRIPT_DIR=$(cd `dirname $0` && pwd -P) +source $SCRIPT_DIR/functions if [ ! -f fixtures/${SCENARIO}.pp ]; then echo "fixtures/${SCENARIO}.pp file does not exist. Please define a valid scenario." @@ -45,17 +47,6 @@ function run_puppet() { return $res } -function is_fedora { - # note we consider CentOS 7 as fedora for now - lsb_release -i 2>/dev/null | grep -iq "fedora" || \ - lsb_release -i 2>/dev/null | grep -iq "CentOS" -} - -function uses_debs { - # check if apt-get is installed, valid for debian based - type "apt-get" 2>/dev/null -} - if uses_debs; then wget https://apt.puppetlabs.com/puppetlabs-release-trusty.deb -O /tmp/puppet.deb $SUDO dpkg -i /tmp/puppet.deb