Merge pull request #119 from markgoddard/dev-automation

Add scripting around the development environment
This commit is contained in:
Mark Goddard 2018-02-06 14:27:01 +00:00 committed by GitHub
commit 58efaee1b7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 574 additions and 44 deletions

12
Vagrantfile vendored
View File

@ -43,22 +43,16 @@ NM_CONTROLLED=no
EOF
sudo ifup eth1
sudo yum -y install gcc git vim python-virtualenv
/vagrant/dev/install.sh
# Configure the legacy development environment. This has been retained
# while transitioning to the new development environment.
cat > /vagrant/kayobe-env << EOF
export KAYOBE_CONFIG_PATH=/vagrant/etc/kayobe
export KOLLA_CONFIG_PATH=/vagrant/etc/kolla
EOF
source /vagrant/kayobe-env
cp /vagrant/dev/dev-vagrant.yml /vagrant/etc/kayobe/
cp /vagrant/dev/dev-hosts /vagrant/etc/kayobe/inventory
cp /vagrant/dev/dev-vagrant-network-allocation.yml /vagrant/etc/kayobe/network-allocation.yml
virtualenv ~/kayobe-venv
source ~/kayobe-venv/bin/activate
pip install -U pip
pip install /vagrant
deactivate
SHELL
end

19
dev/config.sh Normal file
View File

@ -0,0 +1,19 @@
# Configuration for kayobe development environment.
# Path to the kayobe source code repository. Typically this will be the Vagrant
# shared directory.
#export KAYOBE_SOURCE_PATH=/vagrant
# Path to the kayobe-config repository checkout.
#export KAYOBE_CONFIG_SOURCE_PATH=${KAYOBE_SOURCE_PATH}/config/src/kayobe-config
# Path to the kayobe virtual environment.
#export KAYOBE_VENV_PATH=~/kayobe-venv
# Whether to build container images for the seed services. If 0, they will be
# pulled.
#export KAYOBE_SEED_CONTAINER_IMAGE_BUILD=0
# Whether to build container images for the overcloud services. If 0, they will
# be pulled.
#export KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD=0

21
dev/environment-setup.sh Executable file
View File

@ -0,0 +1,21 @@
#!/bin/bash
set -eu
set -o pipefail
# This script can be used to prepare the environment for use with kayobe. This
# includes setting environment variables and activating the python virtual
# environment. This script should be sourced rather than executed in a
# subprocess. e.g. source dev/environment-setup.sh
PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${PARENT}/functions"
function main {
config_init
environment_setup
}
main

203
dev/functions Normal file
View File

@ -0,0 +1,203 @@
#!/bin/bash
set -eu
set -o pipefail
# Library of functions for the kayobe development environment.
# Configuration
function config_defaults {
# Set default values for kayobe development configuration.
# Try to detect if we are running in a vagrant VM.
if [[ -e /vagrant ]]; then
KAYOBE_SOURCE_PATH_DEFAULT=/vagrant
else
KAYOBE_SOURCE_PATH_DEFAULT="$(pwd)"
fi
# Path to the kayobe source code repository. Typically this will be the
# Vagrant shared directory.
export KAYOBE_SOURCE_PATH="${KAYOBE_SOURCE_PATH:-$KAYOBE_SOURCE_PATH_DEFAULT}"
# Path to the kayobe-config repository checkout.
export KAYOBE_CONFIG_SOURCE_PATH="${KAYOBE_CONFIG_SOURCE_PATH:-${KAYOBE_SOURCE_PATH}/config/src/kayobe-config}"
# Path to the kayobe virtual environment.
export KAYOBE_VENV_PATH="${KAYOBE_VENV_PATH:-~/kayobe-venv}"
# Whether to build container images for the seed services. If 0, they will
# be pulled.
export KAYOBE_SEED_CONTAINER_IMAGE_BUILD=${KAYOBE_SEED_CONTAINER_IMAGE_BUILD:-0}
# Whether to build container images for the overcloud services. If 0, they
# will be pulled.
export KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD=${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD:-0}
}
function config_set {
# Source the configuration file, config.sh
PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${PARENT}/config.sh"
}
function config_check {
# Check the configuration environment variables.
if [[ ! -e "$KAYOBE_CONFIG_SOURCE_PATH" ]]; then
if [[ ${KAYOBE_CONFIG_REQUIRED:-1} -eq 1 ]]; then
echo "Kayobe configuration path $KAYOBE_CONFIG_SOURCE_PATH does not exist"
return 1
fi
fi
if [[ ! -e "$KAYOBE_SOURCE_PATH" ]]; then
echo "Kayobe source path $KAYOBE_SOURCE_PATH does not exist"
return 1
fi
}
function config_init {
config_defaults
config_set
config_check
}
# Installation
function install_dependencies {
echo "Installing package dependencies for kayobe"
if [[ -e /etc/centos-release ]]; then
sudo yum -y install gcc git vim python-virtualenv
else
sudo apt install -y python-dev python-virtualenv gcc git
fi
}
function install_venv {
local venv_parent="$(dirname ${KAYOBE_VENV_PATH})"
if [[ ! -d "$venv_parent" ]]; then
mkdir -p "$venv_parent"
fi
if [[ ! -f "${KAYOBE_VENV_PATH}/bin/activate" ]]; then
echo "Creating kayobe virtual environment in ${KAYOBE_VENV_PATH}"
virtualenv "${KAYOBE_VENV_PATH}"
# NOTE: Virtualenv's activate and deactivate scripts reference an
# unbound variable.
set +u
source "${KAYOBE_VENV_PATH}/bin/activate"
pip install -U pip
pip install "${KAYOBE_SOURCE_PATH}"
deactivate
set -u
else
echo "Using existing kayobe virtual environment in ${KAYOBE_VENV_PATH}"
fi
}
# Deployment
function is_deploy_image_built_locally {
ipa_build_images=$(kayobe configuration dump --host controllers[0] --var-name ipa_build_images)
[[ $ipa_build_images =~ ^true$ ]]
}
function environment_setup {
# NOTE: Virtualenv's activate script references an unbound variable.
set +u
source "${KAYOBE_VENV_PATH}/bin/activate"
set -u
source "${KAYOBE_CONFIG_SOURCE_PATH}/kayobe-env"
cd "${KAYOBE_SOURCE_PATH}"
}
function seed_hypervisor_deploy {
# Deploy a seed hypervisor.
environment_setup
echo "Bootstrapping the ansible control host"
kayobe control host bootstrap
echo "Configuring the seed hypervisor"
kayobe seed hypervisor host configure
}
function seed_deploy {
# Deploy a kayobe seed in a VM.
environment_setup
echo "Bootstrapping the ansible control host"
kayobe control host bootstrap
echo "Provisioning the seed VM"
kayobe seed vm provision
echo "Configuring the seed host"
kayobe seed host configure
# Note: This must currently be before host configure, because host
# configure runs kolla-ansible.yml, which validates the presence of the
# built deploy images.
if is_deploy_image_built_locally; then
echo "Building seed deployment images"
kayobe seed deployment image build
else
echo "Not building seed deployment images"
fi
if [[ ${KAYOBE_SEED_CONTAINER_IMAGE_BUILD} = 1 ]]; then
echo "Building seed container images"
kayobe seed container image build
else
echo "Not pulling seed container images - no such command yet"
#kayobe seed container image pull
fi
echo "Deploying containerised seed services"
kayobe seed service deploy
}
function overcloud_deploy {
# Deploy a kayobe control plane.
echo "Deploying a kayobe development environment. This consists of a "
echo "single node OpenStack control plane."
environment_setup
echo "Bootstrapping the ansible control host"
kayobe control host bootstrap
echo "Configuring the controller host"
kayobe overcloud host configure
# Note: This must currently be before host configure, because host
# configure runs kolla-ansible.yml, which validates the presence of the
# built deploy images.
if is_deploy_image_built_locally; then
echo "Building overcloud deployment images"
kayobe overcloud deployment image build
else
echo "Not building overcloud deployment images"
fi
if [[ ${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD} = 1 ]]; then
echo "Building overcloud container images"
kayobe overcloud container image build
else
echo "Pulling overcloud container images"
kayobe overcloud container image pull
fi
echo "Deploying containerised overcloud services"
kayobe overcloud service deploy
echo "Performing post-deployment configuration"
source "${KOLLA_CONFIG_PATH:-/etc/kolla}/admin-openrc.sh"
kayobe overcloud post configure
echo "Control plane deployment complete"
}

22
dev/install.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/bash
set -eu
set -o pipefail
# Install kayobe and its dependencies in a virtual environment.
PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${PARENT}/functions"
function main {
# Don't require kayobe configuration to exist for installation - it is not
# required for the legacy manual deployment procedure.
KAYOBE_CONFIG_REQUIRED=0
config_init
install_dependencies
install_venv
}
main

20
dev/overcloud-deploy.sh Executable file
View File

@ -0,0 +1,20 @@
#!/bin/bash
set -eu
set -o pipefail
# Simple script to stand up a development environment for an OpenStack
# controller in a Vagrant VM using kayobe. This should be executed from within
# the VM.
PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${PARENT}/functions"
function main {
config_init
overcloud_deploy
}
main

19
dev/seed-deploy.sh Executable file
View File

@ -0,0 +1,19 @@
#!/bin/bash
set -eu
set -o pipefail
# Simple script to stand up a development environment for a seed VM using
# kayobe. This should be executed from the hypervisor.
PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${PARENT}/functions"
function main {
config_init
seed_deploy
}
main

19
dev/seed-hypervisor-deploy.sh Executable file
View File

@ -0,0 +1,19 @@
#!/bin/bash
set -eu
set -o pipefail
# Simple script to stand up a development environment for a seed hypervisor
# using kayobe. This should be executed from the hypervisor.
PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${PARENT}/functions"
function main {
config_init
seed_hypervisor_deploy
}
main

View File

@ -0,0 +1,152 @@
.. _development-automated:
===============
Automated Setup
===============
This section provides information on the development tools provided by kayobe
to automate the deployment of various development environments.
For a manual procedure, see :ref:`development-manual`.
Overview
========
The kayobe development environment automation tooling is built using simple
shell scripts. Some minimal configuration can be applied by setting the
environment variables in `dev/config.sh`. Control plane configuration is
typically provided via the `dev-kayobe-config
<https://github.com/stackhpc/dev-kayobe-config/>`_ repository, although it is
also possible to use your own kayobe configuration. This allows us to build a
development environment that is as close to production as possible.
Environments
============
The following development environments are supported:
* Overcloud (single OpenStack controller)
* Seed hypervisor
* Seed VM
The seed VM environment may be used in an environment already deployed as a
seed hypervisor.
Overcloud
=========
Preparation
-----------
Clone the kayobe repository::
git clone https://github.com/stackhpc/kayobe
Change the current directory to the kayobe repository::
cd kayobe
Clone the ``dev-kayobe-config`` repository to ``config/src/kayobe-config``::
mkdir -p config/src
git clone https://github.com/stackhpc/dev-kayobe-config config/src/kayobe-config
Inspect the kayobe configuration and make any changes necessary for your
environment.
If using Vagrant, follow the steps in :ref:`development-vagrant` to prepare
your environment for use with Vagrant and bring up a Vagrant VM.
Usage
-----
If using Vagrant, SSH into the Vagrant VM and change to the shared directory::
vagrant ssh
cd /vagrant
Run the ``dev/overcloud-deploy.sh`` script to deploy the OpenStack control
plane::
./dev/overcloud-deploy.sh
Upon successful completion of this script, the control plane will be active.
Seed Hypervisor
===============
The seed hypervisor development environment is supported for CentOS 7. The
system must be either bare metal, or a VM on a system with nested
virtualisation enabled.
Preparation
-----------
The following commands should be executed on the seed hypervisor.
Clone the kayobe repository::
git clone https://github.com/stackhpc/kayobe
Change the current directory to the kayobe repository::
cd kayobe
Clone the ``add-seed-and-hv`` branch of the ``dev-kayobe-config`` repository to
``config/src/kayobe-config``::
mkdir -p config/src
git clone https://github.com/stackhpc/dev-kayobe-config -b add-seed-and-hv config/src/kayobe-config
Inspect the kayobe configuration and make any changes necessary for your
environment.
Usage
-----
Run the ``dev/seed-hypervisor-deploy.sh`` script to deploy the seed
hypervisor::
./dev/seed-hypervisor-deploy.sh
Upon successful completion of this script, the seed hypervisor will be active.
Seed VM
=======
The seed VM should be deployed on a system configured as a libvirt/KVM
hypervisor, using the kayobe seed hypervisor support or otherwise.
Preparation
-----------
The following commands should be executed on the seed hypervisor.
Change the current directory to the kayobe repository::
git clone https://github.com/stackhpc/kayobe
Change to the ``kayobe`` directory::
cd kayobe
Clone the ``add-seed-and-hv`` branch of the ``dev-kayobe-config`` repository to
``config/src/kayobe-config``::
mkdir -p config/src
git clone https://github.com/stackhpc/dev-kayobe-config -b add-seed-and-hv config/src/kayobe-config
Inspect the kayobe configuration and make any changes necessary for your
environment.
Usage
=====
Run the ``dev/seed-deploy.sh`` script to deploy the seed VM::
./dev/seed-deploy.sh
Upon successful completion of this script, the seed VM will be active. The
seed VM may be accessed via SSH as the ``stack`` user::
ssh stack@192.168.33.5

View File

@ -1,4 +1,4 @@
=================
How to Contribute
=================
.. include:: ../../CONTRIBUTING.rst
.. include:: ../../../CONTRIBUTING.rst

View File

@ -0,0 +1,11 @@
========================
Kayobe Development Guide
========================
.. toctree::
:maxdepth: 2
vagrant
manual
automated
contributing

View File

@ -1,42 +1,26 @@
===========
Development
===========
.. _development-manual:
This section describes how to set up an OpenStack controller in a virtual
machine using `Vagrant <https://www.vagrantup.com/>`_ and Kayobe.
============
Manual Setup
============
This section provides a set of manual steps to set up a development environment
for an OpenStack controller in a virtual machine using `Vagrant
<https://www.vagrantup.com/>`_ and Kayobe.
For a more automated and flexible procedure, see :ref:`development-automated`.
Preparation
===========
First, ensure that Vagrant is installed and correctly configured to use
virtual box. Also install the following vagrant plugins:
Follow the steps in :ref:`development-vagrant` to prepare your environment for
use with Vagrant and bring up a Vagrant VM.
vagrant plugin install vagrant-vbguest
vagrant plugin install vagrant-reload
Manual Installation
===================
Note: if using Ubuntu 16.04 LTS, you may be unable to install any plugins. To
work around this install the upstream version from www.virtualbox.org.
Next, clone kayobe::
git clone https://github.com/stackhpc/kayobe
Change the current directory to the kayobe repository::
cd kayobe
Inspect kayobe's ``Vagrantfile``, noting the provisioning steps::
less Vagrantfile
Bring up a virtual machine::
vagrant up
Wait for the VM to boot.
Installation
============
Sometimes the best way to learn a tool is to ditch the scripts and perform a
manual installation.
SSH into the controller VM::

View File

@ -0,0 +1,58 @@
.. _development-vagrant:
=======
Vagrant
=======
Kayobe provides a Vagrantfile that can be used to bring up a virtual machine
for use as a development environment. The VM is based on the `stackhpc/centos-7
<https://app.vagrantup.com/stackhpc/boxes/centos-7>`_ CentOS 7 image, and
supports the following providers:
* VirtualBox
* VMWare Fusion
The VM is configured with 4GB RAM. It has a single private network in addition
to the standard Vagrant NAT network.
Preparation
===========
First, ensure that Vagrant is installed and correctly configured to use
the required provider. Also install the following vagrant plugin::
vagrant plugin install vagrant-reload
If using the VirtualBox provider, install the following vagrant plugin::
vagrant plugin install vagrant-vbguest
Note: if using Ubuntu 16.04 LTS, you may be unable to install any plugins. To
work around this install the upstream version from www.virtualbox.org.
Usage
=====
Later sections in the development guide cover in more detail how to use the
development VM in different configurations. These steps cover bringing up and
accessing the VM.
Clone the kayobe repository::
git clone https://github.com/stackhpc/kayobe
Change the current directory to the kayobe repository::
cd kayobe
Inspect kayobe's ``Vagrantfile``, noting the provisioning steps::
less Vagrantfile
Bring up a virtual machine::
vagrant up
Wait for the VM to boot, then SSH in::
vagrant ssh

View File

@ -42,8 +42,7 @@ Developer Documentation
.. toctree::
:maxdepth: 2
contributing
development
development/index
Release Notes
-------------

View File

@ -4,6 +4,7 @@
hacking>=0.12.0,<0.13 # Apache-2.0
bashate>=0.2 # Apache-2.0
coverage>=4.0 # Apache-2.0
doc8 # Apache-2.0
sphinx>=1.5.1 # BSD

7
tools/run-bashate.sh Executable file
View File

@ -0,0 +1,7 @@
#!/bin/bash
# Ignore E006 -- line length greater than 80 char
ROOT=$(readlink -fn $(dirname $0)/.. )
find $ROOT -not -wholename \*.tox/\* -and -not -wholename \*.test/\* \
-and -name \*.sh -print0 | xargs -0 bashate -v --ignore E006

View File

@ -19,6 +19,7 @@ commands = unit2 discover {posargs}
[testenv:pep8]
commands =
{toxinidir}/tools/run-bashate.sh
flake8 {posargs} kayobe
# Check the *.rst files
# We use a thin wrapper around doc8 currently, which has support for sphinx