Merge pull request #119 from markgoddard/dev-automation

Add scripting around the development environment

Vagrantfile

@@ -43,22 +43,16 @@ NM_CONTROLLED=no
 EOF
     sudo ifup eth1
 
-    sudo yum -y install gcc git vim python-virtualenv
+    /vagrant/dev/install.sh
 
+    # Configure the legacy development environment. This has been retained
+    # while transitioning to the new development environment.
     cat > /vagrant/kayobe-env << EOF
 export KAYOBE_CONFIG_PATH=/vagrant/etc/kayobe
 export KOLLA_CONFIG_PATH=/vagrant/etc/kolla
 EOF
-    source /vagrant/kayobe-env
-
     cp /vagrant/dev/dev-vagrant.yml /vagrant/etc/kayobe/
     cp /vagrant/dev/dev-hosts /vagrant/etc/kayobe/inventory
     cp /vagrant/dev/dev-vagrant-network-allocation.yml /vagrant/etc/kayobe/network-allocation.yml
-
-    virtualenv ~/kayobe-venv
-    source ~/kayobe-venv/bin/activate
-    pip install -U pip
-    pip install /vagrant
-    deactivate
   SHELL
 end

dev/config.sh

@@ -0,0 +1,19 @@
+# Configuration for kayobe development environment.
+
+# Path to the kayobe source code repository. Typically this will be the Vagrant
+# shared directory.
+#export KAYOBE_SOURCE_PATH=/vagrant
+
+# Path to the kayobe-config repository checkout.
+#export KAYOBE_CONFIG_SOURCE_PATH=${KAYOBE_SOURCE_PATH}/config/src/kayobe-config
+
+# Path to the kayobe virtual environment.
+#export KAYOBE_VENV_PATH=~/kayobe-venv
+
+# Whether to build container images for the seed services. If 0, they will be
+# pulled.
+#export KAYOBE_SEED_CONTAINER_IMAGE_BUILD=0
+
+# Whether to build container images for the overcloud services. If 0, they will
+# be pulled.
+#export KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD=0

dev/environment-setup.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+# This script can be used to prepare the environment for use with kayobe. This
+# includes setting environment variables and activating the python virtual
+# environment. This script should be sourced rather than executed in a
+# subprocess. e.g. source dev/environment-setup.sh
+
+PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "${PARENT}/functions"
+
+
+function main {
+    config_init
+    environment_setup
+}
+
+main

dev/functions

@@ -0,0 +1,203 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+# Library of functions for the kayobe development environment.
+
+# Configuration
+
+function config_defaults {
+    # Set default values for kayobe development configuration.
+
+    # Try to detect if we are running in a vagrant VM.
+    if [[ -e /vagrant ]]; then
+        KAYOBE_SOURCE_PATH_DEFAULT=/vagrant
+    else
+        KAYOBE_SOURCE_PATH_DEFAULT="$(pwd)"
+    fi
+
+    # Path to the kayobe source code repository. Typically this will be the
+    # Vagrant shared directory.
+    export KAYOBE_SOURCE_PATH="${KAYOBE_SOURCE_PATH:-$KAYOBE_SOURCE_PATH_DEFAULT}"
+
+    # Path to the kayobe-config repository checkout.
+    export KAYOBE_CONFIG_SOURCE_PATH="${KAYOBE_CONFIG_SOURCE_PATH:-${KAYOBE_SOURCE_PATH}/config/src/kayobe-config}"
+
+    # Path to the kayobe virtual environment.
+    export KAYOBE_VENV_PATH="${KAYOBE_VENV_PATH:-~/kayobe-venv}"
+
+    # Whether to build container images for the seed services. If 0, they will
+    # be pulled.
+    export KAYOBE_SEED_CONTAINER_IMAGE_BUILD=${KAYOBE_SEED_CONTAINER_IMAGE_BUILD:-0}
+
+    # Whether to build container images for the overcloud services. If 0, they
+    # will be pulled.
+    export KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD=${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD:-0}
+}
+
+function config_set {
+    # Source the configuration file, config.sh
+
+    PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+    source "${PARENT}/config.sh"
+}
+
+function config_check {
+    # Check the configuration environment variables.
+
+    if [[ ! -e "$KAYOBE_CONFIG_SOURCE_PATH" ]]; then
+        if [[ ${KAYOBE_CONFIG_REQUIRED:-1} -eq 1 ]]; then
+            echo "Kayobe configuration path $KAYOBE_CONFIG_SOURCE_PATH does not exist"
+            return 1
+        fi
+    fi
+
+    if [[ ! -e "$KAYOBE_SOURCE_PATH" ]]; then
+        echo "Kayobe source path $KAYOBE_SOURCE_PATH does not exist"
+        return 1
+    fi
+}
+
+function config_init {
+    config_defaults
+    config_set
+    config_check
+}
+
+# Installation
+
+function install_dependencies {
+    echo "Installing package dependencies for kayobe"
+    if [[ -e /etc/centos-release ]]; then
+        sudo yum -y install gcc git vim python-virtualenv
+    else
+        sudo apt install -y python-dev python-virtualenv gcc git
+    fi
+}
+
+function install_venv {
+    local venv_parent="$(dirname ${KAYOBE_VENV_PATH})"
+    if [[ ! -d "$venv_parent" ]]; then
+        mkdir -p "$venv_parent"
+    fi
+    if [[ ! -f "${KAYOBE_VENV_PATH}/bin/activate" ]]; then
+        echo "Creating kayobe virtual environment in ${KAYOBE_VENV_PATH}"
+        virtualenv "${KAYOBE_VENV_PATH}"
+        # NOTE: Virtualenv's activate and deactivate scripts reference an
+        # unbound variable.
+        set +u
+        source "${KAYOBE_VENV_PATH}/bin/activate"
+        pip install -U pip
+        pip install "${KAYOBE_SOURCE_PATH}"
+        deactivate
+        set -u
+    else
+        echo "Using existing kayobe virtual environment in ${KAYOBE_VENV_PATH}"
+    fi
+}
+
+# Deployment
+
+function is_deploy_image_built_locally {
+    ipa_build_images=$(kayobe configuration dump --host controllers[0] --var-name ipa_build_images)
+    [[ $ipa_build_images =~ ^true$ ]]
+}
+
+function environment_setup {
+    # NOTE: Virtualenv's activate script references an unbound variable.
+    set +u
+    source "${KAYOBE_VENV_PATH}/bin/activate"
+    set -u
+    source "${KAYOBE_CONFIG_SOURCE_PATH}/kayobe-env"
+
+    cd "${KAYOBE_SOURCE_PATH}"
+}
+
+function seed_hypervisor_deploy {
+    # Deploy a seed hypervisor.
+    environment_setup
+
+    echo "Bootstrapping the ansible control host"
+    kayobe control host bootstrap
+
+    echo "Configuring the seed hypervisor"
+    kayobe seed hypervisor host configure
+}
+
+function seed_deploy {
+    # Deploy a kayobe seed in a VM.
+    environment_setup
+
+    echo "Bootstrapping the ansible control host"
+    kayobe control host bootstrap
+
+    echo "Provisioning the seed VM"
+    kayobe seed vm provision
+
+    echo "Configuring the seed host"
+    kayobe seed host configure
+
+    # Note: This must currently be before host configure, because host
+    # configure runs kolla-ansible.yml, which validates the presence of the
+    # built deploy images.
+    if is_deploy_image_built_locally; then
+        echo "Building seed deployment images"
+        kayobe seed deployment image build
+    else
+        echo "Not building seed deployment images"
+    fi
+
+    if [[ ${KAYOBE_SEED_CONTAINER_IMAGE_BUILD} = 1 ]]; then
+        echo "Building seed container images"
+        kayobe seed container image build
+    else
+        echo "Not pulling seed container images - no such command yet"
+        #kayobe seed container image pull
+    fi
+
+    echo "Deploying containerised seed services"
+    kayobe seed service deploy
+}
+
+function overcloud_deploy {
+    # Deploy a kayobe control plane.
+    echo "Deploying a kayobe development environment. This consists of a "
+    echo "single node OpenStack control plane."
+
+    environment_setup
+
+    echo "Bootstrapping the ansible control host"
+    kayobe control host bootstrap
+
+    echo "Configuring the controller host"
+    kayobe overcloud host configure
+
+    # Note: This must currently be before host configure, because host
+    # configure runs kolla-ansible.yml, which validates the presence of the
+    # built deploy images.
+    if is_deploy_image_built_locally; then
+        echo "Building overcloud deployment images"
+        kayobe overcloud deployment image build
+    else
+        echo "Not building overcloud deployment images"
+    fi
+
+    if [[ ${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD} = 1 ]]; then
+        echo "Building overcloud container images"
+        kayobe overcloud container image build
+    else
+        echo "Pulling overcloud container images"
+        kayobe overcloud container image pull
+    fi
+
+    echo "Deploying containerised overcloud services"
+    kayobe overcloud service deploy
+
+    echo "Performing post-deployment configuration"
+    source "${KOLLA_CONFIG_PATH:-/etc/kolla}/admin-openrc.sh"
+    kayobe overcloud post configure
+
+    echo "Control plane deployment complete"
+}

dev/install.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+# Install kayobe and its dependencies in a virtual environment.
+
+PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "${PARENT}/functions"
+
+
+function main {
+    # Don't require kayobe configuration to exist for installation - it is not
+    # required for the legacy manual deployment procedure.
+    KAYOBE_CONFIG_REQUIRED=0
+    config_init
+    install_dependencies
+    install_venv
+}
+
+main

dev/overcloud-deploy.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+# Simple script to stand up a development environment for an OpenStack
+# controller in a Vagrant VM using kayobe.  This should be executed from within
+# the VM.
+
+PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "${PARENT}/functions"
+
+
+function main {
+    config_init
+    overcloud_deploy
+}
+
+main

dev/seed-deploy.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+# Simple script to stand up a development environment for a seed VM using
+# kayobe.  This should be executed from the hypervisor.
+
+PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "${PARENT}/functions"
+
+
+function main {
+    config_init
+    seed_deploy
+}
+
+main

dev/seed-hypervisor-deploy.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+# Simple script to stand up a development environment for a seed hypervisor
+# using kayobe.  This should be executed from the hypervisor.
+
+PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "${PARENT}/functions"
+
+
+function main {
+    config_init
+    seed_hypervisor_deploy
+}
+
+main

doc/source/development/automated.rst

@@ -0,0 +1,152 @@
+.. _development-automated:
+
+===============
+Automated Setup
+===============
+
+This section provides information on the development tools provided by kayobe
+to automate the deployment of various development environments.
+
+For a manual procedure, see :ref:`development-manual`.
+
+Overview
+========
+
+The kayobe development environment automation tooling is built using simple
+shell scripts.  Some minimal configuration can be applied by setting the
+environment variables in `dev/config.sh`.  Control plane configuration is
+typically provided via the `dev-kayobe-config
+<https://github.com/stackhpc/dev-kayobe-config/>`_ repository, although it is
+also possible to use your own kayobe configuration.  This allows us to build a
+development environment that is as close to production as possible.
+
+Environments
+============
+
+The following development environments are supported:
+
+* Overcloud (single OpenStack controller)
+* Seed hypervisor
+* Seed VM
+
+The seed VM environment may be used in an environment already deployed as a
+seed hypervisor.
+
+Overcloud
+=========
+
+Preparation
+-----------
+
+Clone the kayobe repository::
+
+    git clone https://github.com/stackhpc/kayobe
+
+Change the current directory to the kayobe repository::
+
+    cd kayobe
+
+Clone the ``dev-kayobe-config`` repository to ``config/src/kayobe-config``::
+
+    mkdir -p config/src
+    git clone https://github.com/stackhpc/dev-kayobe-config config/src/kayobe-config
+
+Inspect the kayobe configuration and make any changes necessary for your
+environment.
+
+If using Vagrant, follow the steps in :ref:`development-vagrant` to prepare
+your environment for use with Vagrant and bring up a Vagrant VM.
+
+Usage
+-----
+
+If using Vagrant, SSH into the Vagrant VM and change to the shared directory::
+
+    vagrant ssh
+    cd /vagrant
+
+Run the ``dev/overcloud-deploy.sh`` script to deploy the OpenStack control
+plane::
+
+    ./dev/overcloud-deploy.sh
+
+Upon successful completion of this script, the control plane will be active.
+
+Seed Hypervisor
+===============
+
+The seed hypervisor development environment is supported for CentOS 7.  The
+system must be either bare metal, or a VM on a system with nested
+virtualisation enabled.
+
+Preparation
+-----------
+
+The following commands should be executed on the seed hypervisor.
+
+Clone the kayobe repository::
+
+    git clone https://github.com/stackhpc/kayobe
+
+Change the current directory to the kayobe repository::
+
+    cd kayobe
+
+Clone the ``add-seed-and-hv`` branch of the ``dev-kayobe-config`` repository to
+``config/src/kayobe-config``::
+
+    mkdir -p config/src
+    git clone https://github.com/stackhpc/dev-kayobe-config -b add-seed-and-hv config/src/kayobe-config
+
+Inspect the kayobe configuration and make any changes necessary for your
+environment.
+
+Usage
+-----
+
+Run the ``dev/seed-hypervisor-deploy.sh`` script to deploy the seed
+hypervisor::
+
+    ./dev/seed-hypervisor-deploy.sh
+
+Upon successful completion of this script, the seed hypervisor will be active.
+
+Seed VM
+=======
+
+The seed VM should be deployed on a system configured as a libvirt/KVM
+hypervisor, using the kayobe seed hypervisor support or otherwise.
+
+Preparation
+-----------
+
+The following commands should be executed on the seed hypervisor.
+
+Change the current directory to the kayobe repository::
+
+    git clone https://github.com/stackhpc/kayobe
+
+Change to the ``kayobe`` directory::
+
+    cd kayobe
+
+Clone the ``add-seed-and-hv`` branch of the ``dev-kayobe-config`` repository to
+``config/src/kayobe-config``::
+
+    mkdir -p config/src
+    git clone https://github.com/stackhpc/dev-kayobe-config -b add-seed-and-hv config/src/kayobe-config
+
+Inspect the kayobe configuration and make any changes necessary for your
+environment.
+
+Usage
+=====
+
+Run the ``dev/seed-deploy.sh`` script to deploy the seed VM::
+
+    ./dev/seed-deploy.sh
+
+Upon successful completion of this script, the seed VM will be active.  The
+seed VM may be accessed via SSH as the ``stack`` user::
+
+    ssh stack@192.168.33.5

doc/source/development/contributing.rst

@@ -1,4 +1,4 @@
 =================
 How to Contribute
 =================
-.. include:: ../../CONTRIBUTING.rst
+.. include:: ../../../CONTRIBUTING.rst

doc/source/development/index.rst

@@ -0,0 +1,11 @@
+========================
+Kayobe Development Guide
+========================
+
+.. toctree::
+   :maxdepth: 2
+
+   vagrant
+   manual
+   automated
+   contributing

doc/source/development/manual.rst

@@ -1,42 +1,26 @@
-===========
+.. _development-manual:
-Development
-===========
 
-This section describes how to set up an OpenStack controller in a virtual
+============
-machine using `Vagrant <https://www.vagrantup.com/>`_ and Kayobe.
+Manual Setup
+============
+
+This section provides a set of manual steps to set up a development environment
+for an OpenStack controller in a virtual machine using `Vagrant
+<https://www.vagrantup.com/>`_ and Kayobe.
+
+For a more automated and flexible procedure, see :ref:`development-automated`.
 
 Preparation
 ===========
 
-First, ensure that Vagrant is installed and correctly configured to use
+Follow the steps in :ref:`development-vagrant` to prepare your environment for
-virtual box. Also install the following vagrant plugins:
+use with Vagrant and bring up a Vagrant VM.
 
-   vagrant plugin install vagrant-vbguest
+Manual Installation
-   vagrant plugin install vagrant-reload
+===================
 
-Note: if using Ubuntu 16.04 LTS, you may be unable to install any plugins. To
+Sometimes the best way to learn a tool is to ditch the scripts and perform a
-work around this install the upstream version from www.virtualbox.org.
+manual installation.
-
-Next, clone kayobe::
-
-    git clone https://github.com/stackhpc/kayobe
-
-Change the current directory to the kayobe repository::
-
-    cd kayobe
-
-Inspect kayobe's ``Vagrantfile``, noting the provisioning steps::
-
-    less Vagrantfile
-
-Bring up a virtual machine::
-
-    vagrant up
-
-Wait for the VM to boot.
-
-Installation
-============
 
 SSH into the controller VM::
 

doc/source/development/vagrant.rst

@@ -0,0 +1,58 @@
+.. _development-vagrant:
+
+=======
+Vagrant
+=======
+
+Kayobe provides a Vagrantfile that can be used to bring up a virtual machine
+for use as a development environment. The VM is based on the `stackhpc/centos-7
+<https://app.vagrantup.com/stackhpc/boxes/centos-7>`_ CentOS 7 image, and
+supports the following providers:
+
+* VirtualBox
+* VMWare Fusion
+
+The VM is configured with 4GB RAM. It has a single private network in addition
+to the standard Vagrant NAT network.
+
+Preparation
+===========
+
+First, ensure that Vagrant is installed and correctly configured to use
+the required provider. Also install the following vagrant plugin::
+
+    vagrant plugin install vagrant-reload
+
+If using the VirtualBox provider, install the following vagrant plugin::
+
+    vagrant plugin install vagrant-vbguest
+
+Note: if using Ubuntu 16.04 LTS, you may be unable to install any plugins. To
+work around this install the upstream version from www.virtualbox.org.
+
+Usage
+=====
+
+Later sections in the development guide cover in more detail how to use the
+development VM in different configurations.  These steps cover bringing up and
+accessing the VM.
+
+Clone the kayobe repository::
+
+    git clone https://github.com/stackhpc/kayobe
+
+Change the current directory to the kayobe repository::
+
+    cd kayobe
+
+Inspect kayobe's ``Vagrantfile``, noting the provisioning steps::
+
+    less Vagrantfile
+
+Bring up a virtual machine::
+
+    vagrant up
+
+Wait for the VM to boot, then SSH in::
+
+    vagrant ssh

doc/source/index.rst

@@ -42,8 +42,7 @@ Developer Documentation
 .. toctree::
    :maxdepth: 2
 
-   contributing
+   development/index
-   development
 
 Release Notes
 -------------

test-requirements.txt

@@ -4,6 +4,7 @@
 
 hacking>=0.12.0,<0.13 # Apache-2.0
 
+bashate>=0.2 # Apache-2.0
 coverage>=4.0 # Apache-2.0
 doc8 # Apache-2.0
 sphinx>=1.5.1 # BSD

tools/run-bashate.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Ignore E006 -- line length greater than 80 char
+
+ROOT=$(readlink -fn $(dirname $0)/.. )
+find $ROOT -not -wholename \*.tox/\* -and -not -wholename \*.test/\* \
+    -and -name \*.sh -print0 | xargs -0 bashate -v --ignore E006

tox.ini

@@ -19,6 +19,7 @@ commands = unit2 discover {posargs}
 
 [testenv:pep8]
 commands =
+  {toxinidir}/tools/run-bashate.sh
   flake8 {posargs} kayobe
   # Check the *.rst files
   # We use a thin wrapper around doc8 currently, which has support for sphinx