#!/bin/bash set -eu set -o pipefail # Library of functions for the kayobe development environment. # Configuration function config_defaults { # Set default values for kayobe development configuration. # Try to detect if we are running in a vagrant VM. if [[ -e /vagrant ]]; then KAYOBE_SOURCE_PATH_DEFAULT=/vagrant else KAYOBE_SOURCE_PATH_DEFAULT="$(pwd)" fi # Path to the kayobe source code repository. Typically this will be the # Vagrant shared directory. export KAYOBE_SOURCE_PATH="${KAYOBE_SOURCE_PATH:-$KAYOBE_SOURCE_PATH_DEFAULT}" # Path to the kayobe-config repository checkout. export KAYOBE_CONFIG_SOURCE_PATH="${KAYOBE_CONFIG_SOURCE_PATH:-${KAYOBE_SOURCE_PATH}/config/src/kayobe-config}" # Path to the kayobe virtual environment. export KAYOBE_VENV_PATH="${KAYOBE_VENV_PATH:-${HOME}/kayobe-venv}" # Path to the Tenks virtual environment. export TENKS_VENV_PATH="${TENKS_VENV_PATH:-${HOME}/tenks-test-venv}" # Whether to provision a VM for the seed host. export KAYOBE_SEED_VM_PROVISION=${KAYOBE_SEED_VM_PROVISION:-1} # Whether to build container images for the seed services. If 0, they will # be pulled. export KAYOBE_SEED_CONTAINER_IMAGE_BUILD=${KAYOBE_SEED_CONTAINER_IMAGE_BUILD:-0} # Whether to build container images for the overcloud services. If 0, they # will be pulled. export KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD=${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD:-0} # Additional arguments to pass to kayobe commands. export KAYOBE_EXTRA_ARGS=${KAYOBE_EXTRA_ARGS:-} } function config_set { # Source the configuration file, config.sh PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${PARENT}/config.sh" } function config_check { # Check the configuration environment variables. if [[ ! -e "$KAYOBE_CONFIG_SOURCE_PATH" ]]; then if [[ ${KAYOBE_CONFIG_REQUIRED:-1} -eq 1 ]]; then echo "Kayobe configuration path $KAYOBE_CONFIG_SOURCE_PATH does not exist" return 1 fi fi if [[ ! -e "$KAYOBE_SOURCE_PATH" ]]; then echo "Kayobe source path $KAYOBE_SOURCE_PATH does not exist" return 1 fi } function config_init { config_defaults config_set config_check } # Installation function install_dependencies { echo "Installing package dependencies for kayobe" if [[ -e /etc/centos-release ]]; then sudo yum -y install gcc git vim python-virtualenv else sudo apt install -y python-dev python-virtualenv gcc git fi } function install_venv { # Install a virtualenv at $1. Install all the packages in proceeding # arguments using pip. venv_path="$1" shift pip_paths="$@" local venv_parent="$(dirname ${venv_path})" if [[ ! -d "$venv_parent" ]]; then mkdir -p "$venv_parent" fi if [[ ! -f "${venv_path}/bin/activate" ]]; then echo "Creating virtual environment in ${venv_path}" virtualenv "${venv_path}" # NOTE: Virtualenv's activate and deactivate scripts reference an # unbound variable. set +u source "${venv_path}/bin/activate" pip install -U pip pip install $pip_paths deactivate set -u else echo "Using existing virtual environment in ${venv_path}" fi } function install_kayobe_venv { # Install the Kayobe venv. install_venv "${KAYOBE_VENV_PATH}" "${KAYOBE_SOURCE_PATH}" } function upgrade_kayobe_venv { echo "Upgrading kayobe virtual environment in ${KAYOBE_VENV_PATH}" virtualenv "${KAYOBE_VENV_PATH}" # NOTE: Virtualenv's activate and deactivate scripts reference an # unbound variable. set +u source "${KAYOBE_VENV_PATH}/bin/activate" pip install -U pip pip install -U "${KAYOBE_SOURCE_PATH}" deactivate set -u } # Deployment function is_deploy_image_built_locally { ipa_build_images=$(kayobe configuration dump --host controllers[0] --var-name ipa_build_images) [[ $ipa_build_images =~ ^true$ ]] } function environment_setup { # NOTE: Virtualenv's activate script references an unbound variable. set +u source "${KAYOBE_VENV_PATH}/bin/activate" set -u source "${KAYOBE_CONFIG_SOURCE_PATH}/kayobe-env" cd "${KAYOBE_SOURCE_PATH}" } function run_kayobe { # Run a kayobe command, including extra arguments provided via # $KAYOBE_EXTRA_ARGS. kayobe ${KAYOBE_EXTRA_ARGS} $* } function control_host_bootstrap { echo "Bootstrapping the Ansible control host" for i in $(seq 1 3); do if run_kayobe control host bootstrap; then chb_success=1 break fi echo "Control host bootstrap failed - likely Ansible Galaxy flakiness. Retrying" done if [[ -z ${chb_success+x} ]]; then die $LINENO "Failed to bootstrap control host" exit 1 fi echo "Bootstrapped control host after $i attempts" } function control_host_upgrade { echo "Upgrading the Ansible control host" for i in $(seq 1 3); do if run_kayobe control host upgrade; then chu_success=1 break fi echo "Control host upgrade failed - likely Ansible Galaxy flakiness. Retrying" done if [[ -z ${chu_success+x} ]]; then die $LINENO "Failed to upgrade control host" exit 1 fi echo "Upgraded control host after $i attempts" } function seed_hypervisor_deploy { # Deploy a seed hypervisor. environment_setup echo "Bootstrapping the Ansible control host" run_kayobe control host bootstrap echo "Configuring the seed hypervisor" run_kayobe seed hypervisor host configure } function seed_deploy { # Deploy a kayobe seed in a VM. environment_setup control_host_bootstrap if [[ ${KAYOBE_SEED_VM_PROVISION} = 1 ]]; then echo "Provisioning the seed VM" run_kayobe seed vm provision fi echo "Configuring the seed host" run_kayobe seed host configure # Note: This must currently be done before host configure, because host # configure runs kolla-ansible.yml, which validates the presence of the # built deploy images. if is_deploy_image_built_locally; then echo "Building seed deployment images" run_kayobe seed deployment image build else echo "Not building seed deployment images" fi if [[ ${KAYOBE_SEED_CONTAINER_IMAGE_BUILD} = 1 ]]; then echo "Building seed container images" run_kayobe seed container image build else echo "Not pulling seed container images - no such command yet" #run_kayobe seed container image pull fi echo "Deploying containerised seed services" run_kayobe seed service deploy } function seed_upgrade { # Upgrade a kayobe seed in a VM. echo "Upgrading Kayobe" upgrade_kayobe_venv environment_setup control_host_upgrade echo "Upgrading the seed host" run_kayobe seed host upgrade if is_deploy_image_built_locally; then echo "Building seed deployment images" run_kayobe seed deployment image build --force-rebuild else echo "Not building seed deployment images" fi if [[ ${KAYOBE_SEED_CONTAINER_IMAGE_BUILD} = 1 ]]; then echo "Building seed container images" run_kayobe seed container image build else echo "Not pulling seed container images - no such command yet" #run_kayobe seed container image pull fi echo "Upgrading containerised seed services" run_kayobe seed service upgrade } function overcloud_deploy { # Deploy a kayobe control plane. echo "Deploying a kayobe development environment. This consists of a " echo "single node OpenStack control plane." environment_setup control_host_bootstrap echo "Configuring the controller host" run_kayobe overcloud host configure # FIXME(mgoddard): Perform host upgrade workarounds to ensure hostname # resolves to IP address of API interface for RabbitMQ. This seems to be # required since https://review.openstack.org/#/c/584427 was merged. echo "Workaround: upgrading the controller host" run_kayobe overcloud host upgrade # Note: This must currently be before host configure, because host # configure runs kolla-ansible.yml, which validates the presence of the # built deploy images. if is_deploy_image_built_locally; then echo "Building overcloud deployment images" run_kayobe overcloud deployment image build else echo "Not building overcloud deployment images" fi if [[ ${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD} = 1 ]]; then echo "Building overcloud container images" run_kayobe overcloud container image build else echo "Pulling overcloud container images" run_kayobe overcloud container image pull fi echo "Deploying containerised overcloud services" run_kayobe overcloud service deploy echo "Performing post-deployment configuration" source "${KOLLA_CONFIG_PATH:-/etc/kolla}/admin-openrc.sh" run_kayobe overcloud post configure echo "Control plane deployment complete" } function overcloud_upgrade { # Upgrade a kayobe control plane. echo "Upgrading a kayobe development environment. This consists of a " echo "single node OpenStack control plane." echo "Upgrading Kayobe" upgrade_kayobe_venv environment_setup control_host_upgrade echo "Upgrading the controller host" run_kayobe overcloud host upgrade if is_deploy_image_built_locally; then echo "Building overcloud deployment images" run_kayobe overcloud deployment image build --force-rebuild else echo "Not building overcloud deployment images" fi if [[ ${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD} = 1 ]]; then echo "Building overcloud container images" run_kayobe overcloud container image build else echo "Pulling overcloud container images" run_kayobe overcloud container image pull fi echo "Saving overcloud service configuration" if ! run_kayobe overcloud service configuration save; then # NOTE(mgoddard): This fails in CI due to a memory error while copying # the IPA deployment images. echo "FIXME: Saving service configuration failed. Ignoring for now" fi echo "Deploying containerised overcloud services" run_kayobe overcloud service upgrade echo "Control plane upgrade complete" } function overcloud_test_init { echo "Performing overcloud test init" environment_setup pip install python-openstackclient source "${KOLLA_CONFIG_PATH:-/etc/kolla}/admin-openrc.sh" # This guards init-runonce from running more than once if mkdir /tmp/init-runonce > /dev/null 2>&1; then echo "Running kolla-ansible init-runonce" ${KOLLA_VENV_PATH:-$HOME/kolla-venv}/share/kolla-ansible/init-runonce else echo "Not running kolla-ansible init-runonce - resources exist" fi } function overcloud_test { set -eu # function arguments name="$1" flavor="$2" network="$3" node_config="{ 'name': '$name', 'flavor': '$flavor', 'network': '$network' }" overcloud_test_init # Perform a simple smoke test against the cloud. echo "Performing a simple smoke test with node config: $node_config" echo "$name: Creating a server" openstack server create --wait --image cirros --flavor "$flavor" --key-name mykey --network "$network" "$name" echo "$name: Server created" openstack server show "$name" status=$(openstack server show "$name" -f value -c status) if [[ $status != ACTIVE ]]; then echo "$name: Node creation failed" return 1 fi # TODO(mgoddard): Test SSH connectivity to the VM. echo "$name: Deleting the Node" openstack server delete --wait "$name" } function tenks_deploy { set -eu # Create a simple test Tenks deployment. Assumes that a bridge named # 'breth1' exists. Arguments: # $1: The path to the Tenks repo. local tenks_path="$1" echo "Configuring Tenks" environment_setup # We don't want to use the Kayobe venv. deactivate # Install the Tenks venv. install_venv "${TENKS_VENV_PATH}" "$tenks_path" source ${TENKS_VENV_PATH:-$HOME/tenks-test-venv}/bin/activate ansible-galaxy install \ --role-file="$tenks_path/requirements.yml" \ --roles-path="$tenks_path/ansible/roles/" local parent="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Install a trivial script for ovs-vsctl that talks to containerised Open # vSwitch. sudo cp --no-clobber "$parent/ovs-vsctl" /usr/bin/ovs-vsctl source "${KOLLA_CONFIG_PATH:-/etc/kolla}/admin-openrc.sh" pip install python-openstackclient ansible-playbook \ -vvv \ --inventory "$tenks_path/ansible/inventory" \ --extra-vars=@"$parent/tenks-deploy-config.yml" \ "$tenks_path/ansible/deploy.yml" } # General purpose # Prints backtrace info # filename:lineno:function # backtrace level function backtrace { local level=$1 local deep deep=$((${#BASH_SOURCE[@]} - 1)) echo "[Call Trace]" while [ $level -le $deep ]; do echo "${BASH_SOURCE[$deep]}:${BASH_LINENO[$deep-1]}:${FUNCNAME[$deep-1]}" deep=$((deep - 1)) done } # Prints line number and "message" then exits # die $LINENO "message" function die { local exitcode=$? set +o xtrace local line=$1; shift if [ $exitcode == 0 ]; then exitcode=1 fi backtrace 2 err $line "$*" # Give buffers a second to flush sleep 1 exit $exitcode } # Prints line number and "message" in error format # err $LINENO "message" function err { local exitcode=$? local xtrace xtrace=$(set +o | grep xtrace) set +o xtrace local msg="[ERROR] ${BASH_SOURCE[2]}:$1 $2" echo "$msg" 1>&2; if [[ -n ${LOGDIR} ]]; then echo "$msg" >> "${LOGDIR}/error.log" fi $xtrace return $exitcode } function die_if_module_not_loaded { if ! grep -q $1 /proc/modules; then die $LINENO "$1 kernel module is not loaded" fi } # running_in_container - Returns true otherwise false function running_in_container { [[ $(systemd-detect-virt --container) != 'none' ]] } # enable_kernel_bridge_firewall - Enable kernel support for bridge firewalling function enable_kernel_bridge_firewall { # Load bridge module. This module provides access to firewall for bridged # frames; and also on older kernels (pre-3.18) it provides sysctl knobs to # enable/disable bridge firewalling sudo modprobe bridge # For newer kernels (3.18+), those sysctl settings are split into a separate # kernel module (br_netfilter). Load it too, if present. sudo modprobe br_netfilter 2>> /dev/null || : # Enable bridge firewalling in case it's disabled in kernel (upstream # default is enabled, but some distributions may decide to change it). # This is at least needed for RHEL 7.2 and earlier releases. for proto in ip ip6; do sudo sysctl -w net.bridge.bridge-nf-call-${proto}tables=1 done } function to_bool { if [[ "$1" =~ (y|Y|yes|Yes|YES|true|True|TRUE|on|On|ON) ]]; then true elif [[ "$1" =~ (n|N|no|No|NO|false|False|FALSE|off|Off|OFF) ]]; then false else die $LINENO "$1 was not a valid yaml boolean" fi } function is_ipxe_enabled { flag="$(run_kayobe configuration dump --host controllers[0] --var-name kolla_enable_ironic_ipxe)" to_bool "$flag" } function is_cinder_enabled { flag="$(run_kayobe configuration dump --host controllers[0] --var-name kolla_enable_cinder)" to_bool "$flag" } function configure_iptables { # NOTE(wszumski): adapted from the ironic devstack plugin, see: # https://github.com/openstack/ironic/blob/36e87dc5b472d79470b783fbba9ce396e3cbb96e/devstack/lib/ironic#L2132 set -eu environment_setup # FIXME(wszumski): set these variables with values from kayobe-config HOST_IP='192.168.33.3' INTERNAL_VIP='192.168.33.2' IRONIC_TFTPSERVER_IP="$HOST_IP" IRONIC_SERVICE_PORT=6385 IRONIC_INSPECTOR_PORT=5050 IRONIC_HTTP_SERVER="$INTERNAL_VIP" GLANCE_SERVICE_PORT=9292 IRONIC_HTTP_PORT=8089 ISCSI_SERVICE_PORT=3260 # enable tftp natting for allowing connections to HOST_IP's tftp server if ! running_in_container; then sudo modprobe nf_conntrack_tftp sudo modprobe nf_nat_tftp enable_kernel_bridge_firewall else die_if_module_not_loaded nf_conntrack_tftp die_if_module_not_loaded nf_nat_tftp fi # explicitly allow DHCP - packets are occasionally being dropped here sudo iptables -I INPUT -p udp --dport 67:68 --sport 67:68 -j ACCEPT || true # nodes boot from TFTP and callback to the API server listening on $HOST_IP sudo iptables -I INPUT -d $IRONIC_TFTPSERVER_IP -p udp --dport 69 -j ACCEPT || true sudo iptables -I INPUT -d $HOST_IP -p tcp --dport $IRONIC_SERVICE_PORT -j ACCEPT || true # open ironic API on baremetal network sudo iptables -I INPUT -d $IRONIC_HTTP_SERVER -p tcp --dport $IRONIC_SERVICE_PORT -j ACCEPT || true # allow IPA to connect to ironic API sudo iptables -I FORWARD -p tcp --dport $IRONIC_SERVICE_PORT -j ACCEPT || true # allow IPA to connect to ironic inspector sudo iptables -I FORWARD -p tcp --dport $IRONIC_INSPECTOR_PORT -j ACCEPT || true # agent ramdisk gets instance image from swift sudo iptables -I INPUT -d $INTERNAL_VIP -p tcp --dport ${SWIFT_DEFAULT_BIND_PORT:-8080} -j ACCEPT || true sudo iptables -I INPUT -d $INTERNAL_VIP -p tcp --dport $GLANCE_SERVICE_PORT -j ACCEPT || true if is_ipxe_enabled; then sudo iptables -I INPUT -d $HOST_IP -p tcp --dport $IRONIC_HTTP_PORT -j ACCEPT || true fi if is_cinder_enabled; then sudo iptables -I INPUT -d $HOST_IP -p tcp --dport $ISCSI_SERVICE_PORT -j ACCEPT || true fi }