From ee93901d054b9b4514aa8a2e367bfc88120594d7 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Evrard Date: Fri, 12 May 2017 13:52:02 +0000 Subject: [PATCH] Make possible to start leap from K This change: - discovers the current running version to know what to leap from, because we can't assume Juno only. At the same time it introduces a human verification of the source branch. - removes the useless "-v" of the runs that produced an undesirable output, making the interface less "user friendly". Change-Id: I04e4780bf5f58638addbd992eab7152f288532ae Co-Authored-By: Jean-Philippe Evrard --- leap-upgrades/README.md | 20 ++- leap-upgrades/lib/functions.sh | 137 ++++++++++++++++-- leap-upgrades/lib/vars.sh | 10 +- leap-upgrades/migrations.sh | 2 +- leap-upgrades/prep.sh | 11 +- leap-upgrades/re-deploy.sh | 20 ++- leap-upgrades/run-stages.sh | 2 +- .../neutron-remove-old-containers.yml | 1 + leap-upgrades/upgrade-utilities/pip-unify.yml | 5 + leap-upgrades/upgrade.sh | 16 +- 10 files changed, 179 insertions(+), 45 deletions(-) diff --git a/leap-upgrades/README.md b/leap-upgrades/README.md index 8a2b0e89..e7775da2 100644 --- a/leap-upgrades/README.md +++ b/leap-upgrades/README.md @@ -1,13 +1,13 @@ # OpenStack-Ansible leap upgrade -## Jump upgrade from OpenStack Juno to Newton using OpenStack-Ansible +## Jump upgrade using OpenStack-Ansible for Ubuntu 14.04 ==**This currently a POC**== ### Uses This utility can be used to upgrade any OpenStack-Ansible deployment running -Juno to the latest Newton release. The process will upgrade the OSA system +Juno / Kilo to the Newton release 14.2.3. The process will upgrade the system components, sync the database through the various releases, and then deploy OSA using the Newton release. While this method will help a deployment skip several releases deployers should be aware that skipping releases is not @@ -19,7 +19,7 @@ the shortest possible time while maintaining data-integrity. #### Requirements - * **You must** have a Juno based OpenStack cloud as deployed by + * **You must** have a Juno/Kilo based OpenStack cloud as deployed by OpenStack-Ansible. * If you are running cinder-volume with LVM in an LXC container **you must** migrate the cinder-volume service to the physical host. @@ -61,7 +61,7 @@ Newton. ---- -### Setting up a Test environment. +### Example leap with a multi-node juno environment. Testing on a multi-node environment can be accomplished using the https://github.com/openstack/openstack-ansible-ops/tree/master/multi-node-aio @@ -76,7 +76,7 @@ very well for development. To run the deployment execute the following commands * Start the deployment w/ ubuntu 14.04.2 to ensure the deployment version is limited in terms of package availability. -#### Process +#### Setup a multi-node AIO Clone the ops tooling and change directory to the multi-node-aio tooling @@ -94,6 +94,8 @@ setup-virsh-net.sh deploy-vms.sh ``` +#### Deploy an example Juno config + After the environment has been deployed clone the RPC configurations which support Juno based clouds. @@ -101,6 +103,8 @@ based clouds. git clone https://github.com/os-cloud/leapfrog-juno-config /etc/rpc_deploy ``` +#### Deploy Juno + Now clone the Juno playbooks into place. ``` bash @@ -118,6 +122,8 @@ openstack-ansible playbooks/haproxy-install.yml openstack-ansible playbooks/setup-everything.yml ``` +#### Test your Juno cloud + To test the cloud's functionality you can execute the OpenStack resource test script located in the scripts directory of the playbooks cloned earlier. @@ -143,3 +149,7 @@ Once the cloud is operational it's recommended that images be created so that th reverted to a previous state should there ever be a need. See https://github.com/openstack/openstack-ansible-ops/tree/master/multi-node-aio#snapshotting-an-environment-before-major-testing for more on creating snapshots. + +#### Run the leapfrog + +See the "Process" part on the top of the page diff --git a/leap-upgrades/lib/functions.sh b/leap-upgrades/lib/functions.sh index 67336fcd..af71fa81 100644 --- a/leap-upgrades/lib/functions.sh +++ b/leap-upgrades/lib/functions.sh @@ -27,24 +27,38 @@ function failure { echo -e '[!]'"\t\033[1;31m${1}\033[0m" } +function debug { + if [[ $DEBUG == "TRUE" ]]; then + echo -e "${1}" >> $DEBUG_PATH + fi +} + function tag_leap_success { notice "LEAP ${1} success" touch "/opt/leap42/openstack-ansible-${1}.leap" + debug "LEAP ${1} marked as success" } function run_lock { + set +e run_item="${RUN_TASKS[$1]}" - file_part="$(echo ${run_item} | sed 's/\s/-/g')" + file_part="$(echo ${run_item} | cut -f 1 -d ' ' | xargs basename)" + other_args="$(echo ${run_item} | cut -f 2- -d ' ' -s | sed 's/[^[:alnum:]_]/-/g')" + debug "Run_lock on $run_item" if [ ! -d "/etc/openstack_deploy/upgrade-leap" ]; then mkdir -p "/etc/openstack_deploy/upgrade-leap" fi - upgrade_marker_file=$(basename "${file_part}") + upgrade_marker_file=${file_part}${other_args} upgrade_marker="/etc/openstack_deploy/upgrade-leap/$upgrade_marker_file.complete" + debug "Upgrade marker is $upgrade_marker" if [ ! -f "$upgrade_marker" ];then + debug "Upgrade marker file not found for this run item." + debug "Will run openstack-ansible $2" + # note(sigmavirus24): use eval so that we properly turn strings like # "/tmp/fix_container_interfaces.yml || true" # into a command, otherwise we'll get an error that there's no playbook @@ -54,8 +68,8 @@ function run_lock { notice "Ran: $run_item" if [ "$playbook_status" == "0" ];then - RUN_TASKS=("${RUN_TASKS[@]/$run_item}") touch "${upgrade_marker}" + unset RUN_TASKS[$1] notice "$run_item has been marked as success at ${upgrade_marker}" else FAILURES_LIST=$(seq $1 $((${#RUN_TASKS[@]} - 1))) @@ -77,6 +91,7 @@ function run_lock { exit 99 fi else + debug "Upgrade marker file found for this run item." RUN_TASKS=("${RUN_TASKS[@]/$run_item.*}") fi set -e @@ -113,13 +128,92 @@ function system_bootstrap { popd } +function validate_upgrade_input { + + echo + warning "Please enter the source series to upgrade from." + notice "JUNO, KILO or LIBERTY" + read -p 'Enter "JUNO", "KILO", or "LIBERTY" to continue: ' UPGRADE_FROM + export INPUT_UPGRADE_FROM=${UPGRADE_FROM} + + if [[ ${INPUT_UPGRADE_FROM} == ${CODE_UPGRADE_FROM} ]]; then + notice "Running LEAP Upgrade from ${CODE_UPGRADE_FROM} to NEWTON" + else + notice "Asking to upgrade a ${INPUT_UPGRADE_FROM}, but code is to ${CODE_UPGRADE_FROM}" + read -p 'Are you sure? Enter "YES" to continue:' RUSURE + if [[ "${RUSURE}" != "YES" ]]; then + notice "Quitting..." + exit 99 + fi + # We should let the user decide if he passes through the checks + export CODE_UPGRADE_FROM=${INPUT_UPGRADE_FROM} + fi +} + +function discover_code_version { + if [[ ! -f "/etc/openstack-release" ]]; then + export CODE_UPGRADE_FROM="JUNO" + notice "You seem to be running Juno" + else + source /etc/openstack-release + case "${DISTRIB_RELEASE%%.*}" in + '11') + export CODE_UPGRADE_FROM="KILO" + notice "You seem to be running Kilo" + ;; + '12') + export CODE_UPGRADE_FROM="LIBERTY" + notice "You seem to be running Liberty" + ;; + '13') + export CODE_UPGRADE_FROM="MITAKA" + notice "You seem to be running Mitaka" + ;; + '14') + export CODE_UPGRADE_FROM="NEWTON" + notice "You seem to be running Newton" + ;; + esac + fi +} + +function set_upgrade_vars { + notice "Setting up vars for the LEAP" + case "${CODE_UPGRADE_FROM}" in + JUNO) + export RELEASE="${JUNO_RELEASE}" + export UPGRADES_TO_TODOLIST="KILO LIBERTY MITAKA NEWTON" + export ANSIBLE_INVENTORY="/opt/leap42/openstack-ansible-${RELEASE}/rpc_deployment/inventory" + export CONFIG_DIR="/etc/rpc_deploy" + ;; + KILO) + export RELEASE="${KILO_RELEASE}" + export UPGRADES_TO_TODOLIST="LIBERTY MITAKA NEWTON" + export ANSIBLE_INVENTORY="/opt/leap42/openstack-ansible-${RELEASE}/playbooks/inventory" + export CONFIG_DIR="/etc/openstack_deploy" + ;; + LIBERTY) + export RELEASE="${LIBERTY_RELEASE}" + export UPGRADES_TO_TODOLIST="MITAKA NEWTON" + export ANSIBLE_INVENTORY="/opt/leap42/openstack-ansible-${RELEASE}/playbooks/inventory" + export CONFIG_DIR="/etc/openstack_deploy" + ;; + MITAKA) + export RELEASE="${MITAKA_RELEASE}" + export UPGRADES_TO_TODOLIST="NEWTON" + export ANSIBLE_INVENTORY="/opt/leap42/openstack-ansible-${RELEASE}/playbooks/inventory" + export CONFIG_DIR="/etc/openstack_deploy" + ;; + esac +} + function pre_flight { ## Pre-flight Check ---------------------------------------------------------- # Clear the screen and make sure the user understands whats happening. clear # Notify the user. - warning "This script will perform a LEAP upgrade from Juno to Newton." + warning "This script will perform a LEAP upgrade to Newton." warning "Once you start the upgrade there's no going back." warning "**Note, this is an OFFLINE upgrade**" notice "If you want to run the upgrade in parts please exit this script to do so." @@ -130,25 +224,23 @@ function pre_flight { if [ "${UPGRADE}" == "YES" ]; then notice "Running LEAP Upgrade" else + notice "Exiting, input wasn't YES" exit 99 fi - mkdir -p /opt/leap42/venvs + discover_code_version - pushd /opt/leap42 - # Using this lookup plugin because it allows us to compile exact service releaes and build a complete venv from it - wget https://raw.githubusercontent.com/openstack/openstack-ansible-plugins/e069d558b3d6ae8fc505d406b13a3fb66201a9c7/lookup/py_pkgs.py -O py_pkgs.py - chmod +x py_pkgs.py - popd + if [ "${VALIDATE_UPGRADE_INPUT}" == "TRUE" ]; then + validate_upgrade_input + fi + + set_upgrade_vars + + mkdir -p /opt/leap42/venvs # If the lxc backend store was not set halt and instruct the user to set it. In Juno we did more to detect the backend storage # size than we do in later releases. While the auto-detection should still work it's best to have the deployer set the value # desired before moving forward. - if [[ -d "/etc/rpc_deploy" ]]; then - CONFIG_DIR="/etc/rpc_deploy" - else - CONFIG_DIR="/etc/openstack_deploy" - fi if ! grep -qwrn "^lxc_container_backing_store" $CONFIG_DIR; then failure "ERROR: 'lxc_container_backing_store' is unset leading to an ambiguous container backend store." failure "Before continuing please set the 'lxc_container_backing_store' in your user_variables.yml file." @@ -156,11 +248,25 @@ function pre_flight { exit 99 fi + if ! grep -qwrn "^neutron_legacy_ha_tool_enabled" $CONFIG_DIR; then + failure "ERROR: 'neutron_legacy_ha_tool_enabled' is unset leading to an ambiguous l3ha handling." + failure "Before continuing please set the 'neutron_legacy_ha_tool_enabled' in your user_variables.yml file." + exit 99 + fi + # Install liberasurecode-dev which will be used in the venv creation process if ! grep -n ^ /etc/apt/sources.list /etc/apt/sources.list.d/* | grep -qw "backports"; then failure "The trusty backports repo has not been enabled on this host." exit 99 fi + + # Don't run this over and over again if the variables above are not set! + pushd /opt/leap42 + # Using this lookup plugin because it allows us to compile exact service releaes and build a complete venv from it + wget https://raw.githubusercontent.com/openstack/openstack-ansible-plugins/e069d558b3d6ae8fc505d406b13a3fb66201a9c7/lookup/py_pkgs.py -O py_pkgs.py + chmod +x py_pkgs.py + popd + apt-get update > /dev/null apt-get -y install liberasurecode-dev > /dev/null @@ -203,6 +309,7 @@ function run_items { pushd ${PB_DIR} # Run the tasks in order for item in ${!RUN_TASKS[@]}; do + debug "Run_items of ${item}: ${RUN_TASKS[$item]}. Starting run_lock" run_lock $item "${RUN_TASKS[$item]}" done popd diff --git a/leap-upgrades/lib/vars.sh b/leap-upgrades/lib/vars.sh index 2f7fda0e..68f35da1 100644 --- a/leap-upgrades/lib/vars.sh +++ b/leap-upgrades/lib/vars.sh @@ -14,12 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +## Generic debug var. This will log actions to /tmp/run_debug +export DEBUG="${DEBUG:-TRUE}" +export DEBUG_PATH="${DEBUG_PATH:-/tmp/run_debug}" + ## Script Vars --------------------------------------------------------------- export JUNO_RELEASE="${JUNO_RELEASE:-10.1.14}" export KILO_RELEASE="${KILO_RELEASE:-11.2.17}" export LIBERTY_RELEASE="${LIBERTY_RELEASE:-12.2.8}" export MITAKA_RELEASE="${MITAKA_RELEASE:-13.3.11}" -export NEWTON_RELEASE="${NEWTON_RELEASE:-d47e29b7d8a385773acadb825e37c82d42b3ec27}" # commit used due to packaging bug caused by setuptools +export NEWTON_RELEASE="${NEWTON_RELEASE:-14.2.3}" # commit used due to packaging bug caused by setuptools ## Environment Vars ------------------------------------------------------------------ export MAIN_PATH="${MAIN_PATH:-/opt/openstack-ansible}" @@ -31,3 +35,7 @@ export UPGRADE_UTILS="${UPGRADE_UTILS:-${SYSTEM_PATH}/upgrade-utilities}" # this URL will be used to download the release built VENVS in the following format. # ${VENV_URL}/openstack-ansible-RELEASE_VERSION.tgz export VENV_URL="${VENV_URL:-https://mirror.rackspace.com/rackspaceprivatecloud/venvs/leap-bundles}" + +export VALIDATE_UPGRADE_INPUT="${VALIDATE_UPGRADE_INPUT:-TRUE}" +export UPGRADES_TO_TODOLIST="${UPGRADES_TO_TODOLIST:-''}" +export CODE_UPGRADE_FROM="" diff --git a/leap-upgrades/migrations.sh b/leap-upgrades/migrations.sh index a729b8dd..b2c62cd8 100755 --- a/leap-upgrades/migrations.sh +++ b/leap-upgrades/migrations.sh @@ -18,7 +18,7 @@ # root@HOSTNAME:/opt/openstack-ansible# echo "YES" | bash scripts/run-upgrade.sh ## Shell Opts ---------------------------------------------------------------- -set -e -u -v +set -e -u ## Main ---------------------------------------------------------------------- source lib/vars.sh diff --git a/leap-upgrades/prep.sh b/leap-upgrades/prep.sh index 3afddd7d..939c7e30 100755 --- a/leap-upgrades/prep.sh +++ b/leap-upgrades/prep.sh @@ -18,7 +18,7 @@ # root@HOSTNAME:/opt/openstack-ansible# echo "YES" | bash scripts/run-upgrade.sh ## Shell Opts ---------------------------------------------------------------- -set -e -u -v +set -e -u ## Main ---------------------------------------------------------------------- source lib/vars.sh @@ -60,17 +60,10 @@ if [[ ! -f "/opt/leap42/openstack-ansible-${NEWTON_RELEASE}-prep.leap" ]]; then fi RUN_TASKS=() + RUN_TASKS+=("${UPGRADE_UTILS}/cinder-volume-container-lvm-check.yml") RUN_TASKS+=("${UPGRADE_UTILS}/db-backup.yml") -if [[ -d "/etc/rpc_deploy" ]]; then - RELEASE="${JUNO_RELEASE}" - export ANSIBLE_INVENTORY="/opt/leap42/openstack-ansible-${RELEASE}/rpc_deployment/inventory" -else - RELEASE="${NEWTON_RELEASE}" - export ANSIBLE_INVENTORY="/opt/leap42/openstack-ansible-${RELEASE}/playbooks/inventory" -fi - # temp upgrade ansible is used to ensure 1.9.x compat. PS1="\\u@\h \\W]\\$" . "/opt/ansible-runtime/bin/activate" run_items "/opt/leap42/openstack-ansible-${RELEASE}" diff --git a/leap-upgrades/re-deploy.sh b/leap-upgrades/re-deploy.sh index 00be214b..9593b5f8 100755 --- a/leap-upgrades/re-deploy.sh +++ b/leap-upgrades/re-deploy.sh @@ -18,33 +18,43 @@ # root@HOSTNAME:/opt/openstack-ansible# echo "YES" | bash scripts/run-upgrade.sh ## Shell Opts ---------------------------------------------------------------- -set -e -u -v +set -e -u ## Main ---------------------------------------------------------------------- source lib/vars.sh source lib/functions.sh ### Run the redeploy tasks -# Forget about the old Juno neutron agent container in inventory. +# Forget about the old neutron agent container in inventory. # This is done to maximize uptime by leaving the old systems in # place while the redeployment work is going on. -SCRIPTS_PATH="/opt/leap42/openstack-ansible-${NEWTON_RELEASE}/scripts" \ - MAIN_PATH="/opt/leap42/openstack-ansible-${NEWTON_RELEASE}" \ - ${UPGRADE_UTILS}/neutron-container-forget.sh +# TODO(evrardjp): Move this to a playbook, this way it will follow the +# RUN_TASKS model + +if [ ! -f /etc/openstack_deploy/upgrade-leap/neutron-container-forget.complete ];then + SCRIPTS_PATH="/opt/leap42/openstack-ansible-${NEWTON_RELEASE}/scripts" \ + MAIN_PATH="/opt/leap42/openstack-ansible-${NEWTON_RELEASE}" \ + ${UPGRADE_UTILS}/neutron-container-forget.sh + touch /etc/openstack_deploy/upgrade-leap/neutron-container-forget.complete +fi link_release "/opt/leap42/openstack-ansible-${NEWTON_RELEASE}" RUN_TASKS=() +# Ensure the same pip everywhere, even if requirement met or above RUN_TASKS+=("${UPGRADE_UTILS}/pip-unify.yml -e release_version=\"${NEWTON_RELEASE}\"") RUN_TASKS+=("${UPGRADE_UTILS}/db-stop.yml") RUN_TASKS+=("${UPGRADE_UTILS}/ansible_fact_cleanup.yml") +# Physical host cleanup RUN_TASKS+=("${UPGRADE_UTILS}/destroy-old-containers.yml") +# Permissions for qemu save, because physical host cleanup RUN_TASKS+=("${UPGRADE_UTILS}/nova-libvirt-fix.yml") RUN_TASKS+=("lxc-hosts-setup.yml") RUN_TASKS+=("lxc-containers-create.yml") RUN_TASKS+=("setup-infrastructure.yml") +# MariaDB sync for major maria upgrades and cluster schema sync RUN_TASKS+=("${UPGRADE_UTILS}/db-force-upgrade.yml") RUN_TASKS+=("os-keystone-install.yml") diff --git a/leap-upgrades/run-stages.sh b/leap-upgrades/run-stages.sh index e1bfebde..c3679113 100755 --- a/leap-upgrades/run-stages.sh +++ b/leap-upgrades/run-stages.sh @@ -18,7 +18,7 @@ # root@HOSTNAME:/opt/openstack-ansible# echo "YES" | bash scripts/run-upgrade.sh ## Shell Opts ---------------------------------------------------------------- -set -e -u -v +set -e -u ## Main ---------------------------------------------------------------------- source lib/vars.sh diff --git a/leap-upgrades/upgrade-utilities/neutron-remove-old-containers.yml b/leap-upgrades/upgrade-utilities/neutron-remove-old-containers.yml index 9c2cf47a..3033ba7f 100644 --- a/leap-upgrades/upgrade-utilities/neutron-remove-old-containers.yml +++ b/leap-upgrades/upgrade-utilities/neutron-remove-old-containers.yml @@ -25,6 +25,7 @@ args: executable: /bin/bash +# TODO: Improve l2 convergence too. - name: Remove the reminent neutron agent containers hosts: "neutron_agent[0]" gather_facts: false diff --git a/leap-upgrades/upgrade-utilities/pip-unify.yml b/leap-upgrades/upgrade-utilities/pip-unify.yml index 427d6a72..716d0dd0 100644 --- a/leap-upgrades/upgrade-utilities/pip-unify.yml +++ b/leap-upgrades/upgrade-utilities/pip-unify.yml @@ -76,6 +76,11 @@ args: executable: /bin/bash + - name: apt-get install libssl-dev + apt: + name: libssl-dev + update_cache: yes + - name: Re-install pip packaging requirements raw: | /usr/bin/python /opt/get-pip.py \ diff --git a/leap-upgrades/upgrade.sh b/leap-upgrades/upgrade.sh index d9c9a77f..6e34d221 100755 --- a/leap-upgrades/upgrade.sh +++ b/leap-upgrades/upgrade.sh @@ -18,7 +18,7 @@ # root@HOSTNAME:/opt/openstack-ansible# echo "YES" | bash scripts/run-upgrade.sh ## Shell Opts ---------------------------------------------------------------- -set -e -u -v +set -e -u ## Main ---------------------------------------------------------------------- source lib/vars.sh @@ -28,8 +28,8 @@ source lib/functions.sh # Run tasks UPGRADE_SCRIPTS="${UPGRADE_UTILS}-kilo/scripts" # If the kilo leap has been accomplished, skip. -if [[ ! -f "/opt/leap42/openstack-ansible-${KILO_RELEASE}.leap" ]]; then - notice 'Running kilo leap' +if [[ ! -f "/opt/leap42/openstack-ansible-${KILO_RELEASE}.leap" ]] && [[ "${UPGRADES_TO_TODOLIST}" =~ .*KILO.* ]]; then + notice 'Leaping to Kilo' link_release "/opt/leap42/openstack-ansible-${KILO_RELEASE}" pushd "/opt/leap42/openstack-ansible-${KILO_RELEASE}" if [[ -d "/etc/rpc_deploy" ]]; then @@ -74,8 +74,8 @@ fi ### Liberty System migration # Run tasks -if [[ ! -f "/opt/leap42/openstack-ansible-${LIBERTY_RELEASE}.leap" ]]; then - notice 'Running liberty leap' +if [[ ! -f "/opt/leap42/openstack-ansible-${LIBERTY_RELEASE}.leap" ]] && [[ "${UPGRADES_TO_TODOLIST}" =~ .*LIBERTY.* ]]; then + notice 'Leaping to liberty' link_release "/opt/leap42/openstack-ansible-${LIBERTY_RELEASE}" UPGRADE_PLAYBOOKS="${UPGRADE_UTILS}-liberty/playbooks" RUN_TASKS=() @@ -91,8 +91,8 @@ fi ### Mitaka System migration # Run tasks -if [[ ! -f "/opt/leap42/openstack-ansible-${MITAKA_RELEASE}.leap" ]]; then - notice 'Running mitaka leap' +if [[ ! -f "/opt/leap42/openstack-ansible-${MITAKA_RELEASE}.leap" ]] && [[ "${UPGRADES_TO_TODOLIST}" =~ .*MITAKA.* ]]; then + notice 'Leaping to Mitaka' link_release "/opt/leap42/openstack-ansible-${MITAKA_RELEASE}" UPGRADE_PLAYBOOKS="${UPGRADE_UTILS}-mitaka/playbooks" RUN_TASKS=() @@ -109,7 +109,7 @@ fi ### Newton Deploy # Run tasks -if [[ ! -f "/opt/leap42/openstack-ansible-${NEWTON_RELEASE}.leap" ]]; then +if [[ ! -f "/opt/leap42/openstack-ansible-${NEWTON_RELEASE}.leap" ]] && [[ "${UPGRADES_TO_TODOLIST}" =~ .*NEWTON* ]]; then notice 'Running newton leap' link_release "/opt/leap42/openstack-ansible-${NEWTON_RELEASE}" UPGRADE_PLAYBOOKS="${UPGRADE_UTILS}-newton/playbooks"