Make possible to start leap from K

This change:
- discovers the current running version to know what
  to leap from, because we can't assume Juno only.
  At the same time it introduces a human verification
  of the source branch.
- removes the useless "-v" of the runs that produced
  an undesirable output, making the interface less
  "user friendly".

Change-Id: I04e4780bf5f58638addbd992eab7152f288532ae
Co-Authored-By: Jean-Philippe Evrard <jean-philippe@evrard.me>

leap-upgrades/README.md

@@ -1,13 +1,13 @@
 # OpenStack-Ansible leap upgrade
 
-## Jump upgrade from OpenStack Juno to Newton using OpenStack-Ansible
+## Jump upgrade using OpenStack-Ansible for Ubuntu 14.04
 
 ==**This currently a POC**==
 
 ### Uses
 
 This utility can be used to upgrade any OpenStack-Ansible deployment running
-Juno to the latest Newton release. The process will upgrade the OSA system
+Juno / Kilo to the Newton release 14.2.3. The process will upgrade the system
 components, sync the database through the various releases, and then deploy
 OSA using the Newton release. While this method will help a deployment skip
 several releases deployers should be aware that skipping releases is not
@@ -19,7 +19,7 @@ the shortest possible time while maintaining data-integrity.
 
 #### Requirements
 
-  * **You must** have a Juno based OpenStack cloud as deployed by
+  * **You must** have a Juno/Kilo based OpenStack cloud as deployed by
     OpenStack-Ansible.
   * If you are running cinder-volume with LVM in an LXC container **you must**
     migrate the cinder-volume service to the physical host.
@@ -61,7 +61,7 @@ Newton.
 
 ----
 
-### Setting up a Test environment.
+### Example leap with a multi-node juno environment.
 
 Testing on a multi-node environment can be accomplished using the
 https://github.com/openstack/openstack-ansible-ops/tree/master/multi-node-aio
@@ -76,7 +76,7 @@ very well for development. To run the deployment execute the following commands
   * Start the deployment w/ ubuntu 14.04.2 to ensure the deployment version is
     limited in terms of package availability.
 
-#### Process
+#### Setup a multi-node AIO
 
 Clone the ops tooling and change directory to the multi-node-aio tooling
 
@@ -94,6 +94,8 @@ setup-virsh-net.sh
 deploy-vms.sh
 ```
 
+#### Deploy an example Juno config
+
 After the environment has been deployed clone the RPC configurations which support Juno
 based clouds.
 
@@ -101,6 +103,8 @@ based clouds.
 git clone https://github.com/os-cloud/leapfrog-juno-config /etc/rpc_deploy
 ```
 
+#### Deploy Juno
+
 Now clone the Juno playbooks into place.
 
 ``` bash
@@ -118,6 +122,8 @@ openstack-ansible playbooks/haproxy-install.yml
 openstack-ansible playbooks/setup-everything.yml
 ```
 
+#### Test your Juno cloud
+
 To test the cloud's functionality you can execute the OpenStack resource test script located in the scripts directory
 of the playbooks cloned earlier.
 
@@ -143,3 +149,7 @@ Once the cloud is operational it's recommended that images be created so that th
 reverted to a previous state should there ever be a need. See
 https://github.com/openstack/openstack-ansible-ops/tree/master/multi-node-aio#snapshotting-an-environment-before-major-testing
 for more on creating snapshots.
+
+#### Run the leapfrog
+
+See the "Process" part on the top of the page

leap-upgrades/lib/functions.sh

@@ -27,24 +27,38 @@ function failure {
   echo -e '[!]'"\t\033[1;31m${1}\033[0m"
 }
 
+function debug {
+  if [[ $DEBUG == "TRUE" ]]; then
+    echo -e "${1}" >> $DEBUG_PATH
+  fi
+}
+
 function tag_leap_success {
   notice "LEAP ${1} success"
   touch "/opt/leap42/openstack-ansible-${1}.leap"
+  debug "LEAP ${1} marked as success"
 }
 
 function run_lock {
+
   set +e
   run_item="${RUN_TASKS[$1]}"
-  file_part="$(echo ${run_item} | sed 's/\s/-/g')"
+  file_part="$(echo ${run_item} | cut -f 1 -d ' ' | xargs basename)"
+  other_args="$(echo ${run_item} | cut -f 2- -d ' ' -s | sed 's/[^[:alnum:]_]/-/g')"
+  debug "Run_lock on $run_item"
 
   if [ ! -d  "/etc/openstack_deploy/upgrade-leap" ]; then
       mkdir -p "/etc/openstack_deploy/upgrade-leap"
   fi
 
-  upgrade_marker_file=$(basename "${file_part}")
+  upgrade_marker_file=${file_part}${other_args}
   upgrade_marker="/etc/openstack_deploy/upgrade-leap/$upgrade_marker_file.complete"
+  debug "Upgrade marker is $upgrade_marker"
 
   if [ ! -f "$upgrade_marker" ];then
+    debug "Upgrade marker file not found for this run item."
+    debug "Will run openstack-ansible $2"
+
     # note(sigmavirus24): use eval so that we properly turn strings like
     # "/tmp/fix_container_interfaces.yml || true"
     # into a command, otherwise we'll get an error that there's no playbook
@@ -54,8 +68,8 @@ function run_lock {
     notice "Ran: $run_item"
 
     if [ "$playbook_status" == "0" ];then
-      RUN_TASKS=("${RUN_TASKS[@]/$run_item}")
       touch "${upgrade_marker}"
+      unset RUN_TASKS[$1]
       notice "$run_item has been marked as success at ${upgrade_marker}"
     else
       FAILURES_LIST=$(seq $1 $((${#RUN_TASKS[@]} - 1)))
@@ -77,6 +91,7 @@ function run_lock {
       exit 99
     fi
   else
+    debug "Upgrade marker file found for this run item."
     RUN_TASKS=("${RUN_TASKS[@]/$run_item.*}")
   fi
   set -e
@@ -113,13 +128,92 @@ function system_bootstrap {
     popd
 }
 
+function validate_upgrade_input {
+
+    echo
+    warning "Please enter the source series to upgrade from."
+    notice "JUNO, KILO or LIBERTY"
+    read -p 'Enter "JUNO", "KILO", or "LIBERTY" to continue: ' UPGRADE_FROM
+    export INPUT_UPGRADE_FROM=${UPGRADE_FROM}
+
+    if [[ ${INPUT_UPGRADE_FROM} == ${CODE_UPGRADE_FROM} ]]; then
+      notice "Running LEAP Upgrade from ${CODE_UPGRADE_FROM} to NEWTON"
+    else
+      notice "Asking to upgrade a ${INPUT_UPGRADE_FROM}, but code is to ${CODE_UPGRADE_FROM}"
+      read -p 'Are you sure? Enter "YES" to continue:' RUSURE
+      if [[ "${RUSURE}" != "YES" ]]; then
+          notice "Quitting..."
+          exit 99
+      fi
+      # We should let the user decide if he passes through the checks
+      export CODE_UPGRADE_FROM=${INPUT_UPGRADE_FROM}
+    fi
+}
+
+function discover_code_version {
+    if [[ ! -f "/etc/openstack-release" ]]; then
+        export CODE_UPGRADE_FROM="JUNO"
+        notice "You seem to be running Juno"
+    else
+        source /etc/openstack-release
+        case "${DISTRIB_RELEASE%%.*}" in
+            '11')
+                export CODE_UPGRADE_FROM="KILO"
+                notice "You seem to be running Kilo"
+            ;;
+            '12')
+                export CODE_UPGRADE_FROM="LIBERTY"
+                notice "You seem to be running Liberty"
+            ;;
+            '13')
+                export CODE_UPGRADE_FROM="MITAKA"
+                notice "You seem to be running Mitaka"
+            ;;
+            '14')
+                export CODE_UPGRADE_FROM="NEWTON"
+                notice "You seem to be running Newton"
+            ;;
+        esac
+    fi
+}
+
+function set_upgrade_vars {
+  notice "Setting up vars for the LEAP"
+  case "${CODE_UPGRADE_FROM}" in
+  JUNO)
+    export RELEASE="${JUNO_RELEASE}"
+    export UPGRADES_TO_TODOLIST="KILO LIBERTY MITAKA NEWTON"
+    export ANSIBLE_INVENTORY="/opt/leap42/openstack-ansible-${RELEASE}/rpc_deployment/inventory"
+    export CONFIG_DIR="/etc/rpc_deploy"
+  ;;
+  KILO)
+    export RELEASE="${KILO_RELEASE}"
+    export UPGRADES_TO_TODOLIST="LIBERTY MITAKA NEWTON"
+    export ANSIBLE_INVENTORY="/opt/leap42/openstack-ansible-${RELEASE}/playbooks/inventory"
+    export CONFIG_DIR="/etc/openstack_deploy"
+  ;;
+  LIBERTY)
+    export RELEASE="${LIBERTY_RELEASE}"
+    export UPGRADES_TO_TODOLIST="MITAKA NEWTON"
+    export ANSIBLE_INVENTORY="/opt/leap42/openstack-ansible-${RELEASE}/playbooks/inventory"
+    export CONFIG_DIR="/etc/openstack_deploy"
+  ;;
+  MITAKA)
+    export RELEASE="${MITAKA_RELEASE}"
+    export UPGRADES_TO_TODOLIST="NEWTON"
+    export ANSIBLE_INVENTORY="/opt/leap42/openstack-ansible-${RELEASE}/playbooks/inventory"
+    export CONFIG_DIR="/etc/openstack_deploy"
+  ;;
+  esac
+}
+
 function pre_flight {
     ## Pre-flight Check ----------------------------------------------------------
     # Clear the screen and make sure the user understands whats happening.
     clear
 
     # Notify the user.
-    warning "This script will perform a LEAP upgrade from Juno to Newton."
+    warning "This script will perform a LEAP upgrade to Newton."
     warning "Once you start the upgrade there's no going back."
     warning "**Note, this is an OFFLINE upgrade**"
     notice "If you want to run the upgrade in parts please exit this script to do so."
@@ -130,25 +224,23 @@ function pre_flight {
     if [ "${UPGRADE}" == "YES" ]; then
       notice "Running LEAP Upgrade"
     else
+      notice "Exiting, input wasn't YES"
       exit 99
     fi
 
-    mkdir -p /opt/leap42/venvs
+    discover_code_version
 
-    pushd /opt/leap42
+    if [ "${VALIDATE_UPGRADE_INPUT}" == "TRUE" ]; then
-      # Using this lookup plugin because it allows us to compile exact service releaes and build a complete venv from it
+        validate_upgrade_input
-      wget https://raw.githubusercontent.com/openstack/openstack-ansible-plugins/e069d558b3d6ae8fc505d406b13a3fb66201a9c7/lookup/py_pkgs.py -O py_pkgs.py
+    fi
-      chmod +x py_pkgs.py
+
-    popd
+    set_upgrade_vars
+
+    mkdir -p /opt/leap42/venvs
 
     # If the lxc backend store was not set halt and instruct the user to set it. In Juno we did more to detect the backend storage
     #  size than we do in later releases. While the auto-detection should still work it's best to have the deployer set the value
     #  desired before moving forward.
-    if [[ -d "/etc/rpc_deploy" ]]; then
-      CONFIG_DIR="/etc/rpc_deploy"
-    else
-      CONFIG_DIR="/etc/openstack_deploy"
-    fi
     if ! grep -qwrn "^lxc_container_backing_store" $CONFIG_DIR; then
       failure "ERROR: 'lxc_container_backing_store' is unset leading to an ambiguous container backend store."
       failure "Before continuing please set the 'lxc_container_backing_store' in your user_variables.yml file."
@@ -156,11 +248,25 @@ function pre_flight {
       exit 99
     fi
 
+    if ! grep -qwrn "^neutron_legacy_ha_tool_enabled" $CONFIG_DIR; then
+      failure "ERROR: 'neutron_legacy_ha_tool_enabled' is unset leading to an ambiguous l3ha handling."
+      failure "Before continuing please set the 'neutron_legacy_ha_tool_enabled' in your user_variables.yml file."
+      exit 99
+    fi
+
     # Install liberasurecode-dev which will be used in the venv creation process
     if ! grep -n ^ /etc/apt/sources.list /etc/apt/sources.list.d/* | grep -qw "backports"; then
       failure "The trusty backports repo has not been enabled on this host."
       exit 99
     fi
+
+    # Don't run this over and over again if the variables above are not set!
+    pushd /opt/leap42
+      # Using this lookup plugin because it allows us to compile exact service releaes and build a complete venv from it
+      wget https://raw.githubusercontent.com/openstack/openstack-ansible-plugins/e069d558b3d6ae8fc505d406b13a3fb66201a9c7/lookup/py_pkgs.py -O py_pkgs.py
+      chmod +x py_pkgs.py
+    popd
+
     apt-get update > /dev/null
     apt-get -y install liberasurecode-dev > /dev/null
 
@@ -203,6 +309,7 @@ function run_items {
       pushd ${PB_DIR}
         # Run the tasks in order
         for item in ${!RUN_TASKS[@]}; do
+          debug "Run_items of ${item}: ${RUN_TASKS[$item]}. Starting run_lock"
           run_lock $item "${RUN_TASKS[$item]}"
         done
       popd

leap-upgrades/lib/vars.sh

@@ -14,12 +14,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+## Generic debug var. This will log actions to /tmp/run_debug
+export DEBUG="${DEBUG:-TRUE}"
+export DEBUG_PATH="${DEBUG_PATH:-/tmp/run_debug}"
+
 ## Script Vars ---------------------------------------------------------------
 export JUNO_RELEASE="${JUNO_RELEASE:-10.1.14}"
 export KILO_RELEASE="${KILO_RELEASE:-11.2.17}"
 export LIBERTY_RELEASE="${LIBERTY_RELEASE:-12.2.8}"
 export MITAKA_RELEASE="${MITAKA_RELEASE:-13.3.11}"
-export NEWTON_RELEASE="${NEWTON_RELEASE:-d47e29b7d8a385773acadb825e37c82d42b3ec27}"  # commit used due to packaging bug caused by setuptools
+export NEWTON_RELEASE="${NEWTON_RELEASE:-14.2.3}"  # commit used due to packaging bug caused by setuptools
 
 ## Environment Vars ------------------------------------------------------------------
 export MAIN_PATH="${MAIN_PATH:-/opt/openstack-ansible}"
@@ -31,3 +35,7 @@ export UPGRADE_UTILS="${UPGRADE_UTILS:-${SYSTEM_PATH}/upgrade-utilities}"
 #  this URL will be used to download the release built VENVS in the following format.
 #  ${VENV_URL}/openstack-ansible-RELEASE_VERSION.tgz
 export VENV_URL="${VENV_URL:-https://mirror.rackspace.com/rackspaceprivatecloud/venvs/leap-bundles}"
+
+export VALIDATE_UPGRADE_INPUT="${VALIDATE_UPGRADE_INPUT:-TRUE}"
+export UPGRADES_TO_TODOLIST="${UPGRADES_TO_TODOLIST:-''}"
+export CODE_UPGRADE_FROM=""

leap-upgrades/migrations.sh

@@ -18,7 +18,7 @@
 #   root@HOSTNAME:/opt/openstack-ansible# echo "YES" | bash scripts/run-upgrade.sh
 
 ## Shell Opts ----------------------------------------------------------------
-set -e -u -v
+set -e -u
 
 ## Main ----------------------------------------------------------------------
 source lib/vars.sh

leap-upgrades/prep.sh

@@ -18,7 +18,7 @@
 #   root@HOSTNAME:/opt/openstack-ansible# echo "YES" | bash scripts/run-upgrade.sh
 
 ## Shell Opts ----------------------------------------------------------------
-set -e -u -v
+set -e -u
 
 ## Main ----------------------------------------------------------------------
 source lib/vars.sh
@@ -60,17 +60,10 @@ if [[ ! -f "/opt/leap42/openstack-ansible-${NEWTON_RELEASE}-prep.leap" ]]; then
 fi
 
 RUN_TASKS=()
+
 RUN_TASKS+=("${UPGRADE_UTILS}/cinder-volume-container-lvm-check.yml")
 RUN_TASKS+=("${UPGRADE_UTILS}/db-backup.yml")
 
-if [[ -d "/etc/rpc_deploy" ]]; then
-  RELEASE="${JUNO_RELEASE}"
-  export ANSIBLE_INVENTORY="/opt/leap42/openstack-ansible-${RELEASE}/rpc_deployment/inventory"
-else
-  RELEASE="${NEWTON_RELEASE}"
-  export ANSIBLE_INVENTORY="/opt/leap42/openstack-ansible-${RELEASE}/playbooks/inventory"
-fi
-
 # temp upgrade ansible is used to ensure 1.9.x compat.
 PS1="\\u@\h \\W]\\$" . "/opt/ansible-runtime/bin/activate"
 run_items "/opt/leap42/openstack-ansible-${RELEASE}"

leap-upgrades/re-deploy.sh

@@ -18,33 +18,43 @@
 #   root@HOSTNAME:/opt/openstack-ansible# echo "YES" | bash scripts/run-upgrade.sh
 
 ## Shell Opts ----------------------------------------------------------------
-set -e -u -v
+set -e -u
 
 ## Main ----------------------------------------------------------------------
 source lib/vars.sh
 source lib/functions.sh
 
 ### Run the redeploy tasks
-# Forget about the old Juno neutron agent container in inventory.
+# Forget about the old neutron agent container in inventory.
 #  This is done to maximize uptime by leaving the old systems in
 #  place while the redeployment work is going on.
-SCRIPTS_PATH="/opt/leap42/openstack-ansible-${NEWTON_RELEASE}/scripts" \
+# TODO(evrardjp): Move this to a playbook, this way it will follow the
-  MAIN_PATH="/opt/leap42/openstack-ansible-${NEWTON_RELEASE}" \
+# RUN_TASKS model
-    ${UPGRADE_UTILS}/neutron-container-forget.sh
+
+if [ ! -f /etc/openstack_deploy/upgrade-leap/neutron-container-forget.complete ];then
+  SCRIPTS_PATH="/opt/leap42/openstack-ansible-${NEWTON_RELEASE}/scripts" \
+    MAIN_PATH="/opt/leap42/openstack-ansible-${NEWTON_RELEASE}" \
+      ${UPGRADE_UTILS}/neutron-container-forget.sh
+  touch /etc/openstack_deploy/upgrade-leap/neutron-container-forget.complete
+fi
 
 link_release "/opt/leap42/openstack-ansible-${NEWTON_RELEASE}"
 RUN_TASKS=()
+# Ensure the same pip everywhere, even if requirement met or above
 RUN_TASKS+=("${UPGRADE_UTILS}/pip-unify.yml -e release_version=\"${NEWTON_RELEASE}\"")
 
 RUN_TASKS+=("${UPGRADE_UTILS}/db-stop.yml")
 RUN_TASKS+=("${UPGRADE_UTILS}/ansible_fact_cleanup.yml")
+# Physical host cleanup
 RUN_TASKS+=("${UPGRADE_UTILS}/destroy-old-containers.yml")
+# Permissions for qemu save, because physical host cleanup
 RUN_TASKS+=("${UPGRADE_UTILS}/nova-libvirt-fix.yml")
 
 RUN_TASKS+=("lxc-hosts-setup.yml")
 RUN_TASKS+=("lxc-containers-create.yml")
 
 RUN_TASKS+=("setup-infrastructure.yml")
+# MariaDB sync for major maria upgrades and cluster schema sync
 RUN_TASKS+=("${UPGRADE_UTILS}/db-force-upgrade.yml")
 
 RUN_TASKS+=("os-keystone-install.yml")

leap-upgrades/run-stages.sh

@@ -18,7 +18,7 @@
 #   root@HOSTNAME:/opt/openstack-ansible# echo "YES" | bash scripts/run-upgrade.sh
 
 ## Shell Opts ----------------------------------------------------------------
-set -e -u -v
+set -e -u
 
 ## Main ----------------------------------------------------------------------
 source lib/vars.sh

leap-upgrades/upgrade-utilities/neutron-remove-old-containers.yml

@@ -25,6 +25,7 @@
       args:
         executable: /bin/bash
 
+# TODO: Improve l2 convergence too.
 - name: Remove the reminent neutron agent containers
   hosts: "neutron_agent[0]"
   gather_facts: false

leap-upgrades/upgrade-utilities/pip-unify.yml

@@ -76,6 +76,11 @@
       args:
         executable: /bin/bash
 
+    - name: apt-get install libssl-dev
+      apt:
+        name: libssl-dev
+        update_cache: yes
+
     - name: Re-install pip packaging requirements
       raw: |
         /usr/bin/python /opt/get-pip.py \

leap-upgrades/upgrade.sh

@@ -18,7 +18,7 @@
 #   root@HOSTNAME:/opt/openstack-ansible# echo "YES" | bash scripts/run-upgrade.sh
 
 ## Shell Opts ----------------------------------------------------------------
-set -e -u -v
+set -e -u
 
 ## Main ----------------------------------------------------------------------
 source lib/vars.sh
@@ -28,8 +28,8 @@ source lib/functions.sh
 # Run tasks
 UPGRADE_SCRIPTS="${UPGRADE_UTILS}-kilo/scripts"
 # If the kilo leap has been accomplished, skip.
-if [[ ! -f "/opt/leap42/openstack-ansible-${KILO_RELEASE}.leap" ]]; then
+if [[ ! -f "/opt/leap42/openstack-ansible-${KILO_RELEASE}.leap" ]] && [[ "${UPGRADES_TO_TODOLIST}" =~ .*KILO.* ]]; then
-  notice 'Running kilo leap'
+  notice 'Leaping to Kilo'
   link_release "/opt/leap42/openstack-ansible-${KILO_RELEASE}"
   pushd "/opt/leap42/openstack-ansible-${KILO_RELEASE}"
     if [[ -d "/etc/rpc_deploy" ]]; then
@@ -74,8 +74,8 @@ fi
 
 ### Liberty System migration
 # Run tasks
-if [[ ! -f "/opt/leap42/openstack-ansible-${LIBERTY_RELEASE}.leap" ]]; then
+if [[ ! -f "/opt/leap42/openstack-ansible-${LIBERTY_RELEASE}.leap" ]] && [[ "${UPGRADES_TO_TODOLIST}" =~ .*LIBERTY.*  ]]; then
-  notice 'Running liberty leap'
+  notice 'Leaping to liberty'
   link_release "/opt/leap42/openstack-ansible-${LIBERTY_RELEASE}"
   UPGRADE_PLAYBOOKS="${UPGRADE_UTILS}-liberty/playbooks"
   RUN_TASKS=()
@@ -91,8 +91,8 @@ fi
 
 ### Mitaka System migration
 # Run tasks
-if [[ ! -f "/opt/leap42/openstack-ansible-${MITAKA_RELEASE}.leap" ]]; then
+if [[ ! -f "/opt/leap42/openstack-ansible-${MITAKA_RELEASE}.leap" ]] && [[ "${UPGRADES_TO_TODOLIST}" =~ .*MITAKA.* ]]; then
-  notice 'Running mitaka leap'
+  notice 'Leaping to Mitaka'
   link_release "/opt/leap42/openstack-ansible-${MITAKA_RELEASE}"
   UPGRADE_PLAYBOOKS="${UPGRADE_UTILS}-mitaka/playbooks"
   RUN_TASKS=()
@@ -109,7 +109,7 @@ fi
 
 ### Newton Deploy
 # Run tasks
-if [[ ! -f "/opt/leap42/openstack-ansible-${NEWTON_RELEASE}.leap" ]]; then
+if [[ ! -f "/opt/leap42/openstack-ansible-${NEWTON_RELEASE}.leap" ]] && [[ "${UPGRADES_TO_TODOLIST}" =~ .*NEWTON*  ]]; then
   notice 'Running newton leap'
   link_release "/opt/leap42/openstack-ansible-${NEWTON_RELEASE}"
   UPGRADE_PLAYBOOKS="${UPGRADE_UTILS}-newton/playbooks"