From 6563a3ce76c5ea8e20056350d68c19a6366bca5d Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Tue, 31 Jan 2012 12:11:56 -0600 Subject: [PATCH] Extract common functions into a separate file This is the start of an effort to organize devstack's code to better document the requirements for configuring the individual components. Change-Id: I3476b76b9d1f9ee63687fb9898a98729118cbd84 --- functions | 91 ++++++++++++++++++++++++++++ stack.sh | 75 +---------------------- tools/build_ramdisk.sh | 34 ++--------- tools/build_tempest.sh | 41 ++----------- tools/build_uec.sh | 7 ++- tools/build_uec_ramdisk.sh | 36 ++--------- tools/configure_tempest.sh | 5 +- tools/copy_dev_environment_to_uec.sh | 12 ++-- tools/get_uec_image.sh | 5 +- 9 files changed, 125 insertions(+), 181 deletions(-) create mode 100644 functions diff --git a/functions b/functions new file mode 100644 index 0000000000..01c4758fa0 --- /dev/null +++ b/functions @@ -0,0 +1,91 @@ +# functions - Common functions used by DevStack components + + +# apt-get wrapper to set arguments correctly +# apt_get package [package ...] +function apt_get() { + [[ "$OFFLINE" = "True" ]] && return + local sudo="sudo" + [[ "$(id -u)" = "0" ]] && sudo="env" + $sudo DEBIAN_FRONTEND=noninteractive \ + http_proxy=$http_proxy https_proxy=$https_proxy \ + apt-get --option "Dpkg::Options::=--force-confold" --assume-yes "$@" +} + + +# Gracefully cp only if source file/dir exists +# cp_it source destination +function cp_it { + if [ -e $1 ] || [ -d $1 ]; then + cp -pRL $1 $2 + fi +} + + +# git clone only if directory doesn't exist already. Since ``DEST`` might not +# be owned by the installation user, we create the directory and change the +# ownership to the proper user. +# Set global RECLONE=yes to simulate a clone when dest-dir exists +# git_clone remote dest-dir branch +function git_clone { + [[ "$OFFLINE" = "True" ]] && return + + GIT_REMOTE=$1 + GIT_DEST=$2 + GIT_BRANCH=$3 + + if echo $GIT_BRANCH | egrep -q "^refs"; then + # If our branch name is a gerrit style refs/changes/... + if [[ ! -d $GIT_DEST ]]; then + git clone $GIT_REMOTE $GIT_DEST + fi + cd $GIT_DEST + git fetch $GIT_REMOTE $GIT_BRANCH && git checkout FETCH_HEAD + else + # do a full clone only if the directory doesn't exist + if [[ ! -d $GIT_DEST ]]; then + git clone $GIT_REMOTE $GIT_DEST + cd $GIT_DEST + # This checkout syntax works for both branches and tags + git checkout $GIT_BRANCH + elif [[ "$RECLONE" == "yes" ]]; then + # if it does exist then simulate what clone does if asked to RECLONE + cd $GIT_DEST + # set the url to pull from and fetch + git remote set-url origin $GIT_REMOTE + git fetch origin + # remove the existing ignored files (like pyc) as they cause breakage + # (due to the py files having older timestamps than our pyc, so python + # thinks the pyc files are correct using them) + find $GIT_DEST -name '*.pyc' -delete + git checkout -f origin/$GIT_BRANCH + # a local branch might not exist + git branch -D $GIT_BRANCH || true + git checkout -b $GIT_BRANCH + fi + fi +} + + +# pip install wrapper to set cache and proxy environment variables +# pip_install package [package ...] +function pip_install { + [[ "$OFFLINE" = "True" ]] && return + sudo PIP_DOWNLOAD_CACHE=/var/cache/pip \ + HTTP_PROXY=$http_proxy \ + HTTPS_PROXY=$https_proxy \ + pip install --use-mirrors $@ +} + + +# Normalize config values to True or False +# VAR=`trueorfalse default-value test-value` +function trueorfalse() { + local default=$1 + local testval=$2 + + [[ -z "$testval" ]] && { echo "$default"; return; } + [[ "0 no false False FALSE" =~ "$testval" ]] && { echo "False"; return; } + [[ "1 yes true True TRUE" =~ "$testval" ]] && { echo "True"; return; } + echo "$default" +} diff --git a/stack.sh b/stack.sh index 8dc54ce248..c40b24c996 100755 --- a/stack.sh +++ b/stack.sh @@ -35,6 +35,9 @@ fi # Keep track of the current devstack directory. TOP_DIR=$(cd $(dirname "$0") && pwd) +# Import common functions +. $TOP_DIR/functions + # stack.sh keeps the list of **apt** and **pip** dependencies in external # files, along with config templates and other useful files. You can find these # in the ``files`` directory (next to this script). We will reference this @@ -86,16 +89,6 @@ source ./stackrc # Destination path for installation ``DEST`` DEST=${DEST:-/opt/stack} -# apt-get wrapper to just get arguments set correctly -function apt_get() { - [[ "$OFFLINE" = "True" ]] && return - local sudo="sudo" - [ "$(id -u)" = "0" ] && sudo="env" - $sudo DEBIAN_FRONTEND=noninteractive \ - http_proxy=$http_proxy https_proxy=$https_proxy \ - apt-get --option "Dpkg::Options::=--force-confold" --assume-yes "$@" -} - # Check to see if we are already running a stack.sh if screen -ls | egrep -q "[0-9].stack"; then echo "You are already running a stack.sh session." @@ -155,18 +148,6 @@ else sudo mv $TEMPFILE /etc/sudoers.d/stack_sh_nova fi -# Normalize config values to True or False -# VAR=`trueorfalse default-value test-value` -function trueorfalse() { - local default=$1 - local testval=$2 - - [[ -z "$testval" ]] && { echo "$default"; return; } - [[ "0 no false False FALSE" =~ "$testval" ]] && { echo "False"; return; } - [[ "1 yes true True TRUE" =~ "$testval" ]] && { echo "True"; return; } - echo "$default" -} - # Set True to configure stack.sh to run cleanly without Internet access. # stack.sh must have been previously run with Internet access to install # prerequisites and initialize $DEST. @@ -542,14 +523,6 @@ function get_packages() { done } -function pip_install { - [[ "$OFFLINE" = "True" ]] && return - sudo PIP_DOWNLOAD_CACHE=/var/cache/pip \ - HTTP_PROXY=$http_proxy \ - HTTPS_PROXY=$https_proxy \ - pip install --use-mirrors $@ -} - # install apt requirements apt_get update apt_get install $(get_packages) @@ -557,48 +530,6 @@ apt_get install $(get_packages) # install python requirements pip_install `cat $FILES/pips/* | uniq` -# git clone only if directory doesn't exist already. Since ``DEST`` might not -# be owned by the installation user, we create the directory and change the -# ownership to the proper user. -function git_clone { - [[ "$OFFLINE" = "True" ]] && return - - GIT_REMOTE=$1 - GIT_DEST=$2 - GIT_BRANCH=$3 - - if echo $GIT_BRANCH | egrep -q "^refs"; then - # If our branch name is a gerrit style refs/changes/... - if [ ! -d $GIT_DEST ]; then - git clone $GIT_REMOTE $GIT_DEST - fi - cd $GIT_DEST - git fetch $GIT_REMOTE $GIT_BRANCH && git checkout FETCH_HEAD - else - # do a full clone only if the directory doesn't exist - if [ ! -d $GIT_DEST ]; then - git clone $GIT_REMOTE $GIT_DEST - cd $GIT_DEST - # This checkout syntax works for both branches and tags - git checkout $GIT_BRANCH - elif [[ "$RECLONE" == "yes" ]]; then - # if it does exist then simulate what clone does if asked to RECLONE - cd $GIT_DEST - # set the url to pull from and fetch - git remote set-url origin $GIT_REMOTE - git fetch origin - # remove the existing ignored files (like pyc) as they cause breakage - # (due to the py files having older timestamps than our pyc, so python - # thinks the pyc files are correct using them) - find $GIT_DEST -name '*.pyc' -delete - git checkout -f origin/$GIT_BRANCH - # a local branch might not exist - git branch -D $GIT_BRANCH || true - git checkout -b $GIT_BRANCH - fi - fi -} - # compute service git_clone $NOVA_REPO $NOVA_DIR $NOVA_BRANCH # python client library to nova that horizon (and others) use diff --git a/tools/build_ramdisk.sh b/tools/build_ramdisk.sh index feaa8a97ad..7c1600b1ec 100755 --- a/tools/build_ramdisk.sh +++ b/tools/build_ramdisk.sh @@ -47,7 +47,10 @@ IMG_FILE=$1 # Keep track of the current directory TOOLS_DIR=$(cd $(dirname "$0") && pwd) -TOP_DIR=`cd $TOOLS_DIR/..; pwd` +TOP_DIR=$(cd $TOOLS_DIR/..; pwd) + +# Import common functions +. $TOP_DIR/functions # Store cwd CWD=`pwd` @@ -170,35 +173,6 @@ if [ ! -r "`ls $MNTDIR/boot/vmlinuz-*-generic | head -1`" ]; then chroot $MNTDIR apt-get install -y linux-generic fi -# git clone only if directory doesn't exist already. Since ``DEST`` might not -# be owned by the installation user, we create the directory and change the -# ownership to the proper user. -function git_clone { - - # clone new copy or fetch latest changes - CHECKOUT=${MNTDIR}$2 - if [ ! -d $CHECKOUT ]; then - mkdir -p $CHECKOUT - git clone $1 $CHECKOUT - else - pushd $CHECKOUT - git fetch - popd - fi - - # FIXME(ja): checkout specified version (should works for branches and tags) - - pushd $CHECKOUT - # checkout the proper branch/tag - git checkout $3 - # force our local version to be the same as the remote version - git reset --hard origin/$3 - popd - - # give ownership to the stack user - chroot $MNTDIR chown -R stack $2 -} - git_clone $NOVA_REPO $DEST/nova $NOVA_BRANCH git_clone $GLANCE_REPO $DEST/glance $GLANCE_BRANCH git_clone $KEYSTONE_REPO $DEST/keystone $KEYSTONE_BRANCH diff --git a/tools/build_tempest.sh b/tools/build_tempest.sh index aa44766ae8..230e8f9b72 100755 --- a/tools/build_tempest.sh +++ b/tools/build_tempest.sh @@ -26,7 +26,10 @@ trap cleanup SIGHUP SIGINT SIGTERM SIGQUIT EXIT # Keep track of the current directory TOOLS_DIR=$(cd $(dirname "$0") && pwd) -TOP_DIR=`cd $TOOLS_DIR/..; pwd` +TOP_DIR=$(cd $TOOLS_DIR/..; pwd) + +# Import common functions +. $TOP_DIR/functions # Abort if localrc is not set if [ ! -e $TOP_DIR/localrc ]; then @@ -43,42 +46,8 @@ DEST=${DEST:-/opt/stack} TEMPEST_DIR=$DEST/tempest -DIST_NAME=${DIST_NAME:-oneiric} - -# git clone only if directory doesn't exist already. Since ``DEST`` might not -# be owned by the installation user, we create the directory and change the -# ownership to the proper user. -function git_clone { - - GIT_REMOTE=$1 - GIT_DEST=$2 - GIT_BRANCH=$3 - - # do a full clone only if the directory doesn't exist - if [ ! -d $GIT_DEST ]; then - git clone $GIT_REMOTE $GIT_DEST - cd $2 - # This checkout syntax works for both branches and tags - git checkout $GIT_BRANCH - elif [[ "$RECLONE" == "yes" ]]; then - # if it does exist then simulate what clone does if asked to RECLONE - cd $GIT_DEST - # set the url to pull from and fetch - git remote set-url origin $GIT_REMOTE - git fetch origin - # remove the existing ignored files (like pyc) as they cause breakage - # (due to the py files having older timestamps than our pyc, so python - # thinks the pyc files are correct using them) - find $GIT_DEST -name '*.pyc' -delete - git checkout -f origin/$GIT_BRANCH - # a local branch might not exist - git branch -D $GIT_BRANCH || true - git checkout -b $GIT_BRANCH - fi -} - # Install tests and prerequisites -sudo PIP_DOWNLOAD_CACHE=/var/cache/pip pip install --use-mirrors `cat $TOP_DIR/files/pips/tempest` +pip_install `cat $TOP_DIR/files/pips/tempest` git_clone $TEMPEST_REPO $TEMPEST_DIR $TEMPEST_BRANCH diff --git a/tools/build_uec.sh b/tools/build_uec.sh index 04e1a45913..ed5a017174 100755 --- a/tools/build_uec.sh +++ b/tools/build_uec.sh @@ -8,7 +8,10 @@ fi # Keep track of the current directory TOOLS_DIR=$(cd $(dirname "$0") && pwd) -TOP_DIR=`cd $TOOLS_DIR/..; pwd` +TOP_DIR=$(cd $TOOLS_DIR/..; pwd) + +# Import common functions +. $TOP_DIR/functions cd $TOP_DIR @@ -34,7 +37,7 @@ fi # Install deps if needed DEPS="kvm libvirt-bin kpartx cloud-utils curl" -apt-get install -y --force-yes $DEPS || true # allow this to fail gracefully for concurrent builds +apt_get install -y --force-yes $DEPS || true # allow this to fail gracefully for concurrent builds # Where to store files and instances WORK_DIR=${WORK_DIR:-/opt/uecstack} diff --git a/tools/build_uec_ramdisk.sh b/tools/build_uec_ramdisk.sh index 174eaac72c..32f90c05c0 100755 --- a/tools/build_uec_ramdisk.sh +++ b/tools/build_uec_ramdisk.sh @@ -40,7 +40,10 @@ DEST_FILE=$1 # Keep track of the current directory TOOLS_DIR=$(cd $(dirname "$0") && pwd) -TOP_DIR=`cd $TOOLS_DIR/..; pwd` +TOP_DIR=$(cd $TOOLS_DIR/..; pwd) + +# Import common functions +. $TOP_DIR/functions cd $TOP_DIR @@ -68,7 +71,7 @@ fi # Install deps if needed DEPS="kvm libvirt-bin kpartx cloud-utils curl" -apt-get install -y --force-yes $DEPS +apt_get install -y --force-yes $DEPS # Where to store files and instances CACHEDIR=${CACHEDIR:-/opt/stack/cache} @@ -113,35 +116,6 @@ if [ ! -r "`ls $MNT_DIR/boot/vmlinuz-*-generic | head -1`" ]; then chroot $MNT_DIR apt-get install -y linux-generic fi -# git clone only if directory doesn't exist already. Since ``DEST`` might not -# be owned by the installation user, we create the directory and change the -# ownership to the proper user. -function git_clone { - - # clone new copy or fetch latest changes - CHECKOUT=${MNT_DIR}$2 - if [ ! -d $CHECKOUT ]; then - mkdir -p $CHECKOUT - git clone $1 $CHECKOUT - else - pushd $CHECKOUT - git fetch - popd - fi - - # FIXME(ja): checkout specified version (should works for branches and tags) - - pushd $CHECKOUT - # checkout the proper branch/tag - git checkout $3 - # force our local version to be the same as the remote version - git reset --hard origin/$3 - popd - - # give ownership to the stack user - chroot $MNT_DIR chown -R stack $2 -} - git_clone $NOVA_REPO $DEST/nova $NOVA_BRANCH git_clone $GLANCE_REPO $DEST/glance $GLANCE_BRANCH git_clone $KEYSTONE_REPO $DEST/keystone $KEYSTONE_BRANCH diff --git a/tools/configure_tempest.sh b/tools/configure_tempest.sh index 00add9a336..f6ef0d3b95 100755 --- a/tools/configure_tempest.sh +++ b/tools/configure_tempest.sh @@ -30,7 +30,10 @@ trap cleanup SIGHUP SIGINT SIGTERM SIGQUIT EXIT # Keep track of the current directory TOOLS_DIR=$(cd $(dirname "$0") && pwd) -TOP_DIR=`cd $TOOLS_DIR/..; pwd` +TOP_DIR=$(cd $TOOLS_DIR/..; pwd) + +# Import common functions +. $TOP_DIR/functions # Abort if localrc is not set if [ ! -e $TOP_DIR/localrc ]; then diff --git a/tools/copy_dev_environment_to_uec.sh b/tools/copy_dev_environment_to_uec.sh index c949b329df..d5687dc14d 100755 --- a/tools/copy_dev_environment_to_uec.sh +++ b/tools/copy_dev_environment_to_uec.sh @@ -8,7 +8,10 @@ set -o errexit # Keep track of the current directory TOOLS_DIR=$(cd $(dirname "$0") && pwd) -TOP_DIR=`cd $TOOLS_DIR/..; pwd` +TOP_DIR=$(cd $TOOLS_DIR/..; pwd) + +# Import common functions +. $TOP_DIR/functions # Change dir to top of devstack cd $TOP_DIR @@ -47,13 +50,6 @@ echo stack:pass | chroot $STAGING_DIR chpasswd ( umask 226 && echo "stack ALL=(ALL) NOPASSWD:ALL" \ > $STAGING_DIR/etc/sudoers.d/50_stack_sh ) -# Gracefully cp only if source file/dir exists -function cp_it { - if [ -e $1 ] || [ -d $1 ]; then - cp -pRL $1 $2 - fi -} - # Copy over your ssh keys and env if desired cp_it ~/.ssh $STAGING_DIR/$DEST/.ssh cp_it ~/.ssh/id_rsa.pub $STAGING_DIR/$DEST/.ssh/authorized_keys diff --git a/tools/get_uec_image.sh b/tools/get_uec_image.sh index f66f2bc2fb..09630740dc 100755 --- a/tools/get_uec_image.sh +++ b/tools/get_uec_image.sh @@ -6,7 +6,10 @@ ROOTSIZE=${ROOTSIZE:-2000} # Keep track of the current directory TOOLS_DIR=$(cd $(dirname "$0") && pwd) -TOP_DIR=`cd $TOOLS_DIR/..; pwd` +TOP_DIR=$(cd $TOOLS_DIR/..; pwd) + +# Import common functions +. $TOP_DIR/functions # exit on error to stop unexpected errors set -o errexit