openstack/trove
Code Issues Proposed changes

Add image build experimental CI job

Browse Source 
Story: 2006553
Task: 36637
Change-Id: I3d909082fb887b24ed25c93e11792031d5053b1e
This commit is contained in:
Lingxian Kong 2019-09-16 12:15:34 +12:00
parent f771e939eb
commit 0cdf8b20b6
7 changed files with 128 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
.zuul.yaml
View File

@ -10,43 +10,42 @@
- release-notes-jobs-python3
check:
jobs:
# Non-voting until job is fixed, remove it if it works,
# it's part of openstack-cover-jobs template.
- openstack-tox-cover:
voting: false
- openstack-tox-pylint
- trove-tox-bandit-baseline:
voting: false
- trove-tox-fakemodetests
- trove-functional-mysql
- trove-scenario-mariadb-single:
voting: false
- trove-scenario-mariadb-multi:
voting: false
- trove-scenario-mysql-single
- trove-scenario-mysql-multi
- trove-scenario-postgresql-single:
voting: false
- trove-scenario-postgresql-multi:
voting: false
- trove-scenario-pxc-single:
- trove-scenario-mariadb-single:
voting: false
- trove-scenario-pxc-multi:
- trove-scenario-mariadb-multi:
voting: false
- trove-scenario-redis-single:
voting: false
- trove-scenario-redis-multi:
voting: false
- trove-tox-fakemodetests
- trove-tox-bandit-baseline:
- trove-tempest-ipv6-only:
voting: false
- openstack-tox-pylint
- trove-tempest-ipv6-only
promote:
jobs:
- publish-trove-guest-image-mysql-ubuntu-xenial
gate:
queue: trove
jobs:
- openstack-tox-pylint
- trove-tox-fakemodetests
- trove-functional-mysql
- trove-scenario-mysql-single
- trove-scenario-mysql-multi
- trove-tox-fakemodetests
- openstack-tox-pylint
- trove-tempest-ipv6-only
- trove-tempest-ipv6-only:
voting: false
experimental:
jobs:
- trove-grenade
@ -58,6 +57,8 @@
- trove-scenario-mongodb-multi
- trove-scenario-percona-single
- trove-scenario-percona-multi
- trove-scenario-pxc-single
- trove-scenario-pxc-multi
- job:
name: trove-legacy-dsvm-base
@ -341,3 +342,25 @@
- ^trove/hacking/.*$
- ^trove/tests/scenario/.*$
- ^trove/tests/unittests/.*$
- job:
name: publish-trove-guest-image
parent: publish-openstack-artifacts
run: playbooks/image-build/run.yaml
post-run: playbooks/image-build/post.yaml
required-projects:
- openstack/diskimage-builder
- openstack/trove
- openstack/tripleo-image-elements
- job:
name: publish-trove-guest-image-mysql-ubuntu-xenial
parent: publish-trove-guest-image
description: |
Build and publish Ubuntu Xenial based Trove guest image to
tarballs.openstack.org.
vars:
datastore_type: mysql
guest_os: ubuntu
guest_os_release: xenial
guest_username: ubuntu

72
devstack/plugin.sh
View File

@ -389,69 +389,6 @@ function setup_mgmt_network() {
# fi
}
# Set up Trove management network and make configuration change.
function config_trove_network {
echo "Finalizing Neutron networking for Trove"
echo "Dumping current network parameters:"
echo " SERVICE_HOST: $SERVICE_HOST"
echo " BRIDGE_IP: $BRIDGE_IP"
echo " PUBLIC_NETWORK_GATEWAY: $PUBLIC_NETWORK_GATEWAY"
echo " NETWORK_GATEWAY: $NETWORK_GATEWAY"
echo " IPV4_ADDRS_SAFE_TO_USE: $IPV4_ADDRS_SAFE_TO_USE"
echo " IPV6_ADDRS_SAFE_TO_USE: $IPV6_ADDRS_SAFE_TO_USE"
echo " FIXED_RANGE: $FIXED_RANGE"
echo " FLOATING_RANGE: $FLOATING_RANGE"
echo " SUBNETPOOL_PREFIX_V4: $SUBNETPOOL_PREFIX_V4"
echo " SUBNETPOOL_SIZE_V4: $SUBNETPOOL_SIZE_V4"
echo " SUBNETPOOL_V4_ID: $SUBNETPOOL_V4_ID"
echo " ROUTER_GW_IP: $ROUTER_GW_IP"
echo " TROVE_MGMT_SUBNET_RANGE: ${TROVE_MGMT_SUBNET_RANGE}"
echo "Creating Trove management network/subnet for Trove service project."
trove_service_project_id=$(openstack project show $SERVICE_PROJECT_NAME -c id -f value)
setup_mgmt_network ${trove_service_project_id} ${TROVE_MGMT_NETWORK_NAME} ${TROVE_MGMT_SUBNET_NAME} ${TROVE_MGMT_SUBNET_RANGE}
mgmt_net_id=$(openstack network show ${TROVE_MGMT_NETWORK_NAME} -c id -f value)
echo "Created Trove management network ${TROVE_MGMT_NETWORK_NAME}(${mgmt_net_id})"
# Share the private network to other projects for testing purpose. We make
# the private network accessible to control plane below so that we could
# reach the private network for integration tests without floating ips
# associated, no matter which user the tests are using.
shared=$(openstack network show ${PRIVATE_NETWORK_NAME} -c shared -f value)
if [[ "$shared" == "False" ]]; then
openstack network set ${PRIVATE_NETWORK_NAME} --share
fi
sudo ip route replace ${IPV4_ADDRS_SAFE_TO_USE} via $ROUTER_GW_IP
# Make sure we can reach the management port of the service VM, this
# configuration is only for testing purpose. In production, it's
# recommended to config the router in the cloud infrastructure for the
# communication between Trove control plane and service VMs.
INTERFACE=trove-mgmt
MGMT_PORT_ID=$(openstack port create --project ${trove_service_project_id} --security-group ${TROVE_MGMT_SECURITY_GROUP} --device-owner trove --network ${TROVE_MGMT_NETWORK_NAME} --host=$(hostname) -c id -f value ${INTERFACE}-port)
MGMT_PORT_MAC=$(openstack port show -c mac_address -f value $MGMT_PORT_ID)
MGMT_PORT_IP=$(openstack port show -f value -c fixed_ips $MGMT_PORT_ID | awk '{FS=",| "; gsub(",",""); gsub("'\''",""); for(i = 1; i <= NF; ++i) {if ($i ~ /^ip_address/) {n=index($i, "="); if (substr($i, n+1) ~ "\\.") print substr($i, n+1)}}}')
sudo ovs-vsctl -- --may-exist add-port ${OVS_BRIDGE:-br-int} $INTERFACE -- set Interface $INTERFACE type=internal -- set Interface $INTERFACE external-ids:iface-status=active -- set Interface $INTERFACE external-ids:attached-mac=$MGMT_PORT_MAC -- set Interface $INTERFACE external-ids:iface-id=$MGMT_PORT_ID -- set Interface $INTERFACE external-ids:skip_cleanup=true
sudo ip link set dev $INTERFACE address $MGMT_PORT_MAC
mask=$(echo ${TROVE_MGMT_SUBNET_RANGE} | awk -F'/' '{print $2}')
sudo ip addr add ${MGMT_PORT_IP}/${mask} dev $INTERFACE
sudo ip link set $INTERFACE up
echo "Neutron network list:"
openstack network list
echo "Neutron subnet list:"
openstack subnet list
echo "ip route:"
sudo ip route
# Now make sure the conf settings are right
iniset $TROVE_CONF DEFAULT network_label_regex ${PRIVATE_NETWORK_NAME}
iniset $TROVE_CONF DEFAULT ip_regex ""
iniset $TROVE_CONF DEFAULT black_list_regex ""
iniset $TROVE_CONF DEFAULT management_networks ${mgmt_net_id}
iniset $TROVE_CONF DEFAULT network_driver trove.network.neutron.NeutronDriver
}
# start_trove() - Start running processes, including screen
function start_trove {
if [[ ${TROVE_USE_MOD_WSGI}" == TRUE" ]]; then
@ -501,7 +438,7 @@ function create_guest_image {
$DEST/trove/integration/scripts/trovestack build-image ${TROVE_DATASTORE_TYPE} ${TROVE_IMAGE_OS} ${TROVE_IMAGE_OS_RELEASE} true
image_name=${TROVE_IMAGE_OS}-${TROVE_DATASTORE_TYPE}
image_name="trove-${TROVE_IMAGE_OS}-${TROVE_IMAGE_OS_RELEASE}-${TROVE_DATASTORE_TYPE}"
image_file=$HOME/images/${image_name}.qcow2
if [ ! -f ${image_file} ]; then
echo "Image file was not found at ${image_file}. Probably it was not created."
@ -562,8 +499,8 @@ function config_trove_network {
echo " TROVE_MGMT_SUBNET_RANGE: ${TROVE_MGMT_SUBNET_RANGE}"
# Save xtrace setting
local XTRACE
XTRACE=$(set +o | grep xtrace)
local orig_xtrace
orig_xtrace=$(set +o | grep xtrace)
set -x
echo "Creating Trove management network/subnet for Trove service project."
@ -613,7 +550,7 @@ function config_trove_network {
iniset $TROVE_CONF DEFAULT network_driver trove.network.neutron.NeutronDriver
# Restore xtrace setting
$XTRACE
$orig_xtrace
}
function config_nova_keypair {
@ -646,6 +583,7 @@ function config_cinder_volume_type {
iniset $TROVE_CONF DEFAULT cinder_volume_type ${volume_type}
}
function config_mgmt_security_group {
local sgid

15
doc/source/admin/trovestack.rst
View File

@ -38,11 +38,24 @@ The trove guest agent image could be created by running the following command:
${datastore_type} \
${guest_os} \
${guest_os_release} \
${dev_mode}
${dev_mode} \
${guest_username} \
${imagepath}
* Currently, only ``guest_os=ubuntu`` and ``guest_os_release=xenial`` are fully
tested and supported.
* Default input values:
.. code-block:: ini
datastore_type=mysql
guest_os=ubuntu
guest_os_release=xenial
dev_mode=true
guest_username=ubuntu
imagepath=$HOME/images/trove-${guest_os}-${guest_os_release}-${datastore_type}.qcow2
* ``dev_mode=true`` is mainly for testing purpose for trove developers and it's
necessary to build the image on the trove controller host, because the host
and the guest VM need to ssh into each other without password. In this mode,

58
integration/scripts/functions_qemu
View File

@ -7,19 +7,19 @@ function build_vm() {
exclaim "Actually building the image, this can take up to 15 minutes"
rm -rf ~/.cache/image-create
datastore_type=$1
guest_os=$2
guest_release=$3
dev_mode=$4
guest_username=$5
image_output=$6
local datastore_type=$1
local guest_os=$2
local guest_release=$3
local dev_mode=$4
local guest_username=$5
local image_output=$6
elementes="base vm"
trove_elements_path=${PATH_TROVE}/integration/scripts/files/elements
GUEST_IMAGETYPE=${GUEST_IMAGETYPE:-"qcow2"}
GUEST_IMAGESIZE=${GUEST_IMAGESIZE:-4}
GUEST_CACHEDIR=${GUEST_CACHEDIR:-"$HOME/.cache/image-create"}
GUEST_WORKING_DIR=${GUEST_WORKING_DIR:-"$HOME/images"}
local elementes="base vm"
local trove_elements_path=${PATH_TROVE}/integration/scripts/files/elements
local GUEST_IMAGETYPE=${GUEST_IMAGETYPE:-"qcow2"}
local GUEST_IMAGESIZE=${GUEST_IMAGESIZE:-4}
local GUEST_CACHEDIR=${GUEST_CACHEDIR:-"$HOME/.cache/image-create"}
local working_dir=$(dirname ${image_output})
export GUEST_USERNAME=${guest_username}
@ -33,7 +33,7 @@ function build_vm() {
export GUEST_LOGDIR=${GUEST_LOGDIR:-"/var/log/trove/"}
export ESCAPED_GUEST_LOGDIR=$(echo ${GUEST_LOGDIR} | sed 's/\//\\\//g')
export TROVESTACK_SCRIPTS=${TROVESTACK_SCRIPTS}
export HOST_SCP_USERNAME=$(whoami)
export HOST_SCP_USERNAME=${HOST_SCP_USERNAME:-$(whoami)}
export HOST_USERNAME=${HOST_SCP_USERNAME}
export SSH_DIR=${SSH_DIR:-"$HOME/.ssh"}
manage_ssh_keys
@ -56,12 +56,7 @@ function build_vm() {
export DIB_CLOUD_IMAGES="https://cloud-images.ubuntu.com/releases/${DIB_RELEASE}/release/"
export BASE_IMAGE_FILE="ubuntu-${releasemapping[${DIB_RELEASE}]}-server-cloudimg-amd64-root.tar.gz"
if [ "${GUEST_WORKING_DIR}" ]; then
mkdir -p ${GUEST_WORKING_DIR}
TEMP=$(mktemp -d ${GUEST_WORKING_DIR}/diskimage-create.XXXXXX)
else
TEMP=$(mktemp -d diskimage-create.XXXXXX)
fi
TEMP=$(mktemp -d ${working_dir}/diskimage-create.XXXXXXX)
pushd $TEMP > /dev/null
elementes="$elementes ${guest_os}"
@ -89,19 +84,20 @@ function build_vm() {
# out of $TEMP
popd > /dev/null
rm -rf $TEMP
sudo rm -rf $TEMP
exclaim "Image ${image_output}.${GUEST_IMAGETYPE} was built successfully."
}
function build_guest_image() {
datastore_type=${1:-"mysql"}
guest_os=${2:-"ubuntu"}
guest_release=${3:-"xenial"}
dev_mode=${4:-"true"}
guest_username=${5:-"ubuntu"}
exclaim "Params for build_guest_image function: $@"
exclaim "Building a ${datastore_type} image of trove guest agent for ${guest_os} ${guest_release}, dev_mode=${dev_mode}"
local datastore_type=${1:-"mysql"}
local guest_os=${2:-"ubuntu"}
local guest_release=${3:-"xenial"}
local dev_mode=${4:-"true"}
local guest_username=${5:-"ubuntu"}
local output=$6
VALID_SERVICES='mysql percona mariadb redis cassandra couchbase mongodb postgresql couchdb vertica db2 pxc'
if ! [[ " $VALID_SERVICES " =~ " $datastore_type " ]]; then
@ -109,15 +105,7 @@ function build_guest_image() {
exit 1
fi
image_name=${guest_os}-${datastore_type}
image_folder=$HOME/images
mkdir -p $image_folder
image_path=${image_folder}/${image_name}
# Always rebuild the image.
rm -rf ${image_folder}/*
build_vm ${datastore_type} ${guest_os} ${guest_release} ${dev_mode} ${guest_username} ${image_path}
build_vm ${datastore_type} ${guest_os} ${guest_release} ${dev_mode} ${guest_username} ${output}
}
function clean_instances() {

21
integration/scripts/trovestack
View File

@ -821,18 +821,31 @@ function cmd_test_init() {
}
function cmd_build_image() {
exclaim "Params for cmd_build_image function: $@"
local IMAGE_DATASTORE_TYPE=${1:-'mysql'}
local IMAGE_GUEST_OS=${2:-'ubuntu'}
local IMAGE_GUEST_RELEASE=${3:-'xenial'}
local DEV_MODE=${4:-'true'}
local guest_username=${5:-"ubuntu"}
local output=$6
exclaim "Ensuring we have all packages needed to build image."
if [[ -z "$output" ]]; then
image_name="trove-${IMAGE_GUEST_OS}-${IMAGE_GUEST_RELEASE}-${IMAGE_DATASTORE_TYPE}"
image_folder=$HOME/images
output="${image_folder}/${image_name}"
fi
# Always rebuild the image.
sudo rm -f $output
sudo mkdir -p $(dirname $output); sudo chmod 777 -R $(dirname $output)
echo "Ensuring we have all packages needed to build image."
sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS update
sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS install qemu git kpartx debootstrap
sudo -H $HTTP_PROXY pip install diskimage-builder
exclaim "Use diskimage-builder to actually build the Trove Guest Agent Image."
build_guest_image $IMAGE_DATASTORE_TYPE $IMAGE_GUEST_OS $IMAGE_GUEST_RELEASE $DEV_MODE
build_guest_image $IMAGE_DATASTORE_TYPE $IMAGE_GUEST_OS $IMAGE_GUEST_RELEASE $DEV_MODE ${guest_username} $output
}
function cmd_build_and_upload_image() {
@ -856,7 +869,7 @@ function cmd_build_and_upload_image() {
glance_imageid=$(openstack $CLOUD_ADMIN_ARG image list | grep "$datastore_type" | awk 'NR==1 {print}' | awk '{print $2}')
if [[ -z $glance_imageid ]]; then
build_guest_image ${datastore_type} ${guest_os} ${guest_release} ${dev_mode} ${guest_username}
cmd_build_image ${datastore_type} ${guest_os} ${guest_release} ${dev_mode} ${guest_username}
image_folder=$HOME/images
qcow_image=`find $image_folder -name '*.qcow2'`
@ -1276,7 +1289,7 @@ function cmd_gate_tests() {
local DATASTORE_TYPE=${1:-'mysql'}
local TEST_GROUP=${2:-${DATASTORE_TYPE}}
local HOST_SCP_USERNAME=${3:-$USER}
local HOST_SCP_USERNAME=${3:-$(whoami)}
local GUEST_USERNAME=${4:-'ubuntu'}
local CONTROLLER_IP=${5:-$ACTUAL_HOSTNAME}
# We're not using devstack-gate in Zuul v3 job

12
playbooks/image-build/post.yaml Normal file
View File

@ -0,0 +1,12 @@
- hosts: all
name: Copy image
tasks:
- synchronize:
src: "{{ ansible_user_dir }}/images"
dest: "{{ zuul.executor.work_root }}/artifacts/"
mode: pull
verify_host: true
rsync_opts:
- --include="trove-*.qcow2"
- --exclude=*
- --prune-empty-dirs

19
playbooks/image-build/run.yaml Normal file
View File

@ -0,0 +1,19 @@
- hosts: all
tasks:
- name: Ensure artifacts/images directory exists
file:
path: '{{ ansible_user_dir }}/images'
state: directory
- name: Build Trove guest image
shell: >
./trovestack build-image
{{ datastore_type }}
{{ guest_os }}
{{ guest_os_release }}
false
{{ guest_username }}
{{ ansible_user_dir }}/images/trove-{{ zuul.branch }}-{{ datastore_type }}-{{ guest_os }}-{{ guest_os_release }}.qcow2
args:
chdir: "{{ ansible_user_dir }}/src/opendev.org/openstack/trove/integration/scripts"
tags:
- skip_ansible_lint