diff --git a/.gitignore b/.gitignore index 9f6ff4c3..ff1dff07 100755 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,6 @@ refstack/client/.smbdeleteAAA9ef44.4 openid/* db.sqlite + +# Don't include Credentials if in local directory +openrc.sh \ No newline at end of file diff --git a/doc/README.md b/doc/README.md index edfc5893..5c6075c5 100644 --- a/doc/README.md +++ b/doc/README.md @@ -3,3 +3,12 @@ RefStack Documentation Welcome to RefStack. This project helps OpenStack collect and distribute test and validation information. +Executing Tests +----------------------------- + +The heart of RefStack is running tests. We've done a lot of things to make that easier (like [T-CUP](tcup.my)), but here's the basic things you need to do to run Tempest in RefStack: + +1. Setup a system with all the Operating System dependencies +1. Clone Refstack and Tempest (get the branch you want) +1. Install and pip dependencies from the requirements files +1. Run the Tempest setup.py install \ No newline at end of file diff --git a/doc/tcup.md b/doc/tcup.md index f56c7710..813d9c7e 100644 --- a/doc/tcup.md +++ b/doc/tcup.md @@ -1,22 +1,46 @@ TCUP Configuration =========================== - # Install Docker using [[https://www.docker.io/gettingstarted/#h_installation]] +The following instructions are designs run Refstack/Tempest in a container with minimal setup on your system. - # Get the code - git clone http://github.com/stackforge/refstack +> These steps are _not_ do not install Refstack for contributions or development, they are intended for a user who wants to just run and report test results. - # enter RefStack - cd refstack +1. Make sure you have python and wget installed for your operating system. - # create/copy your OpenStack credentials into openrc.sh an file +1. Install Docker using [[https://www.docker.io/gettingstarted/#h_installation]] + 1. Note: if you are in an environment with a proxy, make sure you configure `/etc/default/docker` to leverage the proxy too! + 1. You may want to prep-the environment using - # Create the TCUP container - docker build t-container +1. Get the code: `wget https://raw.githubusercontent.com/stackforge/refstack/master/scripts/tcup.py` + 1. note: you can also get the code by cloning the Refstack and running the code in `/scripts/tcup.py` - # Run the container - docker run -v `pwd`:/tcup:rw -i -t 32fe2d733d51 /bin/bash +1. Set your environment variables to access the test target cloud + 1. generally, you will `source openrc.sh` to load the cloud credentials and URLs - # Inside the container run the following - source tcup/openrc.sh - tcup/run_in_tcup.py +1. Run TCUP: `sudo python tcup.py` + 1. if you want to work on the code from Refstack, use `sudo python scripts/tcup.py' + + +## Trouble Shooting TCUP + +There are several ways to trouble shoot, TCUP. + +1. Enter your docker container using `run -i -t [image id from tcup.py] /bin/bash ` +1. Check your environment variables include the OS_* values using `export` +1. Make sure you can access Keystone using `curl $OS_AUTH_URL` +1. Make sure your credentials are working using `keystone catalog` + +## Docker Tips +1. You can run Docker without sudo! + 1. `sudo usermod -a -G docker ` (to permanently run Docker + without sudo) + 1. you will need to reboot after this change (but you can wait until we tell you to reboot later) + 1. if you don't want this to be permanent or active before the reboot use, `sudo chmod 666 /var/run/docker.sock` + +1. You can inspect which containers are running! + 1. `sudo docker ps` shows the running containers + 1. `sudo docker attach` allows you to connect to a container (may have to press enter) + 1. exit from inside the container with `Ctrl-p` + `Ctrl-q` +1. Orphaned Containers: Over time, you may end up with [orphaned contaniers](http://jimhoskins.com/2013/07/27/remove-untagged-docker-images.html), use the following to clean them up + 1. `sudo docker rm $(docker ps -a -q)` + 1. `sudo docker rmi $(docker images | grep "^" | awk "{print $3}")` \ No newline at end of file diff --git a/scripts/tcup.py b/scripts/tcup.py new file mode 100755 index 00000000..db9b1243 --- /dev/null +++ b/scripts/tcup.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +# +# Copyright (c) 2014 Dell Computer, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +# This file creates and runs Tempest in a Docker Container +import commands +import logging +import os +import re + +if __name__ == "__main__": + + # Reference Information + DOCKER_FILE_SOURCE = os.path.join("scripts", "tcup") + DOCKER_FILE_URL = "https://raw.githubusercontent.com/stackforge/" \ + "refstack/master/scripts/tcup/Dockerfile" + REFSTACK_API_ADDRESS = "http://refstack.org/replace_with_correct_url" + IGNORED_ENV_VARS = {"LS_COLORS", "HOME", "PATH", "PWD", "OLDPWD", + "LESSCLOSE", "SSH_CONNECTION"} + REQUIRED_ENV_VARS = {'OS_PASSWORD', 'OS_USERNAME', 'OS_AUTH_URL'} + + # Setup the logger + LOG_FORMAT = "%(asctime)s %(name)s %(levelname)s %(message)s" + logger = logging.getLogger("TCUP") + console_log_handle = logging.StreamHandler() + console_log_handle.setFormatter(logging.Formatter(LOG_FORMAT)) + logger.addHandler(console_log_handle) + if os.environ.get("DEBUG"): + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + + # Confirm you've sourced their openrc credentials already + for env_var in REQUIRED_ENV_VARS: + if not os.environ.get(env_var): + exp = 'Env Variable "%s" Missing: ' \ + 'You may need to "source openrc.sh".' % (env_var) + raise Exception(exp) + + # build the container + logger.info("Downloading & Building TCUP Image...(may take a long time)") + if os.path.isfile(DOCKER_FILE_SOURCE + "/Dockerfile"): + docker_builder = DOCKER_FILE_SOURCE + else: + docker_builder = DOCKER_FILE_URL + logger.info("Executing: `docker build %s`" % (docker_builder)) + build_output = commands.getoutput("docker build %s" % (docker_builder)) + + # provide friendly output progress message + search_for = "Successfully built ([0-9a-f]{12})" + try: + image = re.search(search_for, build_output).group(1) + except re.error: + exp = "ERROR building TCUP container. Details: %s" % (build_output) + raise Exception(exp) + logger.info("TCUP Built Docker Image ID: %s" % (image)) + + # collect environment variables to pass, we don't want all of them + user_env_vars = dict(os.environ) + for env_var in IGNORED_ENV_VARS: + user_env_vars.pop(env_var, None) + + # test specific configuration + if not os.environ.get('TEST_ID'): + user_env_vars["test_id"] = "1000" # TODO: generated good value! + + if not os.environ.get('API_SERVER_ADDRESS'): + user_env_vars["api_addr"] = REFSTACK_API_ADDRESS + + # create the docker run command line + docker_run = "docker run -d -i" + for env_var in user_env_vars: + docker_run += ' -e "%s=%s"' % (env_var, user_env_vars[env_var]) + if "DEBUG" in user_env_vars: + docker_run += " -v `pwd`:/dev" + docker_run += ' -t %s' % (image) + if "DEBUG" in user_env_vars: + docker_run += " /bin/bash" + else: + docker_run += " cd refstack; python refstack/tools/execute_test.py" \ + " --tempest-home /tempest" \ + " --callback ${api_addr} ${test_id}" + if "DEBUG" in user_env_vars: + docker_run_log_output = docker_run + else: + # normally we redact the password + clear_password = "OS_PASSWORD=%s" % user_env_vars['OS_PASSWORD'] + docker_run_log_output = docker_run.replace(clear_password, + "OS_PASSWORD=!REDACTED!") + logger.info("Executing: '%s'" % (docker_run_log_output)) + + # start the container and advise the user about how to attach + docker_output = commands.getoutput(docker_run) + logger.debug(docker_output) + logger.info("""You can monitor the TCUP results using the command + 'sudo docker attach %s' + (hint: you may need to press [enter])""" + % docker_output[0:12]) diff --git a/scripts/tcup/Dockerfile b/scripts/tcup/Dockerfile index c4549af0..197cdbbf 100644 --- a/scripts/tcup/Dockerfile +++ b/scripts/tcup/Dockerfile @@ -1,53 +1,49 @@ +#!/usr/bin/env python +# +# Copyright (c) 2014 OpenStack Foundation, All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# This file creates and runs Tempest in a Docker Container + +FROM ubuntu:13.10 +MAINTAINER OpenStack RefStack Project -FROM ubuntu RUN apt-get update -# Downloading git - -RUN apt-get install -y git python-setuptools - +# Install git, python and essential tools +RUN apt-get install -y git python-setuptools curl ftp RUN easy_install pip -#Downloading dependencies -RUN apt-get install -y libxml2-dev libxslt-dev lib32z1-dev python2.7-dev libssl-dev - -#other dependencies -RUN apt-get install -y python-dev libxslt1-dev libsasl2-dev libsqlite3-dev libldap2-dev libffi-dev ftp -# RUN pip install ftplib -RUN pip install http://gd.tuwien.ac.at/pub/libxml/python/libxml2-python-2.6.9.tar.gz -# http://gd.tuwien.ac.at/pub/libxml/python/libxml2-python-2.4.16.tar.gz -# RUN pip install ftplib libxml2-python - -# Cloning keystone? - - -# Cloning tempest - -RUN git clone https://github.com/openstack/tempest - -#running setup -RUN cd tempest && python setup.py install - -#making file to add exterior files +# Install dependencies (small bites are easier to troubleshoot) +RUN apt-get install -y libxml2-dev libxslt-dev lib32z1-dev +RUN apt-get install -y python2.7-dev python-dev libssl-dev +RUN apt-get install -y python-libxml2 libxslt1-dev libsasl2-dev +RUN apt-get install -y libsqlite3-dev libldap2-dev libffi-dev +# Setup Environment by Cloning refstack & tempest (choose right branch) RUN mkdir temp +RUN git clone https://github.com/stackforge/refstack.git +RUN git clone https://github.com/openstack/tempest.git -#Tempest config creator -ADD https://raw.github.com/dlenwell/refstack/master/refstack/common/tempest_config.py /temp/ +# Version of TCUP +# Changing REBUILDs the container from this point forward +ENV TCUP 0.1.0 -#MORE DEPENDENCIES (for tempest config builder) -ADD https://raw.github.com/openstack/keystone/master/requirements.txt /temp/ -ADD https://raw.github.com/openstack/keystone/master/test-requirements.txt /temp/ - -#using dependencies -RUN pip install -r /temp/requirements.txt -RUN pip install -r /temp/test-requirements.txt - -#Running tempest config creator - - -#Publishing to refstack -#ADD /temp/ - -#rest of file +# Using refstack & tempest dependencies +RUN pip install -r /refstack/requirements.txt +RUN pip install -r /refstack/test-requirements.txt +# Running tempest setup +RUN cd tempest && git checkout stable/havana +RUN cd tempest && python setup.py install