Clean up the image functional test

1. Move setup_docker.sh to tools directory

2. Make a setup_gate.sh that installs necessary packages including
   docker and starts docker.

3. Add logging output.

4. Add default test timeout of 2 hours.

5. Add user to the docker group before running test cases.

6. Run image build as dockerroot group.

This patch has to be one commit to fix the gate in one go.

Co-Authored-By: Steven Dake <stdake@cisco.com>

Change-Id: I83f3cdb1dabf0dfface589c581cb22c155467acc

.testr.conf

@@ -1,4 +1,7 @@
 [DEFAULT]
-test_command=python -m subunit.run discover tests $LISTOPT $IDOPTION
+test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
+             OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
+             OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-7200} \
+             ${PYTHON:-python} -m subunit.run discover ${OS_TEST_PATH:-./tests} $LISTOPT $IDOPTION
 test_id_option=--load-list $IDFILE
 test_list_option=--list

test-requirements.txt

@@ -1,3 +1,5 @@
+oslo.log>=1.0.0  # Apache-2.0
+oslotest>=1.5.1  # Apache-2.0
 PyYAML
 python-barbicanclient>=3.0.1
 python-ceilometerclient>=1.0.6

tests/setup_gate.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+sudo yum install -y libffi-devel openssl-devel docker
+sudo /usr/sbin/usermod -a -G dockerroot ${SUDO_USER:-$USER}
+sudo systemctl start docker
+sleep 1
+
+group_str="jenkins ALL=(:dockerroot) NOPASSWD: ALL"
+sudo grep -x "$group_str" /etc/sudoers > /dev/null || sudo bash -c "echo \"$group_str\" >> /etc/sudoers"
+sudo chown root:dockerroot /var/run/docker.sock
+
+echo "Completed $0."

tests/test_images.py

@@ -10,33 +10,53 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import testtools
+from oslo_log import fixture as log_fixture
-from subprocess import check_output
+from oslo_log import log as logging
+from oslotest import base
 
-class ImagesTest(testtools.TestCase):
+from subprocess import Popen, PIPE, STDOUT
+
+LOG = logging.getLogger(__name__)
+
+class ImagesTest(base.BaseTestCase):
     def setUp(self):
         super(ImagesTest, self).setUp()
+        self.useFixture(log_fixture.SetLogLevel([__name__],
+                        logging.logging.INFO))
 
     def test_builds(self):
-        build_output = check_output(["tools/build-all-docker-images",
+        proc = Popen(['tools/build-all-docker-images',
-                                    "--release",
+                      '--testmode'],
-                                    "--pull",
+                     stdout=PIPE, stderr=STDOUT, bufsize=1)
-                                    "--testmode"])
+        with proc.stdout:
+            for line in iter(proc.stdout.readline, b''):
+                LOG.info(line.strip())
+        proc.wait()
 
         # these are images that are known to not build properly
-        excluded_images = ["kollaglue/centos-rdo-swift-proxy-server",
+        excluded_images = ["kollaglue/centos-rdo-swift-base",
-                           "kollaglue/centos-rdo-swift-container",
-                           "kollaglue/centos-rdo-swift-base",
-                           "kollaglue/centos-rdo-swift-account",
                            "kollaglue/centos-rdo-swift-object",
-                           "kollaglue/centos-rdo-barbican",
+                           "kollaglue/centos-rdo-swift-proxy-server",
-                           "kollaglue/fedora-rdo-base",
+                           "kollaglue/centos-rdo-swift-container",
+                           "kollaglue/centos-rdo-swift-account",
+                           "kollaglue/centos-rdo-zaqar",
                            "kollaglue/centos-rdo-rhel-osp-base"]
 
-        results = eval(build_output.splitlines()[-1])
+        results = eval(line)
 
+        failures = 0
         for image, result in results.iteritems():
             if image in excluded_images:
-                self.assertEqual(result, 'fail')
+                if result is 'fail':
+                    continue
+                failures = failures + 1
+                LOG.warning(">>> Expected image '%s' to fail, please update"
+                            " the excluded_images in source file above if the"
+                            " image build has been fixed." % image)
             else:
-                self.assertNotEqual(result, 'fail')
+                if result is not 'fail':
+                    continue
+                failures = failures + 1
+                LOG.critical(">>> Expected image '%s' to succeed!" % image)
+
+        self.assertEqual(failures, 0, "%d failure(s) occurred" % failures)

tools/build-docker-image

@@ -35,7 +35,7 @@ EOF
 [ -f $TOPDIR/.buildconf ] && . $TOPDIR/.buildconf
 [ -f $IMGDIR/.buildconf ] && . $IMGDIR/.buildconf
 
-ARGS=$(getopt -o hr:n:t:pfuN -l help,namespace:,push,private-registry:,release,tag:,force-rm,no-cache,no-use-released-parent -- "$@") || { usage >&2; exit 2; }
+ARGS=$(getopt -o hr:n:t:pfuN -l help,namespace:,push,pull,private-registry:,release,tag:,force-rm,no-cache,no-use-released-parent -- "$@") || { usage >&2; exit 2; }
 
 eval set -- "$ARGS"
 

tox.ini

@@ -22,16 +22,17 @@ commands =
 
 [testenv:setupenv]
 whitelist_externals = bash
-commands = bash -c tests/setup_docker.sh
+commands = bash -c tests/setup_gate.sh
 
 [testenv:images]
 deps = -r{toxinidir}/test-requirements.txt
 whitelist_externals = find
                       bash
+                      sudo
 commands =
    find . -type f -name "*.pyc" -delete
    bash -c "if [ ! -d .testrepository ]; then testr init; fi"
-   testr run ^(test_images).*
+   sudo -g dockerroot testr run ^(test_images).*
 
 [testenv:startenv]
 whitelist_externals = bash