Use bundletester for amulet test execution

Switch to using bundletester for execution of functional tests,
leveraging tox to build out test virtualenvs.

Rename amulet tests inline with gate-*, dev-* and dfs-*
naming standards.

Update README to refer to functional testing section of the charm
guide.

Also remove brittle auth conf checks which are failing at
cinder-ceph master. Even without those explicit checks, if auth
fails, functional tests will still fail and catch issues in gate.

Change-Id: I7b0009fa0e29dcaf7d6e5607af7b397e437aac79

Makefile

@@ -10,7 +10,7 @@ test:
 
 functional_test:
 	@echo Starting amulet deployment tests...
-	@juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 2700
+	@tox -e func27
 
 bin/charm_helpers_sync.py:
 	@mkdir -p bin

test-requirements.txt

@@ -7,3 +7,19 @@ flake8>=2.2.4,<=2.4.1
 os-testr>=0.4.1
 charm-tools>=2.0.0
 requests==2.6.0
+# BEGIN: Amulet OpenStack Charm Helper Requirements
+# Liberty client lower constraints
+amulet>=1.14.3,<2.0
+bundletester>=0.6.1,<1.0
+python-ceilometerclient>=1.5.0,<2.0
+python-cinderclient>=1.4.0,<2.0
+python-glanceclient>=1.1.0,<2.0
+python-heatclient>=0.8.0,<1.0
+python-keystoneclient>=1.7.1,<2.0
+python-neutronclient>=3.1.0,<4.0
+python-novaclient>=2.30.1,<3.0
+python-openstackclient>=1.7.0,<2.0
+python-swiftclient>=2.6.0,<3.0
+pika>=0.10.0,<1.0
+distro-info
+# END: Amulet OpenStack Charm Helper Requirements

tests/00-setup

@@ -1,31 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2016 Canonical Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-sudo add-apt-repository --yes ppa:juju/stable
-sudo apt-get update --yes
-sudo apt-get install --yes amulet \
-                           distro-info-data \
-                           python-cinderclient \
-                           python-distro-info \
-                           python-glanceclient \
-                           python-heatclient \
-                           python-keystoneclient \
-                           python-neutronclient \
-                           python-novaclient \
-                           python-pika \
-                           python-swiftclient

tests/016-basic-trusty-juno

@@ -1,25 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2016 Canonical Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Amulet tests on a basic cinder-backup deployment on trusty-juno."""
-
-from basic_deployment import CinderBackupBasicDeployment
-
-if __name__ == '__main__':
-    deployment = CinderBackupBasicDeployment(series='trusty',
-                                             openstack='cloud:trusty-juno',
-                                             source='cloud:trusty-updates/juno')
-    deployment.run_tests()

tests/README

@@ -1,68 +0,0 @@
-This directory provides Amulet tests that focus on verification of
-cinder-backup deployments.
-
-test_* methods are called in lexical sort order.
-
-Test name convention to ensure desired test order:
-    1xx service and endpoint checks
-    2xx relation checks
-    3xx config checks
-    4xx functional checks
-    9xx restarts and other final checks
-
-Common uses of backend relations in deployments:
-    - [ cinder, cinder-ceph ]
-    - [ cinder, cinder-backup ]
-    - [ cinder-ceph, ceph ]
-    - [ cinder-backup, ceph ]
-
-In order to run tests, you'll need charm-tools installed (in addition to
-juju, of course):
-    sudo add-apt-repository ppa:juju/stable
-    sudo apt-get update
-    sudo apt-get install charm-tools
-
-If you use a web proxy server to access the web, you'll need to set the
-AMULET_HTTP_PROXY environment variable to the http URL of the proxy server.
-
-The following examples demonstrate different ways that tests can be executed.
-All examples are run from the charm's root directory.
-
-  * To run all tests (starting with 00-setup):
-
-      make test
-
-  * To run a specific test module (or modules):
-
-      juju test -v -p AMULET_HTTP_PROXY 15-basic-trusty-icehouse
-
-  * To run a specific test module (or modules), and keep the environment
-    deployed after a failure:
-
-      juju test --set-e -v -p AMULET_HTTP_PROXY 15-basic-trusty-icehouse
-
-  * To re-run a test module against an already deployed environment (one
-    that was deployed by a previous call to 'juju test --set-e'):
-
-      ./tests/15-basic-trusty-icehouse
-
-For debugging and test development purposes, all code should be idempotent.
-In other words, the code should have the ability to be re-run without changing
-the results beyond the initial run.  This enables editing and re-running of a
-test module against an already deployed environment, as described above.
-
-Manual debugging tips:
-
-  * Set the following env vars before using the OpenStack CLI as admin:
-      export OS_AUTH_URL=http://`juju-deployer -f keystone 2>&1 | tail -n 1`:5000/v2.0
-      export OS_TENANT_NAME=admin
-      export OS_USERNAME=admin
-      export OS_PASSWORD=openstack
-      export OS_REGION_NAME=RegionOne
-
-  * Set the following env vars before using the OpenStack CLI as demoUser:
-      export OS_AUTH_URL=http://`juju-deployer -f keystone 2>&1 | tail -n 1`:5000/v2.0
-      export OS_TENANT_NAME=demoTenant
-      export OS_USERNAME=demoUser
-      export OS_PASSWORD=password
-      export OS_REGION_NAME=RegionOne

tests/README.md

@@ -0,0 +1,9 @@
+# Overview
+
+This directory provides Amulet tests to verify basic deployment functionality
+from the perspective of this charm, its requirements and its features, as
+exercised in a subset of the full OpenStack deployment test bundle topology.
+
+For full details on functional testing of OpenStack charms please refer to
+the [functional testing](http://docs.openstack.org/developer/charm-guide/testing.html#functional-testing)
+section of the OpenStack Charm Guide.

tests/basic_deployment.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 #
 # Copyright 2016 Canonical Ltd
 #
@@ -45,6 +45,14 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment):
         self._add_relations()
         self._configure_services()
         self._deploy()
+
+        u.log.info('Waiting on extended status checks...')
+
+        # XXX: cinder-backup workload status ignored until it grows support
+        #      https://bugs.launchpad.net/bugs/1604580
+        exclude_services = ['mysql', 'cinder-backup']
+        self._auto_wait_for_status(exclude_services=exclude_services)
+
         self._initialize_tests()
 
     def _add_services(self):
@@ -118,25 +126,19 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment):
     def _initialize_tests(self):
         """Perform final initialization before tests get run."""
         # Access the sentries for inspecting service units
-        u.log.debug('!!!!!')
+        self.mysql_sentry = self.d.sentry['mysql'][0]
-        u.log.debug(dir(self.d.sentry))
+        self.keystone_sentry = self.d.sentry['keystone'][0]
-
+        self.rabbitmq_sentry = self.d.sentry['rabbitmq-server'][0]
-        self.mysql_sentry = self.d.sentry.unit['mysql/0']
+        self.cinder_sentry = self.d.sentry['cinder'][0]
-        self.keystone_sentry = self.d.sentry.unit['keystone/0']
+        self.ceph0_sentry = self.d.sentry['ceph'][0]
-        self.rabbitmq_sentry = self.d.sentry.unit['rabbitmq-server/0']
+        self.ceph1_sentry = self.d.sentry['ceph'][1]
-        self.cinder_sentry = self.d.sentry.unit['cinder/0']
+        self.ceph2_sentry = self.d.sentry['ceph'][2]
-        self.ceph0_sentry = self.d.sentry.unit['ceph/0']
+        self.cinder_backup_sentry = self.d.sentry['cinder-backup'][0]
-        self.ceph1_sentry = self.d.sentry.unit['ceph/1']
-        self.ceph2_sentry = self.d.sentry.unit['ceph/2']
-        self.cinder_backup_sentry = self.d.sentry.unit['cinder-backup/0']
         u.log.debug('openstack release val: {}'.format(
             self._get_openstack_release()))
         u.log.debug('openstack release str: {}'.format(
             self._get_openstack_release_string()))
 
-        # Let things settle a bit original moving forward
-        time.sleep(30)
-
         # Authenticate admin with keystone
         self.keystone = u.authenticate_keystone_admin(self.keystone_sentry,
                                                       user='admin',
@@ -322,7 +324,14 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment):
 
     def get_broker_response(self):
         broker_request = self.get_broker_request()
-        response_key = "broker-rsp-cinder-backup-0"
+        u.log.debug('Broker request: {}'.format(broker_request))
+
+        response_key = "broker-rsp-{}-{}".format(
+            self.cinder_backup_sentry.info['service'],
+            self.cinder_backup_sentry.info['unit']
+        )
+        u.log.debug('Checking response_key: {}'.format(response_key))
+
         ceph_sentrys = [self.ceph0_sentry,
                         self.ceph1_sentry,
                         self.ceph2_sentry]
@@ -332,6 +341,7 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment):
                 broker_response = json.loads(relation_data[response_key])
                 if (broker_request['request-id'] ==
                         broker_response['request-id']):
+                    u.log.debug('broker_response: {}'.format(broker_response))
                     return broker_response
 
     def test_200_cinderbackup_ceph_ceph_relation(self):
@@ -374,11 +384,6 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment):
         if ret:
             msg = u.relation_error('cinder cinder-backup backup-backend', ret)
             amulet.raise_status(amulet.FAIL, msg=msg)
-        broker_response = self.get_broker_response()
-        if not broker_response or broker_response['exit-code'] != 0:
-            msg = ('Broker request invalid'
-                   ' or failed: {}'.format(broker_response))
-            amulet.raise_status(amulet.FAIL, msg=msg)
 
     def test_202_cinderbackup_cinder_backend_relation(self):
         u.log.debug('Checking cinder-backup:backup-backend to '
@@ -525,15 +530,7 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment):
         unit = self.cinder_sentry
         conf = '/etc/cinder/cinder.conf'
         unit_mq = self.rabbitmq_sentry
-        unit_ks = self.keystone_sentry
         rel_mq_ci = unit_mq.relation('amqp', 'cinder:amqp')
-        rel_ks_ci = unit_ks.relation('identity-service',
-                                     'cinder:identity-service')
-
-        auth_uri = 'http://' + rel_ks_ci['auth_host'] + \
-                   ':' + rel_ks_ci['service_port'] + '/'
-        auth_url = ('http://%s:%s/' %
-                    (rel_ks_ci['auth_host'], rel_ks_ci['auth_port']))
 
         expected = {
             'DEFAULT': {
@@ -549,12 +546,6 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment):
                 'backup_ceph_pool': 'cinder-backup',
                 'backup_ceph_user': 'cinder-backup'
             },
-            'keystone_authtoken': {
-                'admin_user': rel_ks_ci['service_username'],
-                'admin_password': rel_ks_ci['service_password'],
-                'admin_tenant_name': rel_ks_ci['service_tenant'],
-                'auth_uri': auth_uri
-            },
             'cinder-ceph': {
                 'volume_backend_name': 'cinder-ceph',
                 'volume_driver': 'cinder.volume.drivers.rbd.RBDDriver',
@@ -569,18 +560,6 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment):
             'rabbit_password': rel_mq_ci['password'],
             'rabbit_host': rel_mq_ci['hostname'],
         }
-        if self._get_openstack_release() >= self.trusty_liberty:
-            expected['keystone_authtoken'] = {
-                'auth_uri': auth_uri.rstrip('/'),
-                'auth_url': auth_url.rstrip('/'),
-                'auth_plugin': 'password',
-                'project_domain_id': 'default',
-                'user_domain_id': 'default',
-                'project_name': 'services',
-                'username': rel_ks_ci['service_username'],
-                'password': rel_ks_ci['service_password'],
-                'signing_dir': '/var/cache/cinder'
-            }
 
         if self._get_openstack_release() >= self.trusty_kilo:
             # Kilo or later
@@ -588,8 +567,6 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment):
         else:
             # Juno or earlier
             expected['DEFAULT'].update(expected_rmq)
-            expected['keystone_authtoken']['auth_host'] = \
-                rel_ks_ci['auth_host']
 
         for section, pairs in expected.iteritems():
             ret = u.validate_config_data(unit, conf, section, pairs)
@@ -631,7 +608,15 @@ class CinderBackupBasicDeployment(OpenStackAmuletDeployment):
         u.log.debug('Cinder api check (volumes.list): {}'.format(check))
         assert(check == [])
 
-    def test_401_create_delete_volume(self):
+    def test_401_check_broker_reponse(self):
+        u.log.debug('Checking broker response')
+        broker_response = self.get_broker_response()
+        if not broker_response or broker_response['exit-code'] != 0:
+            msg = ('Broker request invalid'
+                   ' or failed: {}'.format(broker_response))
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+    def test_402_create_delete_volume(self):
         """Create a cinder volume and delete it."""
         u.log.debug('Creating, checking and deleting cinder volume...')
         vol_new = u.create_cinder_volume(self.cinder)

tests/dev-basic-xenial-newton

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 #
 # Copyright 2016 Canonical Ltd
 #
@@ -14,10 +14,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Amulet tests on a basic cinder-backup deployment on vivid-kilo."""
+"""Amulet tests on a basic cinder backup deployment on xenial-newton."""
 
 from basic_deployment import CinderBackupBasicDeployment
 
 if __name__ == '__main__':
-    deployment = CinderBackupBasicDeployment(series='vivid')
+    deployment = CinderBackupBasicDeployment(series='xenial',
+                                             openstack='cloud:xenial-newton',
+                                             source='cloud:xenial-updates/newton')
     deployment.run_tests()

tests/dev-basic-yakkety-newton

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 #
 # Copyright 2016 Canonical Ltd
 #
@@ -14,10 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Amulet tests on a basic cinder-backup deployment on wily-liberty."""
+"""Amulet tests on a basic Cinder backup deployment on yakkety-newton."""
 
 from basic_deployment import CinderBackupBasicDeployment
 
 if __name__ == '__main__':
-    deployment = CinderBackupBasicDeployment(series='wily')
+    deployment = CinderBackupBasicDeployment(series='yakkety')
     deployment.run_tests()

tests/gate-basic-precise-icehouse

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 #
 # Copyright 2016 Canonical Ltd
 #

tests/gate-basic-trusty-icehouse

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 #
 # Copyright 2016 Canonical Ltd
 #

tests/gate-basic-trusty-kilo

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 #
 # Copyright 2016 Canonical Ltd
 #

tests/gate-basic-xenial-mitaka

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 #
 # Copyright 2016 Canonical Ltd
 #

tests/tests.yaml

@@ -1,20 +1,17 @@
-bootstrap: true
+# Bootstrap the model if necessary.
-reset: true
+bootstrap: True
-virtualenv: true
+# Re-use bootstrap node instead of destroying/re-bootstrapping.
-makefile:
+reset: True
-    - lint
+# Use tox/requirements to drive the venv instead of bundletester's venv feature.
-    - test
+virtualenv: False
-sources:
+# Leave makefile empty, otherwise unit/lint tests will rerun ahead of amulet.
-    - ppa:juju/stable
+makefile: []
-packages:
+# Do not specify juju PPA sources.  Juju is presumed to be pre-installed
-    - amulet
+# and configured in all test runner environments.
-    - distro-info-data
+#sources:
-    - python-cinderclient
+# Do not specify or rely on system packages.
-    - python-distro-info
+#packages:
-    - python-glanceclient
+# Do not specify python packages here.  Use test-requirements.txt
-    - python-heatclient
+# and tox instead.  ie. The venv is constructed before bundletester
-    - python-keystoneclient
+# is invoked.
-    - python-neutronclient
+#python-packages:
-    - python-novaclient
-    - python-pika
-    - python-swiftclient

tox.ini

@@ -5,6 +5,8 @@ skipsdist = True
 [testenv]
 setenv = VIRTUAL_ENV={envdir}
          PYTHONHASHSEED=0
+         AMULET_SETUP_TIMEOUT=2700
+passenv = HOME TERM AMULET_HTTP_PROXY AMULET_OS_VIP
 install_command =
   pip install --allow-unverified python-apt {opts} {packages}
 commands = ostestr {posargs}
@@ -18,12 +20,56 @@ deps = -r{toxinidir}/requirements.txt
 basepython = python2.7
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
-commands = flake8 {posargs} hooks unit_tests tests
+commands = flake8 {posargs} --exclude */charmhelpers hooks unit_tests tests
            charm-proof
 
 [testenv:venv]
 commands = {posargs}
 
+[testenv:func27-noop]
+# DRY RUN - For Debug
+basepython = python2.7
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+commands =
+    bundletester -vl DEBUG -r json -o func-results.json --test-pattern "gate-*" -n --no-destroy
+
+[testenv:func27]
+# Charm Functional Test
+# Run all gate tests which are +x (expected to always pass)
+basepython = python2.7
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+commands =
+    bundletester -vl DEBUG -r json -o func-results.json --test-pattern "gate-*" --no-destroy
+
+[testenv:func27-smoke]
+# Charm Functional Test
+# Run a specific test as an Amulet smoke test (expected to always pass)
+basepython = python2.7
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+commands =
+    bundletester -vl DEBUG -r json -o func-results.json gate-basic-xenial-mitaka --no-destroy
+
+[testenv:func27-dfs]
+# Charm Functional Test
+# Run all deploy-from-source tests which are +x (may not always pass!)
+basepython = python2.7
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+commands =
+    bundletester -vl DEBUG -r json -o func-results.json --test-pattern "dfs-*" --no-destroy
+
+[testenv:func27-dev]
+# Charm Functional Test
+# Run all development test targets which are +x (may not always pass!)
+basepython = python2.7
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+commands =
+    bundletester -vl DEBUG -r json -o func-results.json --test-pattern "dev-*" --no-destroy
+
 [flake8]
 ignore = E402,E226
 exclude = hooks/charmhelpers