Use new manila-service-image with public-key auth

- Update devstack plugin
- Update tempest scenario tests

Implements bp reproducible-server-image

Change-Id: I7c71ee2f20074e04021b5c95d8c18096d21b0404
Closes-Bug: #1421104

contrib/ci/pre_test_hook.sh

@@ -45,6 +45,14 @@ else
     echo "MANILA_MULTI_BACKEND=False" >> $localrc_path
 fi
 
+# Enabling isolated metadata in Neutron is required because
+# Tempest creates isolated networks and created vm's in scenario tests don't
+# have access to Nova Metadata service. This leads to unavailability of
+# created vm's in scenario tests.
+echo '[[post-config|$Q_DHCP_CONF_FILE]]' >> $localrc_path
+echo '[DEFAULT]' >> $localrc_path
+echo "enable_isolated_metadata=True" >> $localrc_path
+
 # Go to Tempest dir and checkout stable commit to avoid possible
 # incompatibilities for plugin stored in Manila repo.
 TEMPEST_COMMIT="489f5e62"  # 15 June, 2015

contrib/tempest/tempest/config_share.py

@@ -114,15 +114,17 @@ ShareGroup = [
                      "These test may leave orphaned resources, so be careful "
                      "enabling this opt."),
     cfg.StrOpt("image_with_share_tools",
-               default="ubuntu_1204_nfs_cifs",
+               default="manila-service-image",
                help="Image name for vm booting with nfs/smb clients tool."),
     cfg.StrOpt("image_username",
-               default="ubuntu",
+               default="manila",
                help="Image username."),
-    # HINT(mkoderer): workaround for bug #1421104
     cfg.StrOpt("image_password",
-               default="ubuntu",
+               help="Image password. Should be used for "
-               help="Image password."),
+                    "'image_with_share_tools' without Nova Metadata support."),
+    cfg.StrOpt("client_vm_flavor_ref",
+               default="100",
+               help="Flavor used for client vm in scenario tests."),
     cfg.BoolOpt("run_extend_tests",
                 default=True,
                 help="Defines whether to run share extend tests or not."

contrib/tempest/tempest/scenario/manager_share.py

@@ -155,8 +155,9 @@ class ShareScenarioTest(manager.NetworkScenarioTest):
         if not CONF.share.image_with_share_tools:
             return super(ShareScenarioTest,
                          self).get_remote_client(*args, **kwargs)
-        # HINT(mkoderer): as workaround for bug #1421104 we have to ignore the
+        # NOTE(u_glide): We need custom implementation of this method until
-        # keypair and use the configured username and password
+        # original implementation depends on CONF.compute.ssh_auth_method
+        # option.
         server_or_ip = kwargs['server_or_ip']
         if isinstance(server_or_ip, six.string_types):
             ip = server_or_ip
@@ -164,11 +165,15 @@ class ShareScenarioTest(manager.NetworkScenarioTest):
             addr = server_or_ip['addresses'][CONF.compute.network_for_ssh][0]
             ip = addr['addr']
 
-        username = CONF.share.image_username
+        # NOTE(u_glide): Both options (pkey and password) are required here to
-        password = CONF.share.image_password
+        # support service images without Nova metadata support
+        client_params = {
+            'username': kwargs['username'],
+            'password': CONF.share.image_password,
+            'pkey': kwargs.get('private_key'),
+        }
 
-        linux_client = remote_client.RemoteClient(ip, username=username,
+        linux_client = remote_client.RemoteClient(ip, **client_params)
-                                                  password=password, pkey=None)
         try:
             linux_client.validate_authentication()
         except Exception:

contrib/tempest/tempest/scenario/test_share_basic_ops.py

@@ -26,7 +26,7 @@ CONF = config.CONF
 LOG = logging.getLogger(__name__)
 
 
-class TestShareBasicOps(manager.ShareScenarioTest):
+class ShareBasicOpsBase(manager.ShareScenarioTest):
 
     """This smoke test case follows this basic set of operations:
 
@@ -38,14 +38,14 @@ class TestShareBasicOps(manager.ShareScenarioTest):
      * Mount share
      * Terminate the instance
     """
-    protocol = "NFS"
+    protocol = None
 
     def setUp(self):
-        super(TestShareBasicOps, self).setUp()
+        super(ShareBasicOpsBase, self).setUp()
         # Setup image and flavor the test instance
         # Support both configured and injected values
         if not hasattr(self, 'flavor_ref'):
-            self.flavor_ref = CONF.compute.flavor_ref
+            self.flavor_ref = CONF.share.client_vm_flavor_ref
         if CONF.share.image_with_share_tools:
             images = self.images_client.list_images()
             for img in images:
@@ -56,7 +56,7 @@ class TestShareBasicOps(manager.ShareScenarioTest):
                 msg = ("Image %s not found" %
                        CONF.share.image_with_share_tools)
                 raise exceptions.InvalidConfiguration(message=msg)
-        self.ssh_user = CONF.compute.image_ssh_user
+        self.ssh_user = CONF.share.image_username
         LOG.debug('Starting test for i:{image}, f:{flavor}. '
                   'user: {ssh_user}'.format(
                       image=self.image_ref, flavor=self.flavor_ref,
@@ -73,7 +73,8 @@ class TestShareBasicOps(manager.ShareScenarioTest):
             'security_groups': security_groups,
         }
         instance = self.create_server(image=self.image_ref,
-                                      create_kwargs=create_kwargs)
+                                      create_kwargs=create_kwargs,
+                                      flavor=self.flavor_ref)
         return instance
 
     def init_ssh(self, instance, do_ping=False):
@@ -90,6 +91,10 @@ class TestShareBasicOps(manager.ShareScenarioTest):
             server_or_ip=floating_ip['ip'],
             username=self.ssh_user,
             private_key=self.keypair['private_key'])
+
+        # NOTE(u_glide): Workaround for bug #1465682
+        ssh_client = ssh_client.ssh_client
+
         self.share = self.shares_client.get_share(self.share['id'])
         if do_ping:
             server_ip = self.share['export_location'].split(":")[0]
@@ -97,7 +102,7 @@ class TestShareBasicOps(manager.ShareScenarioTest):
         return ssh_client
 
     def mount_share(self, location, ssh_client):
-        ssh_client.exec_command("sudo mount \"%s\" /mnt" % location)
+        raise NotImplementedError
 
     def umount_share(self, ssh_client):
         ssh_client.exec_command("sudo umount /mnt")
@@ -180,3 +185,33 @@ class TestShareBasicOps(manager.ShareScenarioTest):
                         ssh_client_inst2)
         data = self.read_data(ssh_client_inst2)
         self.assertEqual(test_data, data)
+
+
+class TestShareBasicOpsNFS(ShareBasicOpsBase):
+    protocol = "NFS"
+
+    def mount_share(self, location, ssh_client):
+        ssh_client.exec_command("sudo mount \"%s\" /mnt" % location)
+
+
+class TestShareBasicOpsCIFS(ShareBasicOpsBase):
+    protocol = "CIFS"
+
+    def mount_share(self, location, ssh_client):
+        location = location.replace("\\", "/")
+        ssh_client.exec_command(
+            "sudo mount.cifs \"%s\" /mnt -o guest" % location
+        )
+
+
+# NOTE(u_glide): this function is required to exclude ShareBasicOpsBase from
+# executed test cases.
+# See: https://docs.python.org/2/library/unittest.html#load-tests-protocol
+# for details.
+def load_tests(loader, tests, _):
+    result = []
+    for test_case in tests:
+        if type(test_case._tests[0]) is ShareBasicOpsBase:
+            continue
+        result.append(test_case)
+    return loader.suiteClass(result)

devstack/plugin.sh

@@ -85,15 +85,15 @@ SHARE_DRIVER=${SHARE_DRIVER:-manila.share.drivers.generic.GenericShareDriver}
 eval USER_HOME=~
 MANILA_PATH_TO_PUBLIC_KEY=${MANILA_PATH_TO_PUBLIC_KEY:-"$USER_HOME/.ssh/id_rsa.pub"}
 MANILA_PATH_TO_PRIVATE_KEY=${MANILA_PATH_TO_PRIVATE_KEY:-"$USER_HOME/.ssh/id_rsa"}
+MANILA_SERVICE_KEYPAIR_NAME=${MANILA_SERVICE_KEYPAIR_NAME:-"manila-service"}
 
-MANILA_SERVICE_INSTANCE_USER=${MANILA_SERVICE_INSTANCE_USER:-"ubuntu"}
+MANILA_SERVICE_INSTANCE_USER=${MANILA_SERVICE_INSTANCE_USER:-"manila"}
-MANILA_SERVICE_INSTANCE_PASSWORD=${MANILA_SERVICE_INSTANCE_PASSWORD:-"ubuntu"}
+MANILA_SERVICE_IMAGE_URL=${MANILA_SERVICE_IMAGE_URL:-"https://github.com/uglide/manila-image-elements/releases/download/0.1.0/manila-service-image.qcow2"}
-MANILA_SERVICE_IMAGE_URL=${MANILA_SERVICE_IMAGE_URL:-"https://www.dropbox.com/s/vi5oeh10q1qkckh/ubuntu_1204_nfs_cifs.qcow2"}
+MANILA_SERVICE_IMAGE_NAME=${MANILA_SERVICE_IMAGE_NAME:-"manila-service-image"}
-MANILA_SERVICE_IMAGE_NAME=${MANILA_SERVICE_IMAGE_NAME:-"ubuntu_1204_nfs_cifs"}
 
 MANILA_SERVICE_VM_FLAVOR_REF=${MANILA_SERVICE_VM_FLAVOR_REF:-100}
 MANILA_SERVICE_VM_FLAVOR_NAME=${MANILA_SERVICE_VM_FLAVOR_NAME:-"manila-service-flavor"}
-MANILA_SERVICE_VM_FLAVOR_RAM=${MANILA_SERVICE_VM_FLAVOR_RAM:-64}
+MANILA_SERVICE_VM_FLAVOR_RAM=${MANILA_SERVICE_VM_FLAVOR_RAM:-128}
 MANILA_SERVICE_VM_FLAVOR_DISK=${MANILA_SERVICE_VM_FLAVOR_DISK:-0}
 MANILA_SERVICE_VM_FLAVOR_VCPUS=${MANILA_SERVICE_VM_FLAVOR_VCPUS:-1}
 
@@ -141,7 +141,6 @@ function configure_default_backends {
         iniset $MANILA_CONF $group_name path_to_private_key $MANILA_PATH_TO_PRIVATE_KEY
         iniset $MANILA_CONF $group_name service_image_name $MANILA_SERVICE_IMAGE_NAME
         iniset $MANILA_CONF $group_name service_instance_user $MANILA_SERVICE_INSTANCE_USER
-        iniset $MANILA_CONF $group_name service_instance_password $MANILA_SERVICE_INSTANCE_PASSWORD
         iniset $MANILA_CONF $group_name driver_handles_share_servers True
     done
 }
@@ -273,6 +272,8 @@ function configure_manila {
         ssh-keygen -N "" -t rsa -f $MANILA_PATH_TO_PRIVATE_KEY;
     fi
 
+    iniset $MANILA_CONF DEFAULT manila_service_keypair_name $MANILA_SERVICE_KEYPAIR_NAME
+
     if is_service_enabled tls-proxy; then
         # Set the service port for a proxy to take the original
         iniset $MANILA_CONF DEFAULT osapi_share_listen_port $MANILA_SERVICE_PORT_INT
@@ -315,6 +316,11 @@ function configure_manila_ui {
 }
 
 
+function create_manila_service_keypair {
+    openstack keypair create $MANILA_SERVICE_KEYPAIR_NAME --public-key $MANILA_PATH_TO_PUBLIC_KEY
+}
+
+
 # create_service_share_servers - creates service Nova VMs, one per generic
 # driver, and only if it is configured to mode without handling of share servers.
 function create_service_share_servers {
@@ -329,7 +335,8 @@ function create_service_share_servers {
                 --flavor $MANILA_SERVICE_VM_FLAVOR_NAME \
                 --image $MANILA_SERVICE_IMAGE_NAME \
                 --nic net-id=$private_net_id \
-                --security-groups $MANILA_SERVICE_SECGROUP
+                --security-groups $MANILA_SERVICE_SECGROUP \
+                --key-name $MANILA_SERVICE_KEYPAIR_NAME
 
             vm_id=$(nova show $vm_name | grep ' id ' | get_field 2)
 
@@ -527,6 +534,9 @@ elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
     echo_summary "Creating Manila service image"
     create_manila_service_image
 
+    echo_summary "Creating Manila service keypair"
+    create_manila_service_keypair
+
     echo_summary "Creating Manila service VMs for generic driver \
         backends for which handlng of share servers is disabled."
     create_service_share_servers