From eaaa86f1f42c776e93709057262d17dcb63f0c3d Mon Sep 17 00:00:00 2001 From: Rodrigo Barbieri Date: Thu, 28 Jan 2016 16:51:28 -0200 Subject: [PATCH] Introduced Data Service Added a new service defined as data, whose purpose is to receive requests, process data operations such as copying, migration, backup, and send back the response after operation has been completed. In this patch the service has no methods, it is empty, the operation it should perform will be added in subsequent patches. Implements: blueprint data-copy-service Change-Id: I31365c8ac4197541af175c82f8f18a3d11740a99 --- devstack/plugin.sh | 3 +- devstack/settings | 4 +- manila/cmd/all.py | 3 +- manila/cmd/data.py | 49 +++++++++++++++++++++ manila/common/config.py | 6 +++ manila/data/__init__.py | 0 manila/data/manager.py | 38 ++++++++++++++++ manila/data/rpcapi.py | 41 +++++++++++++++++ manila/tests/cmd/__init__.py | 0 manila/tests/cmd/test_data.py | 46 +++++++++++++++++++ manila/tests/data/__init__.py | 0 manila/tests/data/test_manager.py | 41 +++++++++++++++++ manila/tests/data/test_rpcapi.py | 73 +++++++++++++++++++++++++++++++ setup.cfg | 1 + 14 files changed, 302 insertions(+), 3 deletions(-) create mode 100755 manila/cmd/data.py create mode 100644 manila/data/__init__.py create mode 100644 manila/data/manager.py create mode 100644 manila/data/rpcapi.py create mode 100644 manila/tests/cmd/__init__.py create mode 100644 manila/tests/cmd/test_data.py create mode 100644 manila/tests/data/__init__.py create mode 100644 manila/tests/data/test_manager.py create mode 100644 manila/tests/data/test_rpcapi.py diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 3ae35829a4..886e46adfd 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -586,6 +586,7 @@ function start_manila { screen_it m-api "cd $MANILA_DIR && $MANILA_BIN_DIR/manila-api --config-file $MANILA_CONF" screen_it m-shr "cd $MANILA_DIR && $MANILA_BIN_DIR/manila-share --config-file $MANILA_CONF" screen_it m-sch "cd $MANILA_DIR && $MANILA_BIN_DIR/manila-scheduler --config-file $MANILA_CONF" + screen_it m-dat "cd $MANILA_DIR && $MANILA_BIN_DIR/manila-data --config-file $MANILA_CONF" # Start proxies if enabled if is_service_enabled tls-proxy; then @@ -596,7 +597,7 @@ function start_manila { # stop_manila - Stop running processes function stop_manila { # Kill the manila screen windows - for serv in m-api m-sch m-shr; do + for serv in m-api m-sch m-shr m-dat; do screen -S $SCREEN_NAME -p $serv -X kill done } diff --git a/devstack/settings b/devstack/settings index 7a546fad65..5b0f47b857 100644 --- a/devstack/settings +++ b/devstack/settings @@ -157,9 +157,11 @@ MANILA_ZFSONLINUX_REPLICATION_DOMAIN=${MANILA_ZFSONLINUX_REPLICATION_DOMAIN:-"ZF # Enable manila services # ---------------------- # We have to add Manila to enabled services for screen_it to work -# It consists of 3 parts: m-api (API), m-shr (Share) and m-sch (Scheduler). +# It consists of 4 parts: m-api (API), m-shr (Share), m-sch (Scheduler) +# and m-dat (Data). enable_service manila enable_service m-api enable_service m-shr enable_service m-sch +enable_service m-dat diff --git a/manila/cmd/all.py b/manila/cmd/all.py index c8bd24d6b3..bcd427179b 100755 --- a/manila/cmd/all.py +++ b/manila/cmd/all.py @@ -62,7 +62,8 @@ def main(): except (Exception, SystemExit): LOG.exception(_LE('Failed to load osapi_share')) - for binary in ['manila-share', 'manila-scheduler', 'manila-api']: + for binary in ['manila-share', 'manila-scheduler', 'manila-api', + 'manila-data']: try: launcher.launch_service(service.Service.create(binary=binary)) except (Exception, SystemExit): diff --git a/manila/cmd/data.py b/manila/cmd/data.py new file mode 100755 index 0000000000..8519046425 --- /dev/null +++ b/manila/cmd/data.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +# Copyright 2015, Hitachi Data Systems. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Starter script for manila data copy service.""" + +import eventlet +eventlet.monkey_patch() + +import sys + +from oslo_config import cfg +from oslo_log import log + +from manila import i18n +i18n.enable_lazy() +from manila.common import config # Need to register global_opts # noqa +from manila import service +from manila import utils +from manila import version + +CONF = cfg.CONF + + +def main(): + log.register_options(CONF) + CONF(sys.argv[1:], project='manila', + version=version.version_string()) + log.setup(CONF, "manila") + utils.monkey_patch() + server = service.Service.create(binary='manila-data') + service.serve(server) + service.wait() + + +if __name__ == '__main__': + main() diff --git a/manila/common/config.py b/manila/common/config.py index 47ff990e12..988c438746 100644 --- a/manila/common/config.py +++ b/manila/common/config.py @@ -63,6 +63,9 @@ global_opts = [ cfg.StrOpt('share_topic', default='manila-share', help='The topic share nodes listen on.'), + cfg.StrOpt('data_topic', + default='manila-data', + help='The topic data nodes listen on.'), cfg.BoolOpt('enable_v1_api', default=False, help=_('Deploy v1 of the Manila API. This option is ' @@ -106,6 +109,9 @@ global_opts = [ cfg.StrOpt('share_manager', default='manila.share.manager.ShareManager', help='Full class name for the share manager.'), + cfg.StrOpt('data_manager', + default='manila.data.manager.DataManager', + help='Full class name for the data manager.'), cfg.StrOpt('host', default=socket.gethostname(), help='Name of this node. This can be an opaque identifier. ' diff --git a/manila/data/__init__.py b/manila/data/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/manila/data/manager.py b/manila/data/manager.py new file mode 100644 index 0000000000..4339ddcd2d --- /dev/null +++ b/manila/data/manager.py @@ -0,0 +1,38 @@ +# Copyright 2015, Hitachi Data Systems. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Data Service +""" + +from oslo_config import cfg +from oslo_log import log + +from manila import manager + +LOG = log.getLogger(__name__) + +CONF = cfg.CONF + + +class DataManager(manager.Manager): + """Receives requests to handle data and sends responses.""" + + RPC_API_VERSION = '1.0' + + def __init__(self, service_name=None, *args, **kwargs): + super(DataManager, self).__init__(*args, **kwargs) + + def init_host(self): + pass diff --git a/manila/data/rpcapi.py b/manila/data/rpcapi.py new file mode 100644 index 0000000000..ae7e9dfcbc --- /dev/null +++ b/manila/data/rpcapi.py @@ -0,0 +1,41 @@ +# Copyright 2015, Hitachi Data Systems. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Client side of the data manager RPC API. +""" + +from oslo_config import cfg +import oslo_messaging as messaging + +from manila import rpc + +CONF = cfg.CONF + + +class DataAPI(object): + """Client side of the data RPC API. + + API version history: + + 1.0 - Initial version. + """ + + BASE_RPC_API_VERSION = '1.0' + + def __init__(self): + super(DataAPI, self).__init__() + target = messaging.Target(topic=CONF.data_topic, + version=self.BASE_RPC_API_VERSION) + self.client = rpc.get_client(target, version_cap='1.0') diff --git a/manila/tests/cmd/__init__.py b/manila/tests/cmd/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/manila/tests/cmd/test_data.py b/manila/tests/cmd/test_data.py new file mode 100644 index 0000000000..9ed00b602b --- /dev/null +++ b/manila/tests/cmd/test_data.py @@ -0,0 +1,46 @@ +# Copyright 2015, Hitachi Data Systems. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys + +from manila.cmd import data as manila_data +from manila import test +from manila import version + +CONF = manila_data.CONF + + +class ManilaCmdDataTestCase(test.TestCase): + + def test_main(self): + sys.argv = ['manila-data'] + self.mock_object(manila_data.log, 'setup') + self.mock_object(manila_data.log, 'register_options') + self.mock_object(manila_data.utils, 'monkey_patch') + self.mock_object(manila_data.service.Service, 'create') + self.mock_object(manila_data.service, 'serve') + self.mock_object(manila_data.service, 'wait') + + manila_data.main() + + self.assertEqual('manila', CONF.project) + self.assertEqual(version.version_string(), CONF.version) + manila_data.log.setup.assert_called_once_with(CONF, "manila") + manila_data.log.register_options.assert_called_once_with(CONF) + manila_data.utils.monkey_patch.assert_called_once_with() + manila_data.service.Service.create.assert_called_once_with( + binary='manila-data') + manila_data.service.wait.assert_called_once_with() + manila_data.service.serve.assert_called_once_with( + manila_data.service.Service.create.return_value) diff --git a/manila/tests/data/__init__.py b/manila/tests/data/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/manila/tests/data/test_manager.py b/manila/tests/data/test_manager.py new file mode 100644 index 0000000000..02716caa26 --- /dev/null +++ b/manila/tests/data/test_manager.py @@ -0,0 +1,41 @@ +# Copyright 2015, Hitachi Data Systems. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Tests For Data Manager +""" + +from oslo_config import cfg + +from manila import context +from manila.data import manager +from manila import test + +CONF = cfg.CONF + + +class DataManagerTestCase(test.TestCase): + """Test case for data manager.""" + + manager_cls = manager.DataManager + + def setUp(self): + super(DataManagerTestCase, self).setUp() + self.manager = self.manager_cls() + self.context = context.RequestContext('fake_user', 'fake_project') + self.topic = 'fake_topic' + + def test_init(self): + manager = self.manager + self.assertIsNotNone(manager) diff --git a/manila/tests/data/test_rpcapi.py b/manila/tests/data/test_rpcapi.py new file mode 100644 index 0000000000..52d2951993 --- /dev/null +++ b/manila/tests/data/test_rpcapi.py @@ -0,0 +1,73 @@ +# Copyright 2015, Hitachi Data Systems. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Unit Tests for manila.data.rpcapi +""" + +import copy + +import mock +from oslo_config import cfg + +from manila import context +from manila.data import rpcapi as data_rpcapi +from manila import test + +CONF = cfg.CONF + + +class DataRpcAPITestCase(test.TestCase): + + def setUp(self): + super(DataRpcAPITestCase, self).setUp() + + def tearDown(self): + super(DataRpcAPITestCase, self).tearDown() + + def _test_data_api(self, method, rpc_method, fanout=False, **kwargs): + ctxt = context.RequestContext('fake_user', 'fake_project') + rpcapi = data_rpcapi.DataAPI() + expected_retval = 'foo' if method == 'call' else None + + target = { + "fanout": fanout, + "version": kwargs.pop('version', '1.0'), + } + expected_msg = copy.deepcopy(kwargs) + + self.fake_args = None + self.fake_kwargs = None + + def _fake_prepare_method(*args, **kwds): + for kwd in kwds: + self.assertEqual(target[kwd], kwds[kwd]) + return rpcapi.client + + def _fake_rpc_method(*args, **kwargs): + self.fake_args = args + self.fake_kwargs = kwargs + if expected_retval: + return expected_retval + + with mock.patch.object(rpcapi.client, "prepare") as mock_prepared: + mock_prepared.side_effect = _fake_prepare_method + + with mock.patch.object(rpcapi.client, rpc_method) as mock_method: + mock_method.side_effect = _fake_rpc_method + retval = getattr(rpcapi, method)(ctxt, **kwargs) + self.assertEqual(expected_retval, retval) + expected_args = [ctxt, method, expected_msg] + for arg, expected_arg in zip(self.fake_args, expected_args): + self.assertEqual(expected_arg, arg) diff --git a/setup.cfg b/setup.cfg index 7ada079508..0bd48cc41f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,7 @@ packages = console_scripts = manila-all = manila.cmd.all:main manila-api = manila.cmd.api:main + manila-data = manila.cmd.data:main manila-manage = manila.cmd.manage:main manila-rootwrap = oslo_rootwrap.cmd:main manila-scheduler = manila.cmd.scheduler:main