From 403102e8e9d41c88d46e3860cd2f520bdc44906c Mon Sep 17 00:00:00 2001 From: Doron Chen Date: Thu, 6 Aug 2015 11:48:50 +0300 Subject: [PATCH] Initial Cookiecutter Commit. Change-Id: I58c1914ca033c2c40ba2b63a3c07e0a9a8397ba4 --- .coveragerc | 7 + .gitignore | 54 +++ .gitreview | 4 + .mailmap | 3 + .testr.conf | 7 + CONTRIBUTING.rst | 17 + Deploy/playbook/common.yml | 2 +- .../files/get_image_id.py | 27 +- .../files/pull_tenant_image_main.yml | 2 +- .../roles/common_files/swift_config.py | 136 +++--- Engine/SBus/SBusPythonFacade/SBus.py | 91 ++-- Engine/SBus/SBusPythonFacade/SBusDatagram.py | 118 ++--- .../SBusPythonFacade/SBusFileDescription.py | 15 +- .../SBusPythonFacade/SBusStorletCommand.py | 21 +- Engine/SBus/SBusPythonFacade/setup.py | 10 +- .../send_halt_cmd_to_daemon_factory.py | 64 +-- .../storlet_daemon_factory/daemon_factory.py | 308 +++++++------ Engine/storlet_daemon_factory/setup.py | 9 +- Engine/swift/setup.py | 5 +- Engine/swift/storlet_gateway/__init__.py | 3 - .../storlet_gateway/storlet_docker_gateway.py | 111 +++-- .../swift/storlet_gateway/storlet_runtime.py | 434 ++++++++++-------- .../storlet_gateway/storlet_stub_gateway.py | 18 +- .../storlet_middleware/storlet_common.py | 66 +-- .../storlet_middleware/storlet_handler.py | 215 ++++----- HACKING.rst | 4 + LICENSE | 28 +- MANIFEST.in | 6 + README.rst | 19 + SystemTests/RH_and_CSS_test.py | 137 +++--- SystemTests/SLO_test.py | 195 ++++---- SystemTests/execdep_test.py | 110 +++-- SystemTests/half_storlet_test.py | 175 +++---- SystemTests/identity_storlet_test.py | 251 +++++----- SystemTests/metadata_storlet_test.py | 126 ++--- SystemTests/storlets_test_utils.py | 82 ++-- SystemTests/sys_test.py | 117 ++--- SystemTests/sys_test_params.py | 2 +- SystemTests/textui_progress_bar.py | 49 +- babel.cfg | 2 + doc/source/conf.py | 75 +++ doc/source/contributing.rst | 4 + doc/source/index.rst | 25 + doc/source/installation.rst | 12 + doc/source/readme.rst | 1 + doc/source/usage.rst | 7 + openstack-common.conf | 6 + requirements.txt | 6 + setup.cfg | 47 ++ setup.py | 29 ++ storlets/__init__.py | 18 + storlets/tests/__init__.py | 0 storlets/tests/base.py | 23 + storlets/tests/test_storlets.py | 28 ++ test-requirements.txt | 15 + tox.ini | 35 ++ 56 files changed, 2028 insertions(+), 1353 deletions(-) create mode 100644 .coveragerc create mode 100644 .gitignore create mode 100644 .gitreview create mode 100644 .mailmap create mode 100644 .testr.conf create mode 100644 CONTRIBUTING.rst create mode 100644 HACKING.rst create mode 100644 MANIFEST.in create mode 100644 README.rst create mode 100644 babel.cfg create mode 100755 doc/source/conf.py create mode 100644 doc/source/contributing.rst create mode 100644 doc/source/index.rst create mode 100644 doc/source/installation.rst create mode 100644 doc/source/readme.rst create mode 100644 doc/source/usage.rst create mode 100644 openstack-common.conf create mode 100644 requirements.txt create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 storlets/__init__.py create mode 100644 storlets/tests/__init__.py create mode 100644 storlets/tests/base.py create mode 100644 storlets/tests/test_storlets.py create mode 100644 test-requirements.txt create mode 100644 tox.ini diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..c4727306 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,7 @@ +[run] +branch = True +source = storlets +omit = storlets/openstack/* + +[report] +ignore-errors = True diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..e6257802 --- /dev/null +++ b/.gitignore @@ -0,0 +1,54 @@ +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg +*.egg-info +dist +build +.eggs +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml +.testrepository +.venv + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Complexity +output/*.html +output/*/index.html + +# Sphinx +doc/build + +# pbr generates these +AUTHORS +ChangeLog + +# Editors +*~ +.*.swp +.*sw? diff --git a/.gitreview b/.gitreview new file mode 100644 index 00000000..1d90109f --- /dev/null +++ b/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=review.openstack.org +port=29418 +project=openstack/storlets.git diff --git a/.mailmap b/.mailmap new file mode 100644 index 00000000..516ae6fe --- /dev/null +++ b/.mailmap @@ -0,0 +1,3 @@ +# Format is: +# +# diff --git a/.testr.conf b/.testr.conf new file mode 100644 index 00000000..6d83b3c4 --- /dev/null +++ b/.testr.conf @@ -0,0 +1,7 @@ +[DEFAULT] +test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ + OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ + OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \ + ${PYTHON:-python} -m subunit.run discover -t ./ . $LISTOPT $IDOPTION +test_id_option=--load-list $IDFILE +test_list_option=--list diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 00000000..f4dd3a40 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,17 @@ +If you would like to contribute to the development of OpenStack, you must +follow the steps in this page: + + http://docs.openstack.org/infra/manual/developers.html + +If you already have a good understanding of how the system works and your +OpenStack accounts are set up, you can skip to the development workflow +section of this documentation to learn how changes to OpenStack should be +submitted for review via the Gerrit tool: + + http://docs.openstack.org/infra/manual/developers.html#development-workflow + +Pull requests submitted through GitHub will be ignored. + +Bugs should be filed on Launchpad, not GitHub: + + https://bugs.launchpad.net/storlets diff --git a/Deploy/playbook/common.yml b/Deploy/playbook/common.yml index 0e945912..31a76466 100644 --- a/Deploy/playbook/common.yml +++ b/Deploy/playbook/common.yml @@ -1,6 +1,6 @@ ibm_container_install_dir: opt/ibm lxc_device: /home/docker_device -storlet_source_dir: ~/swift-storlets +storlet_source_dir: ~/storlets python_dist_packages_dir: usr/local/lib/python2.7/dist-packages storlet_gateway_conf_file: /etc/swift/storlet_docker_gateway.conf diff --git a/Deploy/playbook/roles/ansible_storlet_management/files/get_image_id.py b/Deploy/playbook/roles/ansible_storlet_management/files/get_image_id.py index e60d1515..c66fcb62 100755 --- a/Deploy/playbook/roles/ansible_storlet_management/files/get_image_id.py +++ b/Deploy/playbook/roles/ansible_storlet_management/files/get_image_id.py @@ -19,50 +19,51 @@ Limitations under the License. @author: cdoron ''' -import sys -import subprocess import json +import subprocess +import sys + def extractId(tar_file_name, repository, tag): subprocess.call(['tar', 'xf', tar_file_name, 'repositories']) repository_file = open('repositories') j = json.loads(repository_file.read()) - if not repository in j: - print "Not Found" + if repository not in j: + print("Not Found") else: pairs = j[repository] if tag: if tag not in pairs: - print "Not Found" + print("Not Found") else: - print pairs[tag] + print(pairs[tag]) else: if len(pairs) != 1: - print "No tag supplied. Ambiguous" + print("No tag supplied. Ambiguous") else: - print pairs.values()[0] + print(pairs.values()[0]) repository_file.close() subprocess.call(['rm', '-f', 'repositories']) + def usage(argv): - print argv[0] + " [tag]" + print(argv[0] + " [tag]") + def main(argv): if len(argv) < 3 or len(argv) > 4: usage(argv) return - + tar_file_name = argv[1] repository = argv[2] tag = None if len(argv) >= 4: tag = argv[3] - + extractId(tar_file_name, repository, tag) if __name__ == "__main__": main(sys.argv) - - diff --git a/Deploy/playbook/roles/ansible_storlet_management/files/pull_tenant_image_main.yml b/Deploy/playbook/roles/ansible_storlet_management/files/pull_tenant_image_main.yml index fed0245b..dc481f5f 100644 --- a/Deploy/playbook/roles/ansible_storlet_management/files/pull_tenant_image_main.yml +++ b/Deploy/playbook/roles/ansible_storlet_management/files/pull_tenant_image_main.yml @@ -21,5 +21,5 @@ command: docker pull {{ hostvars[groups['docker'][0]]['inventory_hostname'] }}:{{ docker_registry_port }}/{{ tenant_id.stdout_lines[0] }} - name: shutdown_container - shell: "{{ lxc_device }}/scripts/send_halt_cmd_to_daemon_factory.py + shell: "/usr/bin/python {{ lxc_device }}/scripts/send_halt_cmd_to_daemon_factory.py {{ lxc_device }}/pipes/scopes/AUTH_{{ tenant_id.stdout_lines[0] }}/factory_pipe" diff --git a/Deploy/playbook/roles/common_files/swift_config.py b/Deploy/playbook/roles/common_files/swift_config.py index e6afadb7..2a340327 100644 --- a/Deploy/playbook/roles/common_files/swift_config.py +++ b/Deploy/playbook/roles/common_files/swift_config.py @@ -13,21 +13,22 @@ See the License for the specific language governing permissions and Limitations under the License. -------------------------------------------------------------------------''' - ''' @author: cdoron ''' +import ConfigParser +import fileinput import os -import sys import pwd import shutil -import fileinput -import ConfigParser +import sys + def _chown_to_swift(path): uc = pwd.getpwnam('swift') os.chown(path, uc.pw_uid, uc.pw_gid) + def _unpatch_pipeline_line(orig_line, storlet_middleware): mds = list() for md in orig_line.split(): @@ -37,13 +38,14 @@ def _unpatch_pipeline_line(orig_line, storlet_middleware): if storlet_middleware in mds: mds.remove(storlet_middleware) - + new_line = 'pipeline =' for md in mds: new_line += ' ' + md return new_line + '\n' - + + def _patch_proxy_pipeline_line(orig_line, storlet_middleware): mds = list() for md in orig_line.split(): @@ -53,25 +55,26 @@ def _patch_proxy_pipeline_line(orig_line, storlet_middleware): if storlet_middleware in mds: return orig_line - + try: slo_index = mds.index('slo') except Exception: slo_index = -1 - + if slo_index != -1: mds.insert(slo_index, storlet_middleware) else: proxy_index = mds.index('proxy-server') mds.insert(proxy_index, storlet_middleware) - + new_line = 'pipeline =' for md in mds: new_line += ' ' + md return new_line + '\n' -def _patch_object_pipeline_line(orig_line,storlet_middleware): + +def _patch_object_pipeline_line(orig_line, storlet_middleware): mds = list() for md in orig_line.split(): if md == 'pipeline' or md == '=': @@ -80,125 +83,144 @@ def _patch_object_pipeline_line(orig_line,storlet_middleware): if storlet_middleware in mds: return orig_line - + object_index = mds.index('object-server') mds.insert(object_index, storlet_middleware) - + new_line = 'pipeline =' for md in mds: new_line += ' ' + md return new_line + '\n' -def unpatch_swift_config_file(conf, conf_file): - storlet_middleware = conf.get('common-confs','storlet_middleware') - filter_block_first_line = '[filter:%s]\n' % storlet_middleware - for line in fileinput.input(conf_file, inplace = 1): +def unpatch_swift_config_file(conf, conf_file): + storlet_middleware = conf.get('common-confs', 'storlet_middleware') + + for line in fileinput.input(conf_file, inplace=1): if line.startswith('pipeline'): new_line = _unpatch_pipeline_line(line, storlet_middleware) line = new_line - print line, + sys.stdout.write(line) _chown_to_swift(conf_file) - + + def patch_swift_config_file(conf, conf_file, service): - storlet_middleware = conf.get('common-confs','storlet_middleware') - storlet_gateway_implementation_class = conf.get('common-confs','storlet_gateway_module') + storlet_middleware = conf.get('common-confs', 'storlet_middleware') filter_block_first_line = '[filter:%s]\n' % storlet_middleware - + filter_in_file = False - for line in fileinput.input(conf_file, inplace = 1): + for line in fileinput.input(conf_file, inplace=1): if line.startswith('pipeline'): if service == 'proxy': - new_line = _patch_proxy_pipeline_line(line,storlet_middleware) + new_line = _patch_proxy_pipeline_line(line, storlet_middleware) else: - new_line = _patch_object_pipeline_line(line,storlet_middleware) + new_line = _patch_object_pipeline_line(line, + storlet_middleware) line = new_line if filter_block_first_line in line: filter_in_file = True - print line, - - if filter_in_file == False: + sys.stdout.write(line) + + if filter_in_file is False: with open(conf_file, 'a') as f: f.write('\n') f.write(filter_block_first_line) f.write('use = egg:storlets#%s\n' % storlet_middleware) - f.write('storlet_container = %s\n' % conf.get('common-confs','storlet_container')) - f.write('storlet_dependency = %s\n' % conf.get('common-confs','storlet_dependency')) - f.write('storlet_timeout = %s\n' % conf.get('common-confs','storlet_timeout')) - f.write('storlet_gateway_module = %s\n' % conf.get('common-confs','storlet_gateway_module')) - f.write('storlet_gateway_conf = %s\n' % conf.get('common-confs','storlet_gateway_conf')) - f.write('storlet_execute_on_proxy_only = %s\n' % conf.get('common-confs','storlet_proxy_execution')) + f.write('storlet_container = %s\n' % + conf.get('common-confs', 'storlet_container')) + f.write('storlet_dependency = %s\n' % + conf.get('common-confs', 'storlet_dependency')) + f.write('storlet_timeout = %s\n' % + conf.get('common-confs', 'storlet_timeout')) + f.write('storlet_gateway_module = %s\n' % + conf.get('common-confs', 'storlet_gateway_module')) + f.write('storlet_gateway_conf = %s\n' % + conf.get('common-confs', 'storlet_gateway_conf')) + f.write('storlet_execute_on_proxy_only = %s\n' % conf.get( + 'common-confs', 'storlet_proxy_execution')) f.write('execution_server = %s\n' % service) - + _chown_to_swift(conf_file) + def unpatch_swift_storlet_proxy_file(conf): - storlet_proxy_server_conf_file = conf.get('proxy-confs','storlet_proxy_server_conf_file') + storlet_proxy_server_conf_file = conf.get('proxy-confs', + 'storlet_proxy_server_conf_file') if os.path.exists(storlet_proxy_server_conf_file): os.remove(storlet_proxy_server_conf_file) - + + def patch_swift_storlet_proxy_file(conf): - storlet_proxy_server_conf_file = conf.get('proxy-confs','storlet_proxy_server_conf_file') + storlet_proxy_server_conf_file = conf.get('proxy-confs', + 'storlet_proxy_server_conf_file') proxy_server_conf_file = conf.get('proxy-confs', 'proxy_server_conf_file') - + source_file = proxy_server_conf_file target_file = storlet_proxy_server_conf_file shutil.copyfile(source_file, target_file) - for line in fileinput.input(storlet_proxy_server_conf_file, inplace = 1): - if line.startswith('pipeline'): - line= 'pipeline = proxy-logging cache storlet_handler slo proxy-logging proxy-server\n' - print line, - + for line in fileinput.input(storlet_proxy_server_conf_file, inplace=1): + if line.startswith('pipeline'): + line = 'pipeline = proxy-logging cache storlet_handler slo ' + \ + 'proxy-logging proxy-server\n' + sys.stdout.write(line) + _chown_to_swift(storlet_proxy_server_conf_file) - + + def remove_gateway_conf_file(conf): gateway_conf_file = conf.get('common-confs', 'storlet_gateway_conf') if os.path.exists(gateway_conf_file): os.remove(gateway_conf_file) -def remove(conf): - object_server_conf_files = conf.get('object-confs', 'object_server_conf_files').split(',') + +def remove(conf): + object_server_conf_files = conf.get('object-confs', + 'object_server_conf_files').split(',') for f in object_server_conf_files: if os.path.exists(f): unpatch_swift_config_file(conf, f) - proxy_server_conf_file = conf.get('proxy-confs','proxy_server_conf_file') + proxy_server_conf_file = conf.get('proxy-confs', 'proxy_server_conf_file') unpatch_swift_config_file(conf, proxy_server_conf_file) - + unpatch_swift_storlet_proxy_file(conf) remove_gateway_conf_file(conf) + def install(conf): - object_server_conf_files = conf.get('object-confs', 'object_server_conf_files').split(',') + object_server_conf_files = conf.get('object-confs', + 'object_server_conf_files').split(',') for f in object_server_conf_files: - if os.path.exists(f): + if os.path.exists(f): patch_swift_config_file(conf, f, 'object') - - proxy_server_conf_file = conf.get('proxy-confs','proxy_server_conf_file') + + proxy_server_conf_file = conf.get('proxy-confs', 'proxy_server_conf_file') patch_swift_config_file(conf, proxy_server_conf_file, 'proxy') patch_swift_storlet_proxy_file(conf) - + + def usage(argv): - print "Usage: " + argv[0] + " install/remove conf_file" + print("Usage: " + argv[0] + " install/remove conf_file") + def main(argv): if len(argv) != 3: usage(argv) exit(-1) - + conf = ConfigParser.ConfigParser() conf.read(argv[2]) - + if argv[1] == 'install': install(conf) elif argv[1] == 'remove': remove(conf) else: usage(argv) - + if __name__ == "__main__": main(sys.argv) diff --git a/Engine/SBus/SBusPythonFacade/SBus.py b/Engine/SBus/SBusPythonFacade/SBus.py index 64543f12..2677eb9d 100644 --- a/Engine/SBus/SBusPythonFacade/SBus.py +++ b/Engine/SBus/SBusPythonFacade/SBus.py @@ -29,16 +29,17 @@ from ctypes import POINTER class SBus(object): - ''' - @summary: This class wraps low level C-API for SBus functionality + '''@summary: This class wraps low level C-API for SBus functionality + to be used with Python ''' SBUS_SO_NAME = '/usr/local/lib/python2.7/dist-packages/sbus.so' '''--------------------------------------------------------------------''' + def __init__(self): - ''' - @summary: CTOR + '''@summary: CTOR + Setup argument types mappings. ''' @@ -46,12 +47,12 @@ class SBus(object): self.sbus_back_ = ctypes.CDLL(SBus.SBUS_SO_NAME) # create SBus - self.sbus_back_.sbus_create.argtypes = [c_char_p] - self.sbus_back_.sbus_create.restype = c_int + self.sbus_back_.sbus_create.argtypes = [c_char_p] + self.sbus_back_.sbus_create.restype = c_int # listen to SBus - self.sbus_back_.sbus_listen.argtypes = [c_int] - self.sbus_back_.sbus_listen.restype = c_int + self.sbus_back_.sbus_listen.argtypes = [c_int] + self.sbus_back_.sbus_listen.restype = c_int # send message self.sbus_back_.sbus_send_msg.argtypes = [c_char_p, @@ -61,7 +62,7 @@ class SBus(object): c_int, c_char_p, c_int] - self.sbus_back_.sbus_send_msg.restype = c_int + self.sbus_back_.sbus_send_msg.restype = c_int # receive message self.sbus_back_.sbus_recv_msg.argtypes = [c_int, @@ -71,13 +72,13 @@ class SBus(object): POINTER(c_int), POINTER(c_char_p), POINTER(c_int)] - self.sbus_back_.sbus_recv_msg.restype = c_int + self.sbus_back_.sbus_recv_msg.restype = c_int '''--------------------------------------------------------------------''' + @staticmethod def start_logger(str_log_level='DEBUG', container_id=None): - ''' - @summary: Start logger. + '''@summary: Start logger. @param str_log_level: The level of verbosity in log records. Default value - 'DEBUG'. @@ -92,10 +93,10 @@ class SBus(object): sbus_back_.sbus_start_logger(str_log_level, container_id) '''--------------------------------------------------------------------''' + @staticmethod def stop_logger(): - ''' - @summary: Stop logger. + '''@summary: Stop logger. @rtype: void ''' @@ -104,9 +105,9 @@ class SBus(object): sbus_back_.sbus_stop_logger() '''--------------------------------------------------------------------''' + def create(self, sbus_name): - ''' - @summary: Instantiate an SBus. A wrapper for C function. + '''@summary: Instantiate an SBus. A wrapper for C function. @param sbus_name: Path to domain socket "file". @type sbus_name: string @@ -117,9 +118,10 @@ class SBus(object): return self.sbus_back_.sbus_create(sbus_name) '''--------------------------------------------------------------------''' + def listen(self, sbus_handler): - ''' - @summary: Listen to the SBus. + '''@summary: Listen to the SBus. + Suspend the executing thread. @param sbus_handler: Handler to SBus to listen. @@ -131,9 +133,10 @@ class SBus(object): return self.sbus_back_.sbus_listen(sbus_handler) '''--------------------------------------------------------------------''' + def receive(self, sbus_handler): - ''' - @summary: Read the data from SBus. + '''@summary: Read the data from SBus. + Create a datagram. @param sbus_handler: Handler to SBus to read data from. @@ -142,12 +145,12 @@ class SBus(object): @return: An object with the obtained data. Null-able. @rtype: SBusDatagram ''' - ph_files = POINTER(c_int)() + ph_files = POINTER(c_int)() pp_metadata = (c_char_p)() - pp_params = (c_char_p)() - pn_files = (c_int)() + pp_params = (c_char_p)() + pn_files = (c_int)() pn_metadata = (c_int)() - pn_params = (c_int)() + pn_params = (c_int)() # Invoke C function n_status = self.sbus_back_.sbus_recv_msg(sbus_handler, @@ -157,7 +160,7 @@ class SBus(object): pn_metadata, pp_params, pn_params) - result_dtg = None + result_dtg = None if 0 <= n_status: # The invocation was successful. # De-serialize the data @@ -169,10 +172,10 @@ class SBus(object): h_files.append(ph_files[i]) # Extract Python strings - n_metadata = pn_metadata.value + n_metadata = pn_metadata.value str_metadata = pp_metadata.value - n_params = pn_params.value - str_params = pp_params.value + n_params = pn_params.value + str_params = pp_params.value # Trim the junk out if 0 < n_metadata: @@ -187,10 +190,11 @@ class SBus(object): return result_dtg '''--------------------------------------------------------------------''' + @staticmethod def send(sbus_name, datagram): - ''' - @summary: Send the datagram through SBus. + '''@summary: Send the datagram through SBus. + Serialize dictionaries into JSON strings. @param sbus_name: Path to domain socket "file". @@ -204,30 +208,30 @@ class SBus(object): # Serialize the datagram into JSON strings and C integer array str_json_params = datagram.get_params_and_cmd_as_json() - p_params = c_char_p(str_json_params) - n_params = c_int(len(str_json_params)) + p_params = c_char_p(str_json_params) + n_params = c_int(len(str_json_params)) - n_files = c_int(0) - h_files = None - n_metadata = c_int(0) - p_metadata = None + n_files = c_int(0) + h_files = None + n_metadata = c_int(0) + p_metadata = None if datagram.get_num_files() > 0: str_json_metadata = datagram.get_files_metadata_as_json() - p_metadata = c_char_p(str_json_metadata) - n_metadata = c_int(len(str_json_metadata)) + p_metadata = c_char_p(str_json_metadata) + n_metadata = c_int(len(str_json_metadata)) - n_fds = datagram.get_num_files() - n_files = c_int(n_fds) + n_fds = datagram.get_num_files() + n_files = c_int(n_fds) - file_fds = datagram.get_files() - h_files = (c_int * n_fds)() + file_fds = datagram.get_files() + h_files = (c_int * n_fds)() for i in range(n_fds): h_files[i] = file_fds[i] # Invoke C function - sbus = SBus() + sbus = SBus() n_status = sbus.sbus_back_.sbus_send_msg(sbus_name, h_files, n_files, @@ -237,5 +241,4 @@ class SBus(object): n_params) return n_status - '''============================ END OF FILE ===============================''' diff --git a/Engine/SBus/SBusPythonFacade/SBusDatagram.py b/Engine/SBus/SBusPythonFacade/SBusDatagram.py index 8a186053..73dd9159 100644 --- a/Engine/SBus/SBusPythonFacade/SBusDatagram.py +++ b/Engine/SBus/SBusPythonFacade/SBusDatagram.py @@ -21,19 +21,19 @@ Limitations under the License. dictionary of dictionaries ===========================================================================''' -import os import json +import os import syslog -from SBusStorletCommand import SBUS_CMD_NOP from SBusFileDescription import SBUS_FD_OUTPUT_OBJECT +from SBusStorletCommand import SBUS_CMD_NOP '''------------------------------------------------------------------------''' class SBusDatagram(object): - ''' - @summary: This class aggregates data to be transferred + '''@summary: This class aggregates data to be transferred + using SBus functionality. ''' @@ -41,9 +41,9 @@ class SBusDatagram(object): task_id_dict_key_name_ = 'taskId' '''--------------------------------------------------------------------''' + def __init__(self): - ''' - @summary: CTOR + '''@summary: CTOR @ivar e_command_ : A command to Storlet Daemon. @type e_command_ : Integer. SBusStorletCommand enumerated value. @@ -59,19 +59,20 @@ class SBusDatagram(object): @invariant: Quantity of entries in files_metadata_ list is the same as in h_files_, i.e. n_files_. ''' - self.e_command_ = SBUS_CMD_NOP - self.task_id_ = None - self.h_files_ = None - self.n_files_ = 0 + self.e_command_ = SBUS_CMD_NOP + self.task_id_ = None + self.h_files_ = None + self.n_files_ = 0 self.files_metadata_ = None - self.exec_params_ = None + self.exec_params_ = None '''--------------------------------------------------------------------''' + @staticmethod def create_service_datagram(command, outfd): - ''' - @summary: Datagram static factory. + '''@summary: Datagram static factory. + Create "service" datagram, i.e. - command shall be one of {PING, START/STOP/STATUS-DAEMON} @@ -99,12 +100,13 @@ class SBusDatagram(object): return dtg '''--------------------------------------------------------------------''' + def from_raw_data(self, h_files, str_json_metadata, str_json_params): - ''' - @summary: CTOR + '''@summary: CTOR + Construct object from file list and two JSON-encoded strings. @@ -122,10 +124,11 @@ class SBusDatagram(object): self.extract_params(str_json_params) '''--------------------------------------------------------------------''' + def extract_metadata(self, str_json_metadata): - ''' - @summary: Extract files_metadata array + '''@summary: Extract files_metadata array + of dictionaries form a JSON string @requires: n_files_ has to be se @@ -142,9 +145,10 @@ class SBusDatagram(object): self.files_metadata_.append(json.loads(str_curr_metadata)) '''--------------------------------------------------------------------''' + def extract_params(self, str_json_params): - ''' - @summary: Extract command field and exec_params + '''@summary: Extract command field and exec_params + dictionary form a JSON string @param str_json_params: JSON encoding for the execution parameters. @type str_json_params: string. @@ -169,9 +173,10 @@ class SBusDatagram(object): self.exec_params_ = None '''--------------------------------------------------------------------''' + def get_params_and_cmd_as_json(self): - ''' - @summary: Convert command field and execution parameters + '''@summary: Convert command field and execution parameters + dictionary into JSON as the following - 1. Copy exec_params_. Initialize the combined dictionary. 2. Push the next pair into the combined dictionary @@ -193,9 +198,10 @@ class SBusDatagram(object): return str_result '''--------------------------------------------------------------------''' + def get_files_metadata_as_json(self): - ''' - @summary: Encode the list of dictionaries into JSON as the following - + '''@summary: Encode the list of dictionaries into JSON as the following + 1. Create a combined dictionary (Integer-to-String) Key - index in the original list Value - JSON encoding of the certain dictionary @@ -213,9 +219,9 @@ class SBusDatagram(object): return str_result '''--------------------------------------------------------------------''' + def get_num_files(self): - ''' - @summary: Getter. + '''@summary: Getter. @return: The quantity of file descriptors. @rtype: integer @@ -223,9 +229,9 @@ class SBusDatagram(object): return self.n_files_ '''--------------------------------------------------------------------''' + def get_files(self): - ''' - @summary: Getter. + '''@summary: Getter. @return: The list of file descriptors. @rtype: List of integers @@ -233,9 +239,10 @@ class SBusDatagram(object): return self.h_files_ '''--------------------------------------------------------------------''' + def set_files(self, h_files): - ''' - @summary: Setter. + '''@summary: Setter. + Assign file handlers list and update n_files_ field @param h_files: File descriptors. @@ -257,9 +264,10 @@ class SBusDatagram(object): self.h_files_.append(h_files[i]) '''--------------------------------------------------------------------''' + def get_first_file_of_type(self, file_type): - ''' - @summary: Iterate through file list and metadata. + '''@summary: Iterate through file list and metadata. + Find the first file with the required type @param file_type: The file type to look for @@ -273,15 +281,15 @@ class SBusDatagram(object): if (self.get_metadata()[i])['type'] == file_type: try: required_file = os.fdopen(self.get_files()[i], 'w') - except IOError, err: + except IOError as err: syslog.syslog(syslog.LOG_DEBUG, 'Failed to open file: %s' % err.strerror) return required_file '''--------------------------------------------------------------------''' + def get_metadata(self): - ''' - @summary: Getter. + '''@summary: Getter. @return: The list of meta-data dictionaries. @rtype: List of dictionaries @@ -289,9 +297,10 @@ class SBusDatagram(object): return self.files_metadata_ '''--------------------------------------------------------------------''' + def set_metadata(self, metadata): - ''' - @summary: Setter. + '''@summary: Setter. + Assign file_metadata_ field @param metadata: File descriptors meta-data dictionaries. @@ -302,9 +311,9 @@ class SBusDatagram(object): self.files_metadata_ = metadata '''--------------------------------------------------------------------''' + def get_exec_params(self): - ''' - @summary: Getter. + '''@summary: Getter. @return: The execution parameters dictionary. @rtype: Dictionary @@ -312,9 +321,10 @@ class SBusDatagram(object): return self.exec_params_ '''--------------------------------------------------------------------''' + def set_exec_params(self, params): - ''' - @summary: Setter. + '''@summary: Setter. + Assign execution parameters dictionary. @param params: Execution parameters to assign @@ -326,9 +336,10 @@ class SBusDatagram(object): self.exec_params_ = params '''--------------------------------------------------------------------''' + def add_exec_param(self, param_name, param_value): - ''' - @summary: Add a single pair to the exec_params_ dictionary + '''@summary: Add a single pair to the exec_params_ dictionary + Don't change if the parameter exists already @param param_name: Execution parameter name to be added @@ -351,9 +362,9 @@ class SBusDatagram(object): return b_status '''--------------------------------------------------------------------''' + def get_command(self): - ''' - @summary: Getter. + '''@summary: Getter. @return: The Storlet Daemon command. @rtype: SBusStorletCommand @@ -361,9 +372,10 @@ class SBusDatagram(object): return self.e_command_ '''--------------------------------------------------------------------''' + def set_command(self, cmd): - ''' - @summary: Setter. + '''@summary: Setter. + Assign Storlet Daemon command. @param cmd: Command to assign @@ -374,9 +386,9 @@ class SBusDatagram(object): self.e_command_ = cmd '''--------------------------------------------------------------------''' + def get_task_id(self): - ''' - @summary: Getter. + '''@summary: Getter. @return: The task id. @rtype: string @@ -384,9 +396,10 @@ class SBusDatagram(object): return self.task_id_ '''--------------------------------------------------------------------''' + def set_task_id(self, taskId): - ''' - @summary: Setter. + '''@summary: Setter. + Assign task id @param taskId: Command to assign @@ -397,10 +410,11 @@ class SBusDatagram(object): self.task_id_ = taskId '''--------------------------------------------------------------------''' + @staticmethod def dictionaies_equal(d1, d2): - ''' - @summary: Check whether two dictionaries has the same content. + '''@summary: Check whether two dictionaries has the same content. + The order of the entries is not considered. @return: The answer to the above diff --git a/Engine/SBus/SBusPythonFacade/SBusFileDescription.py b/Engine/SBus/SBusPythonFacade/SBusFileDescription.py index dd110c4e..4820ec2c 100644 --- a/Engine/SBus/SBusPythonFacade/SBusFileDescription.py +++ b/Engine/SBus/SBusPythonFacade/SBusFileDescription.py @@ -17,7 +17,6 @@ Limitations under the License. 21-Jul-2014 evgenyl Initial implementation. ===========================================================================''' - '''------------------------------------------------------------------------''' ''' @summary: Enumerate file usage intents. @@ -25,12 +24,12 @@ Limitations under the License. with its Java counterpart. ''' -SBUS_FD_INPUT_OBJECT = 0 -SBUS_FD_OUTPUT_OBJECT = 1 -SBUS_FD_OUTPUT_OBJECT_METADATA = 2 -SBUS_FD_OUTPUT_OBJECT_AND_METADATA = 3 -SBUS_FD_LOGGER = 4 -SBUS_FD_OUTPUT_CONTAINER = 5 -SBUS_FD_OUTPUT_TASK_ID = 6 +SBUS_FD_INPUT_OBJECT = 0 +SBUS_FD_OUTPUT_OBJECT = 1 +SBUS_FD_OUTPUT_OBJECT_METADATA = 2 +SBUS_FD_OUTPUT_OBJECT_AND_METADATA = 3 +SBUS_FD_LOGGER = 4 +SBUS_FD_OUTPUT_CONTAINER = 5 +SBUS_FD_OUTPUT_TASK_ID = 6 '''============================ END OF FILE ===============================''' diff --git a/Engine/SBus/SBusPythonFacade/SBusStorletCommand.py b/Engine/SBus/SBusPythonFacade/SBusStorletCommand.py index 097dc5e2..1809b625 100644 --- a/Engine/SBus/SBusPythonFacade/SBusStorletCommand.py +++ b/Engine/SBus/SBusPythonFacade/SBusStorletCommand.py @@ -17,7 +17,6 @@ Limitations under the License. 21-Jul-2014 evgenyl Initial implementation. ===========================================================================''' - '''------------------------------------------------------------------------''' ''' @summary: Enumerate Storlet Daemon commands. @@ -25,15 +24,15 @@ Limitations under the License. with its Java counterpart. ''' -SBUS_CMD_HALT = 0 -SBUS_CMD_EXECUTE = 1 -SBUS_CMD_START_DAEMON = 2 -SBUS_CMD_STOP_DAEMON = 3 -SBUS_CMD_DAEMON_STATUS = 4 -SBUS_CMD_STOP_DAEMONS = 5 -SBUS_CMD_PING = 6 -SBUS_CMD_DESCRIPTOR = 7 -SBUS_CMD_CANCEL = 8 -SBUS_CMD_NOP = 9 +SBUS_CMD_HALT = 0 +SBUS_CMD_EXECUTE = 1 +SBUS_CMD_START_DAEMON = 2 +SBUS_CMD_STOP_DAEMON = 3 +SBUS_CMD_DAEMON_STATUS = 4 +SBUS_CMD_STOP_DAEMONS = 5 +SBUS_CMD_PING = 6 +SBUS_CMD_DESCRIPTOR = 7 +SBUS_CMD_CANCEL = 8 +SBUS_CMD_NOP = 9 '''============================ END OF FILE ===============================''' diff --git a/Engine/SBus/SBusPythonFacade/setup.py b/Engine/SBus/SBusPythonFacade/setup.py index 1fa8fb69..c920b146 100755 --- a/Engine/SBus/SBusPythonFacade/setup.py +++ b/Engine/SBus/SBusPythonFacade/setup.py @@ -1,5 +1,4 @@ #!/usr/bin/python -#----------------------------------------------------------------------------------------------- '''------------------------------------------------------------------------- Copyright IBM Corp. 2015, 2015 All Rights Reserved Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,8 +15,7 @@ Limitations under the License. -------------------------------------------------------------------------''' from setuptools import setup -setup( name = 'SBusPythonFacade', - version = '1.0', - package_dir={'SBusPythonFacade':''}, - packages=['SBusPythonFacade'] ) - +setup(name='SBusPythonFacade', + version='1.0', + package_dir={'SBusPythonFacade': ''}, + packages=['SBusPythonFacade']) diff --git a/Engine/SMScripts/send_halt_cmd_to_daemon_factory.py b/Engine/SMScripts/send_halt_cmd_to_daemon_factory.py index 9c11f794..fbf0e614 100755 --- a/Engine/SMScripts/send_halt_cmd_to_daemon_factory.py +++ b/Engine/SMScripts/send_halt_cmd_to_daemon_factory.py @@ -1,54 +1,58 @@ -#!/usr/bin/python -#----------------------------------------------------------------------------------------------- -# Copyright IBM Corp. 2015, 2015 All Rights Reserved -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# Limitations under the License. -#----------------------------------------------------------------------------------------------- +'''------------------------------------------------------------------------- +Copyright IBM Corp. 2015, 2015 All Rights Reserved +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +Limitations under the License. +-------------------------------------------------------------------------''' + '''=========================================================================== 02-Dec-2014 evgenyl Initial implementation. ===========================================================================''' -import sys import os +import sys -from SBusPythonFacade.SBus import SBus -from SBusPythonFacade.SBusDatagram import SBusDatagram -from SBusPythonFacade.SBusStorletCommand import SBUS_CMD_HALT +from SBusPythonFacade.SBus import SBus +from SBusPythonFacade.SBusDatagram import SBusDatagram +from SBusPythonFacade.SBusStorletCommand import SBUS_CMD_HALT '''------------------------------------------------------------------------''' + + def print_usage(argv): - print argv[0] + ' /path/to/daemon/factory_pipe' - print 'Example:' - print argv[0] + ' ', - print '/home/lxc_device/pipes/scopes/'\ - 'AUTH_fb8b63c579054c48816ca8acd090b3d9/factory_pipe' + print(argv[0] + ' /path/to/daemon/factory_pipe') + print('Example:') + sys.stdout.write(argv[0] + ' ') + print('/home/lxc_device/pipes/scopes/' + 'AUTH_fb8b63c579054c48816ca8acd090b3d9/factory_pipe') '''------------------------------------------------------------------------''' + + def main(argv): if 2 > len(argv): print_usage(argv) return daemon_factory_pipe_name = argv[1] - fi,fo = os.pipe() + fi, fo = os.pipe() halt_dtg = SBusDatagram.create_service_datagram(SBUS_CMD_HALT, fo) n_status = SBus.send(daemon_factory_pipe_name, halt_dtg) if 0 > n_status: - print 'Sending failed' - else: - print 'Sending succeeded' - cmd_response = os.read( fi, 256 ) - print cmd_response + print('Sending failed') + else: + print('Sending succeeded') + cmd_response = os.read(fi, 256) + print(cmd_response) os.close(fi) os.close(fo) diff --git a/Engine/storlet_daemon_factory/daemon_factory.py b/Engine/storlet_daemon_factory/daemon_factory.py index 79e157e2..8fe4746a 100644 --- a/Engine/storlet_daemon_factory/daemon_factory.py +++ b/Engine/storlet_daemon_factory/daemon_factory.py @@ -20,38 +20,42 @@ XX-XXX-2014 eranr Initial implementation. 01-Dec-2014 evgenyl Dropping multi-threaded monitoring ===========================================================================''' +import errno +import logging +from logging.handlers import SysLogHandler import os import pwd +import signal +import subprocess import sys import time -import errno -import signal -import logging -import subprocess -from logging.handlers import SysLogHandler -from SBusPythonFacade.SBusStorletCommand import SBUS_CMD_START_DAEMON,\ - SBUS_CMD_STOP_DAEMON, SBUS_CMD_DAEMON_STATUS, SBUS_CMD_STOP_DAEMONS,\ - SBUS_CMD_PING, SBUS_CMD_HALT -from SBusPythonFacade.SBusFileDescription import SBUS_FD_OUTPUT_OBJECT from SBusPythonFacade.SBus import SBus -from SBusPythonFacade.SBusDatagram import * +from SBusPythonFacade.SBusDatagram import SBusDatagram +from SBusPythonFacade.SBusFileDescription import SBUS_FD_OUTPUT_OBJECT +from SBusPythonFacade.SBusStorletCommand import SBUS_CMD_DAEMON_STATUS +from SBusPythonFacade.SBusStorletCommand import SBUS_CMD_HALT +from SBusPythonFacade.SBusStorletCommand import SBUS_CMD_PING +from SBusPythonFacade.SBusStorletCommand import SBUS_CMD_START_DAEMON +from SBusPythonFacade.SBusStorletCommand import SBUS_CMD_STOP_DAEMON +from SBusPythonFacade.SBusStorletCommand import SBUS_CMD_STOP_DAEMONS '''========================================================================''' -class daemon_factory(): - ''' - @summary: This class acts as the manager for storlet daemons. +class daemon_factory(object): + '''@summary: This class acts as the manager for storlet daemons. + It listens to commands and reacts on them in an internal loop. As for now (01-Dec-2014) it is a single thread, synchronous processing. ''' '''--------------------------------------------------------------------''' + def __init__(self, path, logger): - ''' - @summary: CTOR + '''@summary: CTOR + Prepare the auxiliary data structures @param path: Path to the pipe file internal SBus listens to @@ -65,10 +69,11 @@ class daemon_factory(): self.storlet_name_to_pipe_name = dict() # Dictionary: map storlet name to daemon process PID self.storlet_name_to_pid = dict() - + self.NUM_OF_TRIES_PINGING_STARTING_DAEMON = 5 '''--------------------------------------------------------------------''' + def get_jvm_args(self, daemon_language, storlet_path, @@ -77,8 +82,8 @@ class daemon_factory(): uds_path, log_level, container_id): - ''' - @summary: get_jvm_args + '''@summary: get_jvm_args + Check the input parameters, produce the list of arguments for JVM process launch @@ -136,8 +141,8 @@ class daemon_factory(): pargs = [] if daemon_language == "java": self.logger.debug('START_DAEMON:preparing arguments') - #Setting two environmental variables - #The path strings are corrupted if passed is pargs list below + # Setting two environmental variables + # The path strings are corrupted if passed is pargs list below os.environ['CLASSPATH'] = str_dmn_clspth os.environ['LD_LIBRARY_PATH'] = str_library_path pargs = [str('/usr/bin/java'), @@ -157,9 +162,10 @@ class daemon_factory(): return n_error_id, error_text, pargs '''--------------------------------------------------------------------''' + def spawn_subprocess(self, pargs): - ''' - @summary: spawn_subprocess + '''@summary: spawn_subprocess + Launch a JVM process for some storlet daemon @param pargs: Arguments for the JVM @@ -192,8 +198,8 @@ class daemon_factory(): format(storlet_name, jvm_pid)) # Keep JVM PID self.storlet_name_to_pid[storlet_name] = jvm_pid - b_status, error_text = self.wait_for_daemon_to_initialize( - storlet_name) + b_status, error_text = \ + self.wait_for_daemon_to_initialize(storlet_name) if not b_status: raise 'No response from Daemon' self.logger.debug('START_DAEMON: just occurred') @@ -207,9 +213,10 @@ class daemon_factory(): return b_status, error_text '''--------------------------------------------------------------------''' + def wait_for_daemon_to_initialize(self, storlet_name): - ''' - @summary: wait_for_daemon_to_initialize + '''@summary: wait_for_daemon_to_initialize + Send a Ping service datagram. Validate that Daemon response is correct. Give up after the predefined number of attempts (5) @@ -222,9 +229,9 @@ class daemon_factory(): @return: Description text of possible error @rtype: String ''' - storlet_pipe_name = self.storlet_name_to_pipe_name[storlet_name] - self.logger.debug('Send PING command to {0} via {1}'.\ - format(storlet_name,storlet_pipe_name)) + storlet_pipe_name = self.storlet_name_to_pipe_name[storlet_name] + self.logger.debug('Send PING command to {0} via {1}'. + format(storlet_name, storlet_pipe_name)) read_fd, write_fd = os.pipe() dtg = SBusDatagram.create_service_datagram(SBUS_CMD_PING, write_fd) b_status = False @@ -241,8 +248,9 @@ class daemon_factory(): os.close(read_fd) os.close(write_fd) return b_status, error_text - + '''--------------------------------------------------------------------''' + def process_start_daemon(self, daemon_language, storlet_path, @@ -251,8 +259,8 @@ class daemon_factory(): uds_path, log_level, container_id): - ''' - @summary: process_start_daemon + '''@summary: process_start_daemon + Start storlet daemon process @see: get_jvm_args for the list of parameters @@ -294,17 +302,18 @@ class daemon_factory(): error_text = '{0} is already running'.format(storlet_name) self.logger.debug(error_text) else: - error_text = '{0} is not running. About to spawn process'.\ + error_text = '{0} is not running. About to spawn process'. \ format(storlet_name) self.logger.debug(error_text) b_status, error_text = self.spawn_subprocess(pargs) - + return b_status, error_text '''--------------------------------------------------------------------''' + def get_process_status_by_name(self, storlet_name): - ''' - @summary: get_process_status_by_name + '''@summary: get_process_status_by_name + Check if the daemon runs for the specific storlet @param storlet_name: Storlet name we are checking the daemon for @@ -327,16 +336,17 @@ class daemon_factory(): b_status, error_text = self.get_process_status_by_pid( daemon_pid, storlet_name) else: - error_text = 'Storlet name {0} not found in map'.\ + error_text = 'Storlet name {0} not found in map'. \ format(storlet_name) self.logger.debug(error_text) return b_status, error_text '''--------------------------------------------------------------------''' + def get_process_status_by_pid(self, daemon_pid, storlet_name): - ''' - @summary: get_process_status_by_pid + '''@summary: get_process_status_by_pid + Check if a process with specific ID runs @param daemon_pid: Storlet daemon process ID @@ -355,14 +365,14 @@ class daemon_factory(): obtained_code = 0 try: obtained_pid, obtained_code = os.waitpid(daemon_pid, os.WNOHANG) - error_text = 'Storlet {0}, PID = {1}, ErrCode = {2}'.\ + error_text = 'Storlet {0}, PID = {1}, ErrCode = {2}'. \ format(storlet_name, obtained_pid, obtained_code) self.logger.debug(error_text) - except OSError, err: + except OSError as err: if err.errno == errno.ESRCH: error_text = 'No running daemon for {0}'.format(storlet_name) elif err.errno == errno.EPERM: - error_text = 'No permission to access daemon for {0}'.\ + error_text = 'No permission to access daemon for {0}'. \ format(storlet_name) else: error_text = 'Unknown error' @@ -376,9 +386,10 @@ class daemon_factory(): return b_status, error_text '''--------------------------------------------------------------------''' + def process_kill(self, storlet_name): - ''' - @summary: process_kill + '''@summary: process_kill + Kill the storlet daemon immediately (kill -9 $DMN_PID) @@ -399,10 +410,10 @@ class daemon_factory(): try: os.kill(dmn_pid, signal.SIGKILL) obtained_pid, obtained_code = os.waitpid(dmn_pid, os.WNOHANG) - error_text = 'Storlet {0}, PID = {1}, ErrCode = {2}'.\ + error_text = 'Storlet {0}, PID = {1}, ErrCode = {2}'. \ format(storlet_name, obtained_pid, obtained_code) self.logger.debug(error_text) - except: + except Exception: self.logger.debug('Crash while killing storlet') self.storlet_name_to_pid.pop(storlet_name) else: @@ -412,10 +423,11 @@ class daemon_factory(): return b_success, error_text '''--------------------------------------------------------------------''' + def process_kill_all(self): - ''' - @summary: process_kill_all - Iterate through storlet daemons. Kill every one. + '''@summary: process_kill_all Iterate through storlet daemons. + + Kill every one. @return: Status (True) @rtype: Boolean @@ -427,9 +439,10 @@ class daemon_factory(): return True, 'OK' '''--------------------------------------------------------------------''' + def shutdown_all_processes(self): - ''' - @summary: shutdown_all_processes + '''@summary: shutdown_all_processes + send HALT command to every spawned process ''' answer = '' @@ -441,9 +454,9 @@ class daemon_factory(): return True, answer '''--------------------------------------------------------------------''' + def shutdown_process(self, storlet_name): - ''' - @summary: send HALT command to storlet daemon + '''@summary: send HALT command to storlet daemon @param storlet_name: Storlet name we are checking the daemon for @type storlet_name: String @@ -452,45 +465,46 @@ class daemon_factory(): @rtype: Boolean @return: Description text of possible error @rtype: String - ''' + ''' + b_status = False - error_text = '' - self.logger.debug('Inside shutdown_process {0}'.format(storlet_name)) - storlet_pipe_name = self.storlet_name_to_pipe_name[storlet_name] - self.logger.debug('Send HALT command to {0} via {1}'.\ - format(storlet_name,storlet_pipe_name)) + self.logger.debug('Inside shutdown_process {0}'.format(storlet_name)) + storlet_pipe_name = self.storlet_name_to_pipe_name[storlet_name] + self.logger.debug('Send HALT command to {0} via {1}'. + format(storlet_name, storlet_pipe_name)) read_fd, write_fd = os.pipe() dtg = SBusDatagram.create_service_datagram(SBUS_CMD_HALT, write_fd) SBus.send(storlet_pipe_name, dtg) os.close(read_fd) os.close(write_fd) dmn_pid = self.storlet_name_to_pid.get(storlet_name, -1) - self.logger.debug('Storlet Daemon PID is {0}'.\ - format(dmn_pid)) + self.logger.debug('Storlet Daemon PID is {0}'.format(dmn_pid)) if -1 != dmn_pid: - os.waitpid(dmn_pid,0) + os.waitpid(dmn_pid, 0) self.storlet_name_to_pid.pop(storlet_name) b_status = True return b_status - + '''--------------------------------------------------------------------''' + def dispatch_command(self, dtg, container_id): - ''' - @summary: dispatch_command - Parse datagram. React on the request. + '''@summary: dispatch_command + + Parse datagram. React on the request. @param dtg: Datagram to process @type dtg: SBus python facade Datagram @param container_id: container id @type container_id: String - + @return: Status @rtype: Boolean @return: Description text of possible error @rtype: String @return: Flag - whether we need to continue operating - @rtype: Boolean - ''' + @rtype: Boolean + ''' + b_status = False error_text = '' b_iterate = True @@ -498,141 +512,146 @@ class daemon_factory(): try: command = dtg.get_command() except Exception: - error_text = "Received message does not have command"\ + error_text = "Received message does not have command" \ " identifier. continuing." b_status = False - self.logger.error( error_text ) + self.logger.error(error_text) else: self.logger.debug("Received command {0}".format(command)) - + prms = dtg.get_exec_params() if command == SBUS_CMD_START_DAEMON: - self.logger.debug( 'Do SBUS_CMD_START_DAEMON' ) - self.logger.debug( 'prms = %s'%str(prms) ) + self.logger.debug('Do SBUS_CMD_START_DAEMON') + self.logger.debug('prms = %s' % str(prms)) b_status, error_text = \ self.process_start_daemon(prms['daemon_language'], - prms['storlet_path'], - prms['storlet_name'], - prms['pool_size'], - prms['uds_path'], + prms['storlet_path'], + prms['storlet_name'], + prms['pool_size'], + prms['uds_path'], prms['log_level'], container_id) elif command == SBUS_CMD_STOP_DAEMON: - self.logger.debug( 'Do SBUS_CMD_STOP_DAEMON' ) - b_status, error_text = self.process_kill(\ - prms['storlet_name']) + self.logger.debug('Do SBUS_CMD_STOP_DAEMON') + b_status, error_text = \ + self.process_kill(prms['storlet_name']) elif command == SBUS_CMD_DAEMON_STATUS: - self.logger.debug( 'Do SBUS_CMD_DAEMON_STATUS' ) - b_status, error_text = self.get_process_status_by_name(\ - prms['storlet_name']) + self.logger.debug('Do SBUS_CMD_DAEMON_STATUS') + b_status, error_text = \ + self.get_process_status_by_name(prms['storlet_name']) elif command == SBUS_CMD_STOP_DAEMONS: - self.logger.debug( 'Do SBUS_CMD_STOP_DAEMONS' ) + self.logger.debug('Do SBUS_CMD_STOP_DAEMONS') b_status, error_text = self.process_kill_all() b_iterate = False elif command == SBUS_CMD_HALT: - self.logger.debug( 'Do SBUS_CMD_HALT' ) + self.logger.debug('Do SBUS_CMD_HALT') b_status, error_text = self.shutdown_all_processes() b_iterate = False elif command == SBUS_CMD_PING: - self.logger.debug( 'Do SBUS_CMD_PING' ) + self.logger.debug('Do SBUS_CMD_PING') b_status = True error_text = 'OK' else: b_status = False error_text = "got unknown command %d" % command - self.logger.error( error_text ) - - self.logger.debug( 'Done' ) + self.logger.error(error_text) + + self.logger.debug('Done') return b_status, error_text, b_iterate - + '''--------------------------------------------------------------------''' + def main_loop(self, container_id): - ''' - @summary: main_loop + '''@summary: main_loop + The 'internal' loop. Listen to SBus, receive datagram, dispatch command, report back. ''' + # Create SBus. Listen and process requests sbus = SBus() - fd = sbus.create( self.pipe_path ) + fd = sbus.create(self.pipe_path) if fd < 0: self.logger.error("Failed to create SBus. exiting.") return - + b_iterate = True b_status = True error_text = '' - + while b_iterate: rc = sbus.listen(fd) if rc < 0: self.logger.error("Failed to wait on SBus. exiting.") return self.logger.debug("Wait returned") - + dtg = sbus.receive(fd) if not dtg: self.logger.error("Failed to receive message. exiting.") return - + try: - outfd = dtg.get_first_file_of_type( SBUS_FD_OUTPUT_OBJECT ) + outfd = dtg.get_first_file_of_type(SBUS_FD_OUTPUT_OBJECT) except Exception: - self.logger.error("Received message does not have outfd."\ + self.logger.error("Received message does not have outfd." " continuing.") continue else: self.logger.debug("Received outfd %d" % outfd.fileno()) - b_status, error_text, b_iterate = self.dispatch_command(dtg, container_id) - + b_status, error_text, b_iterate = \ + self.dispatch_command(dtg, container_id) + self.log_and_report(outfd, b_status, error_text) outfd.close() - + # We left the main loop for some reason. Terminating. - self.logger.debug( 'Leaving main loop' ) - + self.logger.debug('Leaving main loop') + '''--------------------------------------------------------------------''' + def log_and_report(self, outfd, b_status, error_text): - ''' - @summary: log_and_report - Send the result description message + '''@summary: log_and_report + + Send the result description message back to swift middlewear - + @param outfd: Output channel to send the message to @type outfd: File descriptor @param b_status: Flag, whether the operation was successful @type: Boolean @param error_text: The result description @type error_text: String - + @rtype: void ''' - num = -1; answer = str(b_status) + ': ' + error_text self.logger.debug(' Just processed command') - self.logger.debug(' Going to answer: %s'%answer) + self.logger.debug(' Going to answer: %s' % answer) try: - num = outfd.write( answer ) + outfd.write(answer) self.logger.debug(" ... and still alive") - except: - self.logger.debug('Problem while writing response %s'%answer) + except Exception: + self.logger.debug('Problem while writing response %s' % answer) '''======================= END OF daemon_factory CLASS ====================''' '''------------------------------------------------------------------------''' + + def start_logger(logger_name, log_level, container_id): - ''' - @summary: start_logger - Initialize logging of this process. + '''@summary: start_logger + + Initialize logging of this process. Set the logger format. - + @param logger_name: The name to report with @type logger_name: String @param log_level: The verbosity level @type log_level: String - - @rtype: void + + @rtype: void ''' logging.raiseExceptions = False log_level = log_level.upper() @@ -648,26 +667,25 @@ def start_logger(logger_name, log_level, container_id): else: level = logging.ERROR - logger = logging.getLogger("CONT #" + container_id + ": " + logger_name) if log_level == 'OFF': logging.disable(logging.CRITICAL) else: logger.setLevel(level) - - for i in range(0,4): + + for i in range(0, 4): try: sysLogh = SysLogHandler('/dev/log') break except Exception as e: - if i<3: + if i < 3: time.sleep(1) else: raise e - - str_format = '%(name)-12s: %(levelname)-8s %(funcName)s'+\ - ' %(lineno)s [%(process)d, %(threadName)s]'+\ + + str_format = '%(name)-12s: %(levelname)-8s %(funcName)s' + \ + ' %(lineno)s [%(process)d, %(threadName)s]' + \ ' %(message)s' formatter = logging.Formatter(str_format) sysLogh.setFormatter(formatter) @@ -676,42 +694,46 @@ def start_logger(logger_name, log_level, container_id): return logger '''------------------------------------------------------------------------''' + + def usage(): - ''' - @summary: usage + '''@summary: usage + Print the expected command line arguments. - + @rtype: void ''' - print "daemon_factory " + print("daemon_factory ") '''------------------------------------------------------------------------''' + + def main(argv): - ''' - @summary: main - The entry point. - - Initialize logger, + '''@summary: main + + The entry point. + - Initialize logger, - impersonate to swift user, - - create an instance of daemon_factory, - - start the main loop. + - create an instance of daemon_factory, + - start the main loop. ''' + if (len(argv) != 3): usage() return - + pipe_path = argv[0] log_level = argv[1] container_id = argv[2] logger = start_logger("daemon_factory", log_level, container_id) logger.debug("Daemon factory started") SBus.start_logger("DEBUG", container_id=container_id) - + # Impersonate the swift user pw = pwd.getpwnam('swift') - os.setresgid(pw.pw_gid,pw.pw_gid,pw.pw_gid) - os.setresuid(pw.pw_uid,pw.pw_uid,pw.pw_uid) + os.setresgid(pw.pw_gid, pw.pw_gid, pw.pw_gid) + os.setresuid(pw.pw_uid, pw.pw_uid, pw.pw_uid) - factory = daemon_factory(pipe_path, logger) factory.main_loop(container_id) diff --git a/Engine/storlet_daemon_factory/setup.py b/Engine/storlet_daemon_factory/setup.py index 3865b05a..0a8fbd22 100644 --- a/Engine/storlet_daemon_factory/setup.py +++ b/Engine/storlet_daemon_factory/setup.py @@ -14,8 +14,9 @@ See the License for the specific language governing permissions and Limitations under the License. -------------------------------------------------------------------------''' -from setuptools import setup, Extension -setup(name = 'storlet_daemon_factory', - version = '1.0', - package_dir={'storlet_daemon_factory':''}, +from setuptools import setup + +setup(name='storlet_daemon_factory', + version='1.0', + package_dir={'storlet_daemon_factory': ''}, packages=['storlet_daemon_factory']) diff --git a/Engine/swift/setup.py b/Engine/swift/setup.py index a8fa904c..9291d066 100644 --- a/Engine/swift/setup.py +++ b/Engine/swift/setup.py @@ -13,10 +13,11 @@ See the License for the specific language governing permissions and Limitations under the License. -------------------------------------------------------------------------''' from setuptools import setup -paste_factory = ['storlet_handler = storlet_middleware.storlet_handler:filter_factory'] +paste_factory = ['storlet_handler = ' + 'storlet_middleware.storlet_handler:filter_factory'] setup(name='storlets', version='1.0', - packages=['storlet_middleware','storlet_gateway'], + packages=['storlet_middleware', 'storlet_gateway'], entry_points={'paste.filter_factory': paste_factory} ) diff --git a/Engine/swift/storlet_gateway/__init__.py b/Engine/swift/storlet_gateway/__init__.py index b28b04f6..e69de29b 100644 --- a/Engine/swift/storlet_gateway/__init__.py +++ b/Engine/swift/storlet_gateway/__init__.py @@ -1,3 +0,0 @@ - - - diff --git a/Engine/swift/storlet_gateway/storlet_docker_gateway.py b/Engine/swift/storlet_gateway/storlet_docker_gateway.py index 69240bcb..7116a16b 100644 --- a/Engine/swift/storlet_gateway/storlet_docker_gateway.py +++ b/Engine/swift/storlet_gateway/storlet_docker_gateway.py @@ -20,18 +20,20 @@ Created on Mar 24, 2015 ''' import os -import sys -import shutil import select -from eventlet import Timeout +import shutil +from eventlet import Timeout +from storlet_middleware.storlet_common import StorletGatewayBase +from storlet_runtime import RunTimePaths +from storlet_runtime import RunTimeSandbox +from storlet_runtime import StorletInvocationGETProtocol +from storlet_runtime import StorletInvocationPUTProtocol +from storlet_runtime import StorletInvocationSLOProtocol from swift.common.internal_client import InternalClient as ic from swift.common.swob import Request -from storlet_runtime import RunTimeSandbox, RunTimePaths -from storlet_runtime import StorletInvocationGETProtocol,\ - StorletInvocationPUTProtocol, StorletInvocationSLOProtocol from swift.common.utils import config_true_value -from storlet_middleware.storlet_common import StorletGatewayBase + '''--------------------------------------------------------------------------- The Storlet Gateway API @@ -52,24 +54,25 @@ The API is made of: ---------------------------------------------------------------------------''' -class DockerStorletRequest(): - ''' - The StorletRequest class represents a request to be processed by the +class DockerStorletRequest(object): + '''The StorletRequest class represents a request to be processed by the + storlet the request is derived from the Swift request and essentially consists of: 1. A data stream to be processed 2. Metadata identifying the stream ''' + def user_metadata(self, headers): metadata = {} for key in headers: if (key.startswith('X-Storlet') or key.startswith('X-Object-Meta-Storlet')): - pass + pass elif (key.startswith('X-Object-Meta-') or key.startswith('X-Object-Meta-'.lower())): - short_key = key[len('X-Object-Meta-'):] - metadata[short_key] = headers[key] + short_key = key[len('X-Object-Meta-'):] + metadata[short_key] = headers[key] return metadata def _getInitialRequest(self): @@ -130,7 +133,7 @@ class StorletGatewayDocker(StorletGatewayBase): def __iter__(self): return self - + def read_with_timeout(self, size): timeout = Timeout(self.timeout) try: @@ -148,10 +151,10 @@ class StorletGatewayDocker(StorletGatewayBase): timeout.cancel() return chunk - - def next(self, size = 1024): + + def next(self, size=1024): chunk = None - r, w, e = select.select([ self.obj_data ], [], [ ], self.timeout) + r, w, e = select.select([self.obj_data], [], [], self.timeout) if len(r) == 0: self.close() if self.obj_data in r: @@ -161,25 +164,25 @@ class StorletGatewayDocker(StorletGatewayBase): else: return chunk raise StopIteration('Stopped iterator ex') - + def read(self, size=1024): return self.next(size) - + def readline(self, size=-1): return '' + def readlines(self, sizehint=-1): - pass; + pass def close(self): - if self.closed == True: + if self.closed is True: return self.closed = True os.close(self.obj_data) - + def __del__(self): self.close() - def validateStorletUpload(self, req): if (self.container == self.sconf['storlet_container']): @@ -223,9 +226,9 @@ class StorletGatewayDocker(StorletGatewayBase): self._clean_storlet_stuff_from_request(req.headers) req.headers.pop('X-Run-Storlet') - slog_path = self.\ + slog_path = self. \ paths.slog_path(self.idata['storlet_main_class']) - storlet_pipe_path = self.\ + storlet_pipe_path = self. \ paths.host_storlet_pipe(self.idata['storlet_main_class']) sprotocol = StorletInvocationPUTProtocol(sreq, @@ -237,7 +240,9 @@ class StorletGatewayDocker(StorletGatewayBase): self._set_metadata_in_headers(req.headers, out_md) self._upload_storlet_logs(slog_path) - return out_md, StorletGatewayDocker.IterLike(self.data_read_fd, self.storlet_timeout, sprotocol._cancel) + return out_md, StorletGatewayDocker.IterLike(self.data_read_fd, + self.storlet_timeout, + sprotocol._cancel) def gatewayProxyGETFlow(self, req, container, obj, orig_resp): # Flow for running the GET computation on the proxy @@ -250,9 +255,9 @@ class StorletGatewayDocker(StorletGatewayBase): docker_updated) self._add_system_params(req.params) - slog_path = self.\ + slog_path = self. \ paths.slog_path(self.idata['storlet_main_class']) - storlet_pipe_path = self.\ + storlet_pipe_path = self. \ paths.host_storlet_pipe(self.idata['storlet_main_class']) sprotocol = StorletInvocationSLOProtocol(sreq, @@ -264,7 +269,9 @@ class StorletGatewayDocker(StorletGatewayBase): self._set_metadata_in_headers(orig_resp.headers, out_md) self._upload_storlet_logs(slog_path) - return out_md, StorletGatewayDocker.IterLike(self.data_read_fd, self.storlet_timeout, sprotocol._cancel) + return out_md, StorletGatewayDocker.IterLike(self.data_read_fd, + self.storlet_timeout, + sprotocol._cancel) def gatewayObjectGetFlow(self, req, container, obj, orig_resp): sreq = StorletGETRequest(self.account, orig_resp, req.params) @@ -276,9 +283,9 @@ class StorletGatewayDocker(StorletGatewayBase): docker_updated) self._add_system_params(req.params) - slog_path = self.\ + slog_path = self. \ paths.slog_path(self.idata['storlet_main_class']) - storlet_pipe_path = self.paths.\ + storlet_pipe_path = self.paths. \ host_storlet_pipe(self.idata['storlet_main_class']) sprotocol = StorletInvocationGETProtocol(sreq, storlet_pipe_path, @@ -290,7 +297,9 @@ class StorletGatewayDocker(StorletGatewayBase): self._set_metadata_in_headers(orig_resp.headers, out_md) self._upload_storlet_logs(slog_path) - return out_md, StorletGatewayDocker.IterLike(self.data_read_fd, self.storlet_timeout, sprotocol._cancel) + return out_md, StorletGatewayDocker.IterLike(self.data_read_fd, + self.storlet_timeout, + sprotocol._cancel) def verify_access(self, env, version, account, container, object): self.logger.info('Verify access to {0}/{1}/{2}'.format(account, @@ -343,34 +352,34 @@ class StorletGatewayDocker(StorletGatewayBase): req.headers['X-Storlet-' + key] = val def _add_system_params(self, params): - ''' - Adds Storlet engine specific parameters to the invocation + '''Adds Storlet engine specific parameters to the invocation + currently, this consists only of the execution path of the Storlet within the Docker container. ''' - params['storlet_execution_path'] = self.\ + params['storlet_execution_path'] = self. \ paths.sbox_storlet_exec(self.idata['storlet_main_class']) def _clean_storlet_stuff_from_request(self, headers): for key in headers: if (key.startswith('X-Storlet') or key.startswith('X-Object-Meta-Storlet')): - del headers[key] + del headers[key] return headers def _get_storlet_invocation_data(self, req): data = dict() data['storlet_name'] = req.headers.get('X-Run-Storlet') data['generate_log'] = req.headers.get('X-Storlet-Generate-Log', False) - data['storlet_original_timestamp'] = req.headers.\ + data['storlet_original_timestamp'] = req.headers. \ get('X-Storlet-X-Timestamp') - data['storlet_original_size'] = req.headers.\ + data['storlet_original_size'] = req.headers. \ get('X-Storlet-Content-Length') data['storlet_md'] = {'storlet_original_timestamp': data['storlet_original_timestamp'], 'storlet_original_size': data['storlet_original_size']} - data['storlet_main_class'] = req.headers.\ + data['storlet_main_class'] = req.headers. \ get('X-Object-Meta-Storlet-Main') scope = self.account @@ -378,7 +387,7 @@ class StorletGatewayDocker(StorletGatewayBase): if data['scope'].rfind(':') > 0: data['scope'] = data['scope'][:data['scope'].rfind(':')] - data['storlet_dependency'] = req.headers.\ + data['storlet_dependency'] = req.headers. \ get('X-Object-Meta-Storlet-Dependency') data['request_params'] = req.params return data @@ -395,7 +404,7 @@ class StorletGatewayDocker(StorletGatewayBase): try: headers = dict() headers['CONTENT_TYPE'] = 'text/html' - log_obj_name = '%s.log' %\ + log_obj_name = '%s.log' % \ self.idata['storlet_name'][:self.idata['storlet_name']. find('-')] client.upload_object(logfile, self.account, @@ -405,8 +414,8 @@ class StorletGatewayDocker(StorletGatewayBase): raise e def bring_from_cache(self, obj_name, is_storlet): - ''' - Auxiliary function that: + '''Auxiliary function that: + (1) Brings from Swift obj_name, whether this is a storlet or a storlet dependency. (2) Copies from local cache into the Docker conrainer @@ -425,7 +434,7 @@ class StorletGatewayDocker(StorletGatewayBase): swift_source_container = self.paths.storlet_container if not os.path.exists(cache_dir): - os.makedirs(cache_dir, 0755) + os.makedirs(cache_dir, 0o755) # cache_target_path is the actual object we need to deal with # e.g. a concrete storlet or dependency we need to bring/update @@ -467,7 +476,7 @@ class StorletGatewayDocker(StorletGatewayBase): fn.close() if not is_storlet: - expected_perm = resp.headers.\ + expected_perm = resp.headers. \ get('X-Object-Meta-Storlet-Dependency-Permissions', '') if expected_perm != '': os.chmod(cache_target_path, int(expected_perm, 8)) @@ -479,12 +488,12 @@ class StorletGatewayDocker(StorletGatewayBase): # 1. The Docker container does not hold a copy of the object # 2. The Docker container holds an older version of the object update_docker = False - docker_storlet_path = self.paths.\ + docker_storlet_path = self.paths. \ host_storlet(self.idata['storlet_main_class']) docker_target_path = os.path.join(docker_storlet_path, obj_name) if not os.path.exists(docker_storlet_path): - os.makedirs(docker_storlet_path, 0755) + os.makedirs(docker_storlet_path, 0o755) update_docker = True elif not os.path.isfile(docker_target_path): update_docker = True @@ -493,7 +502,7 @@ class StorletGatewayDocker(StorletGatewayBase): fstat_docker_object = os.stat(docker_target_path) b_size_changed = fstat_cached_object.st_size \ != fstat_docker_object.st_size - b_time_changed = float(fstat_cached_object.st_mtime) <\ + b_time_changed = float(fstat_cached_object.st_mtime) < \ float(fstat_docker_object.st_mtime) if (b_size_changed or b_time_changed): update_docker = True @@ -506,8 +515,8 @@ class StorletGatewayDocker(StorletGatewayBase): return update_docker def update_docker_container_from_cache(self): - ''' - Iterates over the storlet name and its dependencies appearing + '''Iterates over the storlet name and its dependencies appearing + in the invocation data and make sure they are brought to the local cache, and from there to the Docker container. Uses the bring_from_cache auxiliary function. @@ -516,7 +525,7 @@ class StorletGatewayDocker(StorletGatewayBase): # where at the host side, reside the storlet containers storlet_path = self.paths.host_storlet_prefix() if not os.path.exists(storlet_path): - os.makedirs(storlet_path, 0755) + os.makedirs(storlet_path, 0o755) # Iterate over storlet and dependencies, and make sure # they are updated within the Docker container. diff --git a/Engine/swift/storlet_gateway/storlet_runtime.py b/Engine/swift/storlet_gateway/storlet_runtime.py index 07ed3c6c..b73999d5 100644 --- a/Engine/swift/storlet_gateway/storlet_runtime.py +++ b/Engine/swift/storlet_gateway/storlet_runtime.py @@ -20,54 +20,60 @@ Created on Feb 10, 2015 ''' import os -import time -import stat import select -import commands +import stat +import subprocess +import time import eventlet from eventlet.timeout import Timeout import json -import shutil -import sys -from swift.common.constraints import MAX_META_OVERALL_SIZE -from swift.common.swob import HTTPBadRequest, Request,\ - HTTPInternalServerError - -from SBusPythonFacade.SBus import * -from SBusPythonFacade.SBusDatagram import * -from SBusPythonFacade.SBusStorletCommand import * -from SBusPythonFacade.SBusFileDescription import * +from SBusPythonFacade.SBus import SBus +from SBusPythonFacade.SBusDatagram import SBusDatagram +from SBusPythonFacade.SBusFileDescription import SBUS_FD_INPUT_OBJECT +from SBusPythonFacade.SBusFileDescription import SBUS_FD_LOGGER +from SBusPythonFacade.SBusFileDescription import SBUS_FD_OUTPUT_OBJECT +from SBusPythonFacade.SBusFileDescription import SBUS_FD_OUTPUT_OBJECT_METADATA +from SBusPythonFacade.SBusFileDescription import SBUS_FD_OUTPUT_TASK_ID +from SBusPythonFacade.SBusStorletCommand import SBUS_CMD_CANCEL +from SBusPythonFacade.SBusStorletCommand import SBUS_CMD_DAEMON_STATUS +from SBusPythonFacade.SBusStorletCommand import SBUS_CMD_EXECUTE +from SBusPythonFacade.SBusStorletCommand import SBUS_CMD_PING +from SBusPythonFacade.SBusStorletCommand import SBUS_CMD_START_DAEMON +from SBusPythonFacade.SBusStorletCommand import SBUS_CMD_STOP_DAEMON from storlet_middleware.storlet_common import StorletLogger +from swift.common.constraints import MAX_META_OVERALL_SIZE eventlet.monkey_patch() - '''--------------------------------------------------------------------------- Sandbox API ''' -class RunTimePaths(): - ''' - The Storlet Engine need to be access stuff located in many paths: - 1. The various communication channels represented as pipes in the filesystem + +class RunTimePaths(object): + '''The Storlet Engine need to be access stuff located in many paths: + + 1. The various communication channels represented as pipes in the + filesystem 2. Directories where to place Storlets 3. Directories where to place logs - + Communication channels ---------------------- The RunTimeSandbox communicates with the Sandbox via two types of pipes - 1. factory pipe - defined per account, used for communication with the sandbox + 1. factory pipe - defined per account, used for communication with the + sandbox for e.g. start/stop a storlet daemon 2. Storlet pipe - defined per account and Storlet, used for communication with a storlet daemon, e.g. to call the invoke API - + Each pipe type has two paths: 1. A path that is inside the sandbox 2. A path that is outside of the sandbox or at the host side. As such this path is prefixed by 'host_' - + Thus, we have the following 4 paths of interest: 1. sandbox_factory_pipe_path 2. host_factory_pipe_path @@ -75,7 +81,7 @@ class RunTimePaths(): 4. host_storlet_pipe_path Our implementation uses the following path structure for the various pipes: - In the host, all pipes belonging to a given account are prefixed by + In the host, all pipes belonging to a given account are prefixed by /, where comes from the configuration Thus: host_factory_pipe_path is of the form //factory_pipe @@ -87,18 +93,20 @@ class RunTimePaths(): Storlets Locations ------------------ - The Storlet binaries are accessible from the sandbox using a mounted directory. + The Storlet binaries are accessible from the sandbox using a mounted + directory. This directory is called the storlet directories. On the host side it is of the form // On the sandbox side it is of the form /home/swift/ comes from the configuration is the prefix of the jar. - + Logs ---- Logs are located in paths of the form: //.log ''' + def __init__(self, account, conf): self.account = account self.scope = account[5:18] @@ -107,14 +115,13 @@ class RunTimePaths(): self.factory_pipe_suffix = 'factory_pipe' self.sandbox_pipe_prefix = '/mnt/channels' self.storlet_pipe_suffix = '_storlet_pipe' - self.sandbox_storlet_dir_prefix = '/home/swift' + self.sandbox_storlet_dir_prefix = '/home/swift' self.host_storlet_root = conf['storlets_dir'] self.host_log_path_root = conf['log_dir'] self.host_cache_root = conf['cache_dir'] self.storlet_container = conf['storlet_container'] self.storlet_dependency = conf['storlet_dependency'] - def host_pipe_prefix(self): return os.path.join(self.host_pipe_root, self.scope) @@ -126,48 +133,53 @@ class RunTimePaths(): os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) def host_factory_pipe(self): - return os.path.join(self.host_pipe_prefix(), + return os.path.join(self.host_pipe_prefix(), self.factory_pipe_suffix) - + def host_storlet_pipe(self, storlet_id): return os.path.join(self.host_pipe_prefix(), storlet_id) - + def sbox_storlet_pipe(self, storlet_id): return os.path.join(self.sandbox_pipe_prefix, storlet_id) - + def sbox_storlet_exec(self, storlet_id): return os.path.join(self.sandbox_storlet_dir_prefix, storlet_id) - + def host_storlet_prefix(self): return os.path.join(self.host_storlet_root, self.scope) - + def host_storlet(self, storlet_id): return os.path.join(self.host_storlet_prefix(), storlet_id) - + def slog_path(self, storlet_id): log_dir = os.path.join(self.host_log_path_root, self.scope, storlet_id) if not os.path.exists(log_dir): os.makedirs(log_dir) return log_dir - + def get_host_storlet_cache_dir(self): - return os.path.join(self.host_cache_root, self.scope,self.storlet_container) - + return os.path.join(self.host_cache_root, self.scope, + self.storlet_container) + def get_host_dependency_cache_dir(self): - return os.path.join(self.host_cache_root, self.scope,self.storlet_dependency) + return os.path.join(self.host_cache_root, self.scope, + self.storlet_dependency) '''--------------------------------------------------------------------------- Docker Stateful Container API -The RunTimeSandbox serve as an API between the Docker Gateway and +The RunTimeSandbox serve as an API between the Docker Gateway and a re-usable per account sandbox ---------------------------------------------------------------------------''' -class RunTimeSandbox(): - ''' - The RunTimeSandbox represents a re-usable per account sandbox. The sandbox - is re-usable in the sense that it can run several storlet daemons. - + + +class RunTimeSandbox(object): + '''The RunTimeSandbox represents a re-usable per account sandbox. + + The sandbox is re-usable in the sense that it can run several storlet + daemons. + The following methods are supported: ping - pings the sandbox for liveness wait - wait for the sandbox to be ready for processing commands @@ -182,22 +194,24 @@ class RunTimeSandbox(): self.account = account self.sandbox_ping_interval = 0.5 - self.sandbox_wait_timeout = int(conf['restart_linux_container_timeout']) + self.sandbox_wait_timeout = \ + int(conf['restart_linux_container_timeout']) self.docker_repo = conf['docker_repo'] self.docker_image_name_prefix = 'tenant' - # TODO: should come from upper layer Storlet metadata + # TODO(should come from upper layer Storlet metadata) self.storlet_language = 'java' - - # TODO: add line in conf - self.storlet_daemon_thread_pool_size = int(conf.get('storlet_daemon_thread_pool_size',5)) - self.storlet_daemon_debug_level = conf.get('storlet_daemon_debug_level','TRACE') - - # TODO: change logger's route if possible + + # TODO(add line in conf) + self.storlet_daemon_thread_pool_size = \ + int(conf.get('storlet_daemon_thread_pool_size', 5)) + self.storlet_daemon_debug_level = \ + conf.get('storlet_daemon_debug_level', 'TRACE') + + # TODO(change logger's route if possible) self.logger = logger - - + def _parse_sandbox_factory_answer(self, str_answer): two_tokens = str_answer.split(':', 1) b_success = False @@ -207,28 +221,28 @@ class RunTimeSandbox(): def ping(self): pipe_path = self.paths.host_factory_pipe() - + read_fd, write_fd = os.pipe() - dtg = SBusDatagram.create_service_datagram( SBUS_CMD_PING, write_fd ) - rc = SBus.send( pipe_path, dtg ) + dtg = SBusDatagram.create_service_datagram(SBUS_CMD_PING, write_fd) + rc = SBus.send(pipe_path, dtg) if (rc < 0): return -1 - - reply = os.read(read_fd,10) - os.close(read_fd) + + reply = os.read(read_fd, 10) + os.close(read_fd) os.close(write_fd) res, error_txt = self._parse_sandbox_factory_answer(reply) - if res == True: + if res is True: return 1 return 0 - + def wait(self): do_wait = True up = 0 to = Timeout(self.sandbox_wait_timeout) try: - while do_wait == True: + while do_wait is True: rc = self.ping() if (rc != 1): time.sleep(self.sandbox_ping_interval) @@ -237,18 +251,17 @@ class RunTimeSandbox(): to.cancel() do_wait = False up = 1 - except Timeout as t: + except Timeout: self.logger.info("wait for sandbox %s timedout" % self.account) do_wait = False finally: to.cancel() return up - + def restart(self): - ''' - Restarts the account's sandbox - + '''Restarts the account's sandbox + Returned value: True - If the sandbox was started successfully False - Otherwise @@ -259,109 +272,115 @@ class RunTimeSandbox(): account_id = self.account[len('auth_'):] else: account_id = self.account - + self.paths.create_host_pipe_prefix() - + docker_container_name = '%s_%s' % (self.docker_image_name_prefix, - account_id) - docker_image_name = '%s/%s' % (self.docker_repo,account_id) - pipe_mount = '%s:%s' % (self.paths.host_pipe_prefix(), + account_id) + docker_image_name = '%s/%s' % (self.docker_repo, account_id) + pipe_mount = '%s:%s' % (self.paths.host_pipe_prefix(), self.paths.sandbox_pipe_prefix) - - storlet_mount = '%s:%s' % (self.paths.host_storlet_prefix(), + + storlet_mount = '%s:%s' % (self.paths.host_storlet_prefix(), self.paths.sandbox_storlet_dir_prefix) - - cmd = '%s/restart_docker_container %s %s %s %s' % ( - self.paths.host_restart_script_dir, - docker_container_name, - docker_image_name, - pipe_mount, - storlet_mount) - - res = commands.getoutput(cmd) + + cmd = [self.paths.host_restart_script_dir + + '/restart_docker_container', + docker_container_name, docker_image_name, pipe_mount, + storlet_mount] + + subprocess.call(cmd) return self.wait() def start_storlet_daemon(self, spath, storlet_id): prms = {} prms['daemon_language'] = 'java' - prms['storlet_path'] = spath - prms['storlet_name'] = storlet_id - prms['uds_path'] = self.paths.sbox_storlet_pipe(storlet_id) - prms['log_level'] = self.storlet_daemon_debug_level - prms['pool_size'] = self.storlet_daemon_thread_pool_size - + prms['storlet_path'] = spath + prms['storlet_name'] = storlet_id + prms['uds_path'] = self.paths.sbox_storlet_pipe(storlet_id) + prms['log_level'] = self.storlet_daemon_debug_level + prms['pool_size'] = self.storlet_daemon_thread_pool_size + read_fd, write_fd = os.pipe() - dtg = SBusDatagram.create_service_datagram( SBUS_CMD_START_DAEMON, - write_fd ) - dtg.set_exec_params( prms ) - + dtg = SBusDatagram.create_service_datagram(SBUS_CMD_START_DAEMON, + write_fd) + dtg.set_exec_params(prms) + pipe_path = self.paths.host_factory_pipe() - rc = SBus.send( pipe_path, dtg ) + rc = SBus.send(pipe_path, dtg) if (rc < 0): return -1 - reply = os.read(read_fd,10) - os.close(read_fd) + reply = os.read(read_fd, 10) + os.close(read_fd) os.close(write_fd) res, error_txt = self._parse_sandbox_factory_answer(reply) - if res == True: + if res is True: return 1 return 0 - + def stop_storlet_daemon(self, storlet_id): read_fd, write_fd = os.pipe() - dtg = SBusDatagram.create_service_datagram( SBUS_CMD_STOP_DAEMON, - write_fd ) + dtg = SBusDatagram.create_service_datagram(SBUS_CMD_STOP_DAEMON, + write_fd) dtg.add_exec_param('storlet_name', storlet_id) pipe_path = self.paths.host_factory_pipe() - rc = SBus.send( pipe_path, dtg ) + rc = SBus.send(pipe_path, dtg) if (rc < 0): - self.logger.info("Failed to send status command to %s %s" % (self.account, storlet_id)) + self.logger.info("Failed to send status command to %s %s" % + (self.account, storlet_id)) return -1 - - reply = os.read(read_fd,10) - os.close(read_fd) + + reply = os.read(read_fd, 10) + os.close(read_fd) os.close(write_fd) res, error_txt = self._parse_sandbox_factory_answer(reply) - if res == True: + if res is True: return 1 return 0 def get_storlet_daemon_status(self, storlet_id): read_fd, write_fd = os.pipe() - dtg = SBusDatagram.create_service_datagram( SBUS_CMD_DAEMON_STATUS, - write_fd ) - dtg.add_exec_param( 'storlet_name', storlet_id) + dtg = SBusDatagram.create_service_datagram(SBUS_CMD_DAEMON_STATUS, + write_fd) + dtg.add_exec_param('storlet_name', storlet_id) pipe_path = self.paths.host_factory_pipe() rc = SBus.send(pipe_path, dtg) if (rc < 0): - self.logger.info("Failed to send status command to %s %s" % (self.account, storlet_id)) + self.logger.info("Failed to send status command to %s %s" % + (self.account, storlet_id)) return -1 - reply = os.read(read_fd,10) - os.close(read_fd) + reply = os.read(read_fd, 10) + os.close(read_fd) os.close(write_fd) - res, error_txt = self._parse_sandbox_factory_answer(reply) - if res == True: + res, error_txt = self._parse_sandbox_factory_answer(reply) + if res is True: return 1 return 0 - def activate_storlet_daemon(self, invocation_data, cache_updated = True): - storlet_daemon_status = self.get_storlet_daemon_status(invocation_data['storlet_main_class']) + def activate_storlet_daemon(self, invocation_data, cache_updated=True): + storlet_daemon_status = \ + self.get_storlet_daemon_status(invocation_data[ + 'storlet_main_class']) if (storlet_daemon_status == -1): # We failed to send a command to the factory. # Best we can do is execute the container. - self.logger.debug('Failed to check Storlet daemon status, restart Docker container') + self.logger.debug('Failed to check Storlet daemon status, ' + 'restart Docker container') res = self.restart() if (res != 1): raise Exception('Docker container is not responsive') storlet_daemon_status = 0 - - if (cache_updated == True and storlet_daemon_status == 1): - # The cache was updated while the daemon is running we need to stop it. - self.logger.debug('The cache was updated, and the storlet daemon is running. Stopping daemon') - res = self.stop_storlet_daemon( invocation_data['storlet_main_class'] ) + + if (cache_updated is True and storlet_daemon_status == 1): + # The cache was updated while the daemon is running we need to + # stop it. + self.logger.debug('The cache was updated, and the storlet daemon ' + 'is running. Stopping daemon') + res = \ + self.stop_storlet_daemon(invocation_data['storlet_main_class']) if res != 1: res = self.restart() if (res != 1): @@ -369,47 +388,53 @@ class RunTimeSandbox(): else: self.logger.debug('Deamon stopped') storlet_daemon_status = 0 - + if (storlet_daemon_status == 0): self.logger.debug('Going to start storlet daemon!') - class_path = '/home/swift/%s/%s' % (invocation_data['storlet_main_class'], - invocation_data['storlet_name']) + class_path = \ + '/home/swift/%s/%s' % (invocation_data['storlet_main_class'], + invocation_data['storlet_name']) for dep in invocation_data['storlet_dependency'].split(','): - class_path = '%s:/home/swift/%s/%s' %\ - (class_path, - invocation_data['storlet_main_class'], - dep) - - daemon_status = self.start_storlet_daemon( - class_path, - invocation_data['storlet_main_class']) + class_path = '%s:/home/swift/%s/%s' % \ + (class_path, + invocation_data['storlet_main_class'], + dep) + + daemon_status = \ + self.start_storlet_daemon(class_path, + invocation_data[ + 'storlet_main_class']) if daemon_status != 1: - self.logger.error('Daemon start Failed, returned code is %d' % daemon_status) + self.logger.error('Daemon start Failed, returned code is %d' % + daemon_status) raise Exception('Daemon start failed') else: self.logger.debug('Daemon started') '''--------------------------------------------------------------------------- Storlet Daemon API -The StorletInvocationGETProtocol, StorletInvocationPUTProtocol, StorletInvocationSLOProtocol -server as an API between the Docker Gateway and the Storlet Daemon which +The StorletInvocationGETProtocol, StorletInvocationPUTProtocol, +StorletInvocationSLOProtocol +server as an API between the Docker Gateway and the Storlet Daemon which runs inside the Docker container. These classes implement the Storlet execution protocol ---------------------------------------------------------------------------''' -class StorletInvocationProtocol(): + + +class StorletInvocationProtocol(object): def _add_input_stream(self, appendFd): - #self.fds.append(self.srequest.stream + # self.fds.append(self.srequest.stream self.fds.append(appendFd) - # TODO: Break request metadata and systemmetadata + # TODO(Break request metadata and systemmetadata) md = dict() md['type'] = SBUS_FD_INPUT_OBJECT if self.srequest.user_metadata is not None: for key, val in self.srequest.user_metadata.iteritems(): md[key] = val self.fdmd.append(md) - + def _add_output_stream(self): self.fds.append(self.execution_str_write_fd) md = dict() @@ -425,26 +450,26 @@ class StorletInvocationProtocol(): md = dict() md['type'] = SBUS_FD_OUTPUT_OBJECT_METADATA self.fdmd.append(md) - + def _add_logger_stream(self): self.fds.append(self.storlet_logger.getfd()) md = dict() md['type'] = SBUS_FD_LOGGER self.fdmd.append(md) - + def _prepare_invocation_descriptors(self): # Add the input stream self._add_input_stream() - # Add the output stream + # Add the output stream self.data_read_fd, self.data_write_fd = os.pipe() self.execution_str_read_fd, self.execution_str_write_fd = os.pipe() self.metadata_read_fd, self.metadata_write_fd = os.pipe() self._add_output_stream() - + # Add the logger self._add_logger_stream() - + def _close_remote_side_descriptors(self): if self.data_write_fd: os.close(self.data_write_fd) @@ -455,42 +480,42 @@ class StorletInvocationProtocol(): def _cancel(self): read_fd, write_fd = os.pipe() - dtg = SBusDatagram.create_service_datagram( SBUS_CMD_CANCEL, write_fd ) + dtg = SBusDatagram.create_service_datagram(SBUS_CMD_CANCEL, write_fd) dtg.set_task_id(self.task_id) - rc = SBus.send( self.storlet_pipe_path, dtg ) + rc = SBus.send(self.storlet_pipe_path, dtg) if (rc < 0): return -1 - reply = os.read(read_fd,10) + os.read(read_fd, 10) os.close(read_fd) os.close(write_fd) - def _invoke(self): - dtg = SBusDatagram() - dtg.set_files( self.fds ) - dtg.set_metadata( self.fdmd ) - dtg.set_exec_params( self.srequest.params ) + dtg = SBusDatagram() + dtg.set_files(self.fds) + dtg.set_metadata(self.fdmd) + dtg.set_exec_params(self.srequest.params) dtg.set_command(SBUS_CMD_EXECUTE) - rc = SBus.send( self.storlet_pipe_path, dtg ) - + rc = SBus.send(self.storlet_pipe_path, dtg) + if (rc < 0): raise Exception("Failed to send execute command") self._wait_for_read_with_timeout(self.execution_str_read_fd) self.task_id = os.read(self.execution_str_read_fd, 10) - def __init__(self, srequest, storlet_pipe_path, storlet_logger_path, timeout): + def __init__(self, srequest, storlet_pipe_path, storlet_logger_path, + timeout): self.srequest = srequest self.storlet_pipe_path = storlet_pipe_path self.storlet_logger_path = storlet_logger_path self.timeout = timeout - + # remote side file descriptors and their metadata lists # to be sent as part of invocation self.fds = list() self.fdmd = list() - + # local side file descriptors self.data_read_fd = None self.data_write_fd = None @@ -499,38 +524,42 @@ class StorletInvocationProtocol(): self.execution_str_read_fd = None self.execution_str_write_fd = None self.task_id = None - + if not os.path.exists(storlet_logger_path): os.makedirs(storlet_logger_path) def _wait_for_read_with_timeout(self, fd): - r, w, e = select.select([ fd ], [], [ ], self.timeout) + r, w, e = select.select([fd], [], [], self.timeout) if len(r) == 0: if self.task_id: self._cancel() raise Timeout('Timeout while waiting for storlet output') if fd in r: return - + def _read_metadata(self): self._wait_for_read_with_timeout(self.metadata_read_fd) flat_json = os.read(self.metadata_read_fd, MAX_META_OVERALL_SIZE) if flat_json is not None: md = json.loads(flat_json) return md - + + class StorletInvocationGETProtocol(StorletInvocationProtocol): - + def _add_input_stream(self): StorletInvocationProtocol._add_input_stream(self, self.srequest.stream) - def __init__(self, srequest, storlet_pipe_path, storlet_logger_path, timeout): - StorletInvocationProtocol.__init__(self, srequest, storlet_pipe_path, storlet_logger_path, timeout) - - def communicate(self): - self.storlet_logger = StorletLogger(self.storlet_logger_path, 'storlet_invoke') + def __init__(self, srequest, storlet_pipe_path, storlet_logger_path, + timeout): + StorletInvocationProtocol.__init__(self, srequest, storlet_pipe_path, + storlet_logger_path, timeout) + + def communicate(self): + self.storlet_logger = StorletLogger(self.storlet_logger_path, + 'storlet_invoke') self.storlet_logger.open() - + self._prepare_invocation_descriptors() try: self._invoke() @@ -539,33 +568,37 @@ class StorletInvocationGETProtocol(StorletInvocationProtocol): finally: self._close_remote_side_descriptors() self.storlet_logger.close() - + out_md = self._read_metadata() - os.close(self.metadata_read_fd) + os.close(self.metadata_read_fd) self._wait_for_read_with_timeout(self.data_read_fd) - os.close(self.execution_str_read_fd) - + os.close(self.execution_str_read_fd) + return out_md, self.data_read_fd + class StorletInvocationProxyProtocol(StorletInvocationProtocol): - def __init__(self, srequest, storlet_pipe_path, storlet_logger_path, timeout): - StorletInvocationProtocol.__init__(self, srequest, storlet_pipe_path, storlet_logger_path, timeout) + def __init__(self, srequest, storlet_pipe_path, storlet_logger_path, + timeout): + StorletInvocationProtocol.__init__(self, srequest, storlet_pipe_path, + storlet_logger_path, timeout) self.input_data_read_fd, self.input_data_write_fd = os.pipe() - # YM this pipe permits to take data from srequest.stream to input_data_write_fd + # YM this pipe permits to take data from srequest.stream to + # input_data_write_fd # YM the write side stays with us, the read side is sent to storlet - def _add_input_stream(self): - StorletInvocationProtocol._add_input_stream(self, self.input_data_read_fd) + StorletInvocationProtocol._add_input_stream(self, + self.input_data_read_fd) - def _wait_for_write_with_timeout(self,fd): - r, w, e = select.select([ ], [ fd ], [ ], self.timeout) + def _wait_for_write_with_timeout(self, fd): + r, w, e = select.select([], [fd], [], self.timeout) if len(w) == 0: raise Timeout('Timeout while waiting for storlet to read') if fd in w: return - + def _write_with_timeout(self, writer, chunk): timeout = Timeout(self.timeout) try: @@ -580,9 +613,10 @@ class StorletInvocationProxyProtocol(StorletInvocationProtocol): timeout.cancel() def communicate(self): - self.storlet_logger = StorletLogger(self.storlet_logger_path, 'storlet_invoke') + self.storlet_logger = StorletLogger(self.storlet_logger_path, + 'storlet_invoke') self.storlet_logger.open() - + self._prepare_invocation_descriptors() try: self._invoke() @@ -591,7 +625,7 @@ class StorletInvocationProxyProtocol(StorletInvocationProtocol): finally: self._close_remote_side_descriptors() self.storlet_logger.close() - + self._wait_for_write_with_timeout(self.input_data_write_fd) # We do the writing in a different thread. # Otherwise, we can run into the following deadlock @@ -599,19 +633,23 @@ class StorletInvocationProxyProtocol(StorletInvocationProtocol): # 2. Storlet reads and starts to write metadata and then data # 3. middleware continues writing # 4. Storlet continues writing and gets stuck as middleware - # is busy writing, but still not consuming the reader end + # is busy writing, but still not consuming the reader end # of the Storlet writer. eventlet.spawn_n(self._write_input_data) out_md = self._read_metadata() self._wait_for_read_with_timeout(self.data_read_fd) - + return out_md, self.data_read_fd - + + class StorletInvocationPUTProtocol(StorletInvocationProxyProtocol): - - def __init__(self, srequest, storlet_pipe_path, storlet_logger_path, timeout): - StorletInvocationProxyProtocol.__init__(self, srequest, storlet_pipe_path, storlet_logger_path, timeout) - + + def __init__(self, srequest, storlet_pipe_path, storlet_logger_path, + timeout): + StorletInvocationProxyProtocol.__init__(self, srequest, + storlet_pipe_path, + storlet_logger_path, timeout) + def _write_input_data(self): writer = os.fdopen(self.input_data_write_fd, 'w') reader = self.srequest.stream @@ -621,10 +659,13 @@ class StorletInvocationPUTProtocol(StorletInvocationProxyProtocol): class StorletInvocationSLOProtocol(StorletInvocationProxyProtocol): - - def __init__(self, srequest, storlet_pipe_path, storlet_logger_path, timeout): - StorletInvocationProxyProtocol.__init__(self, srequest, storlet_pipe_path, storlet_logger_path, timeout) - + + def __init__(self, srequest, storlet_pipe_path, storlet_logger_path, + timeout): + StorletInvocationProxyProtocol.__init__(self, srequest, + storlet_pipe_path, + storlet_logger_path, timeout) + def _write_input_data(self): writer = os.fdopen(self.input_data_write_fd, 'w') reader = self.srequest.stream @@ -633,4 +674,3 @@ class StorletInvocationSLOProtocol(StorletInvocationProxyProtocol): self._write_with_timeout(writer, chunk) # print >> sys.stderr, 'next SLO chunk...%d'% len(chunk) writer.close() - diff --git a/Engine/swift/storlet_gateway/storlet_stub_gateway.py b/Engine/swift/storlet_gateway/storlet_stub_gateway.py index 695b7fcb..10950f30 100644 --- a/Engine/swift/storlet_gateway/storlet_stub_gateway.py +++ b/Engine/swift/storlet_gateway/storlet_stub_gateway.py @@ -1,4 +1,20 @@ -class StorletStubBase(): +'''------------------------------------------------------------------------- +Copyright IBM Corp. 2015, 2015 All Rights Reserved +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +Limitations under the License. +-------------------------------------------------------------------------''' + + +class StorletStubBase(object): def __init__(self, storlet_conf, logger, app, version, account, container, obj): diff --git a/Engine/swift/storlet_middleware/storlet_common.py b/Engine/swift/storlet_middleware/storlet_common.py index 4cf5636b..2aa32516 100644 --- a/Engine/swift/storlet_middleware/storlet_common.py +++ b/Engine/swift/storlet_middleware/storlet_common.py @@ -1,67 +1,66 @@ -#----------------------------------------------------------------------------------------------- -# Copyright IBM Corp. 2015, 2015 All Rights Reserved -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# Limitations under the License. -#----------------------------------------------------------------------------------------------- +'''------------------------------------------------------------------------- +Copyright IBM Corp. 2015, 2015 All Rights Reserved +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +Limitations under the License. +-------------------------------------------------------------------------''' ''' Created on Feb 18, 2014 @author: gilv ''' -from eventlet.timeout import Timeout -import traceback import os import sys +import traceback + -import select - class StorletTimeout(Exception): pass + class StorletLogger(object): def __init__(self, path, name): - self.full_path = os.path.join(path,'%s.log' % name) - + self.full_path = os.path.join(path, '%s.log' % name) + def open(self): - self.file = open(self.full_path,'a') - + self.file = open(self.full_path, 'a') + def getfd(self): return self.file.fileno() - + def getsize(self): statinfo = os.stat(self.full_path) return statinfo.st_size - + def close(self): self.file.close() - + def fobj(self): return open(self.full_path, 'r') + class StorletException(object): - ### Print details about the code line which caused the exception + # Print details about the code line which caused the exception @staticmethod - def handle( logger, exc ): - logger.info('-'*60) + def handle(logger, exc): + logger.info('-' * 60) logger.info(exc) - ### logging.exception() + # logging.exception() traceback.print_exc(file=sys.stdout) - logger.info('-'*60) + logger.info('-' * 60) -class StorletGatewayBase(): +class StorletGatewayBase(object): def validateStorletUpload(self, request): raise NotImplementedError("Not implemented: validateStorletUpload") @@ -81,6 +80,7 @@ class StorletGatewayBase(): def gatewayObjectGetFlow(self, request, container, obj, original_response): raise NotImplementedError("Not implemented: gatewayObjectGetFlow") + class StorletStubGateway(StorletGatewayBase): def __init__(self, sconf, logger, app, version, account, container, @@ -93,10 +93,10 @@ class StorletStubGateway(StorletGatewayBase): self.obj = obj self.sconf = sconf self.dummy_stream = os.pipe() - self.dummy_content = sconf.get('dummy_content','aaaa') + self.dummy_content = sconf.get('dummy_content', 'aaaa') def dummy_invocation(self): - os.write(self.dummy_stream[1],self.dummy_content) + os.write(self.dummy_stream[1], self.dummy_content) os.close(self.dummy_stream[1]) return self.dummy_stream[0], {} diff --git a/Engine/swift/storlet_middleware/storlet_handler.py b/Engine/swift/storlet_middleware/storlet_handler.py index 2cd0482c..e67dae5d 100755 --- a/Engine/swift/storlet_middleware/storlet_handler.py +++ b/Engine/swift/storlet_middleware/storlet_handler.py @@ -19,34 +19,38 @@ Created on Feb 18, 2014 @author: Gil Vernik ''' -from storlet_common import StorletTimeout,StorletException - -from swift.common.utils import get_logger, register_swift_info, is_success, config_true_value -from swift.common.swob import Request, Response, wsgify, \ - HTTPBadRequest, HTTPUnauthorized, \ - HTTPInternalServerError -from swift.proxy.controllers.base import get_account_info -from swift.common.exceptions import ConnectionTimeout -from eventlet import Timeout - import ConfigParser -import os -import sys +from eventlet import Timeout +from storlet_common import StorletException +from storlet_common import StorletTimeout +from swift.common.exceptions import ConnectionTimeout +from swift.common.swob import HTTPBadRequest +from swift.common.swob import HTTPInternalServerError +from swift.common.swob import HTTPUnauthorized +from swift.common.swob import Request +from swift.common.swob import Response +from swift.common.swob import wsgify +from swift.common.utils import config_true_value +from swift.common.utils import get_logger +from swift.common.utils import is_success +from swift.common.utils import register_swift_info +from swift.proxy.controllers.base import get_account_info + class StorletHandlerMiddleware(object): - + def __init__(self, app, conf, storlet_conf): self.app = app self.logger = get_logger(conf, log_route='storlet_handler') self.stimeout = int(storlet_conf.get('storlet_timeout')) - self.storlet_containers = [ storlet_conf.get('storlet_container'), + self.storlet_containers = [storlet_conf.get('storlet_container'), storlet_conf.get('storlet_dependency')] self.execution_server = storlet_conf.get('execution_server') self.gateway_module = storlet_conf['gateway_module'] - self.proxy_only_storlet_execution = storlet_conf['storlet_execute_on_proxy_only'] + self.proxy_only_storlet_execution = \ + storlet_conf['storlet_execute_on_proxy_only'] self.gateway_conf = storlet_conf - @wsgify def __call__(self, req): try: @@ -59,25 +63,22 @@ class StorletHandlerMiddleware(object): version = '0' except Exception as e: return req.get_response(self.app) - + self.logger.debug('storlet_handler call in %s: with %s/%s/%s' % - (self.execution_server, - account, - container, - obj)) + (self.execution_server, account, container, obj)) storlet_execution = False if 'X-Run-Storlet' in req.headers: storlet_execution = True - if (storlet_execution == True and account and container and obj) or \ - (container in self.storlet_containers and obj): - gateway = self.gateway_module(self.gateway_conf, - self.logger, self.app, version, account, - container, obj) + if (storlet_execution is True and account and container and obj) or \ + (container in self.storlet_containers and obj): + gateway = self.gateway_module(self.gateway_conf, + self.logger, self.app, version, + account, container, obj) else: - return req.get_response(self.app) + return req.get_response(self.app) - try: + try: if self.execution_server == 'object' and storlet_execution: if req.method == 'GET': self.logger.info('GET. Run storlet') @@ -86,41 +87,38 @@ class StorletHandlerMiddleware(object): if not is_success(orig_resp.status_int): return orig_resp - if self._is_range_request(req) == True or \ - self._is_slo_get_request(req, orig_resp, account, \ - container, obj) or \ - self.proxy_only_storlet_execution == True: + if self._is_range_request(req) is True or \ + self._is_slo_get_request(req, orig_resp, account, + container, obj) or \ + self.proxy_only_storlet_execution is True: # For SLOs, and proxy only mode - # Storlet are executed on the proxy + # Storlet are executed on the proxy # Therefore we return the object part without # Storlet invocation: - self.logger.info( - 'storlet_handler: invocation over %s/%s/%s %s' % - (account, container, obj, - 'to be executed on proxy')) + self.logger.info('storlet_handler: invocation ' + 'over %s/%s/%s %s' % + (account, container, obj, + 'to be executed on proxy')) return orig_resp - else: + else: # We apply here the Storlet: - self.logger.info( - 'storlet_handler: invocation over %s/%s/%s %s' % - (account, container, obj, - 'to be executed locally')) + self.logger.info('storlet_handler: invocation ' + 'over %s/%s/%s %s' % + (account, container, obj, + 'to be executed locally')) old_env = req.environ.copy() orig_req = Request.blank(old_env['PATH_INFO'], old_env) - (out_md, app_iter) = gateway.gatewayObjectGetFlow(req, - container, - obj, - orig_resp) + (out_md, app_iter) = \ + gateway.gatewayObjectGetFlow(req, container, + obj, orig_resp) if 'Content-Length' in orig_resp.headers: orig_resp.headers.pop('Content-Length') if 'Transfer-Encoding' in orig_resp.headers: orig_resp.headers.pop('Transfer-Encoding') - - return Response( - app_iter, - headers = orig_resp.headers, - request=orig_req, - conditional_response=True) + + return Response(app_iter, headers=orig_resp.headers, + request=orig_req, + conditional_response=True) elif (self.execution_server == 'proxy'): if (storlet_execution or container in self.storlet_containers): @@ -137,7 +135,7 @@ class StorletHandlerMiddleware(object): return HTTPUnauthorized('Storlet: no permission') # The get request may be a SLO object GET request. - # Simplest solution would be to invoke a HEAD + # Simplest solution would be to invoke a HEAD # for every GET request to test if we are in SLO case. # In order to save the HEAD overhead we implemented # a slightly more involved flow: @@ -146,69 +144,69 @@ class StorletHandlerMiddleware(object): # At object side, we invoke the plain (non Storlet) # request and test if we are in SLO case. # and invoke Storlet only if non SLO case. - # Back at proxy side, we test if test received - # full object to detect if we are in SLO case, + # Back at proxy side, we test if test received + # full object to detect if we are in SLO case, # and invoke Storlet only if in SLO case. gateway.augmentStorletRequest(req) original_resp = req.get_response(self.app) - if self._is_range_request(req) == True or \ - self._is_slo_get_request(req, original_resp, account, \ - container, obj) or \ - self.proxy_only_storlet_execution == True: - # SLO / proxy only case: + if self._is_range_request(req) is True or \ + self._is_slo_get_request(req, original_resp, + account, + container, obj) or \ + self.proxy_only_storlet_execution is True: + # SLO / proxy only case: # storlet to be invoked now at proxy side: - (out_md, app_iter) = gateway.gatewayProxyGETFlow(req, - container, - obj, - original_resp) + (out_md, app_iter) = \ + gateway.gatewayProxyGETFlow(req, container, obj, + original_resp) # adapted from non SLO GET flow if is_success(original_resp.status_int): old_env = req.environ.copy() - orig_req = Request.blank(old_env['PATH_INFO'], old_env) + orig_req = Request.blank(old_env['PATH_INFO'], + old_env) resp_headers = original_resp.headers - + resp_headers['Content-Length'] = None - return Response( - app_iter=app_iter, - headers=resp_headers, - request=orig_req, - conditional_response=True) + return Response(app_iter=app_iter, + headers=resp_headers, + request=orig_req, + conditional_response=True) return original_resp else: - # Non proxy GET case: Storlet was already invoked at object side + # Non proxy GET case: Storlet was already invoked at + # object side if 'Transfer-Encoding' in original_resp.headers: original_resp.headers.pop('Transfer-Encoding') - + if is_success(original_resp.status_int): old_env = req.environ.copy() - orig_req = Request.blank(old_env['PATH_INFO'], old_env) + orig_req = Request.blank(old_env['PATH_INFO'], + old_env) resp_headers = original_resp.headers - + resp_headers['Content-Length'] = None - return Response( - app_iter=original_resp.app_iter, - headers=resp_headers, - request=orig_req, - conditional_response=True) + return Response(app_iter=original_resp.app_iter, + headers=resp_headers, + request=orig_req, + conditional_response=True) return original_resp elif req.method == 'PUT': if (container in self.storlet_containers): ret = gateway.validateStorletUpload(req) if ret: - return HTTPBadRequest(body = ret) + return HTTPBadRequest(body=ret) else: if not gateway.authorizeStorletExecution(req): return HTTPUnauthorized('Storlet: no permissions') if storlet_execution: gateway.augmentStorletRequest(req) - (out_md, app_iter) = gateway.gatewayProxyPutFlow(req, - container, - obj) + (out_md, app_iter) = \ + gateway.gatewayProxyPutFlow(req, container, obj) req.environ['wsgi.input'] = app_iter if 'CONTENT_LENGTH' in req.environ: req.environ.pop('CONTENT_LENGTH') @@ -216,7 +214,7 @@ class StorletHandlerMiddleware(object): return req.get_response(self.app) except (StorletTimeout, ConnectionTimeout, Timeout) as e: - StorletException.handle(self.logger, e) + StorletException.handle(self.logger, e) return HTTPInternalServerError(body='Storlet execution timed out') except Exception as e: StorletException.handle(self.logger, e) @@ -229,14 +227,15 @@ class StorletHandlerMiddleware(object): args: req: the request ''' + def _is_range_request(self, req): if 'Range' in req.headers: return True return False ''' - Determines from a GET request and its associated response - if the object is a SLO + Determines from a GET request and its associated response + if the object is a SLO args: req: the request resp: the response @@ -244,39 +243,50 @@ class StorletHandlerMiddleware(object): container: the response as extracted from req obj: the response as extracted from req ''' + def _is_slo_get_request(self, req, resp, account, container, obj): if req.method != 'GET': - return False + return False if req.params.get('multipart-manifest') == 'get': return False - self.logger.info( 'Verify if {0}/{1}/{2} is an SLO assembly object'.format(account,container, obj)) + self.logger.info('Verify if {0}/{1}/{2} is an SLO assembly object'. + format(account, container, obj)) - if resp.status_int < 300 and resp.status_int >= 200 : + if resp.status_int < 300 and resp.status_int >= 200: for key in resp.headers: - if (key.lower() == 'x-static-large-object' and - config_true_value(resp.headers[key])): - self.logger.info( '{0}/{1}/{2} is indeed an SLO assembly object'.format(account,container, obj)) + if (key.lower() == 'x-static-large-object' + and config_true_value(resp.headers[key])): + self.logger.info('{0}/{1}/{2} is indeed an SLO assembly ' + 'object'.format(account, container, obj)) return True - self.logger.info( '{0}/{1}/{2} is NOT an SLO assembly object'.format(account,container, obj)) + self.logger.info('{0}/{1}/{2} is NOT an SLO assembly object'. + format(account, container, obj)) return False - self.logger.error( 'Failed to check if {0}/{1}/{2} is an SLO assembly object. Got status {3}'.format(account,container, obj,resp.status)) - raise Exception('Failed to check if {0}/{1}/{2} is an SLO assembly object. Got status {3}'.format(account,container, obj,resp.status)) + self.logger.error('Failed to check if {0}/{1}/{2} is an SLO assembly ' + 'object. Got status {3}'. + format(account, container, obj, resp.status)) + raise Exception('Failed to check if {0}/{1}/{2} is an SLO assembly ' + 'object. Got status {3}'.format(account, container, + obj, resp.status)) + def filter_factory(global_conf, **local_conf): conf = global_conf.copy() conf.update(local_conf) storlet_conf = dict() - storlet_conf['storlet_timeout'] = conf.get('storlet_timeout',40) - storlet_conf['storlet_container'] = conf.get('storlet_container','storlet') + storlet_conf['storlet_timeout'] = conf.get('storlet_timeout', 40) + storlet_conf['storlet_container'] = \ + conf.get('storlet_container', 'storlet') storlet_conf['storlet_dependency'] = conf.get('storlet_dependency', 'dependency') storlet_conf['execution_server'] = conf.get('execution_server', '') - storlet_conf['storlet_execute_on_proxy_only'] = config_true_value(conf.get('storlet_execute_on_proxy_only', 'false')) + storlet_conf['storlet_execute_on_proxy_only'] = \ + config_true_value(conf.get('storlet_execute_on_proxy_only', 'false')) storlet_conf['gateway_conf'] = {} - module_name = conf.get('storlet_gateway_module','') + module_name = conf.get('storlet_gateway_module', '') mo = module_name[:module_name.rfind(':')] cl = module_name[module_name.rfind(':') + 1:] module = __import__(mo, fromlist=[cl]) @@ -288,8 +298,8 @@ def filter_factory(global_conf, **local_conf): additional_items = configParser.items("DEFAULT") for key, val in additional_items: - storlet_conf[key]= val - + storlet_conf[key] = val + swift_info = {} storlet_conf["gateway_module"] = the_class register_swift_info('storlet_handler', False, **swift_info) @@ -297,4 +307,3 @@ def filter_factory(global_conf, **local_conf): def storlet_handler_filter(app): return StorletHandlerMiddleware(app, conf, storlet_conf) return storlet_handler_filter - diff --git a/HACKING.rst b/HACKING.rst new file mode 100644 index 00000000..85fa2913 --- /dev/null +++ b/HACKING.rst @@ -0,0 +1,4 @@ +storlets Style Commandments +=============================================== + +Read the OpenStack Style Commandments http://docs.openstack.org/developer/hacking/ diff --git a/LICENSE b/LICENSE index 8f71f43f..68c771a0 100644 --- a/LICENSE +++ b/LICENSE @@ -1,3 +1,4 @@ + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -173,30 +174,3 @@ incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..c978a52d --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +include AUTHORS +include ChangeLog +exclude .gitignore +exclude .gitreview + +global-exclude *.pyc diff --git a/README.rst b/README.rst new file mode 100644 index 00000000..ac8cc1f1 --- /dev/null +++ b/README.rst @@ -0,0 +1,19 @@ +=============================== +storlets +=============================== + +Middleware and Compute Engine for an OpenStack Swift compute framework that runs compute within a Swift cluster + +Please feel here a long description which must be at least 3 lines wrapped on +80 cols, so that distribution package maintainers can use it in their packages. +Note that this is a hard requirement. + +* Free software: Apache license +* Documentation: http://docs.openstack.org/developer/storlets +* Source: http://git.openstack.org/cgit/openstack/storlets +* Bugs: http://bugs.launchpad.net/storlets + +Features +-------- + +* TODO diff --git a/SystemTests/RH_and_CSS_test.py b/SystemTests/RH_and_CSS_test.py index 3e00d24b..a25a1822 100644 --- a/SystemTests/RH_and_CSS_test.py +++ b/SystemTests/RH_and_CSS_test.py @@ -17,106 +17,121 @@ Limitations under the License. @author: gilv / cdoron / evgenyl ''' -from sys_test_params import * +from storlets_test_utils import put_storlet_object from swiftclient import client as c +from sys_test_params import ACCOUNT +from sys_test_params import AUTH_IP +from sys_test_params import AUTH_PORT +from sys_test_params import PASSWORD +from sys_test_params import USER_NAME -from storlets_test_utils import put_storlet_containers, put_storlet_object - -EXECDEP_PATH_TO_BUNDLE ='../StorletSamples/ExecDepStorlet/bin/' -EXECDEP_STORLET_NAME='execdepstorlet-1.0.jar' -EXECDEP_STORLET_LOG_NAME='execdepstorlet-1.0.log' +EXECDEP_PATH_TO_BUNDLE = '../StorletSamples/ExecDepStorlet/bin/' +EXECDEP_STORLET_NAME = 'execdepstorlet-1.0.jar' +EXECDEP_STORLET_LOG_NAME = 'execdepstorlet-1.0.log' EXECDEP_JUNK_FILE = 'junk.txt' -EXECDEP_DEPS_NAMES=['get42'] +EXECDEP_DEPS_NAMES = ['get42'] '''------------------------------------------------------------------------''' -def put_storlet_dependency(url, token, dependency_name, local_path_to_dependency): + + +def put_storlet_dependency(url, token, dependency_name, + local_path_to_dependency): metadata = {'X-Object-Meta-Storlet-Dependency-Version': '1'} - f = open('%s/%s'% (local_path_to_dependency, dependency_name), 'r') + f = open('%s/%s' % (local_path_to_dependency, dependency_name), 'r') content_length = None response = dict() c.put_object(url, token, 'dependency', dependency_name, f, - content_length, None, None, "application/octet-stream", metadata, None, None, None, response) + content_length, None, None, "application/octet-stream", + metadata, None, None, None, response) f.close() status = response.get('status') assert (status == 200 or status == 201) '''------------------------------------------------------------------------''' + + def put_storlet_input_object(url, token): resp = dict() - f = open('%s/%s' %(EXECDEP_PATH_TO_BUNDLE, EXECDEP_JUNK_FILE),'r') - c.put_object(url, token, 'myobjects', EXECDEP_JUNK_FILE, f, - content_type = "application/octet-stream", - response_dict = resp) + f = open('%s/%s' % (EXECDEP_PATH_TO_BUNDLE, EXECDEP_JUNK_FILE), 'r') + c.put_object(url, token, 'myobjects', EXECDEP_JUNK_FILE, f, + content_type="application/octet-stream", + response_dict=resp) f.close() - status = resp.get('status') + status = resp.get('status') assert (status == 200 or status == 201) - + '''------------------------------------------------------------------------''' -def deploy_storlet(url,token, name, jarName): - #No need to create containers every time - #put_storlet_containers(url, token) + + +def deploy_storlet(url, token, name, jarName): + # No need to create containers every time + # put_storlet_containers(url, token) put_storlet_object(url, token, jarName, - '../StorletSamples/'+ name + '/bin/', - '', - 'com.ibm.storlet.' + name.lower() + '.' + name) - + '../StorletSamples/' + name + '/bin/', + '', + 'com.ibm.storlet.' + name.lower() + '.' + name) + '''------------------------------------------------------------------------''' + + def invoke_storlet(url, token, storletName, jarName, objectName, mode): resp = dict() if mode == 'PUT': - f = open('../StorletSamples/' + storletName + '/sampleData.txt','r') + f = open('../StorletSamples/' + storletName + '/sampleData.txt', 'r') c.put_object(url, token, 'myobjects', objectName, f, - headers = {'X-Run-Storlet':jarName}, - response_dict = resp) + headers={'X-Run-Storlet': jarName}, + response_dict=resp) f.close() if mode == 'GET': - resp_headers, saved_content = c.get_object(url, token, - 'myobjects', - objectName, - headers = {'X-Run-Storlet':jarName}, - response_dict=resp) - + resp_headers, saved_content = \ + c.get_object(url, token, 'myobjects', objectName, + headers={'X-Run-Storlet': jarName}, + response_dict=resp) + assert (resp['status'] == 200 or resp['status'] == 201) - + if mode == 'GET': return resp_headers, saved_content - + '''------------------------------------------------------------------------''' + + def main(): - os_options = {'tenant_name': ACCOUNT} - url, token = c.get_auth( 'http://' + AUTH_IP + ":" - + AUTH_PORT + '/v2.0', - ACCOUNT + ':' + USER_NAME, - PASSWORD, - os_options = os_options, - auth_version = '2.0' ) + os_options = {'tenant_name': ACCOUNT} + url, token = c.get_auth('http://' + AUTH_IP + ":" + + AUTH_PORT + '/v2.0', + ACCOUNT + ':' + USER_NAME, + PASSWORD, + os_options=os_options, + auth_version='2.0') - print 'Deploying ReadHeaders storlet' - deploy_storlet(url, token, 'ReadHeadersStorlet', - 'readheadersstorlet-1.0.jar') + print('Deploying ReadHeaders storlet') + deploy_storlet(url, token, 'ReadHeadersStorlet', + 'readheadersstorlet-1.0.jar') - print 'Deploying ReadHeaders dependency' - put_storlet_dependency(url, token, 'json-simple-1.1.1.jar', - '../StorletSamples/ReadHeadersStorlet/lib') + print('Deploying ReadHeaders dependency') + put_storlet_dependency(url, token, 'json-simple-1.1.1.jar', + '../StorletSamples/ReadHeadersStorlet/lib') - print 'Deploying CSS storlet' - deploy_storlet(url, token, 'CssStorlet', 'cssstorlet-1.0.jar') + print('Deploying CSS storlet') + deploy_storlet(url, token, 'CssStorlet', 'cssstorlet-1.0.jar') - print "Invoking CSS storlet in PUT mode" - invoke_storlet(url, token, 'CssStorlet', 'cssstorlet-1.0.jar', - 'testDataCss', 'PUT') + print("Invoking CSS storlet in PUT mode") + invoke_storlet(url, token, 'CssStorlet', 'cssstorlet-1.0.jar', + 'testDataCss', 'PUT') - print "Invoking ReadHeaders storlet in GET mode" - headers, content = invoke_storlet(url, token, 'ReadHeadersStorlet', - 'readheadersstorlet-1.0.jar', 'testDataCss', 'GET') + print("Invoking ReadHeaders storlet in GET mode") + headers, content = invoke_storlet(url, token, 'ReadHeadersStorlet', + 'readheadersstorlet-1.0.jar', + 'testDataCss', 'GET') - assert '{"Square-Sums":"[2770444.6455999985, 1.9458262030000027E7,' \ - + ' 95.17999999999981]","Lines-Num":"356","Sums":"[27037.0' \ - + '40000000008, 83229.09999999998, 168.39999999999947]"}'\ - == content - - print "ReadHeaders test finished" + assert '{"Square-Sums":"[2770444.6455999985, 1.9458262030000027E7,' \ + + ' 95.17999999999981]","Lines-Num":"356","Sums":"[27037.0' \ + + '40000000008, 83229.09999999998, 168.39999999999947]"}' \ + == content + + print("ReadHeaders test finished") '''------------------------------------------------------------------------''' if __name__ == "__main__": main() diff --git a/SystemTests/SLO_test.py b/SystemTests/SLO_test.py index 5e81d153..9141e784 100644 --- a/SystemTests/SLO_test.py +++ b/SystemTests/SLO_test.py @@ -1,152 +1,183 @@ -import os -import sys +'''------------------------------------------------------------------------- +Copyright IBM Corp. 2015, 2015 All Rights Reserved +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +Limitations under the License. +-------------------------------------------------------------------------''' + import json +import os import random import string -from sys_test_params import * from swiftclient import client as c +from sys_test_params import ACCOUNT +from sys_test_params import AUTH_IP +from sys_test_params import AUTH_PORT +from sys_test_params import PASSWORD +from sys_test_params import USER_NAME -from storlets_test_utils import put_storlet_containers, put_storlet_object, progress, progress_ln, progress_msg from identity_storlet_test import IDENTITY_STORLET_NAME +from storlets_test_utils import progress +from storlets_test_utils import progress_ln +from storlets_test_utils import progress_msg +from storlets_test_utils import put_storlet_object -SLOIDENTITY_PATH_TO_BUNDLE ='../StorletSamples/SLOIdentityStorlet/bin' -SLOIDENTITY_STORLET_NAME='sloidentitystorlet-1.0.jar' +SLOIDENTITY_PATH_TO_BUNDLE = '../StorletSamples/SLOIdentityStorlet/bin' +SLOIDENTITY_STORLET_NAME = 'sloidentitystorlet-1.0.jar' '''------------------------------------------------------------------------''' # Test Constants -#PATH_TO_BUNDLE = -#STORLET_NAME = -#STORLET_LOG_NAME = -#SOURCE_FILE = +# PATH_TO_BUNDLE = +# STORLET_NAME = +# STORLET_LOG_NAME = +# SOURCE_FILE = '''------------------------------------------------------------------------''' - + + def create_local_chunks(): - for i in range(1,10): + for i in range(1, 10): progress() oname = '/tmp/slo_chunk_%d' % i - f = open(oname,'w') - f.write(''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(1048576))) + f = open(oname, 'w') + f.write(''.join(random.choice(string.ascii_uppercase + string.digits) + for _ in range(1048576))) f.close() progress_ln() def delete_local_chunks(): - for i in range(1,10): + for i in range(1, 10): oname = '/tmp/slo_chunk_%d' % i os.remove(oname) - + + def put_SLO(url, token): # Create temp files assembly = [] - for i in range(1,10): - oname = '/tmp/slo_chunk_%d' % i - f = open(oname,'r') + for i in range(1, 10): + oname = '/tmp/slo_chunk_%d' % i + f = open(oname, 'r') content_length = None response = dict() progress() - c.put_object(url, token, 'myobjects', oname, f, - content_length, None, None, "application/octet-stream", - None, None, None, None, response) + c.put_object(url, token, 'myobjects', oname, f, + content_length, None, None, "application/octet-stream", + None, None, None, None, response) f.close() status = response.get('status') assert (status >= 200 and status < 300) - + headers = response.get('headers') segment = dict() segment['path'] = 'myobjects/%s' % oname segment['size_bytes'] = 1048576 segment['etag'] = headers['etag'] assembly.append(segment) - + content_length = None response = dict() - headers = {'x-object-meta-prop1' : 'val1'} + headers = {'x-object-meta-prop1': 'val1'} progress() - c.put_object(url, token, 'myobjects', 'assembly', json.dumps(assembly), - content_length=None, etag=None, chunk_size=None, headers=headers, - query_string='multipart-manifest=put', response_dict=response) + c.put_object(url, token, 'myobjects', 'assembly', json.dumps(assembly), + content_length=None, etag=None, chunk_size=None, + headers=headers, query_string='multipart-manifest=put', + response_dict=response) status = response.get('status') assert (status >= 200 and status < 300) progress_ln() + def get_SLO(url, token): response = dict() - headers, body = c.get_object(url, token, 'myobjects', 'assembly', http_conn=None, - resp_chunk_size=1048576, query_string=None, response_dict=response, headers=None) + headers, body = c.get_object(url, token, 'myobjects', 'assembly', + http_conn=None, resp_chunk_size=1048576, + query_string=None, response_dict=response, + headers=None) i = 1 for chunk in body: oname = '/tmp/slo_chunk_%d' % i - f = open(oname,'r') - file_content=f.read() - #print '%s %s' % (chunk[:10], file_content[:10]) - #print '%d %d' % (len(chunk), len(file_content)) + f = open(oname, 'r') + file_content = f.read() + # print '%s %s' % (chunk[:10], file_content[:10]) + # print '%d %d' % (len(chunk), len(file_content)) progress() assert(chunk == file_content) f.close() - i=i+1 + i = i + 1 progress_ln() + def compare_slo_to_chunks(body): i = 1 for chunk in body: if chunk: - if i<10: + if i < 10: progress() oname = '/tmp/slo_chunk_%d' % i - f = open(oname,'r') - file_content=f.read() - #print '%s %s' % (chunk[:10], file_content[:10]) - #print '%d %d' % (len(chunk), len(file_content)) + f = open(oname, 'r') + file_content = f.read() + # print '%s %s' % (chunk[:10], file_content[:10]) + # print '%d %d' % (len(chunk), len(file_content)) assert(chunk == file_content) f.close() - i=i+1 + i = i + 1 else: aux_content = '' - for j in range(1,4): + for j in range(1, 4): oname = '/tmp/aux_file%d' % j - f = open(oname,'r') - aux_content+=f.read() + f = open(oname, 'r') + aux_content += f.read() f.close() assert(chunk == aux_content) progress_ln() - + + def invoke_identity_on_get_SLO(url, token): - metadata = {'X-Run-Storlet': IDENTITY_STORLET_NAME } + metadata = {'X-Run-Storlet': IDENTITY_STORLET_NAME} response = dict() headers, body = c.get_object(url, token, - 'myobjects','assembly', - query_string = None, + 'myobjects', 'assembly', + query_string=None, response_dict=response, resp_chunk_size=1048576, headers=metadata) compare_slo_to_chunks(body) + def invoke_identity_on_get_SLO_double(url, token): - metadata = {'X-Run-Storlet': IDENTITY_STORLET_NAME } + metadata = {'X-Run-Storlet': IDENTITY_STORLET_NAME} response = dict() headers, body = c.get_object(url, token, 'myobjects', 'assembly', - query_string = 'double=true', + query_string='double=true', response_dict=response, resp_chunk_size=2048, headers=metadata) - + i = 1 progress() oname = '/tmp/slo_chunk_%d' % i - f = open(oname,'r') - file_content=f.read() - - j = 0 # Count chunks in file 1...1024 + f = open(oname, 'r') + file_content = f.read() + + j = 0 # Count chunks in file 1...1024 for chunk in body: - file_fragment = file_content[j*1024:(j+1)*1024] + file_fragment = file_content[j * 1024:(j + 1) * 1024] chunk_framgment_low = chunk[0:1024] chunk_framgment_high = chunk[1024:2048] assert(chunk_framgment_low == file_fragment) assert(chunk_framgment_high == file_fragment) - j = j +1 + j = j + 1 if j == 1024: i = i + 1 if i == 10: @@ -154,21 +185,22 @@ def invoke_identity_on_get_SLO_double(url, token): f.close() progress() oname = '/tmp/slo_chunk_%d' % i - f = open(oname,'r') - file_content=f.read() + f = open(oname, 'r') + file_content = f.read() j = 0 assert i == 10 progress_ln() + def invoke_identity_on_partial_get_SLO(url, token): - metadata = {'X-Run-Storlet': IDENTITY_STORLET_NAME } + metadata = {'X-Run-Storlet': IDENTITY_STORLET_NAME} for i in range(5): progress() response = dict() headers, body = c.get_object(url, token, 'myobjects', 'assembly', - query_string = None, + query_string=None, response_dict=response, resp_chunk_size=1048576, headers=metadata) @@ -179,43 +211,46 @@ def invoke_identity_on_partial_get_SLO(url, token): if j == 5: break progress_ln() - # def delete_files(): # for i in range(1,4): # fname = '/tmp/aux_file%d' % i # os.remove(fname) - + + def create_container(url, token, name): response = dict() - c.put_container(url, token, name, headers=None, response_dict = response) - status = response.get('status') + c.put_container(url, token, name, headers=None, response_dict=response) + status = response.get('status') assert (status >= 200 or status < 300) + def deploy_sloidentity_storlet(url, token): progress() response = dict() c.put_container(url, token, 'mysloobject', None, None, response) - status = response.get('status') + status = response.get('status') assert (status >= 200 or status < 300) progress() - put_storlet_object( url, token, - SLOIDENTITY_STORLET_NAME, - SLOIDENTITY_PATH_TO_BUNDLE, - '', - 'com.ibm.storlet.sloidentity.SLOIdentityStorlet') + put_storlet_object(url, token, + SLOIDENTITY_STORLET_NAME, + SLOIDENTITY_PATH_TO_BUNDLE, + '', + 'com.ibm.storlet.sloidentity.SLOIdentityStorlet') progress_ln() - + '''------------------------------------------------------------------------''' + + def main(): os_options = {'tenant_name': ACCOUNT} - url, token = c.get_auth( 'http://' + AUTH_IP + ":" - + AUTH_PORT + '/v2.0', - ACCOUNT + ':' + USER_NAME, - PASSWORD, - os_options = os_options, - auth_version = '2.0' ) + url, token = c.get_auth('http://' + AUTH_IP + ":" + + AUTH_PORT + '/v2.0', + ACCOUNT + ':' + USER_NAME, + PASSWORD, + os_options=os_options, + auth_version='2.0') # print('Creating containers for auxiliary files') create_container(url, token, 'myobjects') create_container(url, token, 'container1') @@ -234,8 +269,8 @@ def main(): progress_msg("Invoking storlet on SLO in GET with double") invoke_identity_on_get_SLO_double(url, token) - #progress_msg("Invoking storlet on SLO in partial GET") - #invoke_identity_on_partial_get_SLO(url, token) + # progress_msg("Invoking storlet on SLO in partial GET") + # invoke_identity_on_partial_get_SLO(url, token) delete_local_chunks() '''------------------------------------------------------------------------''' diff --git a/SystemTests/execdep_test.py b/SystemTests/execdep_test.py index 7a71c664..8f5319aa 100644 --- a/SystemTests/execdep_test.py +++ b/SystemTests/execdep_test.py @@ -17,80 +17,90 @@ Limitations under the License. @author: gilv / cdoron / evgenyl ''' -from sys_test_params import * +from storlets_test_utils import put_file_as_storlet_input_object +from storlets_test_utils import put_storlet_object from swiftclient import client as c +from sys_test_params import ACCOUNT +from sys_test_params import AUTH_IP +from sys_test_params import AUTH_PORT +from sys_test_params import PASSWORD +from sys_test_params import USER_NAME -from storlets_test_utils import put_storlet_object, \ - put_file_as_storlet_input_object - -EXECDEP_PATH_TO_BUNDLE ='../StorletSamples/ExecDepStorlet/bin/' -EXECDEP_STORLET_NAME='execdepstorlet-1.0.jar' -EXECDEP_STORLET_LOG_NAME='execdepstorlet-1.0.log' +EXECDEP_PATH_TO_BUNDLE = '../StorletSamples/ExecDepStorlet/bin/' +EXECDEP_STORLET_NAME = 'execdepstorlet-1.0.jar' +EXECDEP_STORLET_LOG_NAME = 'execdepstorlet-1.0.log' EXECDEP_JUNK_FILE = 'junk.txt' -EXECDEP_DEPS_NAMES=['get42'] +EXECDEP_DEPS_NAMES = ['get42'] '''------------------------------------------------------------------------''' + + def put_storlet_executable_dependencies(url, token): resp = dict() - for d in EXECDEP_DEPS_NAMES: + for d in EXECDEP_DEPS_NAMES: metadata = {'X-Object-Meta-Storlet-Dependency-Version': '1', - 'X-Object-Meta-Storlet-Dependency-Permissions': '0755' } - - f = open('%s/%s' %(EXECDEP_PATH_TO_BUNDLE, d),'r') - c.put_object(url, token, 'dependency', d, f, - content_type = "application/octet-stream", - headers = metadata, - response_dict = resp) + 'X-Object-Meta-Storlet-Dependency-Permissions': '0755'} + + f = open('%s/%s' % (EXECDEP_PATH_TO_BUNDLE, d), 'r') + c.put_object(url, token, 'dependency', d, f, + content_type="application/octet-stream", + headers=metadata, + response_dict=resp) f.close() - status = resp.get('status') + status = resp.get('status') assert (status == 200 or status == 201) '''------------------------------------------------------------------------''' -def deploy_storlet(url,token): - #No need to create containers every time - #put_storlet_containers(url, token) - put_storlet_object( url, token, - EXECDEP_STORLET_NAME, - EXECDEP_PATH_TO_BUNDLE, - ','.join( str(x) for x in EXECDEP_DEPS_NAMES), - 'com.ibm.storlet.execdep.ExecDepStorlet') + + +def deploy_storlet(url, token): + # No need to create containers every time + # put_storlet_containers(url, token) + put_storlet_object(url, token, + EXECDEP_STORLET_NAME, + EXECDEP_PATH_TO_BUNDLE, + ','.join(str(x) for x in EXECDEP_DEPS_NAMES), + 'com.ibm.storlet.execdep.ExecDepStorlet') put_storlet_executable_dependencies(url, token) - put_file_as_storlet_input_object(url, - token, - EXECDEP_PATH_TO_BUNDLE, - EXECDEP_JUNK_FILE ) - + put_file_as_storlet_input_object(url, + token, + EXECDEP_PATH_TO_BUNDLE, + EXECDEP_JUNK_FILE) + '''------------------------------------------------------------------------''' + + def invoke_storlet(url, token): - metadata = {'X-Run-Storlet': EXECDEP_STORLET_NAME } + metadata = {'X-Run-Storlet': EXECDEP_STORLET_NAME} resp = dict() - resp_headers, gf = c.get_object(url, token, - 'myobjects', - EXECDEP_JUNK_FILE, - response_dict=resp, - headers=metadata) - + resp_headers, gf = c.get_object(url, token, + 'myobjects', + EXECDEP_JUNK_FILE, + response_dict=resp, + headers=metadata) + assert 'x-object-meta-depend-ret-code' in resp_headers assert resp_headers['x-object-meta-depend-ret-code'] == '42' assert resp['status'] == 200 - - + '''------------------------------------------------------------------------''' + + def main(): os_options = {'tenant_name': ACCOUNT} - url, token = c.get_auth( 'http://' + AUTH_IP + ":" - + AUTH_PORT + '/v2.0', - ACCOUNT + ':' + USER_NAME, - PASSWORD, - os_options = os_options, - auth_version = '2.0' ) - - print 'Deploying ExecDep storlet and dependencies' + url, token = c.get_auth('http://' + AUTH_IP + ":" + + AUTH_PORT + '/v2.0', + ACCOUNT + ':' + USER_NAME, + PASSWORD, + os_options=os_options, + auth_version='2.0') + + print('Deploying ExecDep storlet and dependencies') deploy_storlet(url, token) - - print "Invoking ExecDep storlet" + + print("Invoking ExecDep storlet") invoke_storlet(url, token) - + '''------------------------------------------------------------------------''' if __name__ == "__main__": main() diff --git a/SystemTests/half_storlet_test.py b/SystemTests/half_storlet_test.py index 04634ccc..47670128 100644 --- a/SystemTests/half_storlet_test.py +++ b/SystemTests/half_storlet_test.py @@ -13,132 +13,141 @@ See the License for the specific language governing permissions and Limitations under the License. -------------------------------------------------------------------------''' -import os -import json import random import string -from sys_test_params import * from swiftclient import client as c +from sys_test_params import ACCOUNT +from sys_test_params import AUTH_IP +from sys_test_params import AUTH_PORT +from sys_test_params import PASSWORD +from sys_test_params import USER_NAME -from storlets_test_utils import put_storlet_containers, put_storlet_object, \ - progress, progress_ln, progress_msg +from storlets_test_utils import put_storlet_object '''------------------------------------------------------------------------''' # Test Constants -HALF_PATH_TO_BUNDLE ='../StorletSamples/HalfStorlet/bin/' -HALF_STORLET_NAME='halfstorlet-1.0.jar' +HALF_PATH_TO_BUNDLE = '../StorletSamples/HalfStorlet/bin/' +HALF_STORLET_NAME = 'halfstorlet-1.0.jar' HALF_SOURCE_FILE = 'source.txt' '''------------------------------------------------------------------------''' + + def put_storlet_input_object(url, token): resp = dict() - metadata = {'X-Object-Meta-Testkey':'tester'} - f = open('%s/%s' %(HALF_PATH_TO_BUNDLE, HALF_SOURCE_FILE),'r') - c.put_object(url, token, 'myobjects', HALF_SOURCE_FILE, f, - content_type = "application/octet-stream", - headers = metadata, - response_dict = resp) + metadata = {'X-Object-Meta-Testkey': 'tester'} + f = open('%s/%s' % (HALF_PATH_TO_BUNDLE, HALF_SOURCE_FILE), 'r') + c.put_object(url, token, 'myobjects', HALF_SOURCE_FILE, f, + content_type="application/octet-stream", + headers=metadata, + response_dict=resp) f.close() - status = resp.get('status') + status = resp.get('status') assert (status == 200 or status == 201) - + '''------------------------------------------------------------------------''' -def deploy_storlet(url,token): - #No need to create containers every time - #put_storlet_containers(url, token) - put_storlet_object( url, token, - HALF_STORLET_NAME, - HALF_PATH_TO_BUNDLE, - '', - 'com.ibm.storlet.half.HalfStorlet') - put_storlet_input_object( url, token ) - + + +def deploy_storlet(url, token): + # No need to create containers every time + # put_storlet_containers(url, token) + put_storlet_object(url, token, + HALF_STORLET_NAME, + HALF_PATH_TO_BUNDLE, + '', + 'com.ibm.storlet.half.HalfStorlet') + put_storlet_input_object(url, token) + '''------------------------------------------------------------------------''' - -def invoke_storlet(url, token, op, params = None, global_params = None, headers = None): - if params != None: - querystring='' + + +def invoke_storlet(url, token, op, params=None, global_params=None, + headers=None): + if params is not None: + querystring = '' for key in params: querystring += '%s=%s,' % (key, params[key]) querystring = querystring[:-1] else: querystring = None - + metadata = {'X-Run-Storlet': HALF_STORLET_NAME} if headers: metadata.update(headers) if op == 'GET': # Get original object - original_headers, original_content = c.get_object(url, token, - 'myobjects', - HALF_SOURCE_FILE, - response_dict=dict()) - #print original_headers + original_headers, original_content = \ + c.get_object(url, token, 'myobjects', HALF_SOURCE_FILE, + response_dict=dict()) + # print original_headers file_length = int(original_headers['content-length']) - processed_headers, returned_content = c.get_object(url, token, - 'myobjects', - HALF_SOURCE_FILE, - query_string = querystring, - response_dict=dict(), - headers=metadata, - resp_chunk_size = file_length) + processed_headers, returned_content = \ + c.get_object(url, token, 'myobjects', HALF_SOURCE_FILE, + query_string=querystring, response_dict=dict(), + headers=metadata, resp_chunk_size=file_length) processed_content = '' for chunk in returned_content: if chunk: - processed_content+=chunk - - assert(original_headers['X-Object-Meta-Testkey'.lower()] == processed_headers['X-Object-Meta-Testkey'.lower()]) + processed_content += chunk + + assert(original_headers['X-Object-Meta-Testkey'.lower()] == + processed_headers['X-Object-Meta-Testkey'.lower()]) return processed_content - if op == 'PUT': # PUT a random file response = dict() - uploaded_content = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(1024)) - random_md = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(32)) - #content_length = 1024 + uploaded_content = ''.join(random.choice(string.ascii_uppercase + + string.digits) for _ in range(1024)) + random_md = ''.join(random.choice(string.ascii_uppercase + + string.digits) for _ in range(32)) + # content_length = 1024 content_length = None - headers = {'X-Run-Storlet': HALF_STORLET_NAME, - 'X-Object-Meta-Testkey' : random_md } - c.put_object(url, token, 'myobjects', 'half_random_source', uploaded_content, - content_length, None, None, "application/octet-stream", - headers, None, None, querystring, response) - resp_headers, saved_content = c.get_object(url, token, - 'myobjects', - 'half_random_source', - response_dict=dict()) - - if params != None and params.get('double',None) == 'true': - assert(uploaded_content==saved_content[:1024]) - assert(uploaded_content==saved_content[1024:]) + headers = {'X-Run-Storlet': HALF_STORLET_NAME, + 'X-Object-Meta-Testkey': random_md} + c.put_object(url, token, 'myobjects', 'half_random_source', + uploaded_content, content_length, None, None, + "application/octet-stream", headers, None, None, + querystring, response) + resp_headers, saved_content = c.get_object(url, token, 'myobjects', + 'half_random_source', + response_dict=dict()) + + if params is not None and params.get('double', None) == 'true': + assert(uploaded_content == saved_content[:1024]) + assert(uploaded_content == saved_content[1024:]) else: - assert(uploaded_content == saved_content) - - if params != None and params.get('execute',None) != None: - assert(resp_headers['X-Object-Meta-Execution result'.lower()] == '42') - - assert(resp_headers['X-Object-Meta-Testkey'.lower()] == random_md) + assert(uploaded_content == saved_content) + + if params is not None and params.get('execute', None) is not None: + assert(resp_headers['X-Object-Meta-Execution result'.lower()] == + '42') + + assert(resp_headers['X-Object-Meta-Testkey'.lower()] == random_md) '''------------------------------------------------------------------------''' + + def main(): os_options = {'tenant_name': ACCOUNT} - url, token = c.get_auth( 'http://' + AUTH_IP + ":" - + AUTH_PORT + '/v2.0', - ACCOUNT + ':' + USER_NAME, - PASSWORD, - os_options = os_options, - auth_version = '2.0' ) - - print 'Deploying Half storlet and dependencies' - + url, token = c.get_auth('http://' + AUTH_IP + ":" + + AUTH_PORT + '/v2.0', + ACCOUNT + ':' + USER_NAME, + PASSWORD, + os_options=os_options, + auth_version='2.0') + + print('Deploying Half storlet and dependencies') + deploy_storlet(url, token) - - print "Invoking Half storlet on GET" - assert (invoke_storlet(url, token,'GET') == 'acegikmn') - print "Invoking Half storlet on GET with byte ranges" - assert (invoke_storlet(url, token,'GET', headers = {'range': 'bytes=5-10'}) == 'fhj') - + + print("Invoking Half storlet on GET") + assert (invoke_storlet(url, token, 'GET') == 'acegikmn') + print("Invoking Half storlet on GET with byte ranges") + assert (invoke_storlet(url, token, 'GET', + headers={'range': 'bytes=5-10'}) == 'fhj') + '''------------------------------------------------------------------------''' if __name__ == "__main__": main() diff --git a/SystemTests/identity_storlet_test.py b/SystemTests/identity_storlet_test.py index ad8f7538..6d62efd8 100644 --- a/SystemTests/identity_storlet_test.py +++ b/SystemTests/identity_storlet_test.py @@ -14,185 +14,200 @@ Limitations under the License. -------------------------------------------------------------------------''' import os -import json import random import string -from sys_test_params import * from swiftclient import client as c +from sys_test_params import ACCOUNT +from sys_test_params import AUTH_IP +from sys_test_params import AUTH_PORT +from sys_test_params import PASSWORD +from sys_test_params import USER_NAME -from storlets_test_utils import put_storlet_containers, put_storlet_object, \ - progress, progress_ln, progress_msg +from storlets_test_utils import progress +from storlets_test_utils import progress_ln +from storlets_test_utils import progress_msg +from storlets_test_utils import put_storlet_object '''------------------------------------------------------------------------''' # Test Constants -IDENTITY_PATH_TO_BUNDLE ='../StorletSamples/IdentityStorlet/bin/' -IDENTITY_STORLET_NAME='identitystorlet-1.0.jar' -IDENTITY_STORLET_LOG_NAME='identitystorlet-1.0.log' +IDENTITY_PATH_TO_BUNDLE = '../StorletSamples/IdentityStorlet/bin/' +IDENTITY_STORLET_NAME = 'identitystorlet-1.0.jar' +IDENTITY_STORLET_LOG_NAME = 'identitystorlet-1.0.log' IDENTITY_SOURCE_FILE = 'source.txt' -IDENTITY_DEPS_NAMES=['get42'] +IDENTITY_DEPS_NAMES = ['get42'] - '''------------------------------------------------------------------------''' + + def put_storlet_executable_dependencies(url, token): resp = dict() - for d in IDENTITY_DEPS_NAMES: + for d in IDENTITY_DEPS_NAMES: metadata = {'X-Object-Meta-Storlet-Dependency-Version': '1', - 'X-Object-Meta-Storlet-Dependency-Permissions': '0755' } - - f = open('%s/%s' %(IDENTITY_PATH_TO_BUNDLE, d),'r') - c.put_object(url, token, 'dependency', d, f, - content_type = "application/octet-stream", - headers = metadata, - response_dict = resp) + 'X-Object-Meta-Storlet-Dependency-Permissions': '0755'} + + f = open('%s/%s' % (IDENTITY_PATH_TO_BUNDLE, d), 'r') + c.put_object(url, token, 'dependency', d, f, + content_type="application/octet-stream", + headers=metadata, + response_dict=resp) f.close() - status = resp.get('status') + status = resp.get('status') assert (status == 200 or status == 201) '''------------------------------------------------------------------------''' + + def put_storlet_input_object(url, token): resp = dict() - metadata = {'X-Object-Meta-Testkey':'tester'} - f = open('%s/%s' %(IDENTITY_PATH_TO_BUNDLE, IDENTITY_SOURCE_FILE),'r') - c.put_object(url, token, 'myobjects', IDENTITY_SOURCE_FILE, f, - content_type = "application/octet-stream", - headers = metadata, - response_dict = resp) + metadata = {'X-Object-Meta-Testkey': 'tester'} + f = open('%s/%s' % (IDENTITY_PATH_TO_BUNDLE, IDENTITY_SOURCE_FILE), 'r') + c.put_object(url, token, 'myobjects', IDENTITY_SOURCE_FILE, f, + content_type="application/octet-stream", + headers=metadata, + response_dict=resp) f.close() - status = resp.get('status') + status = resp.get('status') assert (status == 200 or status == 201) - + '''------------------------------------------------------------------------''' -def deploy_storlet(url,token): - #No need to create containers every time - #put_storlet_containers(url, token) - put_storlet_object( url, token, - IDENTITY_STORLET_NAME, - IDENTITY_PATH_TO_BUNDLE, - ','.join( str(x) for x in IDENTITY_DEPS_NAMES), - 'com.ibm.storlet.identity.IdentityStorlet') + + +def deploy_storlet(url, token): + # No need to create containers every time + # put_storlet_containers(url, token) + put_storlet_object(url, token, + IDENTITY_STORLET_NAME, + IDENTITY_PATH_TO_BUNDLE, + ','.join(str(x) for x in IDENTITY_DEPS_NAMES), + 'com.ibm.storlet.identity.IdentityStorlet') put_storlet_executable_dependencies(url, token) - put_storlet_input_object( url, token ) - + put_storlet_input_object(url, token) + '''------------------------------------------------------------------------''' + + def invoke_storlet_on_1GB_file(url, token): - GBFile = open('/tmp/1GB_file','w') + GBFile = open('/tmp/1GB_file', 'w') for _ in range(128): progress() - uploaded_content = ''.join('1' for _ in range(8*1024*1024)) + uploaded_content = ''.join('1' for _ in range(8 * 1024 * 1024)) GBFile.write(uploaded_content) GBFile.close() - - headers = {'X-Run-Storlet': IDENTITY_STORLET_NAME } - GBFile = open('/tmp/1GB_file','r') - response=dict() + + headers = {'X-Run-Storlet': IDENTITY_STORLET_NAME} + GBFile = open('/tmp/1GB_file', 'r') + response = dict() progress() - c.put_object(url, token, 'myobjects', '1GBFile', GBFile, - 1024*1024*1024, None, None, "application/octet-stream", + c.put_object(url, token, 'myobjects', '1GBFile', GBFile, + 1024 * 1024 * 1024, None, None, "application/octet-stream", headers, None, None, None, response) progress() - status = response.get('status') + status = response.get('status') assert (status == 200 or status == 201) progress() os.remove('/tmp/1GB_file') progress_ln() - - -def invoke_storlet(url, token, op, params = None, global_params = None): - if params != None: - querystring='' + + +def invoke_storlet(url, token, op, params=None, global_params=None): + if params is not None: + querystring = '' for key in params: querystring += '%s=%s,' % (key, params[key]) querystring = querystring[:-1] else: querystring = None - + metadata = {'X-Run-Storlet': IDENTITY_STORLET_NAME} if op == 'GET': # Get original object - original_headers, original_content = c.get_object(url, token, - 'myobjects', - IDENTITY_SOURCE_FILE, - response_dict=dict()) - #print original_headers + original_headers, original_content = \ + c.get_object(url, token, 'myobjects', IDENTITY_SOURCE_FILE, + response_dict=dict()) + # print original_headers file_length = int(original_headers['content-length']) - processed_headers, returned_content = c.get_object(url, token, - 'myobjects', - IDENTITY_SOURCE_FILE, - query_string = querystring, - response_dict=dict(), - headers=metadata, - resp_chunk_size = file_length) + processed_headers, returned_content = \ + c.get_object(url, token, 'myobjects', IDENTITY_SOURCE_FILE, + query_string=querystring, response_dict=dict(), + headers=metadata, resp_chunk_size=file_length) processed_content = '' for chunk in returned_content: if chunk: - processed_content+=chunk - - if params != None and params.get('execute',None) != None: - assert(processed_headers['X-Object-Meta-Execution result'.lower()] == '42') - if params != None and params.get('double',None) == 'true': - assert(original_content==processed_content[:file_length]) - assert(original_content==processed_content[file_length:]) + processed_content += chunk + + if params is not None and params.get('execute', None) is not None: + assert(processed_headers['X-Object-Meta-Execution result'.lower()] + == '42') + if params is not None and params.get('double', None) == 'true': + assert(original_content == processed_content[:file_length]) + assert(original_content == processed_content[file_length:]) else: assert(original_content == processed_content) - assert(original_headers['X-Object-Meta-Testkey'.lower()] == processed_headers['X-Object-Meta-Testkey'.lower()]) - + assert(original_headers['X-Object-Meta-Testkey'.lower()] == + processed_headers['X-Object-Meta-Testkey'.lower()]) + if op == 'PUT': # PUT a random file response = dict() - uploaded_content = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(1024)) - random_md = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(32)) - #content_length = 1024 + uploaded_content = ''.join(random.choice(string.ascii_uppercase + + string.digits) for _ in range(1024)) + random_md = ''.join(random.choice(string.ascii_uppercase + + string.digits) for _ in range(32)) content_length = None - headers = {'X-Run-Storlet': IDENTITY_STORLET_NAME, - 'X-Object-Meta-Testkey' : random_md } - c.put_object(url, token, 'myobjects', 'identity_random_source', uploaded_content, - content_length, None, None, "application/octet-stream", - headers, None, None, querystring, response) - resp_headers, saved_content = c.get_object(url, token, - 'myobjects', - 'identity_random_source', - response_dict=dict()) - - if params != None and params.get('double',None) == 'true': - assert(uploaded_content==saved_content[:1024]) - assert(uploaded_content==saved_content[1024:]) + headers = {'X-Run-Storlet': IDENTITY_STORLET_NAME, + 'X-Object-Meta-Testkey': random_md} + c.put_object(url, token, 'myobjects', 'identity_random_source', + uploaded_content, content_length, None, None, + "application/octet-stream", headers, None, None, + querystring, response) + resp_headers, saved_content = c.get_object(url, token, + 'myobjects', + 'identity_random_source', + response_dict=dict()) + + if params is not None and params.get('double', None) == 'true': + assert(uploaded_content == saved_content[:1024]) + assert(uploaded_content == saved_content[1024:]) else: - assert(uploaded_content == saved_content) - - if params != None and params.get('execute',None) != None: - assert(resp_headers['X-Object-Meta-Execution result'.lower()] == '42') - - assert(resp_headers['X-Object-Meta-Testkey'.lower()] == random_md) + assert(uploaded_content == saved_content) + + if params is not None and params.get('execute', None) is not None: + assert(resp_headers['X-Object-Meta-Execution result'.lower()] == + '42') + + assert(resp_headers['X-Object-Meta-Testkey'.lower()] == random_md) '''------------------------------------------------------------------------''' + + def main(): os_options = {'tenant_name': ACCOUNT} - url, token = c.get_auth( 'http://' + AUTH_IP + ":" - + AUTH_PORT + '/v2.0', - ACCOUNT + ':' + USER_NAME, - PASSWORD, - os_options = os_options, - auth_version = '2.0' ) - - print 'Deploying Identity storlet and dependencies' - + url, token = c.get_auth('http://' + AUTH_IP + ":" + + AUTH_PORT + '/v2.0', + ACCOUNT + ':' + USER_NAME, + PASSWORD, + os_options=os_options, + auth_version='2.0') + + print('Deploying Identity storlet and dependencies') + deploy_storlet(url, token) - - print "Invoking Identity storlet on PUT" - invoke_storlet(url, token,'PUT') + + print("Invoking Identity storlet on PUT") + invoke_storlet(url, token, 'PUT') progress_msg("Invoking Identity storlet on 1GB file PUT") - invoke_storlet_on_1GB_file(url, token) - print "Invoking Identity storlet on PUT with execution of dependency" - invoke_storlet(url, token,'PUT', {'execute' : 'true'}) - print "Invoking Identity storlet on PUT with double" - invoke_storlet(url, token,'PUT', {'double' : 'true'}) - print "Invoking Identity storlet on GET" - invoke_storlet(url, token,'GET') - print "Invoking Identity storlet on GET with double" - invoke_storlet(url, token,'GET', {'double' : 'true'}) - print "Invoking Identity storlet on GET with execution of dependency" - invoke_storlet(url, token,'GET',{'execute' : 'true'}) - + invoke_storlet_on_1GB_file(url, token) + print("Invoking Identity storlet on PUT with execution of dependency") + invoke_storlet(url, token, 'PUT', {'execute': 'true'}) + print("Invoking Identity storlet on PUT with double") + invoke_storlet(url, token, 'PUT', {'double': 'true'}) + print("Invoking Identity storlet on GET") + invoke_storlet(url, token, 'GET') + print("Invoking Identity storlet on GET with double") + invoke_storlet(url, token, 'GET', {'double': 'true'}) + print("Invoking Identity storlet on GET with execution of dependency") + invoke_storlet(url, token, 'GET', {'execute': 'true'}) + '''------------------------------------------------------------------------''' if __name__ == "__main__": main() diff --git a/SystemTests/metadata_storlet_test.py b/SystemTests/metadata_storlet_test.py index 9fe269e9..0b20645c 100644 --- a/SystemTests/metadata_storlet_test.py +++ b/SystemTests/metadata_storlet_test.py @@ -13,71 +13,76 @@ See the License for the specific language governing permissions and Limitations under the License. -------------------------------------------------------------------------''' -import json -import random -import string -from sys_test_params import * from swiftclient import client as c +from sys_test_params import ACCOUNT +from sys_test_params import AUTH_IP +from sys_test_params import AUTH_PORT +from sys_test_params import PASSWORD +from sys_test_params import USER_NAME -from storlets_test_utils import put_storlet_containers, put_storlet_object +from storlets_test_utils import put_storlet_object '''------------------------------------------------------------------------''' # Test Constants -PATH_TO_BUNDLE ='../StorletSamples/TestMetadataStorlet/bin/' -STORLET_NAME='testmetadatastorlet-1.0.jar' -STORLET_LOG_NAME='testmetadatastorlet-1.0.log' +PATH_TO_BUNDLE = '../StorletSamples/TestMetadataStorlet/bin/' +STORLET_NAME = 'testmetadatastorlet-1.0.jar' +STORLET_LOG_NAME = 'testmetadatastorlet-1.0.log' SOURCE_FILE = 'source.txt' '''------------------------------------------------------------------------''' + + def put_storlet_input_object(url, token): resp = dict() - metadata = {'X-Object-Meta-key1':'1', - 'X-Object-Meta-key2':'2', - 'X-Object-Meta-key3':'3', - 'X-Object-Meta-key4':'4', - 'X-Object-Meta-key5':'5', - 'X-Object-Meta-key6':'6', - 'X-Object-Meta-key7':'7', - 'X-Object-Meta-key8':'8', - 'X-Object-Meta-key9':'9', - 'X-Object-Meta-key10':'10'} - f = open('%s/%s' %(PATH_TO_BUNDLE, SOURCE_FILE),'r') - c.put_object(url, token, 'myobjects', SOURCE_FILE, f, - content_type = "application/octet-stream", - headers = metadata, - response_dict = resp) + metadata = {'X-Object-Meta-key1': '1', + 'X-Object-Meta-key2': '2', + 'X-Object-Meta-key3': '3', + 'X-Object-Meta-key4': '4', + 'X-Object-Meta-key5': '5', + 'X-Object-Meta-key6': '6', + 'X-Object-Meta-key7': '7', + 'X-Object-Meta-key8': '8', + 'X-Object-Meta-key9': '9', + 'X-Object-Meta-key10': '10'} + f = open('%s/%s' % (PATH_TO_BUNDLE, SOURCE_FILE), 'r') + c.put_object(url, token, 'myobjects', SOURCE_FILE, f, + content_type="application/octet-stream", + headers=metadata, + response_dict=resp) f.close() - status = resp.get('status') + status = resp.get('status') assert (status == 200 or status == 201) - + '''------------------------------------------------------------------------''' -def deploy_storlet(url,token): - #No need to create containers every time - #put_storlet_containers(url, token) - put_storlet_object( url, token, - STORLET_NAME, - PATH_TO_BUNDLE, - '', - 'com.ibm.storlet.testmetadatastorlet.MetadataStorlet') - put_storlet_input_object( url, token ) - + + +def deploy_storlet(url, token): + # No need to create containers every time + # put_storlet_containers(url, token) + put_storlet_object(url, token, + STORLET_NAME, + PATH_TO_BUNDLE, + '', + 'com.ibm.storlet.testmetadatastorlet.MetadataStorlet') + put_storlet_input_object(url, token) + '''------------------------------------------------------------------------''' -def invoke_storlet(url, token,op, params = None, global_params = None): - if params != None: - querystring='' + + +def invoke_storlet(url, token, op, params=None, global_params=None): + if params is not None: + querystring = '' for key in params: querystring += '%s=%s,' % (key, params[key]) querystring = querystring[:-1] else: querystring = None - - metadata = {'X-Run-Storlet': STORLET_NAME } + + metadata = {'X-Run-Storlet': STORLET_NAME} if op == 'GET': # Get original object - original_headers, original_content = c.get_object(url, token, - 'myobjects', - SOURCE_FILE, - response_dict=dict(), - headers=metadata) + original_headers, original_content = \ + c.get_object(url, token, 'myobjects', SOURCE_FILE, + response_dict=dict(), headers=metadata) assert(original_headers['X-Object-Meta-key1'.lower()] == '1') assert(original_headers['X-Object-Meta-key2'.lower()] == '2') assert(original_headers['X-Object-Meta-key3'.lower()] == '3') @@ -88,24 +93,27 @@ def invoke_storlet(url, token,op, params = None, global_params = None): assert(original_headers['X-Object-Meta-key8'.lower()] == '8') assert(original_headers['X-Object-Meta-key9'.lower()] == '9') assert(original_headers['X-Object-Meta-key10'.lower()] == '10') - assert(original_headers['X-Object-Meta-override_key'.lower()] == 'new_value') - + assert(original_headers['X-Object-Meta-override_key'.lower()] == + 'new_value') + '''------------------------------------------------------------------------''' + + def main(): os_options = {'tenant_name': ACCOUNT} - url, token = c.get_auth( 'http://' + AUTH_IP + ":" - + AUTH_PORT + '/v2.0', - ACCOUNT + ':' + USER_NAME, - PASSWORD, - os_options = os_options, - auth_version = '2.0' ) - - print 'Deploying storlet and dependencies' + url, token = c.get_auth('http://' + AUTH_IP + ":" + + AUTH_PORT + '/v2.0', + ACCOUNT + ':' + USER_NAME, + PASSWORD, + os_options=os_options, + auth_version='2.0') + + print('Deploying storlet and dependencies') deploy_storlet(url, token) - - print "Invoking storlet on GET" - invoke_storlet(url, token,'GET') - + + print("Invoking storlet on GET") + invoke_storlet(url, token, 'GET') + '''------------------------------------------------------------------------''' if __name__ == "__main__": main() diff --git a/SystemTests/storlets_test_utils.py b/SystemTests/storlets_test_utils.py index 033039ce..02c740ca 100644 --- a/SystemTests/storlets_test_utils.py +++ b/SystemTests/storlets_test_utils.py @@ -14,95 +14,107 @@ Limitations under the License. -------------------------------------------------------------------------''' import sys -import json from swiftclient import client as c -from swiftclient.client import encode_utf8, http_connection + def progress(): sys.stdout.write('.') sys.stdout.flush() + + def progress_ln(): sys.stdout.write('\n') sys.stdout.flush() + + def progress_msg(msg): sys.stdout.write(msg) sys.stdout.flush() '''------------------------------------------------------------------------''' -def enable_account_for_storlets(url,token): + + +def enable_account_for_storlets(url, token): headers = dict() headers['X-Account-Meta-storlet-enabled'] = 'True' c.post_account(url, token, headers) '''------------------------------------------------------------------------''' -def put_storlet_containers(url,token): + + +def put_storlet_containers(url, token): response = dict() c.put_container(url, token, 'storlet', None, None, response) - status = response.get('status') + status = response.get('status') assert (status >= 200 or status < 300) response = dict() c.put_container(url, token, 'dependency', None, None, response) - status = response.get('status') + status = response.get('status') assert (status >= 200 or status < 300) response = dict() c.put_container(url, token, 'storletlog', None, None, response) - status = response.get('status') + status = response.get('status') assert (status >= 200 or status < 300) - + '''------------------------------------------------------------------------''' + + def put_file_as_storlet_input_object(url, token, local_path, local_file): resp = dict() - f = open('%s/%s' %(local_path, local_file),'r') - c.put_object(url, token, 'myobjects', local_file, f, - content_type = "application/octet-stream", - response_dict = resp) + f = open('%s/%s' % (local_path, local_file), 'r') + c.put_object(url, token, 'myobjects', local_file, f, + content_type="application/octet-stream", + response_dict=resp) f.close() - status = resp.get('status') + status = resp.get('status') assert (status == 200 or status == 201) - + '''------------------------------------------------------------------------''' -def put_storlet_object(url, token, storlet_name, storlet_path, + + +def put_storlet_object(url, token, storlet_name, storlet_path, dependency, main_class): # Delete previous storlet - resp = dict() - ''' - try: - c.delete_object(url, token, 'storlet', storlet_name, None, + # resp = dict() + '''try: + + c.delete_object(url, token, 'storlet', storlet_name, None, None, None, None, resp) except Exception as e: if (resp.get('status')== 404): - print 'Nothing to delete' - ''' - metadata = {'X-Object-Meta-Storlet-Language':'Java', - 'X-Object-Meta-Storlet-Interface-Version':'1.0', - 'X-Object-Meta-Storlet-Dependency': dependency, - 'X-Object-Meta-Storlet-Object-Metadata':'no', + print 'Nothing to delete' + ''' + metadata = {'X-Object-Meta-Storlet-Language': 'Java', + 'X-Object-Meta-Storlet-Interface-Version': '1.0', + 'X-Object-Meta-Storlet-Dependency': dependency, + 'X-Object-Meta-Storlet-Object-Metadata': 'no', 'X-Object-Meta-Storlet-Main': main_class} - f = open('%s/%s' % (storlet_path, storlet_name),'r') + f = open('%s/%s' % (storlet_path, storlet_name), 'r') content_length = None response = dict() - c.put_object(url, token, 'storlet', storlet_name, f, - content_length, None, None, - "application/octet-stream", metadata, + c.put_object(url, token, 'storlet', storlet_name, f, + content_length, None, None, + "application/octet-stream", metadata, None, None, None, response) f.close() - status = response.get('status') + status = response.get('status') assert (status == 200 or status == 201) - '''------------------------------------------------------------------------''' + + def put_dependency(url, token, local_path_to_dep, dep_name): metadata = {'X-Object-Meta-Storlet-Dependency-Version': '1'} - f = open('%s/%s' %(local_path_to_dep, dep_name),'r') + f = open('%s/%s' % (local_path_to_dep, dep_name), 'r') content_length = None response = dict() - c.put_object(url, token, 'dependency', dep_name, f, - content_length, None, None, "application/octet-stream", + c.put_object(url, token, 'dependency', dep_name, f, + content_length, None, None, "application/octet-stream", metadata, None, None, None, response) f.close() - status = response.get('status') + status = response.get('status') assert (status == 200 or status == 201) diff --git a/SystemTests/sys_test.py b/SystemTests/sys_test.py index ee22b188..744f6eba 100644 --- a/SystemTests/sys_test.py +++ b/SystemTests/sys_test.py @@ -17,52 +17,50 @@ Limitations under the License. @author: gilv ''' -import threading -import time -import json import os -import random -import string -import tarfile +import threading -from sys_test_params import * from swiftclient import client as c -from swiftclient.client import encode_utf8, http_connection +from sys_test_params import ACCOUNT +from sys_test_params import AUTH_IP +from sys_test_params import AUTH_PORT +from sys_test_params import PASSWORD +from sys_test_params import USER_NAME -from storlets_test_utils import enable_account_for_storlets, \ - put_dependency, put_storlet_containers, put_storlet_object +from storlets_test_utils import put_storlet_object + +TEST_STORLET_NAME = 'test-10.jar' -TEST_STORLET_NAME='test-10.jar' - PATH_TO_STORLET_GIT_MODULE = '' PATH_TO_STORLETS = '' '''------------------------------------------------------------------------''' + + def invokeTestStorlet(url, token, op, withlog=False): - headers = {'X-Run-Storlet':TEST_STORLET_NAME} - if withlog == True: + headers = {'X-Run-Storlet': TEST_STORLET_NAME} + if withlog is True: headers['X-Storlet-Generate-Log'] = 'True' params = 'op={0}¶m2=val2'.format(op) resp_dict = dict() try: - resp_headers, gf = c.get_object(url, token, 'myobjects', - 'test_object', None, None, params, + resp_headers, gf = c.get_object(url, token, 'myobjects', + 'test_object', None, None, params, resp_dict, headers) - #print resp_dict + # print(resp_dict) get_text = gf - #print get_text + # print(get_text) get_response_status = resp_dict.get('status') - - if withlog == True: - resp_headers, gf = c.get_object(url, token, - 'storletlog', 'test.log', + + if withlog is True: + resp_headers, gf = c.get_object(url, token, + 'storletlog', 'test.log', None, None, None, None, headers) assert resp_headers.get('status') == 200 - text = gf.read() + gf.read() assert resp_headers.get('status') == 200 - #print text - + if op == 'print': assert get_response_status == 200 assert 'op' in get_text @@ -73,27 +71,32 @@ def invokeTestStorlet(url, token, op, withlog=False): except Exception as e: get_response_status = resp_dict.get('status') if op == 'crash': - print get_response_status + print(get_response_status) assert get_response_status >= 500 or get_response_status == 404 - + if op == 'hold': - #print get_response_status + # print(get_response_status) assert get_response_status >= 500 or get_response_status == 404 if op == 'print': - #print get_response_status + # print(get_response_status) raise e - + '''------------------------------------------------------------------------''' + + class myTestThread (threading.Thread): def __init__(self, url, token): threading.Thread.__init__(self) self.token = token self.url = url + def run(self): invokeTestStorlet(self.url, self.token, "print", False) '''------------------------------------------------------------------------''' + + def invokeTestStorletinParallel(url, token): mythreads = [] @@ -103,63 +106,67 @@ def invokeTestStorletinParallel(url, token): for t in mythreads: t.start() - + for t in mythreads: t.join() '''------------------------------------------------------------------------''' + + def testTestStorlet(url, token): - print "Deploying test storlet" - put_storlet_object(url, - token, + print("Deploying test storlet") + put_storlet_object(url, + token, TEST_STORLET_NAME, "%s/TestStorlet/bin/" % PATH_TO_STORLETS, - '', + '', 'com.ibm.storlet.test.test1') - print "uploading object to execute test upon" + print("uploading object to execute test upon") c.put_object(url, token, 'myobjects', 'test_object', 'some content') - print "Invoking test storlet to print" + print("Invoking test storlet to print") invokeTestStorlet(url, token, "print", False) - print "Invoking test storlet to crash" + print("Invoking test storlet to crash") invokeTestStorlet(url, token, "crash") - print "Invoking test storlet to hold" + print("Invoking test storlet to hold") invokeTestStorlet(url, token, "hold") - print "Invoking test storlet to print" + print("Invoking test storlet to print") invokeTestStorlet(url, token, "print", False) - print "Invoking test storlet in parallel to print" + print("Invoking test storlet in parallel to print") invokeTestStorletinParallel(url, token) - '''------------------------------------------------------------------------''' + + def init_path_dependant_params(): - global PATH_TO_STORLET_GIT_MODULE + global PATH_TO_STORLET_GIT_MODULE global PATH_TO_STORLETS PATH_TO_STORLET_GIT_MODULE = '../' if PATH_TO_STORLET_GIT_MODULE == '': - PATH_TO_STORLET_GIT_MODULE = os.environ['HOME'] + \ - '/workspace/Storlets' - PATH_TO_STORLETS='%s/StorletSamples' % PATH_TO_STORLET_GIT_MODULE - + PATH_TO_STORLET_GIT_MODULE = os.environ['HOME'] + '/workspace/Storlets' + PATH_TO_STORLETS = '%s/StorletSamples' % PATH_TO_STORLET_GIT_MODULE + '''------------------------------------------------------------------------''' + + def main(): init_path_dependant_params() - print 'Getting token' + print('Getting token') os_options = {'tenant_name': ACCOUNT} - url, token = c.get_auth("http://" + AUTH_IP + ":" + AUTH_PORT \ - + "/v2.0", ACCOUNT + ":" + USER_NAME, - PASSWORD, os_options = os_options, + url, token = c.get_auth("http://" + AUTH_IP + ":" + AUTH_PORT + + "/v2.0", ACCOUNT + ":" + USER_NAME, + PASSWORD, os_options=os_options, auth_version="2.0") - print 'Creating myobjects container' + print('Creating myobjects container') c.put_container(url, token, 'myobjects') - - print 'Invoking test storlet' + + print('Invoking test storlet') testTestStorlet(url, token) os.system('python execdep_test.py') @@ -167,7 +174,7 @@ def main(): os.system('python half_storlet_test.py') os.system('python metadata_storlet_test.py') os.system('python SLO_test.py') - + '''------------------------------------------------------------------------''' if __name__ == "__main__": main() diff --git a/SystemTests/sys_test_params.py b/SystemTests/sys_test_params.py index 94ef51fb..dcde1ae4 100644 --- a/SystemTests/sys_test_params.py +++ b/SystemTests/sys_test_params.py @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and Limitations under the License. -------------------------------------------------------------------------''' ''' -IMPORTANT: Make sure the variables AUTH_PI and KEYSTONE_IP point to the system +IMPORTANT: Make sure the variables AUTH_PI and KEYSTONE_IP point to the system you are testing!!! ''' '''------------------------------------------------------------------------''' diff --git a/SystemTests/textui_progress_bar.py b/SystemTests/textui_progress_bar.py index 4cd86ea2..ac96682a 100755 --- a/SystemTests/textui_progress_bar.py +++ b/SystemTests/textui_progress_bar.py @@ -18,22 +18,21 @@ Limitations under the License. 13-Jan-2015 evgenyl Initial implementation. ===========================================================================''' -import time import sys +import time '''------------------------------------------------------------------------''' -class TextUIProgressBar: - ''' - @summary: This class simulates Progress Bar GUI widget in UNIX terminal. +class TextUIProgressBar(object): + '''@summary: This class simulates Progress Bar GUI widget in UNIX terminal. + ''' '''--------------------------------------------------------------------''' + def __init__(self): - ''' - @summary: CTOR, define some constant mapping - ''' + '''@summary: CTOR, define some constant mapping''' self.colors = {} self.colors['gray'] = '30' self.colors['red'] = '31' @@ -45,12 +44,12 @@ class TextUIProgressBar: self.colors['white'] = '37' '''--------------------------------------------------------------------''' - def update_progress_bar(self, complete, total, caption = '', color='' ): - ''' - @summary: update_progress_bar - Drawing code. The idea is + + def update_progress_bar(self, complete, total, caption='', color=''): + '''@summary: update_progress_bar Drawing code. The idea is + - jump to the beginning of the line - - print the same amount of characters + - print the same amount of characters but in a different proportion (complete/total) @param complete: How many steps were completed? @type complete: Integer, not-negative @@ -58,30 +57,32 @@ class TextUIProgressBar: @type total: Integer, not-negative @param caption: Description to add after the bar @type caption: String - @param color: Which color to use while drawing? + @param color: Which color to use while drawing? Only a predefined set of colors is supported @type color: String ''' - color = self.colors.get(color, self.colors['white']) + color = self.colors.get(color, self.colors['white']) color_start = '\033[01;' + color + 'm' - color_stop = '\033[00m' - print '\r' + color_start + u'\u2591'*complete + \ - u'\u2593'*(total-complete) + color_stop, + color_stop = '\033[00m' + sys.stdout.write('\r' + color_start + u'\u2591' * complete + + u'\u2593' * (total - complete) + color_stop) if 0 < len(caption): - print '{0}'.format(caption) , + sys.stdout.write('{0}'.format(caption)) sys.stdout.flush() '''--------------------------------------------------------------------''' + def test(self): + '''@summary: test + + Unit test. Simulate a process of 10 steps with delay of one + second after each step. ''' - @summary: test - Unit test. Simulate a process of 10 steps with - delay of one second after each step. - ''' + k = self.colors.keys() l = len(k) - for j in range(1, l+1): - self.update_progress_bar(j, l, str(j), k[j-1]) + for j in range(1, l + 1): + self.update_progress_bar(j, l, str(j), k[j - 1]) time.sleep(1) '''============================= END OF FILE ==============================''' diff --git a/babel.cfg b/babel.cfg new file mode 100644 index 00000000..15cd6cb7 --- /dev/null +++ b/babel.cfg @@ -0,0 +1,2 @@ +[python: **.py] + diff --git a/doc/source/conf.py b/doc/source/conf.py new file mode 100755 index 00000000..8e08f673 --- /dev/null +++ b/doc/source/conf.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +sys.path.insert(0, os.path.abspath('../..')) +# -- General configuration ---------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + 'sphinx.ext.autodoc', + #'sphinx.ext.intersphinx', + 'oslosphinx' +] + +# autodoc generation is a bit aggressive and a nuisance when doing heavy +# text edit cycles. +# execute "export SPHINX_DEBUG=1" in your terminal to disable + +# The suffix of source filenames. +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'storlets' +copyright = u'2013, OpenStack Foundation' + +# If true, '()' will be appended to :func: etc. cross-reference text. +add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +add_module_names = True + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# -- Options for HTML output -------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +# html_theme_path = ["."] +# html_theme = '_theme' +# html_static_path = ['static'] + +# Output file base name for HTML help builder. +htmlhelp_basename = '%sdoc' % project + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass +# [howto/manual]). +latex_documents = [ + ('index', + '%s.tex' % project, + u'%s Documentation' % project, + u'OpenStack Foundation', 'manual'), +] + +# Example configuration for intersphinx: refer to the Python standard library. +#intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst new file mode 100644 index 00000000..1728a61c --- /dev/null +++ b/doc/source/contributing.rst @@ -0,0 +1,4 @@ +============ +Contributing +============ +.. include:: ../../CONTRIBUTING.rst diff --git a/doc/source/index.rst b/doc/source/index.rst new file mode 100644 index 00000000..223c78b1 --- /dev/null +++ b/doc/source/index.rst @@ -0,0 +1,25 @@ +.. storlets documentation master file, created by + sphinx-quickstart on Tue Jul 9 22:26:36 2013. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to storlets's documentation! +======================================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + readme + installation + usage + contributing + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/doc/source/installation.rst b/doc/source/installation.rst new file mode 100644 index 00000000..6b01dfa9 --- /dev/null +++ b/doc/source/installation.rst @@ -0,0 +1,12 @@ +============ +Installation +============ + +At the command line:: + + $ pip install storlets + +Or, if you have virtualenvwrapper installed:: + + $ mkvirtualenv storlets + $ pip install storlets diff --git a/doc/source/readme.rst b/doc/source/readme.rst new file mode 100644 index 00000000..a6210d3d --- /dev/null +++ b/doc/source/readme.rst @@ -0,0 +1 @@ +.. include:: ../../README.rst diff --git a/doc/source/usage.rst b/doc/source/usage.rst new file mode 100644 index 00000000..725e8bc5 --- /dev/null +++ b/doc/source/usage.rst @@ -0,0 +1,7 @@ +======== +Usage +======== + +To use storlets in a project:: + + import storlets diff --git a/openstack-common.conf b/openstack-common.conf new file mode 100644 index 00000000..913fdf5f --- /dev/null +++ b/openstack-common.conf @@ -0,0 +1,6 @@ +[DEFAULT] + +# The list of modules to copy from oslo-incubator.git + +# The base module to hold the copy of openstack.common +base=storlets diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..95137a6f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. + +pbr>=0.6,!=0.7,<1.0 +Babel>=1.3 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..cc37b198 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,47 @@ +[metadata] +name = storlets +summary = Middleware and Compute Engine for an OpenStack Swift compute framework that runs compute within a Swift cluster +description-file = + README.rst +author = OpenStack +author-email = openstack-dev@lists.openstack.org +home-page = http://www.openstack.org/ +classifier = + Environment :: OpenStack + Intended Audience :: Information Technology + Intended Audience :: System Administrators + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 2.6 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.3 + Programming Language :: Python :: 3.4 + +[files] +packages = + storlets + +[build_sphinx] +source-dir = doc/source +build-dir = doc/build +all_files = 1 + +[upload_sphinx] +upload-dir = doc/build/html + +[compile_catalog] +directory = storlets/locale +domain = storlets + +[update_catalog] +domain = storlets +output_dir = storlets/locale +input_file = storlets/locale/storlets.pot + +[extract_messages] +keywords = _ gettext ngettext l_ lazy_gettext +mapping_file = babel.cfg +output_file = storlets/locale/storlets.pot diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..056c16c2 --- /dev/null +++ b/setup.py @@ -0,0 +1,29 @@ +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT +import setuptools + +# In python < 2.7.4, a lazy loading of package `pbr` will break +# setuptools if some other modules registered functions in `atexit`. +# solution from: http://bugs.python.org/issue15881#msg170215 +try: + import multiprocessing # noqa +except ImportError: + pass + +setuptools.setup( + setup_requires=['pbr'], + pbr=True) diff --git a/storlets/__init__.py b/storlets/__init__.py new file mode 100644 index 00000000..67e1c78a --- /dev/null +++ b/storlets/__init__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pbr.version + +__version__ = pbr.version.VersionInfo( + 'storlets').version_string() diff --git a/storlets/tests/__init__.py b/storlets/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/storlets/tests/base.py b/storlets/tests/base.py new file mode 100644 index 00000000..1c30cdb5 --- /dev/null +++ b/storlets/tests/base.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# Copyright 2010-2011 OpenStack Foundation +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslotest import base + + +class TestCase(base.BaseTestCase): + + """Test case base class for all unit tests.""" diff --git a/storlets/tests/test_storlets.py b/storlets/tests/test_storlets.py new file mode 100644 index 00000000..9a12540e --- /dev/null +++ b/storlets/tests/test_storlets.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +test_storlets +---------------------------------- + +Tests for `storlets` module. +""" + +from storlets.tests import base + + +class TestStorlets(base.TestCase): + + def test_something(self): + pass diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 00000000..8592bde2 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,15 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. + +hacking<0.11,>=0.10.0 + +coverage>=3.6 +discover +python-subunit>=0.0.18 +sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 +oslosphinx>=2.2.0 # Apache-2.0 +oslotest>=1.2.0 # Apache-2.0 +testrepository>=0.0.18 +testscenarios>=0.4 +testtools>=0.9.36,!=1.2.0 diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..03552bca --- /dev/null +++ b/tox.ini @@ -0,0 +1,35 @@ +[tox] +minversion = 1.6 +envlist = py27,pep8 +skipsdist = True + +[testenv] +usedevelop = True +install_command = pip install -U {opts} {packages} +setenv = + VIRTUAL_ENV={envdir} +deps = -r{toxinidir}/test-requirements.txt +commands = python setup.py test --slowest --testr-args='{posargs}' + +[testenv:pep8] +commands = flake8 + +[testenv:venv] +commands = {posargs} + +[testenv:cover] +commands = python setup.py test --coverage --testr-args='{posargs}' + +[testenv:docs] +commands = python setup.py build_sphinx + +[testenv:debug] +commands = oslo_debug_helper {posargs} + +[flake8] +# E123, E125 skipped as they are invalid PEP-8. + +show-source = True +ignore = E123,E125 +builtins = _ +exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build