Remove iPython extension
This has been unmaintained and its functional tests have been broken for some time. Because the current upper version is not compatible with Python 3.12 and we aren't sure if we can bump the cap safely, let's remove the extension. Also replace the openstack CLI by the swift CLI to resolve the current CI failure. Change-Id: I1a2f908fef4b660686d75ea6e8330287b423cd8b
This commit is contained in:
Takashi Kajinami
parent
21098c840d
commit
86d8f05d51
@ -130,7 +130,7 @@ function configure_swift_and_keystone_for_storlets {
|
|||||||
# Create storlet related containers and set ACLs
|
# Create storlet related containers and set ACLs
|
||||||
start_swift
|
start_swift
|
||||||
_export_swift_os_vars
|
_export_swift_os_vars
|
||||||
openstack object store account set --property Storlet-Enabled=True
|
swift post --meta "Storlet-Enabled:True"
|
||||||
swift post --read-acl $SWIFT_DEFAULT_PROJECT:$SWIFT_MEMBER_USER $STORLETS_STORLET_CONTAINER_NAME
|
swift post --read-acl $SWIFT_DEFAULT_PROJECT:$SWIFT_MEMBER_USER $STORLETS_STORLET_CONTAINER_NAME
|
||||||
swift post --read-acl $SWIFT_DEFAULT_PROJECT:$SWIFT_MEMBER_USER $STORLETS_DEPENDENCY_CONTAINER_NAME
|
swift post --read-acl $SWIFT_DEFAULT_PROJECT:$SWIFT_MEMBER_USER $STORLETS_DEPENDENCY_CONTAINER_NAME
|
||||||
swift post $STORLETS_LOG_CONTAIER_NAME
|
swift post $STORLETS_LOG_CONTAIER_NAME
|
||||||
|
|||||||
@ -4,4 +4,3 @@ reno>=3.1.0 # Apache-2.0
|
|||||||
|
|
||||||
# TOD(takashi): The following items should be migrated to requirements.txt
|
# TOD(takashi): The following items should be migrated to requirements.txt
|
||||||
python-swiftclient>=3.1.0
|
python-swiftclient>=3.1.0
|
||||||
ipython<6.0
|
|
||||||
|
|||||||
@ -51,7 +51,6 @@ Overview and Concepts
|
|||||||
storlets_terminology
|
storlets_terminology
|
||||||
storlet_engine_overview
|
storlet_engine_overview
|
||||||
api/overview_api
|
api/overview_api
|
||||||
ipython_integration
|
|
||||||
installation
|
installation
|
||||||
|
|
||||||
Related Projects
|
Related Projects
|
||||||
|
|||||||
@ -1,164 +0,0 @@
|
|||||||
IPython Notebook Integration
|
|
||||||
============================
|
|
||||||
|
|
||||||
IPython/Jupyter provides a browser based interactive shell that supports data visualization. The storlets integration with IPython allows an easy deployment and invocation of storlets via an IPython notebook. In the below sections we describe how to setup IPython notebook to work with storlets, how to deploy a python storlet and how to invoke a storlet.
|
|
||||||
|
|
||||||
Set up IPython to work with storlets
|
|
||||||
------------------------------------
|
|
||||||
Setting up an IPython notebook to work with storlets involves:
|
|
||||||
|
|
||||||
#. Providing the authentication information of a storlet enabled Swift account.
|
|
||||||
This is done by setting environment variables similar to those used by swift
|
|
||||||
client. The exact variables that need to be set are dependent on the auth middleware
|
|
||||||
used and the auth protocol version. For more details please refer to:
|
|
||||||
`python-swiftclient docs
|
|
||||||
<https://docs.openstack.org/python-swiftclient/latest/cli/index.html#authentication>`_.
|
|
||||||
|
|
||||||
#. Load the storlets IPython extension.
|
|
||||||
|
|
||||||
The below shows environment variables definitions that comply with the
|
|
||||||
default storlets development environment installation (`s2aio <http://storlets.readthedocs.io/en/latest/s2aio.html>`__).
|
|
||||||
|
|
||||||
::
|
|
||||||
|
|
||||||
import os
|
|
||||||
os.environ['OS_AUTH_VERSION'] = '3'
|
|
||||||
os.environ['OS_AUTH_URL'] = 'http://127.0.0.1/v3'
|
|
||||||
os.environ['OS_USERNAME'] = 'tester'
|
|
||||||
os.environ['OS_PASSWORD'] = 'testing'
|
|
||||||
os.environ['OS_USER_DOMAIN_NAME'] = 'default'
|
|
||||||
os.environ['OS_PROJECT_DOMAIN_NAME'] = 'default'
|
|
||||||
os.environ['OS_PROJECT_NAME'] = 'test'
|
|
||||||
|
|
||||||
To load the storlets IPython extension simply enter and execute the below:
|
|
||||||
|
|
||||||
::
|
|
||||||
|
|
||||||
%load_ext storlets.tools.extensions.ipython
|
|
||||||
|
|
||||||
Deploy a Python storlet
|
|
||||||
-----------------------
|
|
||||||
General background on storlets deployment is found `here <http://storlets.readthedocs.io/en/latest/writing_and_deploying_storlets.html#storlet-deployment-guidelines>`__.
|
|
||||||
|
|
||||||
In a new notebook cell, enter the '%%storletapp' directive
|
|
||||||
followed by the storlet name. Following that type the storlet code.
|
|
||||||
Below is an example of a simple 'identitiy' storlet.
|
|
||||||
Executing the cell will deploy the storlet into Swift.
|
|
||||||
|
|
||||||
::
|
|
||||||
|
|
||||||
%%storletapp test.TestStorlet
|
|
||||||
|
|
||||||
class TestStorlet(object):
|
|
||||||
def __init__(self, logger):
|
|
||||||
self.logger = logger
|
|
||||||
|
|
||||||
def __call__(self, in_files, out_files, params):
|
|
||||||
"""
|
|
||||||
The function called for storlet invocation
|
|
||||||
:param in_files: a list of StorletInputFile
|
|
||||||
:param out_files: a list of StorletOutputFile
|
|
||||||
:param params: a dict of request parameters
|
|
||||||
"""
|
|
||||||
self.logger.debug('Returning metadata')
|
|
||||||
metadata = in_files[0].get_metadata()
|
|
||||||
for key in params.keys():
|
|
||||||
metadata[key] = params[key]
|
|
||||||
out_files[0].set_metadata(metadata)
|
|
||||||
|
|
||||||
self.logger.debug('Start to return object data')
|
|
||||||
content = ''
|
|
||||||
while True:
|
|
||||||
buf = in_files[0].read(16)
|
|
||||||
if not buf:
|
|
||||||
break
|
|
||||||
content += buf
|
|
||||||
self.logger.debug('Received %d bytes' % len(content))
|
|
||||||
self.logger.debug('Writing back %d bytes' % len(content))
|
|
||||||
out_files[0].write(content)
|
|
||||||
self.logger.debug('Complete')
|
|
||||||
in_files[0].close()
|
|
||||||
out_files[0].close()
|
|
||||||
|
|
||||||
.. note:: To run the storlet on an actual data set, one can enter the following at
|
|
||||||
the top of the cell
|
|
||||||
|
|
||||||
::
|
|
||||||
|
|
||||||
%%storletapp test.TestStorlet --with-invoke --input path:/<container>/<object> --print-result
|
|
||||||
|
|
||||||
N.B. Useful commands such as 'dry-run' is under development. And more
|
|
||||||
details for options are in the next section.
|
|
||||||
|
|
||||||
Invoke a storlet
|
|
||||||
----------------
|
|
||||||
General information on storlet invocation can be found `here <http://storlets.readthedocs.io/en/latest/api/overview_api.html#storlets-invocation>`__.
|
|
||||||
|
|
||||||
Here is how an invocation works:
|
|
||||||
|
|
||||||
#. Define an optional dictionary variable params that would hold the invocation parameters:
|
|
||||||
|
|
||||||
::
|
|
||||||
|
|
||||||
myparams = {'color' : 'red'}
|
|
||||||
|
|
||||||
#. To invoke test.TestStorlet on a get just type the following:
|
|
||||||
|
|
||||||
::
|
|
||||||
|
|
||||||
%get --storlet test.py --input path:/<container>/<object> -i myparams -o myresult
|
|
||||||
|
|
||||||
The invocation will execute test.py over the specified swift object with parameters read from myparams.
|
|
||||||
The result is placed in myresults.
|
|
||||||
The '-i' argument is optional, however, if specified the supplied value must be a name of a defined dictionary variable.
|
|
||||||
myresults is an instance of storlets.tools.extensions.ipython.Response. This class has the following members:
|
|
||||||
|
|
||||||
#. status - An integer holding the Http response status
|
|
||||||
|
|
||||||
#. headers - A dictionary holding the storlet invocation response headers
|
|
||||||
|
|
||||||
#. iter_content - An iterator over the response body
|
|
||||||
|
|
||||||
#. content - The content of the response body
|
|
||||||
|
|
||||||
|
|
||||||
#. To invoke test.TestStorlet on a put just type the following:
|
|
||||||
|
|
||||||
::
|
|
||||||
|
|
||||||
%put --storlet test.py --input <full path to local file> --output path:/<container>/<object> -i myparams -o myresult
|
|
||||||
|
|
||||||
The invocation will execute test.py over the uploaded file specified with the --input option which must be a full local path.
|
|
||||||
test.py is invoked with parameters read from myparams.
|
|
||||||
The result is placed in myresults.
|
|
||||||
The '-i' argument is optional, however, if specified the supplied value must be a name of a defined variable.
|
|
||||||
myresults is a dictionary with the following keys:
|
|
||||||
|
|
||||||
#. status - An integer holding the Http response status
|
|
||||||
|
|
||||||
#. headers - A dictionary holding the storlet invocation response headers
|
|
||||||
|
|
||||||
#. To invoke test.TestStorlet on a copy just type the following:
|
|
||||||
|
|
||||||
::
|
|
||||||
|
|
||||||
%copy --storlet test.py --input path:/<container>/<object> --output path:/<container>/<object> -i myparams -o myresult
|
|
||||||
|
|
||||||
The invocation will execute test.py over the input object specified with the --input option.
|
|
||||||
The execution result will be saved in the output object specified with the --output option.
|
|
||||||
test.py is invoked with parameters read from myparams.
|
|
||||||
The result is placed in myresults.
|
|
||||||
The '-i' argument is optional, however, if specified the supplied value must be a name of a defined variable.
|
|
||||||
myresults is a dictionary with the following keys:
|
|
||||||
|
|
||||||
#. status - An integer holding the Http response status
|
|
||||||
|
|
||||||
#. headers - A dictionary holding the storlet invocation response headers
|
|
||||||
|
|
||||||
|
|
||||||
Extension docs
|
|
||||||
^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
.. automodule:: storlets.tools.extensions.ipython
|
|
||||||
:members:
|
|
||||||
:show-inheritance:
|
|
||||||
@ -1,446 +0,0 @@
|
|||||||
# Copyright (c) 2015, 2016 OpenStack Foundation.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
||||||
# implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
|
|
||||||
"""Implementation of magic funcs for interaction with the OpenStack Storlets.
|
|
||||||
|
|
||||||
This extension is desined to use os environment variables to set
|
|
||||||
authentication and storage target host. (for now)
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
import os
|
|
||||||
import string
|
|
||||||
from swiftclient.client import Connection
|
|
||||||
|
|
||||||
from IPython.core import magic_arguments
|
|
||||||
# TODO(kota_): we may need some error handing in ipython shell so keep those
|
|
||||||
# errors import as references.
|
|
||||||
# from IPython.core.alias import AliasError, Alias
|
|
||||||
from IPython.core.error import UsageError
|
|
||||||
from IPython.core.magic import Magics, magics_class, cell_magic, line_magic
|
|
||||||
from IPython.utils.py3compat import unicode_type
|
|
||||||
|
|
||||||
|
|
||||||
class Response(object):
|
|
||||||
"""
|
|
||||||
Response object to return the object to ipython cell
|
|
||||||
|
|
||||||
:param status: int for status code
|
|
||||||
:param headers: a dict for response headers
|
|
||||||
:param body_iter: an iterator object which takes the body content from
|
|
||||||
"""
|
|
||||||
def __init__(self, status, headers, body_iter=None):
|
|
||||||
self.status = status
|
|
||||||
self.headers = headers
|
|
||||||
self._body_iter = body_iter or iter([])
|
|
||||||
|
|
||||||
def __iter__(self):
|
|
||||||
return self._body_iter
|
|
||||||
|
|
||||||
def iter_content(self):
|
|
||||||
# TODO(kota_): supports chunk_size like requests.Response
|
|
||||||
return self._body_iter
|
|
||||||
|
|
||||||
@property
|
|
||||||
def content(self):
|
|
||||||
return b''.join([chunk for chunk in self._body_iter])
|
|
||||||
|
|
||||||
|
|
||||||
def get_swift_connection():
|
|
||||||
# find api version
|
|
||||||
for k in ('ST_AUTH_VERSION', 'OS_AUTH_VERSION', 'OS_IDENTITY_API_VERSION'):
|
|
||||||
if k in os.environ:
|
|
||||||
auth_version = os.environ[k]
|
|
||||||
break
|
|
||||||
else:
|
|
||||||
auth_version = 1
|
|
||||||
|
|
||||||
# cast from string to int
|
|
||||||
auth_version = int(float(auth_version))
|
|
||||||
|
|
||||||
if auth_version == 3:
|
|
||||||
# keystone v3
|
|
||||||
try:
|
|
||||||
auth_url = os.environ['OS_AUTH_URL']
|
|
||||||
auth_user = os.environ['OS_USERNAME']
|
|
||||||
auth_password = os.environ['OS_PASSWORD']
|
|
||||||
project_name = os.environ['OS_PROJECT_NAME']
|
|
||||||
except KeyError:
|
|
||||||
raise UsageError(
|
|
||||||
"You need to set OS_AUTH_URL, OS_USERNAME, OS_PASSWORD and "
|
|
||||||
"OS_PROJECT_NAME for Swift authentication")
|
|
||||||
auth_os_options = {
|
|
||||||
'user_domain_name': os.environ.get(
|
|
||||||
'OS_USER_DOMAIN_NAME', 'Default'),
|
|
||||||
'project_domain_name': os.environ.get(
|
|
||||||
'OS_PROJECT_DOMAIN_NAME', 'Default'),
|
|
||||||
'project_name': project_name,
|
|
||||||
}
|
|
||||||
return Connection(auth_url, auth_user, auth_password,
|
|
||||||
os_options=auth_os_options,
|
|
||||||
auth_version='3')
|
|
||||||
|
|
||||||
elif auth_version == 2:
|
|
||||||
# keystone v2 (not implemented)
|
|
||||||
raise NotImplementedError('keystone v2 is not supported')
|
|
||||||
else:
|
|
||||||
try:
|
|
||||||
auth_url = os.environ['ST_AUTH']
|
|
||||||
auth_user = os.environ['ST_USER']
|
|
||||||
auth_password = os.environ['ST_KEY']
|
|
||||||
except KeyError:
|
|
||||||
raise UsageError(
|
|
||||||
"You need to set ST_AUTH, ST_USER, ST_KEY for "
|
|
||||||
"Swift authentication")
|
|
||||||
return Connection(auth_url, auth_user, auth_password)
|
|
||||||
|
|
||||||
|
|
||||||
@magics_class
|
|
||||||
class StorletMagics(Magics):
|
|
||||||
"""Magics to interact with OpenStack Storlets
|
|
||||||
"""
|
|
||||||
|
|
||||||
def _parse_input_path(self, path_str):
|
|
||||||
"""
|
|
||||||
Parse formatted to path to swift container and object names
|
|
||||||
|
|
||||||
:param path_str: path string starts with "path:" prefix
|
|
||||||
:return (container, obj): Both container and obj are formatted
|
|
||||||
as string
|
|
||||||
:raise UsageError: if the path_str is not formatted as expected
|
|
||||||
"""
|
|
||||||
if not path_str.startswith('path:'):
|
|
||||||
raise UsageError(
|
|
||||||
'swift object path must have the format: '
|
|
||||||
'"path:/<container>/<object>"')
|
|
||||||
try:
|
|
||||||
src_container_obj = path_str[len('path:'):]
|
|
||||||
src_container, src_obj = src_container_obj.strip(
|
|
||||||
'/').split('/', 1)
|
|
||||||
return src_container, src_obj
|
|
||||||
except ValueError:
|
|
||||||
raise UsageError(
|
|
||||||
'swift object path must have the format: '
|
|
||||||
'"path:/<container>/<object>"')
|
|
||||||
|
|
||||||
def _generate_params_headers(self, param_dict):
|
|
||||||
"""
|
|
||||||
Parse parameter args dict to swift headers
|
|
||||||
|
|
||||||
:param param_dict: a dict of input parameters
|
|
||||||
:return headers: a dict for swift headers
|
|
||||||
"""
|
|
||||||
headers = {}
|
|
||||||
for i, (key, value) in enumerate(param_dict.items()):
|
|
||||||
headers['X-Storlet-Parameter-%d' % i] =\
|
|
||||||
'%s:%s' % (key, value)
|
|
||||||
|
|
||||||
return headers
|
|
||||||
|
|
||||||
@magic_arguments.magic_arguments()
|
|
||||||
@magic_arguments.argument(
|
|
||||||
'container_obj', type=unicode_type,
|
|
||||||
help='container/object path to upload'
|
|
||||||
)
|
|
||||||
@cell_magic
|
|
||||||
def uploadfile(self, line, cell):
|
|
||||||
"""Upload the contents of the cell to OpenStack Swift.
|
|
||||||
"""
|
|
||||||
args = magic_arguments.parse_argstring(self.uploadfile, line)
|
|
||||||
container, obj = args.container_obj.split('/', 1)
|
|
||||||
conn = get_swift_connection()
|
|
||||||
conn.put_object(container, obj, cell,
|
|
||||||
{'Content-Type': 'application/python'})
|
|
||||||
|
|
||||||
@magic_arguments.magic_arguments()
|
|
||||||
@magic_arguments.argument(
|
|
||||||
'module_class', type=unicode_type,
|
|
||||||
help='module and class name to upload'
|
|
||||||
)
|
|
||||||
@magic_arguments.argument(
|
|
||||||
'-c', '--container', type=unicode_type, default='storlet',
|
|
||||||
help='Storlet container name, "storlet" in default'
|
|
||||||
)
|
|
||||||
@magic_arguments.argument(
|
|
||||||
'-d', '--dependencies', type=unicode_type, default='storlet',
|
|
||||||
help='Storlet container name, "storlet" in default'
|
|
||||||
)
|
|
||||||
@magic_arguments.argument(
|
|
||||||
'--with-invoke', action='store_true', default=False,
|
|
||||||
help='An option to run storlet for testing. '
|
|
||||||
'This requires --input option'
|
|
||||||
)
|
|
||||||
@magic_arguments.argument(
|
|
||||||
'--input', type=unicode_type, default='',
|
|
||||||
help='Specifiy input object path that must be of the form '
|
|
||||||
'"path:/<container>/<object>"'
|
|
||||||
)
|
|
||||||
@magic_arguments.argument(
|
|
||||||
'--print-result', action='store_true', default=False,
|
|
||||||
help='Print result object to stdout. Note that this may be a large'
|
|
||||||
'binary depends on your app'
|
|
||||||
)
|
|
||||||
@cell_magic
|
|
||||||
def storletapp(self, line, cell):
|
|
||||||
args = magic_arguments.parse_argstring(self.storletapp, line)
|
|
||||||
module_path = args.module_class
|
|
||||||
assert module_path.count('.') == 1
|
|
||||||
headers = {
|
|
||||||
'X-Object-Meta-Storlet-Language': 'python',
|
|
||||||
'X-Object-Meta-Storlet-Interface-Version': '1.0',
|
|
||||||
'X-Object-Meta-Storlet-Object-Metadata': 'no',
|
|
||||||
'X-Object-Meta-Storlet-Main': module_path,
|
|
||||||
'Content-Type': 'application/octet-stream',
|
|
||||||
}
|
|
||||||
storlet_obj = '%s.py' % module_path.split('.')[0]
|
|
||||||
|
|
||||||
conn = get_swift_connection()
|
|
||||||
conn.put_object(args.container, storlet_obj, cell, headers=headers)
|
|
||||||
|
|
||||||
print('Upload storlets succeeded /%s/%s'
|
|
||||||
% (args.container, storlet_obj))
|
|
||||||
print('Example command `swift download <container> <object> '
|
|
||||||
'-H X-Run-Storlet:%s`' % storlet_obj)
|
|
||||||
|
|
||||||
if args.with_invoke:
|
|
||||||
if not args.input:
|
|
||||||
raise UsageError(
|
|
||||||
'--with-invoke option requires --input to run the app')
|
|
||||||
|
|
||||||
src_container, src_obj = self._parse_input_path(args.input)
|
|
||||||
|
|
||||||
headers = {'X-Run-Storlet': '%s' % storlet_obj}
|
|
||||||
|
|
||||||
# invoke storlet app
|
|
||||||
resp_headers, resp_content_iter = conn.get_object(
|
|
||||||
src_container, src_obj, resp_chunk_size=64 * 1024,
|
|
||||||
headers=headers)
|
|
||||||
|
|
||||||
print('Invocation Complete')
|
|
||||||
if args.print_result:
|
|
||||||
print('Result Content:')
|
|
||||||
print(''.join(resp_content_iter))
|
|
||||||
else:
|
|
||||||
# drain all resp content stream
|
|
||||||
for x in resp_content_iter:
|
|
||||||
pass
|
|
||||||
|
|
||||||
@magic_arguments.magic_arguments()
|
|
||||||
@magic_arguments.argument(
|
|
||||||
'--input', type=unicode_type,
|
|
||||||
help='The input object for the storlet execution'
|
|
||||||
'this option must be of the form "path:<container>/<object>"'
|
|
||||||
)
|
|
||||||
@magic_arguments.argument(
|
|
||||||
'--storlet', type=unicode_type,
|
|
||||||
help='The storlet to execute over the input'
|
|
||||||
)
|
|
||||||
@magic_arguments.argument(
|
|
||||||
'-i', type=unicode_type,
|
|
||||||
help=('A name of a variable defined in the environment '
|
|
||||||
'holding a dictionary with the storlet invocation '
|
|
||||||
'input parameters')
|
|
||||||
)
|
|
||||||
@magic_arguments.argument(
|
|
||||||
'-o', type=unicode_type,
|
|
||||||
help=('A name of an output variable to hold the invocation result '
|
|
||||||
'The output variable is a dictionary with the fields: '
|
|
||||||
'status, headers, content_iter holding the response status, '
|
|
||||||
'headers, and body iterator accordingly')
|
|
||||||
)
|
|
||||||
@line_magic
|
|
||||||
def get(self, line):
|
|
||||||
args = magic_arguments.parse_argstring(self.get, line)
|
|
||||||
if not args.o:
|
|
||||||
raise UsageError('-o option is mandatory for the invocation')
|
|
||||||
if not args.o[0].startswith(tuple(string.ascii_letters)):
|
|
||||||
raise UsageError('The output variable name must be a valid prefix '
|
|
||||||
'of a python variable, that is, start with a '
|
|
||||||
'letter')
|
|
||||||
if not args.storlet:
|
|
||||||
raise UsageError('--storlet option is mandatory '
|
|
||||||
'for the invocation')
|
|
||||||
if not args.input:
|
|
||||||
raise UsageError('--input option is mandatory for the invocation')
|
|
||||||
|
|
||||||
src_container, src_obj = self._parse_input_path(args.input)
|
|
||||||
|
|
||||||
headers = {'X-Run-Storlet': '%s' % args.storlet}
|
|
||||||
# pick -i option and translate the params to
|
|
||||||
# X-Storlet-Parameter-x headers
|
|
||||||
storlet_headers = self._generate_params_headers(
|
|
||||||
self.shell.user_ns[args.i] if args.i else {})
|
|
||||||
headers.update(storlet_headers)
|
|
||||||
|
|
||||||
# invoke storlet app on get
|
|
||||||
conn = get_swift_connection()
|
|
||||||
response_dict = dict()
|
|
||||||
resp_headers, resp_content_iter = conn.get_object(
|
|
||||||
src_container, src_obj,
|
|
||||||
resp_chunk_size=64 * 1024,
|
|
||||||
headers=headers,
|
|
||||||
response_dict=response_dict)
|
|
||||||
|
|
||||||
res = Response(int(response_dict['status']),
|
|
||||||
resp_headers,
|
|
||||||
resp_content_iter)
|
|
||||||
self.shell.user_ns[args.o] = res
|
|
||||||
|
|
||||||
@magic_arguments.magic_arguments()
|
|
||||||
@magic_arguments.argument(
|
|
||||||
'--input', type=unicode_type,
|
|
||||||
help='The input object for the storlet execution'
|
|
||||||
'this option must be of the form "path:<container>/<object>"'
|
|
||||||
)
|
|
||||||
@magic_arguments.argument(
|
|
||||||
'--output', type=unicode_type,
|
|
||||||
help='The output object for the storlet execution'
|
|
||||||
'this option must be of the form "path:<container>/<object>"'
|
|
||||||
)
|
|
||||||
@magic_arguments.argument(
|
|
||||||
'--storlet', type=unicode_type,
|
|
||||||
help='The storlet to execute over the input'
|
|
||||||
)
|
|
||||||
@magic_arguments.argument(
|
|
||||||
'-i', type=unicode_type,
|
|
||||||
help=('A name of a variable defined in the environment '
|
|
||||||
'holding a dictionary with the storlet invocation '
|
|
||||||
'input parameters')
|
|
||||||
)
|
|
||||||
@magic_arguments.argument(
|
|
||||||
'-o', type=unicode_type,
|
|
||||||
help=('A name of an output variable to hold the invocation result '
|
|
||||||
'The output variable is a dictionary with the fields: '
|
|
||||||
'status, headers, holding the response status and '
|
|
||||||
'headers accordingly')
|
|
||||||
)
|
|
||||||
@line_magic
|
|
||||||
def copy(self, line):
|
|
||||||
args = magic_arguments.parse_argstring(self.copy, line)
|
|
||||||
if not args.o:
|
|
||||||
raise UsageError('-o option is mandatory for the invocation')
|
|
||||||
if not args.o[0].startswith(tuple(string.ascii_letters)):
|
|
||||||
raise UsageError('The output variable name must be a valid prefix '
|
|
||||||
'of a python variable, that is, start with a '
|
|
||||||
'letter')
|
|
||||||
if not args.storlet:
|
|
||||||
raise UsageError('--storlet option is mandatory '
|
|
||||||
'for the invocation')
|
|
||||||
if not args.input:
|
|
||||||
raise UsageError('--input option is mandatory for the invocation')
|
|
||||||
|
|
||||||
if not args.output:
|
|
||||||
raise UsageError('--output option is mandatory for the invocation')
|
|
||||||
|
|
||||||
src_container, src_obj = self._parse_input_path(args.input)
|
|
||||||
dst_container, dst_obj = self._parse_input_path(args.output)
|
|
||||||
destination = '/%s/%s' % (dst_container, dst_obj)
|
|
||||||
|
|
||||||
headers = {'X-Run-Storlet': '%s' % args.storlet}
|
|
||||||
# pick -i option and translate the params to
|
|
||||||
# X-Storlet-Parameter-x headers
|
|
||||||
storlet_headers = self._generate_params_headers(
|
|
||||||
self.shell.user_ns[args.i] if args.i else {})
|
|
||||||
headers.update(storlet_headers)
|
|
||||||
|
|
||||||
# invoke storlet app on copy
|
|
||||||
conn = get_swift_connection()
|
|
||||||
|
|
||||||
response_dict = dict()
|
|
||||||
conn.copy_object(
|
|
||||||
src_container, src_obj,
|
|
||||||
destination=destination,
|
|
||||||
headers=headers,
|
|
||||||
response_dict=response_dict)
|
|
||||||
|
|
||||||
res = Response(int(response_dict['status']),
|
|
||||||
response_dict['headers'])
|
|
||||||
self.shell.user_ns[args.o] = res
|
|
||||||
|
|
||||||
@magic_arguments.magic_arguments()
|
|
||||||
@magic_arguments.argument(
|
|
||||||
'--input', type=unicode_type,
|
|
||||||
help='The local input object for upload'
|
|
||||||
'this option must be a full path of a local file'
|
|
||||||
)
|
|
||||||
@magic_arguments.argument(
|
|
||||||
'--output', type=unicode_type,
|
|
||||||
help='The output object of the storlet execution'
|
|
||||||
'this option must be of the form "path:<container>/<object>"'
|
|
||||||
)
|
|
||||||
@magic_arguments.argument(
|
|
||||||
'--storlet', type=unicode_type,
|
|
||||||
help='The storlet to execute over the input'
|
|
||||||
)
|
|
||||||
@magic_arguments.argument(
|
|
||||||
'-i', type=unicode_type,
|
|
||||||
help=('A name of a variable defined in the environment '
|
|
||||||
'holding a dictionary with the storlet invocation '
|
|
||||||
'input parameters')
|
|
||||||
)
|
|
||||||
@magic_arguments.argument(
|
|
||||||
'-o', type=unicode_type,
|
|
||||||
help=('A name of an output variable to hold the invocation result '
|
|
||||||
'The output variable is a dictionary with the fields: '
|
|
||||||
'status, headers, holding the response status and '
|
|
||||||
'headers accordingly')
|
|
||||||
)
|
|
||||||
@line_magic
|
|
||||||
def put(self, line):
|
|
||||||
args = magic_arguments.parse_argstring(self.put, line)
|
|
||||||
if not args.o:
|
|
||||||
raise UsageError('-o option is mandatory for the invocation')
|
|
||||||
if not args.o[0].startswith(tuple(string.ascii_letters)):
|
|
||||||
raise UsageError('The output variable name must be a valid prefix '
|
|
||||||
'of a python variable, that is, start with a '
|
|
||||||
'letter')
|
|
||||||
if not args.storlet:
|
|
||||||
raise UsageError('--storlet option is mandatory '
|
|
||||||
'for the invocation')
|
|
||||||
if not args.input:
|
|
||||||
raise UsageError('--input option is mandatory for the invocation')
|
|
||||||
if not args.input.startswith('/'):
|
|
||||||
raise UsageError('--input argument must be a full path')
|
|
||||||
|
|
||||||
if not args.output:
|
|
||||||
raise UsageError('--output option is mandatory for the invocation')
|
|
||||||
|
|
||||||
dst_container, dst_obj = self._parse_input_path(args.output)
|
|
||||||
|
|
||||||
headers = {'X-Run-Storlet': '%s' % args.storlet}
|
|
||||||
# pick -i option and translate the params to
|
|
||||||
# X-Storlet-Parameter-x headers
|
|
||||||
storlet_headers = self._generate_params_headers(
|
|
||||||
self.shell.user_ns[args.i] if args.i else {})
|
|
||||||
headers.update(storlet_headers)
|
|
||||||
|
|
||||||
# invoke storlet app on copy
|
|
||||||
conn = get_swift_connection()
|
|
||||||
response_dict = dict()
|
|
||||||
with open(args.input, 'rb') as content:
|
|
||||||
conn.put_object(
|
|
||||||
dst_container, dst_obj,
|
|
||||||
content,
|
|
||||||
headers=headers,
|
|
||||||
response_dict=response_dict)
|
|
||||||
|
|
||||||
res = Response(int(response_dict['status']),
|
|
||||||
response_dict['headers'])
|
|
||||||
self.shell.user_ns[args.o] = res
|
|
||||||
|
|
||||||
|
|
||||||
def load_ipython_extension(ipython):
|
|
||||||
ipython.register_magics(StorletMagics)
|
|
||||||
@ -10,9 +10,5 @@ testscenarios>=0.4
|
|||||||
testtools>=0.9.36
|
testtools>=0.9.36
|
||||||
python-swiftclient>=3.1.0
|
python-swiftclient>=3.1.0
|
||||||
python-keystoneclient
|
python-keystoneclient
|
||||||
ipython<6.0
|
|
||||||
ipywidgets<7.6.0
|
|
||||||
jupyter
|
|
||||||
nbformat
|
|
||||||
|
|
||||||
bashate # Apache-2.0
|
bashate # Apache-2.0
|
||||||
|
|||||||
@ -1,140 +0,0 @@
|
|||||||
# Copyright (c) 2010-2016 OpenStack Foundation
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
||||||
# implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
import os
|
|
||||||
import unittest
|
|
||||||
import tempfile
|
|
||||||
import subprocess
|
|
||||||
import nbformat
|
|
||||||
|
|
||||||
|
|
||||||
COULD_BE_CHANGED = ['x-storlet-generated-from-account',
|
|
||||||
'x-trans-id',
|
|
||||||
'x-openstack-request-id',
|
|
||||||
'x-storlet-generated-from-last-modified',
|
|
||||||
'last-modified',
|
|
||||||
'x-timestamp',
|
|
||||||
'date', ]
|
|
||||||
|
|
||||||
|
|
||||||
class TestJupyterExcecution(unittest.TestCase):
|
|
||||||
def _run_notebook(self, path):
|
|
||||||
"""Execute a notebook via nbconvert and collect output.
|
|
||||||
:returns (parsed nb object, execution errors)
|
|
||||||
"""
|
|
||||||
with tempfile.NamedTemporaryFile(suffix=".ipynb") as fout:
|
|
||||||
args = ["jupyter", "nbconvert", "--to", "notebook", "--execute",
|
|
||||||
"--ExecutePreprocessor.timeout=60",
|
|
||||||
"--output", fout.name, path]
|
|
||||||
try:
|
|
||||||
subprocess.check_output(args, stderr=subprocess.STDOUT)
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
# Note that CalledProcessError will have stdout/stderr in py3
|
|
||||||
# instead of output attribute
|
|
||||||
self.fail('jupyter nbconvert fails with:\n'
|
|
||||||
'STDOUT: %s\n' % (e.output))
|
|
||||||
|
|
||||||
fout.seek(0)
|
|
||||||
nb = nbformat.read(fout, nbformat.current_nbformat)
|
|
||||||
|
|
||||||
# gather all error messages in all cells in the notebook
|
|
||||||
errors = [output for cell in nb.cells if "outputs" in cell
|
|
||||||
for output in cell["outputs"]
|
|
||||||
if output.output_type == "error"]
|
|
||||||
|
|
||||||
return nb, errors
|
|
||||||
|
|
||||||
def _clear_text_list(self, node):
|
|
||||||
"""
|
|
||||||
convert a notebook cell to text list like ["a", "b"]
|
|
||||||
N.B. each text will be striped
|
|
||||||
"""
|
|
||||||
texts = list()
|
|
||||||
if 'text' in node:
|
|
||||||
for line in node['text'].split('\n'):
|
|
||||||
if line:
|
|
||||||
# NOTE: extract bytes lines appeared in the cell
|
|
||||||
# in PY2, it's NOT needed but in PY3 it is required
|
|
||||||
# because PY3 explicitly add 'b' prefix to bytes
|
|
||||||
# objects
|
|
||||||
if line.startswith("b\'"):
|
|
||||||
text = eval(line).decode('ascii')
|
|
||||||
for inline_text in text.split('\n'):
|
|
||||||
if inline_text:
|
|
||||||
texts.append(inline_text.strip())
|
|
||||||
else:
|
|
||||||
texts.append(line.strip())
|
|
||||||
return texts
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
def _flatten_output_text(self, notebook):
|
|
||||||
"""
|
|
||||||
This helper method make the notebook output cells flatten to a single
|
|
||||||
direction list.
|
|
||||||
"""
|
|
||||||
output_text_list = []
|
|
||||||
for cell in notebook.cells:
|
|
||||||
for output in cell.get("outputs", []):
|
|
||||||
output_text_list.extend(self._clear_text_list(output))
|
|
||||||
return output_text_list
|
|
||||||
|
|
||||||
def test_notebook(self):
|
|
||||||
# TODO(takashi): Fix this
|
|
||||||
self.skipTest("This test is currently incompatible with "
|
|
||||||
"the latest Jupyer/iPython")
|
|
||||||
test_path = os.path.abspath(__file__)
|
|
||||||
test_dir = os.path.dirname(test_path)
|
|
||||||
original_notebook = os.path.join(test_dir, 'test_notebook.ipynb')
|
|
||||||
|
|
||||||
with open(original_notebook) as f:
|
|
||||||
original_nb = nbformat.read(f, nbformat.current_nbformat)
|
|
||||||
expected_output = self._flatten_output_text(original_nb)
|
|
||||||
got_nb, errors = self._run_notebook(original_notebook)
|
|
||||||
self.assertFalse(errors)
|
|
||||||
got = self._flatten_output_text(got_nb)
|
|
||||||
self._assert_output(expected_output, got)
|
|
||||||
|
|
||||||
def _assert_output(self, expected_output, got):
|
|
||||||
for expected_line, got_line in zip(expected_output, got):
|
|
||||||
try:
|
|
||||||
expected_line = eval(expected_line)
|
|
||||||
got_line = eval(got_line)
|
|
||||||
except (NameError, SyntaxError, AttributeError):
|
|
||||||
# sanity, both line should be string type
|
|
||||||
self.assertIsInstance(expected_line, str)
|
|
||||||
self.assertIsInstance(got_line, str)
|
|
||||||
# this is for normal text line (NOT json dict)
|
|
||||||
self.assertEqual(expected_line, got_line)
|
|
||||||
else:
|
|
||||||
if isinstance(expected_line, dict) and \
|
|
||||||
isinstance(got_line, dict):
|
|
||||||
expected_and_got = zip(
|
|
||||||
sorted(expected_line.items()),
|
|
||||||
sorted(got_line.items()))
|
|
||||||
for (expected_key, expected_value), (got_key, got_value) \
|
|
||||||
in expected_and_got:
|
|
||||||
self.assertEqual(expected_key, got_key)
|
|
||||||
if expected_key in COULD_BE_CHANGED:
|
|
||||||
# TODO(kota_): make more validation for each format
|
|
||||||
continue
|
|
||||||
else:
|
|
||||||
self.assertEqual(expected_value, got_value)
|
|
||||||
else:
|
|
||||||
self.assertEqual(expected_line, got_line)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
unittest.main()
|
|
||||||
@ -1,219 +0,0 @@
|
|||||||
{
|
|
||||||
"cells": [
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 6,
|
|
||||||
"metadata": {
|
|
||||||
"collapsed": true,
|
|
||||||
"deletable": true,
|
|
||||||
"editable": true
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"%reload_ext storlets.tools.extensions.ipython\n",
|
|
||||||
"import os\n",
|
|
||||||
"os.environ['OS_AUTH_VERSION'] = '3'\n",
|
|
||||||
"os.environ['OS_AUTH_URL'] = 'http://127.0.0.1/identity/v3'\n",
|
|
||||||
"os.environ['OS_USERNAME'] = 'tester'\n",
|
|
||||||
"os.environ['OS_PASSWORD'] = 'testing'\n",
|
|
||||||
"os.environ['OS_USER_DOMAIN_NAME'] = 'default'\n",
|
|
||||||
"os.environ['OS_PROJECT_DOMAIN_NAME'] = 'default'\n",
|
|
||||||
"os.environ['OS_PROJECT_NAME'] = 'test'"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 7,
|
|
||||||
"metadata": {
|
|
||||||
"collapsed": false,
|
|
||||||
"deletable": true,
|
|
||||||
"editable": true
|
|
||||||
},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"Upload storlets succeeded /storlet/test.py\n",
|
|
||||||
"Example command `swift download <container> <object> -H X-Run-Storlet:test.py`\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"%%storletapp test.TestStorlet\n",
|
|
||||||
"\n",
|
|
||||||
"class TestStorlet(object):\n",
|
|
||||||
" def __init__(self, logger):\n",
|
|
||||||
" self.logger = logger\n",
|
|
||||||
"\n",
|
|
||||||
" def __call__(self, in_files, out_files, params):\n",
|
|
||||||
" \"\"\"\n",
|
|
||||||
" The function called for storlet invocation\n",
|
|
||||||
" :param in_files: a list of StorletInputFile\n",
|
|
||||||
" :param out_files: a list of StorletOutputFile\n",
|
|
||||||
" :param params: a dict of request parameters\n",
|
|
||||||
" \"\"\"\n",
|
|
||||||
" self.logger.debug('Returning metadata')\n",
|
|
||||||
" metadata = in_files[0].get_metadata()\n",
|
|
||||||
" for key in params.keys():\n",
|
|
||||||
" metadata[key] = params[key]\n",
|
|
||||||
" out_files[0].set_metadata(metadata)\n",
|
|
||||||
"\n",
|
|
||||||
" self.logger.debug('Start to return object data')\n",
|
|
||||||
" content = b''\n",
|
|
||||||
" while True:\n",
|
|
||||||
" buf = in_files[0].read(16)\n",
|
|
||||||
" if not buf:\n",
|
|
||||||
" break\n",
|
|
||||||
" content += buf\n",
|
|
||||||
" self.logger.debug('Received %d bytes' % len(content))\n",
|
|
||||||
" self.logger.debug('Writing back %d bytes' % len(content))\n",
|
|
||||||
" out_files[0].write(content)\n",
|
|
||||||
" self.logger.debug('Complete')\n",
|
|
||||||
" in_files[0].close()\n",
|
|
||||||
" out_files[0].close()"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 8,
|
|
||||||
"metadata": {
|
|
||||||
"collapsed": false,
|
|
||||||
"deletable": true,
|
|
||||||
"editable": true
|
|
||||||
},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"{u'x-object-meta-storlet-language': u'python', u'x-trans-id': u'tx39de631f32cf42d58f407-0058f89455', u'x-object-meta-storlet-main': u'test.TestStorlet', u'transfer-encoding': u'chunked', u'x-object-meta-storlet-interface-version': u'1.0', u'x-object-meta-storlet-object-metadata': u'no', u'x-object-meta-storlet_execution_path': u'/home/swift/test.TestStorlet', u'last-modified': u'Thu, 20 Apr 2017 10:58:29 GMT', u'etag': u'43106d4a48a1f33745e11bd71596d8c4', u'x-timestamp': u'1492685908.66304', u'x-object-meta-color': u'red', u'date': u'Thu, 20 Apr 2017 10:58:29 GMT', u'x-openstack-request-id': u'tx39de631f32cf42d58f407-0058f89455', u'content-type': u'application/octet-stream', u'accept-ranges': u'bytes'}\n",
|
|
||||||
"\n",
|
|
||||||
"class TestStorlet(object):\n",
|
|
||||||
" def __init__(self, logger):\n",
|
|
||||||
" self.logger = logger\n",
|
|
||||||
"\n",
|
|
||||||
" def __call__(self, in_files, out_files, params):\n",
|
|
||||||
" \"\"\"\n",
|
|
||||||
" The function called for storlet invocation\n",
|
|
||||||
" :param in_files: a list of StorletInputFile\n",
|
|
||||||
" :param out_files: a list of StorletOutputFile\n",
|
|
||||||
" :param params: a dict of request parameters\n",
|
|
||||||
" \"\"\"\n",
|
|
||||||
" self.logger.debug('Returning metadata')\n",
|
|
||||||
" metadata = in_files[0].get_metadata()\n",
|
|
||||||
" for key in params.keys():\n",
|
|
||||||
" metadata[key] = params[key]\n",
|
|
||||||
" out_files[0].set_metadata(metadata)\n",
|
|
||||||
"\n",
|
|
||||||
" self.logger.debug('Start to return object data')\n",
|
|
||||||
" content = b''\n",
|
|
||||||
" while True:\n",
|
|
||||||
" buf = in_files[0].read(16)\n",
|
|
||||||
" if not buf:\n",
|
|
||||||
" break\n",
|
|
||||||
" content += buf\n",
|
|
||||||
" self.logger.debug('Received %d bytes' % len(content))\n",
|
|
||||||
" self.logger.debug('Writing back %d bytes' % len(content))\n",
|
|
||||||
" out_files[0].write(content)\n",
|
|
||||||
" self.logger.debug('Complete')\n",
|
|
||||||
" in_files[0].close()\n",
|
|
||||||
" out_files[0].close()\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"myparams = {'color' : 'red'}\n",
|
|
||||||
"%get --storlet test.py --input path:/storlet/test.py -i myparams -o myresult\n",
|
|
||||||
"print(myresult.headers)\n",
|
|
||||||
"print(myresult.content)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 9,
|
|
||||||
"metadata": {
|
|
||||||
"collapsed": false,
|
|
||||||
"deletable": true,
|
|
||||||
"editable": true
|
|
||||||
},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"{u'content-length': u'0', u'x-storlet-generated-from-last-modified': u'Thu, 20 Apr 2017 10:58:29 GMT', u'x-storlet-generated-from-account': u'AUTH_6dbd182dfa9f4ad6ace88992683ee483', u'last-modified': u'Thu, 20 Apr 2017 10:58:32 GMT', u'etag': u'43106d4a48a1f33745e11bd71596d8c4', u'x-trans-id': u'tx8cc66cd8cda643508bee7-0058f89457', u'date': u'Thu, 20 Apr 2017 10:58:31 GMT', u'content-type': u'text/html; charset=UTF-8', u'x-openstack-request-id': u'tx8cc66cd8cda643508bee7-0058f89457', u'x-storlet-generated-from': u'storlet/test.py'}\n",
|
|
||||||
"201\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"%copy --storlet test.py --input path:/storlet/test.py --output path:/log/test.py -i myparams -o myresult\n",
|
|
||||||
"print(myresult.headers)\n",
|
|
||||||
"print(myresult.status)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 10,
|
|
||||||
"metadata": {
|
|
||||||
"collapsed": false,
|
|
||||||
"deletable": true,
|
|
||||||
"editable": true
|
|
||||||
},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"name": "stdout",
|
|
||||||
"output_type": "stream",
|
|
||||||
"text": [
|
|
||||||
"{u'content-length': u'0', u'last-modified': u'Thu, 20 Apr 2017 10:58:34 GMT', u'etag': u'faccc93089c22cafce9b64e7cc9f2047', u'x-trans-id': u'tx9577116d45ee4e43bc3fd-0058f89458', u'date': u'Thu, 20 Apr 2017 10:58:33 GMT', u'content-type': u'text/html; charset=UTF-8', u'x-openstack-request-id': u'tx9577116d45ee4e43bc3fd-0058f89458'}\n",
|
|
||||||
"201\n"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"import os\n",
|
|
||||||
"try:\n",
|
|
||||||
" os.mkdir('/tmp/tmpZl6teg')\n",
|
|
||||||
"except:\n",
|
|
||||||
" pass\n",
|
|
||||||
"with open('/tmp/tmpZl6teg/storlet_invoke.log', 'w') as f:\n",
|
|
||||||
" for x in range(10):\n",
|
|
||||||
" f.write('INFO: sapmle log line')\n",
|
|
||||||
"%put --storlet test.py --input /tmp/tmpZl6teg/storlet_invoke.log --output path:/log/onvoke.log -i myparams -o myresult\n",
|
|
||||||
"print(myresult.headers)\n",
|
|
||||||
"print(myresult.status)"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"collapsed": true,
|
|
||||||
"deletable": true,
|
|
||||||
"editable": true
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": []
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"kernelspec": {
|
|
||||||
"display_name": "Python",
|
|
||||||
"language": "python",
|
|
||||||
"name": "python"
|
|
||||||
},
|
|
||||||
"language_info": {
|
|
||||||
"codemirror_mode": {
|
|
||||||
"name": "ipython",
|
|
||||||
"version": 2
|
|
||||||
},
|
|
||||||
"file_extension": ".py",
|
|
||||||
"mimetype": "text/x-python",
|
|
||||||
"name": "python",
|
|
||||||
"nbconvert_exporter": "python"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 2
|
|
||||||
}
|
|
||||||
@ -1,502 +0,0 @@
|
|||||||
# Copyright (c) 2010-2016 OpenStack Foundation
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
||||||
# implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
from IPython.core.error import UsageError
|
|
||||||
import itertools
|
|
||||||
from io import FileIO as file
|
|
||||||
from io import StringIO
|
|
||||||
import os
|
|
||||||
import unittest
|
|
||||||
from unittest import mock
|
|
||||||
|
|
||||||
from storlets.tools.extensions.ipython import StorletMagics
|
|
||||||
|
|
||||||
|
|
||||||
class FakeConnection(object):
|
|
||||||
def __init__(self, fake_status=200, fake_headers=None, fake_iter=None):
|
|
||||||
self._fake_status = fake_status
|
|
||||||
self._fake_headers = fake_headers or {}
|
|
||||||
self._fake_iter = fake_iter or iter([])
|
|
||||||
|
|
||||||
def _return_fake_response(self, **kwargs):
|
|
||||||
if 'response_dict' in kwargs:
|
|
||||||
kwargs['response_dict']['status'] = self._fake_status
|
|
||||||
kwargs['response_dict']['headers'] = self._fake_headers
|
|
||||||
kwargs['response_dict']['content_iter'] = self._fake_iter
|
|
||||||
|
|
||||||
if 'resp_chunk_size' in kwargs:
|
|
||||||
resp_body = self._fake_iter
|
|
||||||
else:
|
|
||||||
resp_body = b''.join([chunk for chunk in self._fake_iter])
|
|
||||||
|
|
||||||
return (self._fake_headers, resp_body)
|
|
||||||
|
|
||||||
# Those 3 methods are just for entry point difference from the caller
|
|
||||||
# but all methods returns same response format with updating response_dict
|
|
||||||
def get_object(self, *args, **kwargs):
|
|
||||||
return self._return_fake_response(**kwargs)
|
|
||||||
|
|
||||||
def copy_object(self, *args, **kwargs):
|
|
||||||
return self._return_fake_response(**kwargs)
|
|
||||||
|
|
||||||
def put_object(self, *args, **kwargs):
|
|
||||||
return self._return_fake_response(**kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
class MockShell(object):
|
|
||||||
def __init__(self):
|
|
||||||
self.user_ns = {}
|
|
||||||
|
|
||||||
def register(self, var_name, value):
|
|
||||||
self.user_ns[var_name] = value
|
|
||||||
|
|
||||||
|
|
||||||
class BaseTestIpythonExtension(object):
|
|
||||||
def setUp(self):
|
|
||||||
self.magics = StorletMagics()
|
|
||||||
# set auth info for keystone
|
|
||||||
self.os_original_env = os.environ.copy()
|
|
||||||
self._set_auth_environ()
|
|
||||||
self.magics.shell = MockShell()
|
|
||||||
|
|
||||||
def tearDown(self):
|
|
||||||
os.environ = self.os_original_env.copy()
|
|
||||||
|
|
||||||
def _set_auth_environ(self):
|
|
||||||
# helper method to set auth information for keystone v3 (default)
|
|
||||||
os.environ['OS_AUTH_VERSION'] = '3'
|
|
||||||
os.environ['OS_AUTH_URL'] = 'http://127.0.0.1/identity/v3'
|
|
||||||
os.environ['OS_USERNAME'] = 'tester'
|
|
||||||
os.environ['OS_PASSWORD'] = 'testing'
|
|
||||||
os.environ['OS_USER_DOMAIN_NAME'] = 'default'
|
|
||||||
os.environ['OS_PROJECT_DOMAIN_NAME'] = 'default'
|
|
||||||
os.environ['OS_PROJECT_NAME'] = 'test'
|
|
||||||
|
|
||||||
@mock.patch('storlets.tools.extensions.ipython.Connection')
|
|
||||||
def _call_cell(self, func, line, cell, fake_conn):
|
|
||||||
fake_conn.return_value = self.fake_connection
|
|
||||||
# cell magic
|
|
||||||
func(line, cell)
|
|
||||||
|
|
||||||
@mock.patch('storlets.tools.extensions.ipython.Connection')
|
|
||||||
def _call_line(self, func, line, fake_conn):
|
|
||||||
fake_conn.return_value = self.fake_connection
|
|
||||||
# line magic
|
|
||||||
func(line)
|
|
||||||
|
|
||||||
|
|
||||||
class TestStorletMagicStorletApp(BaseTestIpythonExtension, unittest.TestCase):
|
|
||||||
def setUp(self):
|
|
||||||
super(TestStorletMagicStorletApp, self).setUp()
|
|
||||||
self.fake_connection = FakeConnection()
|
|
||||||
|
|
||||||
def _call_storletapp(self, line, cell):
|
|
||||||
self._call_cell(self.magics.storletapp, line, cell)
|
|
||||||
|
|
||||||
def test_storlet_magics(self):
|
|
||||||
line = 'test.TestStorlet'
|
|
||||||
cell = ''
|
|
||||||
self._call_storletapp(line, cell)
|
|
||||||
|
|
||||||
def test_storlet_magics_usage_error(self):
|
|
||||||
# no line result in UsageError (from ipython default behavior)
|
|
||||||
line = ''
|
|
||||||
cell = ''
|
|
||||||
self.assertRaises(
|
|
||||||
UsageError, self._call_storletapp, line, cell)
|
|
||||||
|
|
||||||
def test_storlet_magics_with_invoke(self):
|
|
||||||
line = 'test.TestStorlet --with-invoke --input path:/foo/bar'
|
|
||||||
cell = ''
|
|
||||||
self._call_storletapp(line, cell)
|
|
||||||
|
|
||||||
def test_storlet_magics_with_invoke_no_input_fail(self):
|
|
||||||
line = 'test.TestStorlet --with-invoke'
|
|
||||||
cell = ''
|
|
||||||
with self.assertRaises(UsageError) as cm:
|
|
||||||
self._call_storletapp(line, cell)
|
|
||||||
self.assertEqual(
|
|
||||||
'--with-invoke option requires --input to run the app',
|
|
||||||
cm.exception.args[0])
|
|
||||||
|
|
||||||
def test_storlet_magics_invalid_input_fail(self):
|
|
||||||
invalid_input_patterns = (
|
|
||||||
'invalid', # no "path:" prefix
|
|
||||||
'path://', # no container object in the slash
|
|
||||||
'path:/container', # only container
|
|
||||||
'path:container', # only container w/o slash prefix
|
|
||||||
)
|
|
||||||
|
|
||||||
for invalid_input in invalid_input_patterns:
|
|
||||||
with self.assertRaises(UsageError) as cm:
|
|
||||||
self.magics._parse_input_path(invalid_input)
|
|
||||||
self.assertEqual(
|
|
||||||
'swift object path must have the format: '
|
|
||||||
'"path:/<container>/<object>"',
|
|
||||||
cm.exception.args[0])
|
|
||||||
|
|
||||||
def test_storlet_magics_stdout(self):
|
|
||||||
line = 'test.TestStorlet --with-invoke --input path:/foo/bar' \
|
|
||||||
'--print-result'
|
|
||||||
cell = ''
|
|
||||||
fake_stdout = StringIO()
|
|
||||||
with mock.patch('sys.stdout', fake_stdout):
|
|
||||||
self._call_storletapp(line, cell)
|
|
||||||
stdout_string = fake_stdout.getvalue()
|
|
||||||
expected_outputs = (
|
|
||||||
'Upload storlets succeeded',
|
|
||||||
'Example command `swift download <container> <object> '
|
|
||||||
'-H X-Run-Storlet:',
|
|
||||||
'Invocation Complete',
|
|
||||||
)
|
|
||||||
|
|
||||||
for expected in expected_outputs:
|
|
||||||
self.assertIn(expected, stdout_string)
|
|
||||||
|
|
||||||
def test_storlet_auth_v3_no_enough_auth_info(self):
|
|
||||||
line = 'test.TestStorlet'
|
|
||||||
cell = ''
|
|
||||||
|
|
||||||
# OS_AUTH_URL, OS_USERNAME, OS_PASSWORD, OS_PROJECT_NAME are required
|
|
||||||
required = ('OS_AUTH_URL', 'OS_USERNAME',
|
|
||||||
'OS_PASSWORD', 'OS_PROJECT_NAME')
|
|
||||||
|
|
||||||
for combi_length in range(1, len(required) + 1):
|
|
||||||
for combination in itertools.combinations(required, combi_length):
|
|
||||||
# set full environ for auth
|
|
||||||
self._set_auth_environ()
|
|
||||||
# and delete specific required keys
|
|
||||||
for key in combination:
|
|
||||||
del os.environ[key]
|
|
||||||
|
|
||||||
with self.assertRaises(UsageError) as e:
|
|
||||||
self._call_storletapp(line, cell)
|
|
||||||
|
|
||||||
self.assertEqual(
|
|
||||||
"You need to set OS_AUTH_URL, OS_USERNAME, OS_PASSWORD "
|
|
||||||
"and OS_PROJECT_NAME for Swift authentication",
|
|
||||||
e.exception.args[0])
|
|
||||||
|
|
||||||
def test_storlet_auth_v2_not_supported(self):
|
|
||||||
line = 'test.TestStorlet'
|
|
||||||
cell = ''
|
|
||||||
os.environ['OS_AUTH_VERSION'] = '2'
|
|
||||||
with self.assertRaises(NotImplementedError) as e:
|
|
||||||
self._call_storletapp(line, cell)
|
|
||||||
self.assertEqual(
|
|
||||||
'keystone v2 is not supported',
|
|
||||||
e.exception.args[0])
|
|
||||||
|
|
||||||
def test_storlet_auth_v1_no_enough_auth_info(self):
|
|
||||||
# NOTE: Now, storlets should work on keystone v3 so that this test
|
|
||||||
# may be deprecated in the future if we don't have to support pure
|
|
||||||
# swift upstream v1 auth (e.g. tempauth).
|
|
||||||
line = 'test.TestStorlet'
|
|
||||||
cell = ''
|
|
||||||
|
|
||||||
# ST_AUTH, ST_USER, ST_KEY are required
|
|
||||||
required = ('ST_AUTH', 'ST_USER', 'ST_KEY')
|
|
||||||
|
|
||||||
# v1 doesn't require OS_AUTH_VERSION
|
|
||||||
del os.environ['OS_AUTH_VERSION']
|
|
||||||
try:
|
|
||||||
del os.environ['OS_IDENTITY_API_VERSION']
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def _set_v1_auth():
|
|
||||||
os.environ['ST_AUTH'] = 'http://localhost/v1/auth'
|
|
||||||
os.environ['ST_USER'] = 'test:tester'
|
|
||||||
os.environ['ST_KEY'] = 'testing'
|
|
||||||
|
|
||||||
for combi_length in range(1, len(required) + 1):
|
|
||||||
for combination in itertools.combinations(required, combi_length):
|
|
||||||
# set full environ for auth
|
|
||||||
_set_v1_auth()
|
|
||||||
# and delete specific required keys
|
|
||||||
for key in combination:
|
|
||||||
del os.environ[key]
|
|
||||||
|
|
||||||
with self.assertRaises(UsageError) as e:
|
|
||||||
self._call_storletapp(line, cell)
|
|
||||||
|
|
||||||
self.assertEqual(
|
|
||||||
"You need to set ST_AUTH, ST_USER, ST_KEY "
|
|
||||||
"for Swift authentication",
|
|
||||||
e.exception.args[0])
|
|
||||||
|
|
||||||
|
|
||||||
class TestStorletMagicGet(BaseTestIpythonExtension, unittest.TestCase):
|
|
||||||
def setUp(self):
|
|
||||||
super(TestStorletMagicGet, self).setUp()
|
|
||||||
self.fake_connection = FakeConnection()
|
|
||||||
|
|
||||||
def _call_get(self, line):
|
|
||||||
self._call_line(self.magics.get, line)
|
|
||||||
|
|
||||||
def test_get_invalid_args(self):
|
|
||||||
scenarios = [
|
|
||||||
{
|
|
||||||
'line': '--input path:/c/o --storlet a.b',
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': '-o option is mandatory for the invocation'
|
|
||||||
}, {
|
|
||||||
'line': '--input path:/c/o -o a1234',
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': '--storlet option is mandatory for the invocation'
|
|
||||||
}, {
|
|
||||||
'line': '--storlet a.b -o a1234',
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': '--input option is mandatory for the invocation'
|
|
||||||
}, {
|
|
||||||
'line': '--input path/c/o --storlet a.b -o a1234',
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': ('swift object path must have the format: '
|
|
||||||
'"path:/<container>/<object>"')
|
|
||||||
}, {
|
|
||||||
'line': '--input path:/c/ --storlet a.b -o a1234',
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': ('swift object path must have the format: '
|
|
||||||
'"path:/<container>/<object>"')
|
|
||||||
}, {
|
|
||||||
'line': '--input path:/c --storlet a.b -o a1234',
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': ('swift object path must have the format: '
|
|
||||||
'"path:/<container>/<object>"')
|
|
||||||
}, {
|
|
||||||
'line': '--input path:/c/o --storlet a.b -o 1234',
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': ('The output variable name must be a valid prefix '
|
|
||||||
'of a python variable, that is, start with a '
|
|
||||||
'letter')
|
|
||||||
}]
|
|
||||||
|
|
||||||
for scenario in scenarios:
|
|
||||||
with self.assertRaises(UsageError) as e:
|
|
||||||
self._call_get(scenario['line'])
|
|
||||||
self.assertEqual(scenario['msg'], e.exception.args[0])
|
|
||||||
|
|
||||||
def _test_get(self, line, outvar_name):
|
|
||||||
self._call_get(line)
|
|
||||||
self.assertTrue(outvar_name in self.magics.shell.user_ns)
|
|
||||||
resp = self.magics.shell.user_ns[outvar_name]
|
|
||||||
self.assertEqual({}, resp.headers)
|
|
||||||
self.assertEqual(200, resp.status)
|
|
||||||
self.assertEqual('', ''.join([chunk for chunk in iter(resp)]))
|
|
||||||
self.assertEqual(b'', resp.content)
|
|
||||||
|
|
||||||
def test_get(self):
|
|
||||||
outvar_name = 'a1234'
|
|
||||||
line = '--input path:/c/o --storlet a.b -o %s' % outvar_name
|
|
||||||
self._test_get(line, outvar_name)
|
|
||||||
|
|
||||||
def test_get_with_input(self):
|
|
||||||
params_name = 'params'
|
|
||||||
outvar_name = 'a1234'
|
|
||||||
line = '--input path:/c/o --storlet a.b -o %s -i %s' % (outvar_name,
|
|
||||||
params_name)
|
|
||||||
# register the variable to user_ns
|
|
||||||
self.magics.shell.register(params_name, {'a': 'b'})
|
|
||||||
self._test_get(line, outvar_name)
|
|
||||||
|
|
||||||
def test_get_with_input_error(self):
|
|
||||||
params_name = 'params'
|
|
||||||
outvar_name = 'a1234'
|
|
||||||
line = '--input path:/c/o --storlet a.b -o %s -i %s' % (outvar_name,
|
|
||||||
params_name)
|
|
||||||
with self.assertRaises(KeyError):
|
|
||||||
self._test_get(line, outvar_name)
|
|
||||||
|
|
||||||
|
|
||||||
class TestStorletMagicCopy(BaseTestIpythonExtension, unittest.TestCase):
|
|
||||||
def setUp(self):
|
|
||||||
super(TestStorletMagicCopy, self).setUp()
|
|
||||||
self.fake_connection = FakeConnection()
|
|
||||||
|
|
||||||
def _call_copy(self, line):
|
|
||||||
self._call_line(self.magics.copy, line)
|
|
||||||
|
|
||||||
def test_copy_invalid_args(self):
|
|
||||||
scenarios = [
|
|
||||||
{
|
|
||||||
'line': '--input path:/c/o --storlet a.b --output path:/c/o',
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': '-o option is mandatory for the invocation'
|
|
||||||
}, {
|
|
||||||
'line': ('--input path:/c/o --storlet a.b -o 1234 '
|
|
||||||
'--output path:/c/o'),
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': ('The output variable name must be a valid prefix '
|
|
||||||
'of a python variable, that is, start with a '
|
|
||||||
'letter')
|
|
||||||
}, {
|
|
||||||
'line': '--input path:/c/o -o a1234 --output path:/c/o',
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': '--storlet option is mandatory for the invocation'
|
|
||||||
}, {
|
|
||||||
'line': '--storlet a.b -o a1234 --output path:/c/o',
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': '--input option is mandatory for the invocation'
|
|
||||||
}, {
|
|
||||||
'line': ('--input path/c/o --storlet a.b -o a1234 '
|
|
||||||
'--output path:/c/o'),
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': ('swift object path must have the format: '
|
|
||||||
'"path:/<container>/<object>"')
|
|
||||||
}, {
|
|
||||||
'line': ('--input path:/c/ --storlet a.b -o a1234 '
|
|
||||||
'--output path:/c/o'),
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': ('swift object path must have the format: '
|
|
||||||
'"path:/<container>/<object>"')
|
|
||||||
}, {
|
|
||||||
'line': ('--input path:/c --storlet a.b -o a1234 '
|
|
||||||
'--output path:/c/o'),
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': ('swift object path must have the format: '
|
|
||||||
'"path:/<container>/<object>"')
|
|
||||||
}, {
|
|
||||||
'line': '--input path:/c --storlet a.b -o a1234 ',
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': ('--output option is mandatory for the invocation')
|
|
||||||
}, {
|
|
||||||
'line': ('--input path:/c --storlet a.b -o a1234 '
|
|
||||||
'--output path/c/o'),
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': ('swift object path must have the format: '
|
|
||||||
'"path:/<container>/<object>"')
|
|
||||||
}]
|
|
||||||
|
|
||||||
for scenario in scenarios:
|
|
||||||
with self.assertRaises(UsageError) as e:
|
|
||||||
self._call_copy(scenario['line'])
|
|
||||||
self.assertEqual(scenario['msg'], e.exception.args[0])
|
|
||||||
|
|
||||||
def _test_copy(self, line, outvar_name):
|
|
||||||
self._call_copy(line)
|
|
||||||
self.assertTrue(outvar_name in self.magics.shell.user_ns)
|
|
||||||
resp = self.magics.shell.user_ns[outvar_name]
|
|
||||||
self.assertEqual({}, resp.headers)
|
|
||||||
self.assertEqual(200, resp.status)
|
|
||||||
# sanity, no body
|
|
||||||
self.assertEqual(b'', resp.content)
|
|
||||||
|
|
||||||
def test_copy(self):
|
|
||||||
outvar_name = 'a1234'
|
|
||||||
line = ('--input path:/c/o --output path:/c/o '
|
|
||||||
'--storlet a.b -o %s' % outvar_name)
|
|
||||||
self._test_copy(line, outvar_name)
|
|
||||||
|
|
||||||
def test_copy_stdout_with_input(self):
|
|
||||||
params_name = 'params'
|
|
||||||
outvar_name = 'a1234'
|
|
||||||
line = ('--input path:/c/o --output path:/c/o '
|
|
||||||
'--storlet a.b -o %s -i %s' % (outvar_name, params_name))
|
|
||||||
self.magics.shell.register(params_name, {'a': 'b'})
|
|
||||||
self._test_copy(line, outvar_name)
|
|
||||||
|
|
||||||
def test_copy_stdout_with_input_error(self):
|
|
||||||
params_name = 'params'
|
|
||||||
outvar_name = 'a1234'
|
|
||||||
line = ('--input path:/c/o --output path:/c/o '
|
|
||||||
'--storlet a.b -o %s -i %s' % (outvar_name, params_name))
|
|
||||||
with self.assertRaises(KeyError):
|
|
||||||
self._test_copy(line, outvar_name)
|
|
||||||
|
|
||||||
|
|
||||||
class TestStorletMagicPut(BaseTestIpythonExtension, unittest.TestCase):
|
|
||||||
def setUp(self):
|
|
||||||
super(TestStorletMagicPut, self).setUp()
|
|
||||||
self.fake_connection = FakeConnection(201)
|
|
||||||
|
|
||||||
def _call_put(self, line):
|
|
||||||
self._call_line(self.magics.put, line)
|
|
||||||
|
|
||||||
def test_put_invalid_args(self):
|
|
||||||
scenarios = [
|
|
||||||
{
|
|
||||||
'line': '--input /c/o --storlet a.b --output path:/c/o',
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': '-o option is mandatory for the invocation'
|
|
||||||
}, {
|
|
||||||
'line': ('--input /c/o --storlet a.b -o 1234 '
|
|
||||||
'--output path:/c/o'),
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': ('The output variable name must be a valid prefix '
|
|
||||||
'of a python variable, that is, start with a '
|
|
||||||
'letter')
|
|
||||||
}, {
|
|
||||||
'line': '--input /c/o -o a1234 --output path:/c/o',
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': '--storlet option is mandatory for the invocation'
|
|
||||||
}, {
|
|
||||||
'line': '--storlet a.b -o a1234 --output path:/c/o',
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': '--input option is mandatory for the invocation'
|
|
||||||
}, {
|
|
||||||
'line': ('--input /c/o --storlet a.b -o a1234 '
|
|
||||||
'--output path/c/o'),
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': ('swift object path must have the format: '
|
|
||||||
'"path:/<container>/<object>"')
|
|
||||||
}, {
|
|
||||||
'line': ('--input path:c/ --storlet a.b -o a1234 '
|
|
||||||
'--output path:/c/o'),
|
|
||||||
'exception': UsageError,
|
|
||||||
'msg': ('--input argument must be a full path')
|
|
||||||
}]
|
|
||||||
|
|
||||||
for scenario in scenarios:
|
|
||||||
with self.assertRaises(UsageError) as e:
|
|
||||||
self._call_put(scenario['line'])
|
|
||||||
self.assertEqual(scenario['msg'], e.exception.args[0])
|
|
||||||
|
|
||||||
def _test_put(self, line, outvar_name):
|
|
||||||
open_name = '%s.open' % 'storlets.tools.extensions.ipython'
|
|
||||||
with mock.patch(open_name, create=True) as mock_open:
|
|
||||||
mock_open.return_value = mock.MagicMock(spec=file)
|
|
||||||
self._call_put(line)
|
|
||||||
self.assertTrue(outvar_name in self.magics.shell.user_ns)
|
|
||||||
resp = self.magics.shell.user_ns[outvar_name]
|
|
||||||
self.assertEqual({}, resp.headers)
|
|
||||||
self.assertEqual(201, resp.status)
|
|
||||||
# sanity, no body
|
|
||||||
self.assertEqual(b'', resp.content)
|
|
||||||
|
|
||||||
def test_put(self):
|
|
||||||
outvar_name = 'a1234'
|
|
||||||
line = ('--input /c/o --storlet a.b '
|
|
||||||
'--output path:a/b -o %s' % outvar_name)
|
|
||||||
self._test_put(line, outvar_name)
|
|
||||||
|
|
||||||
def test_put_stdout_with_input(self):
|
|
||||||
params_name = 'params'
|
|
||||||
outvar_name = 'a1234'
|
|
||||||
line = ('--input /c/o --storlet a.b -o %s -i %s '
|
|
||||||
'--output path:a/b' % (outvar_name, params_name))
|
|
||||||
self.magics.shell.register(params_name, {'a': 'b'})
|
|
||||||
self._test_put(line, outvar_name)
|
|
||||||
|
|
||||||
def test_put_stdout_with_input_error(self):
|
|
||||||
params_name = 'params'
|
|
||||||
outvar_name = 'a1234'
|
|
||||||
line = ('--input /c/o --storlet a.b -o %s -i %s '
|
|
||||||
'--output path:a/b' % (outvar_name, params_name))
|
|
||||||
with self.assertRaises(KeyError):
|
|
||||||
self._test_put(line, outvar_name)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
unittest.main()
|
|
||||||
Loading…
Reference in New Issue
Block a user