From f5815f065a8bfb91fe21dbd1e11a0ea3592673bd Mon Sep 17 00:00:00 2001 From: Takashi Kajinami Date: Wed, 18 Dec 2019 23:30:12 +0900 Subject: [PATCH] Use the same python version to run storlet agents This patch refactors package installations in storlets devstack plugin, so that we can run storlets agents with the same python version as the one we have in host to run storlets middleware. This patch also bumps up openjdk version from 8 to 11, and python3 version from 3.5 to 3.6, so that we use the latest versin provided in distro. Change-Id: I6cb87976ff4fa6caaa58cf555685f24d95f214a8 --- devstack/plugin.sh | 110 +++++++++--------- doc/source/engine_dev_installation.rst | 33 +++--- install_libs.sh | 11 +- playbooks/storlets-functional/pre.yaml | 2 + s2aio.sh | 1 + setup.cfg | 9 +- src/java/SBus/build.xml | 15 +-- .../storlet/sbus/ServerSBusInDatagram.java | 6 +- storlets/agent/common/server.py | 4 +- storlets/agent/common/utils.py | 3 +- storlets/agent/daemon/server.py | 30 ++--- storlets/agent/daemon_factory/server.py | 32 +++-- storlets/gateway/gateways/docker/runtime.py | 3 +- storlets/sbus/client/cli.py | 21 ++-- storlets/sbus/sbus.py | 3 +- storlets/swift_middleware/handlers/proxy.py | 4 +- .../functional/java/test_thumbnail_storlet.py | 6 +- 17 files changed, 150 insertions(+), 143 deletions(-) diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 97846a3e..ca77a014 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -75,25 +75,25 @@ TMP_REGISTRY_PREFIX=/tmp/registry # Functions # --------- -_storlets_swift_start() { +function _storlets_swift_start { swift-init --run-dir=${SWIFT_DATA_DIR}/run all start || true } -_storlets_swift_stop() { +function _storlets_swift_stop { swift-init --run-dir=${SWIFT_DATA_DIR}/run all stop || true } -_storlets_swift_restart() { +function _storlets_swift_restart { swift-init --run-dir=${SWIFT_DATA_DIR}/run all restart || true } -_export_os_vars() { +function _export_os_vars { export OS_IDENTITY_API_VERSION=3 export OS_AUTH_URL="http://$KEYSTONE_IP/identity/v3" export OS_REGION_NAME=RegionOne } -_export_keystone_os_vars() { +function _export_keystone_os_vars { _export_os_vars export OS_USERNAME=$ADMIN_USER export OS_USER_DOMAIN_ID=$STORLETS_DEFAULT_USER_DOMAIN_ID @@ -102,7 +102,7 @@ _export_keystone_os_vars() { export OS_PROJECT_DOMAIN_ID=$STORLETS_DEFAULT_PROJECT_DOMAIN_ID } -_export_swift_os_vars() { +function _export_swift_os_vars { _export_os_vars export OS_USERNAME=$SWIFT_DEFAULT_USER export OS_USER_DOMAIN_ID=$STORLETS_DEFAULT_USER_DOMAIN_ID @@ -111,7 +111,7 @@ _export_swift_os_vars() { export OS_PROJECT_DOMAIN_ID=$STORLETS_DEFAULT_PROJECT_DOMAIN_ID } -configure_swift_and_keystone_for_storlets() { +function configure_swift_and_keystone_for_storlets { # Add project and users to Keystone _export_keystone_os_vars project_test_created=$(openstack project list | grep -w $SWIFT_DEFAULT_PROJECT | wc -l) @@ -149,7 +149,7 @@ configure_swift_and_keystone_for_storlets() { swift post $STORLETS_LOG_CONTAIER_NAME } -_install_docker() { +function _install_docker { # TODO: Add other dirstors. # This one is geared towards Ubuntu # See other projects that install docker @@ -190,47 +190,50 @@ _install_docker() { fi } -prepare_storlets_install() { +function prepare_storlets_install { sudo mkdir -p "$STORLETS_DOCKER_DEVICE"/docker sudo chmod 777 $STORLETS_DOCKER_DEVICE _install_docker - sudo add-apt-repository -y ppa:openjdk-r/ppa - sudo apt-get update - sudo apt-get install -y openjdk-8-jdk-headless - sudo apt-get install -y ant - sudo apt-get install -y python - sudo apt-get install -y python-setuptools - sudo apt-get install -y python3.5 - sudo apt-get install -y python3-setuptools + + if is_ubuntu; then + install_package openjdk-11-jdk-headless ant + else + die $LINENO "Unsupported distro" + fi + + if python3_enabled; then + install_python3 + fi } -_generate_jre_dockerfile() { - cat < ${TMP_REGISTRY_PREFIX}/repositories/${STORLETS_DOCKER_BASE_IMG_NAME}_jre8/Dockerfile +function _generate_jre_dockerfile { + local PYTHON_PACKAGES='python2.7 python-dev python3.6 python3.6-dev' + if python3_enabled; then + PYTHON_PACKAGES="python2.7 python${PYTHON3_VERSION} python${PYTHON3_VERSION}-dev" + fi + + cat < ${TMP_REGISTRY_PREFIX}/repositories/${STORLETS_DOCKER_BASE_IMG_NAME}_jre11/Dockerfile FROM $STORLETS_DOCKER_BASE_IMG MAINTAINER root RUN apt-get update && \ - apt-get install python -y && \ - apt-get install python3.5 -y && \ - apt-get install git -y && \ - apt-get update && \ - apt-get install openjdk-8-jre-headless -y && \ + apt-get install ${PYTHON_PACKAGES} openjdk-11-jre-headless -y && \ apt-get clean EOF } -create_base_jre_image() { +function create_base_jre_image { echo "Create base jre image" docker pull $STORLETS_DOCKER_BASE_IMG - mkdir -p ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre8 + mkdir -p ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11 _generate_jre_dockerfile - cd ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre8 - docker build -q -t ${STORLETS_DOCKER_BASE_IMG_NAME}_jre8 . + cd ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11 + docker build -q -t ${STORLETS_DOCKER_BASE_IMG_NAME}_jre11 . cd - } -_generate_logback_xml() { - cat < ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre8_storlets/logback.xml +function _generate_logback_xml { + cat < ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11_storlets/logback.xml /tmp/SDaemon.log @@ -257,9 +260,9 @@ _generate_logback_xml() { EOF } -_generate_jre_storlet_dockerfile() { - cat < ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre8_storlets/Dockerfile -FROM ${STORLETS_DOCKER_BASE_IMG_NAME}_jre8 +function _generate_jre_storlet_dockerfile { + cat < ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11_storlets/Dockerfile +FROM ${STORLETS_DOCKER_BASE_IMG_NAME}_jre11 MAINTAINER root RUN [ "groupadd", "-g", "$STORLETS_DOCKER_SWIFT_GROUP_ID", "swift" ] RUN [ "useradd", "-u" , "$STORLETS_DOCKER_SWIFT_USER_ID", "-g", "$STORLETS_DOCKER_SWIFT_GROUP_ID", "swift" ] @@ -275,25 +278,29 @@ ENTRYPOINT ["/usr/local/libexec/storlets/init_container.sh"] EOF } -create_storlet_engine_image() { +function create_storlet_engine_image { echo "Create Storlet engine image" - mkdir -p ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre8_storlets + mkdir -p ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11_storlets _generate_logback_xml _generate_jre_storlet_dockerfile - cd ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre8_storlets - docker build -q -t ${STORLETS_DOCKER_BASE_IMG_NAME}_jre8_storlets . + cd ${TMP_REGISTRY_PREFIX}/repositories/"$STORLETS_DOCKER_BASE_IMG_NAME"_jre11_storlets + docker build -q -t ${STORLETS_DOCKER_BASE_IMG_NAME}_jre11_storlets . cd - } -install_storlets_code() { +function install_storlets_code { echo "Installing storlets" cd $REPO_DIR sudo ./install_libs.sh - sudo pip install -r requirements.txt - sudo python setup.py install - sudo pip3 install -r requirements.txt - sudo python3 setup.py install - sudo chown -R ${STORLETS_SWIFT_RUNTIME_USER} storlets.egg-info* + pip_install . + + # Also install code to library directory so that we can import them + # from docker container. + sudo mkdir -p -m 755 /usr/local/lib/storlets/python + pip_install . -t /usr/local/lib/storlets/python --no-compile + for bin_file in storlets-daemon storlets-daemon-factory ; do + sudo cp `which ${bin_file}` /usr/local/libexec/storlets/ + done sudo mkdir -p $STORLETS_DOCKER_DEVICE/scripts sudo chown "$STORLETS_SWIFT_RUNTIME_USER":"$STORLETS_SWIFT_RUNTIME_GROUP" "$STORLETS_DOCKER_DEVICE"/scripts @@ -305,7 +312,7 @@ install_storlets_code() { cd - } -_generate_swift_middleware_conf() { +function _generate_swift_middleware_conf { cat < /tmp/swift_middleware_conf [proxy-confs] proxy_server_conf_file = /etc/swift/proxy-server.conf @@ -327,7 +334,7 @@ storlet_proxy_execution = $STORLETS_PROXY_EXECUTION_ONLY EOF } -_generate_storlet-docker-gateway() { +function _generate_storlet-docker-gateway { cat < /tmp/storlet-docker-gateway.conf [DEFAULT] storlet_logcontainer = $STORLETS_LOG_CONTAIER_NAME @@ -341,14 +348,14 @@ storlet_timeout = $STORLETS_RUNTIME_TIMEOUT EOF } -_generate_default_tenant_dockerfile() { +function _generate_default_tenant_dockerfile { cat < ${TMP_REGISTRY_PREFIX}/repositories/"$SWIFT_DEFAULT_PROJECT_ID"/Dockerfile -FROM ${STORLETS_DOCKER_BASE_IMG_NAME}_jre8_storlets +FROM ${STORLETS_DOCKER_BASE_IMG_NAME}_jre11_storlets MAINTAINER root EOF } -create_default_tenant_image() { +function create_default_tenant_image { SWIFT_DEFAULT_PROJECT_ID=`openstack project list | grep -w $SWIFT_DEFAULT_PROJECT | awk '{ print $2 }'` mkdir -p ${TMP_REGISTRY_PREFIX}/repositories/$SWIFT_DEFAULT_PROJECT_ID _generate_default_tenant_dockerfile @@ -357,7 +364,7 @@ create_default_tenant_image() { cd - } -create_test_config_file() { +function create_test_config_file { testfile=${REPO_DIR}/test.conf iniset ${testfile} general keystone_default_domain $STORLETS_DEFAULT_PROJECT_DOMAIN_ID iniset ${testfile} general keystone_public_url $KEYSTONE_PUBLIC_URL @@ -369,8 +376,7 @@ create_test_config_file() { iniset ${testfile} general region } - -install_storlets() { +function install_storlets { echo "Install storlets dependencies" prepare_storlets_install @@ -392,7 +398,7 @@ install_storlets() { _storlets_swift_restart } -uninstall_storlets() { +function uninstall_storlets { sudo service docker stop sudo sed -r 's#^.*DOCKER_OPTS=.*$#DOCKER_OPTS="--debug --storage-opt dm.override_udev_sync_check=true"#' /etc/default/docker diff --git a/doc/source/engine_dev_installation.rst b/doc/source/engine_dev_installation.rst index 29a81f29..3df7288e 100644 --- a/doc/source/engine_dev_installation.rst +++ b/doc/source/engine_dev_installation.rst @@ -119,10 +119,7 @@ We need the following for the Java parts :: - sudo add-apt-repository ppa:openjdk-r/ppa - sudo apt-get update - sudo apt-get install openjdk-8-jdk - sudo apt-get install ant + sudo apt-get install openjdk-11-jdk ant We need the following for Docker @@ -160,9 +157,9 @@ Step 2: Create a Docker image with Java :: - mkdir -p $HOME/docker_repos/ubuntu_18.04_jre8 + mkdir -p $HOME/docker_repos/ubuntu_18.04_jre11 -Create the file: $HOME/docker_repos/ubuntu_18.04_jre8/Dockerfile +Create the file: $HOME/docker_repos/ubuntu_18.04_jre11/Dockerfile with the following content: :: @@ -175,17 +172,15 @@ with the following content: RUN apt-get update && \ apt-get install python && \ apt-get install software-properties-common && \ - add-apt-repository ppa:openjdk-r/ppa && \ - apt-get update && \ - apt-get install openjdk-8-jre && \ + apt-get install openjdk-11-jre-headless ant && \ apt-get clean Build the image :: - cd $HOME/docker_repos/ubuntu_18.04_jre8 - sudo docker build -q -t ubuntu_18.04_jre8 . + cd $HOME/docker_repos/ubuntu_18.04_jre11 + sudo docker build -q -t ubuntu_18.04_jre11 . cd - @@ -193,16 +188,16 @@ Step 3: Augment the above created image with the storlets stuff :: - mkdir -p $HOME/docker_repos/ubuntu_18.04_jre8_storlets + mkdir -p $HOME/docker_repos/ubuntu_18.04_jre11_storlets cp $HOME/storlets/install/storlets/roles/docker_storlet_engine_image/files/logback.xml . cd - -Create the file: $HOME/docker_repos/ubuntu_18.04_jre8_storlets/Dockerfile +Create the file: $HOME/docker_repos/ubuntu_18.04_jre11_storlets/Dockerfile with the following content: :: - FROM ubuntu_18.04_jre8 + FROM ubuntu_18.04_jre11 MAINTAINER root @@ -222,8 +217,8 @@ Build the image :: - cd $HOME/docker_repos/ubuntu_18.04_jre8_storlets - sudo docker build -q -t ubuntu_18.04_jre8_storlets . + cd $HOME/docker_repos/ubuntu_18.04_jre11_storlets + sudo docker build -q -t ubuntu_18.04_jre11_storlets . cd - Step 4: Create a tenant specific image. The engine looks for images @@ -242,12 +237,12 @@ The response from the above contains the account line, e.g.: The account id is the number following the 'AUTH\_' prefix. -Next create the file $HOME/docker_repos/ubuntu_18.04_jre8_storlets_/Dockerfile +Next create the file $HOME/docker_repos/ubuntu_18.04_jre11_storlets_/Dockerfile with the following content: :: - FROM ubuntu_18.04_jre8_storlets + FROM ubuntu_18.04_jre11_storlets MAINTAINER root @@ -255,7 +250,7 @@ Build the image :: - cd $HOME/docker_repos/ubuntu_18.04_jre8_storlets_ + cd $HOME/docker_repos/ubuntu_18.04_jre11_storlets_ sudo docker build -q -t . cd - diff --git a/install_libs.sh b/install_libs.sh index bd13a3e2..685053d3 100755 --- a/install_libs.sh +++ b/install_libs.sh @@ -23,14 +23,5 @@ ant install # Install container side scripts SRC=bin DST=/usr/local/libexec/storlets -mkdir -p $DST -chmod 755 $DST +mkdir -p -m 755 $DST cp "$SRC/init_container.sh" $DST -cp "$SRC/storlets-daemon" $DST -cp "$SRC/storlets-daemon-factory" $DST - -# Install python library to be mouted by containers -DST=/usr/local/lib/storlets/python -mkdir -p $DST -chmod 755 $DST -pip install . -t $DST --no-compile diff --git a/playbooks/storlets-functional/pre.yaml b/playbooks/storlets-functional/pre.yaml index 53efb61c..8e806271 100644 --- a/playbooks/storlets-functional/pre.yaml +++ b/playbooks/storlets-functional/pre.yaml @@ -8,3 +8,5 @@ ./s2aio.sh install executable: /bin/bash chdir: 'src/{{ zuul.project.canonical_name }}' + environment: + USE_PYTHON3: "False" diff --git a/s2aio.sh b/s2aio.sh index 28773e54..45f7054d 100755 --- a/s2aio.sh +++ b/s2aio.sh @@ -37,6 +37,7 @@ _prepare_devstack_env() { cp devstack/localrc.sample $DEVSTACK_DIR/localrc fi + source $DEVSTACK_DIR/stackrc source $DEVSTACK_DIR/functions source $DEVSTACK_DIR/functions-common source $DEVSTACK_DIR/lib/swift diff --git a/setup.cfg b/setup.cfg index f4f2f432..bd99a115 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,10 +26,6 @@ skip_changelog = True [files] packages = storlets -scripts = - bin/storlets-daemon - bin/storlets-daemon-factory - bin/sbus [entry_points] paste.filter_factory = @@ -39,6 +35,11 @@ storlets.gateways = stub = storlets.gateway.gateways.stub:StorletGatewayStub docker = storlets.gateway.gateways.docker:StorletGatewayDocker +console_scripts = + sbus = storlets.sbus.cli:main + storlets-daemon = storlets.agent.daemon.server:main + storlets-daemon-factory = storlets.agent.daemon_factory.server:main + [build_sphinx] source-dir = doc/source build-dir = doc/build diff --git a/src/java/SBus/build.xml b/src/java/SBus/build.xml index 2e18c251..77b1bba5 100644 --- a/src/java/SBus/build.xml +++ b/src/java/SBus/build.xml @@ -28,20 +28,15 @@ srcdir="src/main" destdir="bin" classpath="../dependencies/json_simple-1.1.jar" - includeantruntime="false"/> + includeantruntime="false" + nativeheaderdir='.'/> - - - - - - - + - - + + diff --git a/src/java/SBus/src/main/org/openstack/storlet/sbus/ServerSBusInDatagram.java b/src/java/SBus/src/main/org/openstack/storlet/sbus/ServerSBusInDatagram.java index 4abd79ec..f8932030 100644 --- a/src/java/SBus/src/main/org/openstack/storlet/sbus/ServerSBusInDatagram.java +++ b/src/java/SBus/src/main/org/openstack/storlet/sbus/ServerSBusInDatagram.java @@ -114,15 +114,15 @@ public class ServerSBusInDatagram { } String strMD = msg.getMetadata(); - this.metadata = (HashMap>[])new HashMap[getNFiles()]; + this.metadata = (HashMap>[])new HashMap[getNFiles()]; JSONArray jsonarray = (JSONArray)(new JSONParser().parse(strMD)); - Iterator it = jsonarray.iterator(); + Iterator it = jsonarray.iterator(); int i=0; while (it.hasNext()) { this.metadata[i] = new HashMap>(); HashMap storletsMetadata = new HashMap(); HashMap storageMetadata = new HashMap(); - JSONObject jsonobject = (JSONObject)it.next(); + JSONObject jsonobject = it.next(); if (jsonobject.containsKey("storage")) { populateMetadata(storageMetadata, (JSONObject)jsonobject.get("storage")); } diff --git a/storlets/agent/common/server.py b/storlets/agent/common/server.py index 9ac6545d..d8d8fe6d 100644 --- a/storlets/agent/common/server.py +++ b/storlets/agent/common/server.py @@ -108,7 +108,7 @@ class SBusServer(object): handler = self.get_handler(command) resp = handler(dtg) except ValueError as err: - self.logger.error(err.message) + self.logger.exception('Failed to handle request') resp = CommandFailure(str(err)) except CommandResponse as err: resp = err @@ -140,7 +140,7 @@ class SBusServer(object): :param resp: CommandResponse instance """ try: - outfile.write(resp.report_message) + outfile.write(resp.report_message.encode('utf-8')) except IOError: self.logger.exception('Unable to return response to client') diff --git a/storlets/agent/common/utils.py b/storlets/agent/common/utils.py index cd4b4241..3a0df4d7 100644 --- a/storlets/agent/common/utils.py +++ b/storlets/agent/common/utils.py @@ -15,8 +15,9 @@ import logging from logging.handlers import SysLogHandler +# TODO(takashi): Make the following parameters configurable DEFAULT_PY2 = 2.7 -DEFAULT_PY3 = 3.5 +DEFAULT_PY3 = 3.6 def get_logger(logger_name, log_level, container_id): diff --git a/storlets/agent/daemon/server.py b/storlets/agent/daemon/server.py index 7dfc0e08..f3b6e65f 100644 --- a/storlets/agent/daemon/server.py +++ b/storlets/agent/daemon/server.py @@ -14,12 +14,12 @@ # limitations under the License. import argparse import errno +import importlib import os import pwd +import signal import sys import uuid -import signal -import importlib from storlets.sbus import SBus from storlets.agent.common.server import command_handler, EXIT_FAILURE, \ CommandSuccess, CommandFailure, SBusServer @@ -254,20 +254,22 @@ def main(): # Initialize logger logger = get_logger("storlets-daemon", opts.log_level, opts.container_id) logger.debug("Storlet Daemon started") - SBus.start_logger("DEBUG", container_id=opts.container_id) - # Impersonate the swift user - pw = pwd.getpwnam('swift') - os.setresgid(pw.pw_gid, pw.pw_gid, pw.pw_gid) - os.setresuid(pw.pw_uid, pw.pw_uid, pw.pw_uid) - - # create an instance of storlet daemon try: + SBus.start_logger("DEBUG", container_id=opts.container_id) + + # Impersonate the swift user + pw = pwd.getpwnam('swift') + os.setresgid(pw.pw_gid, pw.pw_gid, pw.pw_gid) + os.setresuid(pw.pw_uid, pw.pw_uid, pw.pw_uid) + + # create an instance of storlet daemon daemon = StorletDaemon(opts.storlet_name, opts.sbus_path, logger, opts.pool_size) - except Exception as err: - logger.error(err.message) - return EXIT_FAILURE - # Start the main loop - return daemon.main_loop() + # Start the main loop + sys.exit(daemon.main_loop()) + + except Exception: + logger.error('Unhandled exception') + sys.exit(EXIT_FAILURE) diff --git a/storlets/agent/daemon_factory/server.py b/storlets/agent/daemon_factory/server.py index 18558f6f..b696fb8b 100644 --- a/storlets/agent/daemon_factory/server.py +++ b/storlets/agent/daemon_factory/server.py @@ -18,14 +18,15 @@ import os import pwd import signal import subprocess +import sys import time from storlets.sbus import SBus from storlets.sbus.client import SBusClient from storlets.sbus.client.exceptions import SBusClientException, \ SBusClientSendError -from storlets.agent.common.server import command_handler, CommandSuccess, \ - CommandFailure, SBusServer +from storlets.agent.common.server import command_handler, EXIT_FAILURE, \ + CommandSuccess, CommandFailure, SBusServer from storlets.agent.common.utils import get_logger, DEFAULT_PY2, DEFAULT_PY3 @@ -107,6 +108,8 @@ class StorletDaemonFactory(SBusServer): if int(float(daemon_language_version)) == 3: daemon_language_version = DEFAULT_PY3 else: + # TODO(takashi): Switch default python version to 3 once we drop + # python2 support. daemon_language_version = DEFAULT_PY2 python_interpreter = '/usr/bin/python%s' % daemon_language_version @@ -521,15 +524,22 @@ def main(): # Initialize logger logger = get_logger("daemon-factory", opts.log_level, opts.container_id) logger.debug("Daemon factory started") - SBus.start_logger("DEBUG", container_id=opts.container_id) - # Impersonate the swift user - pw = pwd.getpwnam('swift') - os.setresgid(pw.pw_gid, pw.pw_gid, pw.pw_gid) - os.setresuid(pw.pw_uid, pw.pw_uid, pw.pw_uid) + try: + SBus.start_logger("DEBUG", container_id=opts.container_id) - # create an instance of daemon_factory - factory = StorletDaemonFactory(opts.sbus_path, logger, opts.container_id) + # Impersonate the swift user + pw = pwd.getpwnam('swift') + os.setresgid(pw.pw_gid, pw.pw_gid, pw.pw_gid) + os.setresuid(pw.pw_uid, pw.pw_uid, pw.pw_uid) - # Start the main loop - return factory.main_loop() + # create an instance of daemon_factory + factory = StorletDaemonFactory(opts.sbus_path, logger, + opts.container_id) + + # Start the main loop + sys.exit(factory.main_loop()) + + except Exception: + logger.eception('Unhandled exception') + sys.exit(EXIT_FAILURE) diff --git a/storlets/gateway/gateways/docker/runtime.py b/storlets/gateway/gateways/docker/runtime.py index 0296cdb7..3f8770e7 100644 --- a/storlets/gateway/gateways/docker/runtime.py +++ b/storlets/gateway/gateways/docker/runtime.py @@ -232,7 +232,8 @@ class RunTimeSandbox(object): self.logger = logger self.default_docker_image_name = \ - conf.get('default_docker_image_name', 'ubuntu_18.04_jre8_storlets') + conf.get('default_docker_image_name', + 'ubuntu_18.04_jre11_storlets') def ping(self): """ diff --git a/storlets/sbus/client/cli.py b/storlets/sbus/client/cli.py index cff7d011..f298a18d 100644 --- a/storlets/sbus/client/cli.py +++ b/storlets/sbus/client/cli.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -from sys import exit +import sys from storlets.sbus.client import SBusClient from storlets.sbus.client.exceptions import SBusClientException @@ -21,19 +21,20 @@ EXIT_SUCCESS = 0 EXIT_ERROR = 1 -def main(argv): - # TODO(takashi): Add more detailed help message +def main(): + argv = sys.argv + # TODO(takashi): Add more detailed help message if len(argv) < 3: print('sbus ') - exit(EXIT_ERROR) + sys.exit(EXIT_ERROR) command = argv[1] pipe_path = argv[2] if not os.path.exists(pipe_path): print('ERROR: Pipe file %s does not exist' % pipe_path) - exit(EXIT_ERROR) + sys.exit(EXIT_ERROR) client = SBusClient(pipe_path) try: @@ -45,19 +46,19 @@ def main(argv): resp = handler() except (AttributeError, NotImplementedError): print('ERROR: Command %s is not supported' % command) - exit(EXIT_ERROR) + sys.exit(EXIT_ERROR) except SBusClientException as err: print('ERROR: Failed to send sbus command %s to %s: %s' % (command, pipe_path, err)) - exit(EXIT_ERROR) + sys.exit(EXIT_ERROR) except Exception as err: print('ERROR: Unknown error: %s' % err) - exit(EXIT_ERROR) + sys.exit(EXIT_ERROR) print('Response: %s: %s' % (resp.status, resp.message)) if resp.status: print('OK') - exit(EXIT_SUCCESS) + sys.exit(EXIT_SUCCESS) else: print('ERROR: Got error response') - exit(EXIT_ERROR) + sys.exit(EXIT_ERROR) diff --git a/storlets/sbus/sbus.py b/storlets/sbus/sbus.py index 4e71a0a0..e6665bcf 100644 --- a/storlets/sbus/sbus.py +++ b/storlets/sbus/sbus.py @@ -62,7 +62,8 @@ class SBus(object): sbus_back_ = CDLL(SBus.SBUS_SO_NAME) sbus_back_.sbus_start_logger.argtypes = [c_char_p, c_char_p] - sbus_back_.sbus_start_logger(str_log_level, container_id) + sbus_back_.sbus_start_logger(str_log_level.encode("utf-8"), + container_id.encode("utf-8")) @staticmethod def stop_logger(): diff --git a/storlets/swift_middleware/handlers/proxy.py b/storlets/swift_middleware/handlers/proxy.py index ee01105b..5741c6c4 100644 --- a/storlets/swift_middleware/handlers/proxy.py +++ b/storlets/swift_middleware/handlers/proxy.py @@ -195,9 +195,9 @@ class StorletProxyHandler(StorletBaseHandler): acl_string = '.r:%s' % self._build_acl_string(user_name, storlet_name) try: clean_acl('X-Container-Read', acl_string) - except ValueError as e: + except ValueError as err: msg = ('storlet ACL update request has invalid values %s' - % e.message) + % str(err)) raise HTTPBadRequest(msg.encode('utf8')) # Make sure the resulting acl permits a single entity diff --git a/tests/functional/java/test_thumbnail_storlet.py b/tests/functional/java/test_thumbnail_storlet.py index b2aa8a9b..fed96a77 100644 --- a/tests/functional/java/test_thumbnail_storlet.py +++ b/tests/functional/java/test_thumbnail_storlet.py @@ -61,7 +61,7 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest): headers = c.head_object(self.url, self.token, self.container, 'gen_thumb_on_put.jpg') - self.assertEqual('49032', headers['content-length']) + self.assertLess(int(headers['content-length']), 1087318) self.assertEqual('thumbnail', headers['x-object-meta-name']) def invoke_storlet_on_copy_from(self): @@ -88,7 +88,7 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest): headers = c.head_object(self.url, self.token, self.container, 'gen_thumb_on_copy.jpg') - self.assertEqual('49032', headers['content-length']) + self.assertLess(int(headers['content-length']), 1087318) self.assertEqual('thumbnail', headers['x-object-meta-name']) self.assertTrue('x-object-meta-x-timestamp' not in headers) self.assertTrue('x-timestamp' in headers) @@ -109,7 +109,7 @@ class TestThumbnailStorlet(StorletJavaFunctionalTest): headers = c.head_object(self.url, self.token, self.container, 'gen_thumb_on_copy_.jpg') - self.assertEqual('49032', headers['content-length']) + self.assertLess(int(headers['content-length']), 1087318) self.assertEqual('thumbnail', headers['x-object-meta-name']) self.assertTrue('x-object-meta-x-timestamp' not in headers) self.assertTrue('x-timestamp' in headers)